teradataml 17.20.0.6__py3-none-any.whl → 20.0.0.0__py3-none-any.whl

teradataml/data/scripts/sklearn/sklearn_fit.py

@@ -0,0 +1,175 @@
+import sys
+import numpy as np
+import pickle
+import base64
+
+DELIMITER = '\t'
+
+def get_value(value):
+    ret_val = value
+    try:
+        ret_val = float(value.replace(' ', ''))
+    except Exception as ex:
+        # If the value can't be converted to float, then it is string.
+        pass
+    return ret_val
+
+
+def get_values_list(values, ignore_none=True):
+    ret_vals = []
+    for val in values:
+        if val == "" and ignore_none:
+            # Empty cell value in the database table.
+            continue
+        ret_vals.append(get_value(val))
+
+    return ret_vals
+
+def convert_to_type(val, typee):
+    if typee == 'int':
+        return int(val)
+    if typee == 'float':
+        val = get_value(val)
+        return float(val)
+    if typee == 'bool':
+        return bool(val)
+    return str(val)
+
+def get_classes_as_list(classes, actual_type):
+    if classes == "None":
+        return None
+    if actual_type == "None":
+        sys.exit("type of class elements is None where class elements exists.")
+
+    # separated by '--'
+    classes = classes.split("--")
+
+    for idx, cls in enumerate(classes):
+        classes[idx] = convert_to_type(cls, actual_type)
+    
+    return classes
+
+
+def splitter(strr, delim=",", convert_to="str"):
+    """
+    Split the string based on delimiter and convert to the type specified.
+    """
+    if strr == "None":
+        return []
+    return [convert_to_type(i, convert_to) for i in strr.split(delim)]
+
+# Arguments to the Script
+if len(sys.argv) != 10:
+    # 10 arguments command line arguments should be passed to this file.
+    # 1: file to be run
+    # 2. function name
+    # 3. No of feature columns.
+    # 4. No of class labels.
+    # 5. Comma separated indices of partition columns.
+    # 6. Comma separated types of the partition columns.
+    # 7. Model file prefix to generated model file using partition columns.
+    # 8. classes (separated by '--') - should be converted to list. "None" if no classes exists.
+    # 9. type of elements in passed in classes. "None" if no classes exists.
+    # 10. Flag to check the system type. True, means Lake, Enterprise otherwise
+    sys.exit("10 arguments command line arguments should be passed: file to be run,"
+             " function name, no of feature columns, no of class labels, comma separated indices"
+             " and types of partition columns, model file prefix ,"
+             " classes, type of elements in classes and flag to check lake or enterprise.")
+
+is_lake_system = eval(sys.argv[9])
+if not is_lake_system:
+    db = sys.argv[0].split("/")[1]
+function_name = sys.argv[1]
+n_f_cols = int(sys.argv[2])
+n_c_labels = int(sys.argv[3])
+data_partition_column_types = splitter(sys.argv[5])
+data_partition_column_indices = splitter(sys.argv[4], convert_to="int") # indices are integers.
+model_file_prefix = sys.argv[6]
+class_type = sys.argv[8]
+classes = get_classes_as_list(sys.argv[7], class_type)
+
+model = None
+
+# Data Format (n_features, k_labels, one data_partition_column):
+# feature1, feature2, ..., featuren, label1, label2, ... labelk, data_partition_column1, ..., 
+# data_partition_columnn
+# There can be no labels also.
+
+# Read data from table through STO and build features and labels.
+features = []
+labels = []
+data_partition_column_values = []
+
+
+while 1:
+    try:
+        line = input()
+        if line == '':  # Exit if user provides blank line
+            break
+        else:
+            values = line.split(DELIMITER)
+            features.append(get_values_list(values[:n_f_cols]))
+            if n_c_labels > 0:
+                labels.append(get_values_list(values[n_f_cols:(n_f_cols+n_c_labels)]))
+            if not data_partition_column_values:
+                # Partition column values is same for all rows. Hence, only read once.
+                for i, val in enumerate(data_partition_column_indices):
+                    data_partition_column_values.append(
+                        convert_to_type(values[val], typee=data_partition_column_types[i])
+                        )
+
+                # Prepare the corresponding model file name and extract model.
+                partition_join = "_".join([str(x) for x in data_partition_column_values])
+                # Replace '-' with '_' as '-' because partition_columns can be negative.
+                partition_join = partition_join.replace("-", "_")
+
+                model_file_path = f"{model_file_prefix}_{partition_join}"\
+                    if is_lake_system else \
+                    f"./{db}/{model_file_prefix}_{partition_join}"
+
+                with open(model_file_path, "rb") as fp:
+                    model = pickle.loads(fp.read())
+
+                if model is None:
+                    sys.exit("Model file is not installed in Vantage.")
+
+    except EOFError:  # Exit if reached EOF or CTRL-D
+        break
+
+if not len(features):
+    sys.exit(0)
+
+# Fit/partial_fit the model to the data.
+if function_name == "partial_fit":
+    if labels and classes:
+        model.partial_fit(np.array(features), np.array(labels), classes=classes)
+    elif labels:
+        model.partial_fit(np.array(features), np.array(labels))
+    elif classes:
+        model.partial_fit(np.array(features), classes=classes)
+    else:
+        model.partial_fit(np.array(features))
+elif function_name == "fit":
+    # For IsotonicRegression, fit() accepts training target as
+    # y: array-like of shape (n_samples,).
+    if labels:
+        labels = np.array(labels).reshape(-1) \
+            if model.__class__.__name__ == "IsotonicRegression" else np.array(labels)
+        model.fit(np.array(features), labels)
+    else:
+        model.fit(np.array(features))
+
+model_str = pickle.dumps(model)
+
+if is_lake_system:
+    model_file_path = f"/tmp/{model_file_prefix}_{partition_join}.pickle"
+
+# Write to file in Vantage, to be used in predict/scoring.
+with open(model_file_path, "wb") as fp:
+    fp.write(model_str)
+
+model_data = model_file_path if is_lake_system \
+    else base64.b64encode(model_str)
+
+# Print the model to be read from script.
+print(*(data_partition_column_values + [model_data]), sep=DELIMITER)

teradataml/data/scripts/sklearn/sklearn_fit_predict.py

@@ -0,0 +1,135 @@
+import sys
+import numpy as np
+import pickle
+import math
+
+DELIMITER = '\t'
+
+def get_value(value):
+    ret_val = value
+    try:
+        ret_val = float(value.replace(' ', ''))
+    except Exception as ex:
+        # If the value can't be converted to float, then it is string.
+        pass
+    return ret_val
+
+def get_values_list(values, ignore_none=True):
+    ret_vals = []
+    for val in values:
+        if val == "" and ignore_none:
+            # Empty cell value in the database table.
+            continue
+        ret_vals.append(get_value(val))
+
+    return ret_vals
+
+def convert_to_type(val, typee):
+    if typee == 'int':
+        return int(val)
+    if typee == 'float':
+        return float(val)
+    if typee == 'bool':
+        return bool(val)
+    return str(val)
+
+def splitter(strr, delim=",", convert_to="str"):
+    """
+    Split the string based on delimiter and convert to the type specified.
+    """
+    if strr == "None":
+        return []
+    return [convert_to_type(i, convert_to) for i in strr.split(delim)]
+
+# Arguments to the Script
+if len(sys.argv) != 7:
+    # 6 arguments command line arguments should be passed to this file.
+    # 1: file to be run
+    # 2. No of feature columns.
+    # 3. No of class labels.
+    # 4. Comma separated indices of partition columns.
+    # 5. Comma separated types of the partition columns.
+    # 6. Model file prefix to generated model file using partition columns.
+    # 7. Flag to check the system type. True, means Lake, Enterprise otherwise.
+    sys.exit("7 arguments should be passed to this file - file to be run, "\
+             "no of feature columns, no of class labels, comma separated indices and types of "\
+             "partition columns, model file prefix to generate model file using partition "\
+             "columns and flag to check lake or enterprise.")
+
+is_lake_system = eval(sys.argv[6])
+if not is_lake_system:
+    db = sys.argv[0].split("/")[1]
+n_f_cols = int(sys.argv[1])
+n_c_labels = int(sys.argv[2])
+model_file_prefix = sys.argv[5]
+data_partition_column_types = splitter(sys.argv[4])
+data_partition_column_indices = splitter(sys.argv[3], convert_to="int") # indices are integers.
+
+model = None
+
+# Data Format (n_features, k_labels, one data_partition_columns):
+# feature1, feature2, ..., featuren, label1, label2, ... labelk, data_partition_column1, ..., 
+# data_partition_columnn.
+# There can be no labels also.
+
+# Read data from table through STO and build features and labels.
+features = []
+labels = []
+data_partition_column_values = []
+
+
+while 1:
+    try:
+        line = input()
+        if line == '':  # Exit if user provides blank line
+            break
+        else:
+            values = line.split(DELIMITER)
+            features.append(get_values_list(values[:n_f_cols]))
+            if n_c_labels > 0:
+                labels.append(get_values_list(values[n_f_cols:(n_f_cols+n_c_labels)]))
+            if not data_partition_column_values:
+                # Partition column values is same for all rows. Hence, only read once.
+                for i, val in enumerate(data_partition_column_indices):
+                    data_partition_column_values.append(
+                        convert_to_type(values[val], typee=data_partition_column_types[i])
+                        )
+
+                # Prepare the corresponding model file name and extract model.
+                partition_join = "_".join([str(x) for x in data_partition_column_values])
+                # Replace '-' with '_' as '-' because partition_columns can be negative.
+                partition_join = partition_join.replace("-", "_")
+
+                model_file_path = f"{model_file_prefix}_{partition_join}" \
+                    if is_lake_system else \
+                    f"./{db}/{model_file_prefix}_{partition_join}"
+
+                with open(model_file_path, "rb") as fp:
+                    model = pickle.loads(fp.read())
+
+                if model is None:
+                    sys.exit("Model file is not installed in Vantage.")
+
+    except EOFError:  # Exit if reached EOF or CTRL-D
+        break
+
+if not len(features):
+    sys.exit(0)
+
+# write code to call fit_predict with features and labels when n_c_labels > 0
+if n_c_labels > 0:
+    predictions = model.fit_predict(np.array(features), np.array(labels))
+else:
+    predictions = model.fit_predict(np.array(features))
+
+# Export results to to the Databse through standard output
+for i in range(len(predictions)):
+    if n_c_labels > 0:
+        # Add labels into output, if user passes it.
+        result_list = features[i] + labels[i] + [predictions[i]]
+    else:
+        result_list = features[i] + [predictions[i]]
+    print(*(data_partition_column_values +
+            ['' if (val is None or math.isnan(val) or math.isinf(val))
+             else val for val in result_list]),
+             sep= DELIMITER)

teradataml/data/scripts/sklearn/sklearn_function.template

@@ -0,0 +1,113 @@
+import sys, json
+import pickle, base64, importlib, numpy as np
+from collections import OrderedDict
+
+func_name = "<func_name>"
+module_name = "<module_name>"
+params = json.loads('<params>')
+
+DELIMITER = '\t'
+
+def get_value(value):
+    ret_val = value
+    try:
+        ret_val = float(value.replace(' ', ''))
+    except Exception as ex:
+        # If the value can't be converted to float, then it is string.
+        pass
+    return ret_val
+
+def convert_to_type(val, typee):
+    if typee == 'int':
+        return int(val)
+    if typee == 'float':
+        return get_value(val)
+    if typee == 'bool':
+        return bool(val)
+    return str(val)
+
+def splitter(strr, delim=",", convert_to="str"):
+    """
+    Split the string based on delimiter and convert to the type specified.
+    """
+    if strr == "None":
+        return []
+    return [convert_to_type(i, convert_to) for i in strr.split(delim)]
+
+# Arguments to the Script.
+if len(sys.argv) != 4:
+    # 4 arguments command line arguments should be passed to this file.
+    # 1: file to be run
+    # 2. Comma separated indices of partition columns.
+    # 3. Comma separated types of the partition columns.
+    # 4. Data columns information separted by "--" where each data column information is in the form
+    #    "<arg_name>-<comma separated data indices>-<comma separated data types>".
+    sys.exit("4 arguments command line arguments should be passed: file to be run,"
+             " comma separated indices and types of partition columns, data columns information"
+             " separated by '--' where each data column information is in the form"
+             " '<arg_name>-<comma separated data indices>-<comma separated data types>'.")
+
+db = sys.argv[0].split("/")[1]
+data_partition_column_indices = splitter(sys.argv[1], convert_to="int") # indices are integers.
+data_partition_column_types = splitter(sys.argv[2])
+
+# Data related arguments information of indices and types.
+data_args_indices_types = OrderedDict()
+
+# Data related arguments values - prepare dictionary and populate data later.
+data_args_values = {}
+
+for data_arg in sys.argv[3].split("--"):
+    arg_name, indices, types = data_arg.split("-")
+    indices = splitter(indices, convert_to="int")
+    types = splitter(types)
+
+    data_args_indices_types[arg_name] = {"indices": indices, "types": types}
+    data_args_values[arg_name] = [] # Keeping empty for each data arg name and populate data later.
+
+data_partition_column_values = []
+data_present = False
+
+# Read data - columns information is passed as command line argument and stored in
+# data_args_indices_types dictionary. 
+while 1:
+    try:
+        line = input()
+        if line == '':  # Exit if user provides blank line
+            break
+        else:
+            data_present = True
+            values = line.split(DELIMITER)
+            if not data_partition_column_values:
+                # Partition column values is same for all rows. Hence, only read once.
+                for i, val in enumerate(data_partition_column_indices):
+                    data_partition_column_values.append(
+                        convert_to_type(values[val], typee=data_partition_column_types[i])
+                        )
+            
+            # Prepare data dictionary containing only arguments related to data.
+            for arg_name in data_args_values:
+                data_indices = data_args_indices_types[arg_name]["indices"]
+                types = data_args_indices_types[arg_name]["types"]
+                cur_row = []
+                for idx, data_idx in enumerate(data_indices):
+                    cur_row.append(convert_to_type(values[data_idx], types[idx]))
+                data_args_values[arg_name].append(cur_row)
+    except EOFError:  # Exit if reached EOF or CTRL-D
+        break
+
+if not data_present:
+    sys.exit(0)
+
+# Update data as numpy arrays.
+for arg_name in data_args_values:
+    np_values = np.array(data_args_values[arg_name])
+    data_args_values[arg_name] = np_values
+
+# Combine all arguments.
+all_args = {**data_args_values, **params}
+
+module_ = importlib.import_module(module_name)
+sklearn_model = getattr(module_, func_name)(**all_args)
+
+print(*(data_partition_column_values + [base64.b64encode(pickle.dumps(sklearn_model))]), sep=DELIMITER)

teradataml/data/scripts/sklearn/sklearn_model_selection_split.py

@@ -0,0 +1,158 @@
+import pickle
+import math
+import sys
+import numpy as np
+import base64
+
+DELIMITER = '\t'
+
+
+def get_value(value):
+    ret_val = value
+    try:
+        ret_val = round(float("".join(value.split())), 2)
+    except Exception as ex:
+        # If the value can't be converted to float, then it is string.
+        pass
+    return ret_val
+
+
+def get_values_list(values, ignore_none=True):
+    ret_vals = []
+    for val in values:
+        if val == "" and ignore_none:
+            # Empty cell value in the database table.
+            continue
+        ret_vals.append(get_value(val))
+
+    return ret_vals
+
+def convert_to_type(val, typee):
+    if typee == 'int':
+        return int(val)
+    if typee == 'float':
+        return float(val)
+    if typee == 'bool':
+        return eval(val)
+    return str(val)
+
+def splitter(strr, delim=",", convert_to="str"):
+    """
+    Split the string based on delimiter and convert to the type specified.
+    """
+    if strr == "None":
+        return []
+    return [convert_to_type(i, convert_to) for i in strr.split(delim)]
+
+
+# Arguments to the Script
+if len(sys.argv) != 9:
+    # 9 arguments command line arguments should be passed to this file.
+    # 1: file to be run
+    # 2. function name
+    # 3. No of feature columns.
+    # 4. No of class labels.
+    # 5. No of group columns.
+    # 6. Comma separated indices of partition columns.
+    # 7. Comma separated types of the partition columns.
+    # 8. Model file prefix to generated model file using partition columns.
+    # 9. Flag to check the system type. True, means Lake, Enterprise otherwise.
+    sys.exit("9 arguments command line arguments should be passed: file to be run,"
+             " function name, no of feature columns, no of class labels, no of group columns,"
+             " comma separated indices and types of partition columns, model file prefix to"
+             " generated model file using partition columns and flag to check lake or enterprise.")
+
+
+is_lake_system = eval(sys.argv[8])
+if not is_lake_system:
+    db = sys.argv[0].split("/")[1]
+function_name = sys.argv[1]
+n_f_cols = int(sys.argv[2])
+n_c_labels = int(sys.argv[3])
+n_g_cols = int(sys.argv[4])
+data_partition_column_types = splitter(sys.argv[6])
+data_partition_column_indices = splitter(sys.argv[5], convert_to="int") # indices are integers.
+model_file_prefix = sys.argv[7]
+
+
+model = None
+data_partition_column_values = []
+
+# Data Format (n_features, k_labels, one data_partition_column):
+# feature1, feature2, ..., featuren, label1, label2, ... labelk, data_partition_column1, ..., 
+# data_partition_columnn.
+# labels are optional.
+
+features = []
+labels = []
+groups = []
+while 1:
+    try:
+        line = input()
+        if line == '':  # Exit if user provides blank line
+            break
+        else:
+            values = line.split(DELIMITER)
+            if not data_partition_column_values:
+                # Partition column values is same for all rows. Hence, only read once.
+                for i, val in enumerate(data_partition_column_indices):
+                    data_partition_column_values.append(
+                        convert_to_type(values[val], typee=data_partition_column_types[i])
+                        )
+
+                # Prepare the corresponding model file name and extract model.
+                partition_join = "_".join([str(x) for x in data_partition_column_values])
+                # Replace '-' with '_' as '-' because partition_columns can be negative.
+                partition_join = partition_join.replace("-", "_")
+
+                model_file_path = f"{model_file_prefix}_{partition_join}" \
+                    if is_lake_system else \
+                    f"./{db}/{model_file_prefix}_{partition_join}"
+
+                with open(model_file_path, "rb") as fp:
+                    model = pickle.loads(fp.read())
+
+                if not model:
+                    sys.exit("Model file is not installed in Vantage.")
+
+            start = 0
+            if n_f_cols > 0:
+                features.append(get_values_list(values[:n_f_cols]))
+                start = start + n_f_cols
+            if n_c_labels > 0:
+                labels.append(get_values_list(values[start:(start+n_c_labels)]))
+                start = start + n_c_labels
+            if n_g_cols > 0:
+                groups.append(get_values_list(values[start:(start+n_g_cols)]))
+
+    except EOFError:  # Exit if reached EOF or CTRL-D
+        break
+
+if len(features) == 0:
+    sys.exit(0)
+
+features = np.array(features) if len(features) > 0 else None
+labels = np.array(labels).flatten() if len(labels) > 0 else None
+groups = np.array(groups).flatten() if len(groups) > 0 else None
+
+if function_name == "split":
+    # Printing both train and test data instead of just indices unlike sklearn.
+    # Generator is created based on split_id and type of split (train/test) in client.
+    split_id = 1
+    for train_idx, test_idx in model.split(features, labels, groups):
+        X_train, X_test = features[train_idx], features[test_idx]
+        y_train, y_test = labels[train_idx], labels[test_idx]
+        for X, y in zip(X_train, y_train):
+            print(*(data_partition_column_values + [split_id, "train"] +
+                    ['' if (val is None or math.isnan(val) or math.isinf(val)) else val 
+                     for val in X] + [y]
+                    ),sep=DELIMITER)
+        for X, y in zip(X_test, y_test):
+            print(*(data_partition_column_values + [split_id, "test"] +
+                    ['' if (val is None or math.isnan(val) or math.isinf(val)) else val 
+                     for val in X] + [y]
+                    ),sep=DELIMITER)
+        split_id += 1
+else:
+    val = getattr(model, function_name)(features, labels, groups)
+    print(*(data_partition_column_values + [val]), sep=DELIMITER)