teradataml 17.20.0.7__py3-none-any.whl → 20.0.0.1__py3-none-any.whl

teradataml/data/scripts/deploy_script.py

@@ -0,0 +1,84 @@
+import sys
+import numpy as np
+from sklearn.pipeline import make_pipeline
+from sklearn.preprocessing import StandardScaler
+from sklearn.svm import SVC
+import base64, pickle
+
+DELIMITER = "\t"
+
+def get_value(value):
+    ret_val = value
+    try:
+        ret_val = float(value.replace(' ', ''))
+    except Exception as ex:
+        # If the value can't be converted to float, then it is string.
+        pass
+    return ret_val
+
+
+def get_values_list(values, ignore_none=True):
+    ret_vals = []
+    for val in values:
+        if val == "" and ignore_none:
+            # Empty cell value in the database table.
+            continue
+        ret_vals.append(get_value(val))
+
+    return ret_vals
+
+if len(sys.argv) != 2:
+    sys.exit("Script command format: python deploy_script.py <enterprise/lake>")
+
+vantage_type = sys.argv[1]
+
+data_partition_column_values = []
+data_partition_column_indices = [5, 6]
+
+features = []
+labels = []
+
+while 1:
+    try:
+        line = input()
+        if line == '':  # Exit if user provides blank line
+            break
+        else:
+            values = line.split(DELIMITER)
+            
+            features.append(get_values_list(values[:4]))
+            labels.append(get_values_list([values[4]]))
+            if not data_partition_column_values:
+                # Partition column values is same for all rows. Hence, only read once.
+                for i, val in enumerate(data_partition_column_indices):
+                    data_partition_column_values.append(int(values[val]))
+
+    except EOFError:  # Exit if reached EOF or CTRL-D
+        break
+
+if not len(features):
+    sys.exit(0)
+
+X = np.array(features)
+y = np.array(labels)
+
+clf = make_pipeline(StandardScaler(), SVC(gamma='auto'))
+clf.fit(X, y)
+
+model_str = pickle.dumps(clf)
+
+# Prepare the corresponding model file name and extract model.
+partition_join = "_".join([str(x) for x in data_partition_column_values])
+# Replace '-' with '_' as '-' because partition_columns can be negative.
+partition_join = partition_join.replace("-", "_")
+
+if vantage_type == "lake":
+    model = f"/tmp/sklearn_model_{partition_join}.pickle"
+    with open(model, "wb") as fp:
+        fp.write(model_str)
+elif vantage_type == "enterprise":
+    model = base64.b64encode(model_str)
+else:
+    sys.exit("Invalid vantage type. Use either 'lake' or 'enterprise'.")
+
+print(*(data_partition_column_values + [model]), sep=DELIMITER)

teradataml/data/scripts/mapper.R

@@ -0,0 +1,20 @@
+#!/usr/bin/Rscript
+
+# Read input from STDIN (standard input)
+input <- file("stdin", open = "r")
+while (length(line <- readLines(input, n = 1)) > 0) {
+  # Remove leading and trailing whitespace
+  line <- trimws(line)
+  # Split the line into words
+  words <- unlist(strsplit(line, "\\s+"))
+  # Increase counters
+  for (word in words) {
+    # Write the results to STDOUT (standard output);
+    # what we output here will be the input for the
+    # Reduce step, i.e., the input for reducer.py
+    #
+    # tab-delimited; the trivial word count is 1
+    cat(paste(word,"\t","1\n"))
+  }
+}
+close(input)

teradataml/data/scripts/mapper.py

@@ -1,16 +1,16 @@
-#!/usr/bin/python
-import sys
-# input comes from STDIN (standard input)
-for line in sys.stdin:
-    # remove leading and trailing whitespace
-    line = line.strip()
-    # split the line into words
-    words = line.split()
-    # increase counters
-    for word in words:
-        # write the results to STDOUT (standard output);
-        # what we output here will be the input for the
-        # Reduce step, i.e. the input for reducer.py
-        #
-        # tab-delimited; the trivial word count is 1
+#!/usr/bin/python
+import sys
+# input comes from STDIN (standard input)
+for line in sys.stdin:
+    # remove leading and trailing whitespace
+    line = line.strip()
+    # split the line into words
+    words = line.split()
+    # increase counters
+    for word in words:
+        # write the results to STDOUT (standard output);
+        # what we output here will be the input for the
+        # Reduce step, i.e. the input for reducer.py
+        #
+        # tab-delimited; the trivial word count is 1
         print ('%s\t%s' % (word, 1))

teradataml/data/scripts/mapper_replace.py

@@ -1,16 +1,16 @@
-#!/usr/bin/python
-import sys
-# input comes from STDIN (standard input)
-for line in sys.stdin:
-    # remove leading and trailing whitespace
-    line = line.strip()
-    # split the line into words
-    words = line.split()
-    # increase counters
-    for newword in words:
-        # write the results to STDOUT (standard output);
-        # what we output here will be the input for the
-        # Reduce step, i.e. the input for reducer.py
-        #
-        # tab-delimited; the trivial word count is 1
+#!/usr/bin/python
+import sys
+# input comes from STDIN (standard input)
+for line in sys.stdin:
+    # remove leading and trailing whitespace
+    line = line.strip()
+    # split the line into words
+    words = line.split()
+    # increase counters
+    for newword in words:
+        # write the results to STDOUT (standard output);
+        # what we output here will be the input for the
+        # Reduce step, i.e. the input for reducer.py
+        #
+        # tab-delimited; the trivial word count is 1
         print ('%s,%s' % (newword, 1))

teradataml/data/scripts/sklearn/sklearn_fit.py

@@ -0,0 +1,171 @@
+import sys
+import numpy as np
+import pickle
+import base64
+
+DELIMITER = '\t'
+
+def get_values_list(values, types, model_obj):
+    ret_vals = []
+    for i, val in enumerate(values):
+        if type(model_obj).__name__ == "MultiLabelBinarizer" and val == "":
+            continue
+        ret_vals.append(convert_to_type(val, types[i]))
+    return ret_vals
+
+def convert_to_type(val, typee):
+    if typee == 'int':
+        return int(val) if val != "" else np.nan
+    if typee == 'float':
+        if isinstance(val, str):
+            val = val.replace(' ', '')
+        return float(val) if val != "" else np.nan
+    if typee == 'bool':
+        return eval(val) if val != "" else None
+    return str(val) if val != "" else None
+
+def get_classes_as_list(classes, actual_type):
+    if classes == "None":
+        return None
+    if actual_type == "None":
+        sys.exit("type of class elements is None where class elements exists.")
+
+    # separated by '--'
+    classes = classes.split("--")
+
+    for idx, cls in enumerate(classes):
+        classes[idx] = convert_to_type(cls, actual_type)
+    
+    return classes
+
+
+def splitter(strr, delim=",", convert_to="str"):
+    """
+    Split the string based on delimiter and convert to the type specified.
+    """
+    if strr == "None":
+        return []
+    return [convert_to_type(i, convert_to) for i in strr.split(delim)]
+
+# Arguments to the Script
+if len(sys.argv) != 10:
+    # 10 arguments command line arguments should be passed to this file.
+    # 1: file to be run
+    # 2. function name
+    # 3. No of feature columns.
+    # 4. No of class labels.
+    # 5. Comma separated indices of partition columns.
+    # 6. Comma separated types of all the data columns.
+    # 7. Model file prefix to generated model file using partition columns.
+    # 8. classes (separated by '--') - should be converted to list. "None" if no classes exists.
+    # 9. type of elements in passed in classes. "None" if no classes exists.
+    # 10. Flag to check the system type. True, means Lake, Enterprise otherwise
+    sys.exit("10 arguments command line arguments should be passed: file to be run,"
+             " function name, no of feature columns, no of class labels, comma separated indices"
+             " of partition columns, comma separated types of all columns, model file prefix ,"
+             " classes, type of elements in classes and flag to check lake or enterprise.")
+
+is_lake_system = eval(sys.argv[9])
+if not is_lake_system:
+    db = sys.argv[0].split("/")[1]
+function_name = sys.argv[1]
+n_f_cols = int(sys.argv[2])
+n_c_labels = int(sys.argv[3])
+data_column_types = splitter(sys.argv[5], delim="--")
+data_partition_column_indices = splitter(sys.argv[4], convert_to="int") # indices are integers.
+model_file_prefix = sys.argv[6]
+class_type = sys.argv[8]
+classes = get_classes_as_list(sys.argv[7], class_type)
+
+data_partition_column_types = [data_column_types[idx] for idx in data_partition_column_indices]
+
+model = None
+
+# Data Format (n_features, k_labels, one data_partition_column):
+# feature1, feature2, ..., featuren, label1, label2, ... labelk, data_partition_column1, ..., 
+# data_partition_columnn
+# There can be no labels also.
+
+# Read data from table through STO and build features and labels.
+features = []
+labels = []
+data_partition_column_values = []
+
+
+while 1:
+    try:
+        line = input()
+        if line == '':  # Exit if user provides blank line
+            break
+        else:
+            values = line.split(DELIMITER)
+
+            if not data_partition_column_values:
+                # Partition column values is same for all rows. Hence, only read once.
+                for i, val in enumerate(data_partition_column_indices):
+                    data_partition_column_values.append(
+                        convert_to_type(values[val], typee=data_partition_column_types[i])
+                        )
+
+                # Prepare the corresponding model file name and extract model.
+                partition_join = "_".join([str(x) for x in data_partition_column_values])
+                # Replace '-' with '_' as '-' because partition_columns can be negative.
+                partition_join = partition_join.replace("-", "_")
+
+                model_file_path = f"{model_file_prefix}_{partition_join}"\
+                    if is_lake_system else \
+                    f"./{db}/{model_file_prefix}_{partition_join}"
+
+                with open(model_file_path, "rb") as fp:
+                    model = pickle.loads(fp.read())
+
+                if model is None:
+                    sys.exit("Model file is not installed in Vantage.")
+
+            values = get_values_list(values, data_column_types, model)
+            values = values[:-len(data_partition_column_indices)] # Already processed partition columns.
+            features.append(values[:n_f_cols])
+            if n_c_labels > 0:
+                labels.append(values[n_f_cols:(n_f_cols+n_c_labels)])
+
+
+    except EOFError:  # Exit if reached EOF or CTRL-D
+        break
+
+if not len(features):
+    sys.exit(0)
+
+# Fit/partial_fit the model to the data.
+if function_name == "partial_fit":
+    if labels and classes:
+        model.partial_fit(np.array(features), np.array(labels), classes=classes)
+    elif labels:
+        model.partial_fit(np.array(features), np.array(labels))
+    elif classes:
+        model.partial_fit(np.array(features), classes=classes)
+    else:
+        model.partial_fit(np.array(features))
+elif function_name == "fit":
+    # For IsotonicRegression, fit() accepts training target as
+    # y: array-like of shape (n_samples,).
+    if labels:
+        labels = np.array(labels).reshape(-1) \
+            if model.__class__.__name__ == "IsotonicRegression" else np.array(labels)
+        model.fit(np.array(features), labels)
+    else:
+        model.fit(np.array(features))
+
+model_str = pickle.dumps(model)
+
+if is_lake_system:
+    model_file_path = f"/tmp/{model_file_prefix}_{partition_join}.pickle"
+
+# Write to file in Vantage, to be used in predict/scoring.
+with open(model_file_path, "wb") as fp:
+    fp.write(model_str)
+
+model_data = model_file_path if is_lake_system \
+    else base64.b64encode(model_str)
+
+# Print the model to be read from script.
+print(*(data_partition_column_values + [model_data]), sep=DELIMITER)

teradataml/data/scripts/sklearn/sklearn_fit_predict.py

@@ -0,0 +1,127 @@
+import sys
+import numpy as np
+import pickle
+import math
+
+DELIMITER = '\t'
+
+def get_values_list(values, types):
+    ret_vals = []
+    for i, val in enumerate(values):
+        ret_vals.append(convert_to_type(val, types[i]))
+    return ret_vals
+
+def convert_to_type(val, typee):
+    if typee == 'int':
+        return int(val) if val != "" else np.nan
+    if typee == 'float':
+        if isinstance(val, str):
+            val = val.replace(' ', '')
+        return float(val) if val != "" else np.nan
+    if typee == 'bool':
+        return eval(val) if val != "" else None
+    return str(val) if val != "" else None
+
+def splitter(strr, delim=",", convert_to="str"):
+    """
+    Split the string based on delimiter and convert to the type specified.
+    """
+    if strr == "None":
+        return []
+    return [convert_to_type(i, convert_to) for i in strr.split(delim)]
+
+# Arguments to the Script
+if len(sys.argv) != 7:
+    # 6 arguments command line arguments should be passed to this file.
+    # 1: file to be run
+    # 2. No of feature columns.
+    # 3. No of class labels.
+    # 4. Comma separated indices of partition columns.
+    # 5. Comma separated types of all the data columns.
+    # 6. Model file prefix to generated model file using partition columns.
+    # 7. Flag to check the system type. True, means Lake, Enterprise otherwise.
+    sys.exit("7 arguments should be passed to this file - file to be run, "\
+             "no of feature columns, no of class labels, comma separated indices of partition "
+             "columns, comma separated types of all columns, model file prefix to generate model "
+             "file using partition columns and flag to check lake or enterprise.")
+
+is_lake_system = eval(sys.argv[6])
+if not is_lake_system:
+    db = sys.argv[0].split("/")[1]
+n_f_cols = int(sys.argv[1])
+n_c_labels = int(sys.argv[2])
+model_file_prefix = sys.argv[5]
+data_column_types = splitter(sys.argv[4], delim="--")
+data_partition_column_indices = splitter(sys.argv[3], convert_to="int") # indices are integers.
+
+data_partition_column_types = [data_column_types[idx] for idx in data_partition_column_indices]
+
+model = None
+
+# Data Format (n_features, k_labels, one data_partition_columns):
+# feature1, feature2, ..., featuren, label1, label2, ... labelk, data_partition_column1, ..., 
+# data_partition_columnn.
+# There can be no labels also.
+
+# Read data from table through STO and build features and labels.
+features = []
+labels = []
+data_partition_column_values = []
+
+
+while 1:
+    try:
+        line = input()
+        if line == '':  # Exit if user provides blank line
+            break
+        else:
+            values = line.split(DELIMITER)
+            values = get_values_list(values, data_column_types)
+            features.append(values[:n_f_cols])
+            if n_c_labels > 0:
+                labels.append(values[n_f_cols:(n_f_cols+n_c_labels)])
+            if not data_partition_column_values:
+                # Partition column values is same for all rows. Hence, only read once.
+                for i, val in enumerate(data_partition_column_indices):
+                    data_partition_column_values.append(
+                        convert_to_type(values[val], typee=data_partition_column_types[i])
+                        )
+
+                # Prepare the corresponding model file name and extract model.
+                partition_join = "_".join([str(x) for x in data_partition_column_values])
+                # Replace '-' with '_' as '-' because partition_columns can be negative.
+                partition_join = partition_join.replace("-", "_")
+
+                model_file_path = f"{model_file_prefix}_{partition_join}" \
+                    if is_lake_system else \
+                    f"./{db}/{model_file_prefix}_{partition_join}"
+
+                with open(model_file_path, "rb") as fp:
+                    model = pickle.loads(fp.read())
+
+                if model is None:
+                    sys.exit("Model file is not installed in Vantage.")
+
+    except EOFError:  # Exit if reached EOF or CTRL-D
+        break
+
+if not len(features):
+    sys.exit(0)
+
+# write code to call fit_predict with features and labels when n_c_labels > 0
+if n_c_labels > 0:
+    predictions = model.fit_predict(np.array(features), np.array(labels))
+else:
+    predictions = model.fit_predict(np.array(features))
+
+# Export results to to the Databse through standard output
+for i in range(len(predictions)):
+    if n_c_labels > 0:
+        # Add labels into output, if user passes it.
+        result_list = features[i] + labels[i] + [predictions[i]]
+    else:
+        result_list = features[i] + [predictions[i]]
+    print(*(data_partition_column_values +
+            ['' if (val is None or (not isinstance(val, str) and (math.isnan(val) or math.isinf(val))))
+             else val for val in result_list]),
+             sep= DELIMITER)

teradataml/data/scripts/sklearn/sklearn_function.template

@@ -0,0 +1,108 @@
+import sys, json
+import pickle, base64, importlib, numpy as np
+from collections import OrderedDict
+
+func_name = "<func_name>"
+module_name = "<module_name>"
+params = json.loads('<params>')
+
+DELIMITER = '\t'
+
+def convert_to_type(val, typee):
+    if typee == 'int':
+        return int(val) if val != "" else np.nan
+    if typee == 'float':
+        if isinstance(val, str):
+            val = val.replace(' ', '')
+        return float(val) if val != "" else np.nan
+    if typee == 'bool':
+        return eval(val) if val != "" else None
+    return str(val) if val != "" else None
+
+def splitter(strr, delim=",", convert_to="str"):
+    """
+    Split the string based on delimiter and convert to the type specified.
+    """
+    if strr == "None":
+        return []
+    return [convert_to_type(i, convert_to) for i in strr.split(delim)]
+
+# Arguments to the Script.
+if len(sys.argv) != 4:
+    # 4 arguments command line arguments should be passed to this file.
+    # 1: file to be run
+    # 2. Comma separated indices of partition columns.
+    # 3. Comma separated types of all the data columns.
+    # 4. Data columns information separted by "--" where each data column information is in the form
+    #    "<arg_name>-<comma separated data indices>-<comma separated data types>".
+    sys.exit("4 arguments command line arguments should be passed: file to be run,"
+             " comma separated indices of partition columns, comma separated types of all columns,"
+             " data columns information separated by '--' where each data column information is"
+             " in the form '<arg_name>-<comma separated data indices>-<comma separated data types>'.")
+
+db = sys.argv[0].split("/")[1]
+data_partition_column_indices = splitter(sys.argv[1], convert_to="int") # indices are integers.
+data_column_types = splitter(sys.argv[2], delim="--")
+
+data_partition_column_types = [data_column_types[idx] for idx in data_partition_column_indices]
+
+# Data related arguments information of indices and types.
+data_args_indices_types = OrderedDict()
+
+# Data related arguments values - prepare dictionary and populate data later.
+data_args_values = {}
+
+for data_arg in sys.argv[3].split("--"):
+    arg_name, indices, types = data_arg.split("-")
+    indices = splitter(indices, convert_to="int")
+    types = splitter(types)
+
+    data_args_indices_types[arg_name] = {"indices": indices, "types": types}
+    data_args_values[arg_name] = [] # Keeping empty for each data arg name and populate data later.
+
+data_partition_column_values = []
+data_present = False
+
+# Read data - columns information is passed as command line argument and stored in
+# data_args_indices_types dictionary. 
+while 1:
+    try:
+        line = input()
+        if line == '':  # Exit if user provides blank line
+            break
+        else:
+            data_present = True
+            values = line.split(DELIMITER)
+            if not data_partition_column_values:
+                # Partition column values is same for all rows. Hence, only read once.
+                for i, val in enumerate(data_partition_column_indices):
+                    data_partition_column_values.append(
+                        convert_to_type(values[val], typee=data_partition_column_types[i])
+                        )
+            
+            # Prepare data dictionary containing only arguments related to data.
+            for arg_name in data_args_values:
+                data_indices = data_args_indices_types[arg_name]["indices"]
+                types = data_args_indices_types[arg_name]["types"]
+                cur_row = []
+                for idx, data_idx in enumerate(data_indices):
+                    cur_row.append(convert_to_type(values[data_idx], types[idx]))
+                data_args_values[arg_name].append(cur_row)
+    except EOFError:  # Exit if reached EOF or CTRL-D
+        break
+
+if not data_present:
+    sys.exit(0)
+
+# Update data as numpy arrays.
+for arg_name in data_args_values:
+    np_values = np.array(data_args_values[arg_name])
+    data_args_values[arg_name] = np_values
+
+# Combine all arguments.
+all_args = {**data_args_values, **params}
+
+module_ = importlib.import_module(module_name)
+sklearn_model = getattr(module_, func_name)(**all_args)
+
+print(*(data_partition_column_values + [base64.b64encode(pickle.dumps(sklearn_model))]), sep=DELIMITER)