teradataml 20.0.0.0__py3-none-any.whl → 20.0.0.1__py3-none-any.whl

teradataml/data/scripts/sklearn/sklearn_fit.py

@@ -5,35 +5,24 @@ import base64
 
 DELIMITER = '\t'
 
-def get_value(value):
-    ret_val = value
-    try:
-        ret_val = float(value.replace(' ', ''))
-    except Exception as ex:
-        # If the value can't be converted to float, then it is string.
-        pass
-    return ret_val
-
-
-def get_values_list(values, ignore_none=True):
+def get_values_list(values, types, model_obj):
     ret_vals = []
-    for val in values:
-        if val == "" and ignore_none:
-            # Empty cell value in the database table.
+    for i, val in enumerate(values):
+        if type(model_obj).__name__ == "MultiLabelBinarizer" and val == "":
             continue
-        ret_vals.append(get_value(val))
-
+        ret_vals.append(convert_to_type(val, types[i]))
     return ret_vals
 
 def convert_to_type(val, typee):
     if typee == 'int':
-        return int(val)
+        return int(val) if val != "" else np.nan
     if typee == 'float':
-        val = get_value(val)
-        return float(val)
+        if isinstance(val, str):
+            val = val.replace(' ', '')
+        return float(val) if val != "" else np.nan
     if typee == 'bool':
-        return bool(val)
-    return str(val)
+        return eval(val) if val != "" else None
+    return str(val) if val != "" else None
 
 def get_classes_as_list(classes, actual_type):
     if classes == "None":
@@ -66,14 +55,14 @@ if len(sys.argv) != 10:
     # 3. No of feature columns.
     # 4. No of class labels.
     # 5. Comma separated indices of partition columns.
-    # 6. Comma separated types of the partition columns.
+    # 6. Comma separated types of all the data columns.
     # 7. Model file prefix to generated model file using partition columns.
     # 8. classes (separated by '--') - should be converted to list. "None" if no classes exists.
     # 9. type of elements in passed in classes. "None" if no classes exists.
     # 10. Flag to check the system type. True, means Lake, Enterprise otherwise
     sys.exit("10 arguments command line arguments should be passed: file to be run,"
              " function name, no of feature columns, no of class labels, comma separated indices"
-             " and types of partition columns, model file prefix ,"
+             " of partition columns, comma separated types of all columns, model file prefix ,"
              " classes, type of elements in classes and flag to check lake or enterprise.")
 
 is_lake_system = eval(sys.argv[9])
@@ -82,12 +71,14 @@ if not is_lake_system:
 function_name = sys.argv[1]
 n_f_cols = int(sys.argv[2])
 n_c_labels = int(sys.argv[3])
-data_partition_column_types = splitter(sys.argv[5])
+data_column_types = splitter(sys.argv[5], delim="--")
 data_partition_column_indices = splitter(sys.argv[4], convert_to="int") # indices are integers.
 model_file_prefix = sys.argv[6]
 class_type = sys.argv[8]
 classes = get_classes_as_list(sys.argv[7], class_type)
 
+data_partition_column_types = [data_column_types[idx] for idx in data_partition_column_indices]
+
 model = None
 
 # Data Format (n_features, k_labels, one data_partition_column):
@@ -108,9 +99,7 @@ while 1:
             break
         else:
             values = line.split(DELIMITER)
-            features.append(get_values_list(values[:n_f_cols]))
-            if n_c_labels > 0:
-                labels.append(get_values_list(values[n_f_cols:(n_f_cols+n_c_labels)]))
+
             if not data_partition_column_values:
                 # Partition column values is same for all rows. Hence, only read once.
                 for i, val in enumerate(data_partition_column_indices):
@@ -133,6 +122,13 @@ while 1:
                 if model is None:
                     sys.exit("Model file is not installed in Vantage.")
 
+            values = get_values_list(values, data_column_types, model)
+            values = values[:-len(data_partition_column_indices)] # Already processed partition columns.
+            features.append(values[:n_f_cols])
+            if n_c_labels > 0:
+                labels.append(values[n_f_cols:(n_f_cols+n_c_labels)])
+
+
     except EOFError:  # Exit if reached EOF or CTRL-D
         break
 

teradataml/data/scripts/sklearn/sklearn_fit_predict.py

@@ -5,33 +5,22 @@ import math
 
 DELIMITER = '\t'
 
-def get_value(value):
-    ret_val = value
-    try:
-        ret_val = float(value.replace(' ', ''))
-    except Exception as ex:
-        # If the value can't be converted to float, then it is string.
-        pass
-    return ret_val
-
-def get_values_list(values, ignore_none=True):
+def get_values_list(values, types):
     ret_vals = []
-    for val in values:
-        if val == "" and ignore_none:
-            # Empty cell value in the database table.
-            continue
-        ret_vals.append(get_value(val))
-
+    for i, val in enumerate(values):
+        ret_vals.append(convert_to_type(val, types[i]))
     return ret_vals
 
 def convert_to_type(val, typee):
     if typee == 'int':
-        return int(val)
+        return int(val) if val != "" else np.nan
     if typee == 'float':
-        return float(val)
+        if isinstance(val, str):
+            val = val.replace(' ', '')
+        return float(val) if val != "" else np.nan
     if typee == 'bool':
-        return bool(val)
-    return str(val)
+        return eval(val) if val != "" else None
+    return str(val) if val != "" else None
 
 def splitter(strr, delim=",", convert_to="str"):
     """
@@ -48,13 +37,13 @@ if len(sys.argv) != 7:
     # 2. No of feature columns.
     # 3. No of class labels.
     # 4. Comma separated indices of partition columns.
-    # 5. Comma separated types of the partition columns.
+    # 5. Comma separated types of all the data columns.
     # 6. Model file prefix to generated model file using partition columns.
     # 7. Flag to check the system type. True, means Lake, Enterprise otherwise.
     sys.exit("7 arguments should be passed to this file - file to be run, "\
-             "no of feature columns, no of class labels, comma separated indices and types of "\
-             "partition columns, model file prefix to generate model file using partition "\
-             "columns and flag to check lake or enterprise.")
+             "no of feature columns, no of class labels, comma separated indices of partition "
+             "columns, comma separated types of all columns, model file prefix to generate model "
+             "file using partition columns and flag to check lake or enterprise.")
 
 is_lake_system = eval(sys.argv[6])
 if not is_lake_system:
@@ -62,9 +51,11 @@ if not is_lake_system:
 n_f_cols = int(sys.argv[1])
 n_c_labels = int(sys.argv[2])
 model_file_prefix = sys.argv[5]
-data_partition_column_types = splitter(sys.argv[4])
+data_column_types = splitter(sys.argv[4], delim="--")
 data_partition_column_indices = splitter(sys.argv[3], convert_to="int") # indices are integers.
 
+data_partition_column_types = [data_column_types[idx] for idx in data_partition_column_indices]
+
 model = None
 
 # Data Format (n_features, k_labels, one data_partition_columns):
@@ -85,9 +76,10 @@ while 1:
             break
         else:
             values = line.split(DELIMITER)
-            features.append(get_values_list(values[:n_f_cols]))
+            values = get_values_list(values, data_column_types)
+            features.append(values[:n_f_cols])
             if n_c_labels > 0:
-                labels.append(get_values_list(values[n_f_cols:(n_f_cols+n_c_labels)]))
+                labels.append(values[n_f_cols:(n_f_cols+n_c_labels)])
             if not data_partition_column_values:
                 # Partition column values is same for all rows. Hence, only read once.
                 for i, val in enumerate(data_partition_column_indices):
@@ -130,6 +122,6 @@ for i in range(len(predictions)):
     else:
         result_list = features[i] + [predictions[i]]
     print(*(data_partition_column_values +
-            ['' if (val is None or math.isnan(val) or math.isinf(val))
+            ['' if (val is None or (not isinstance(val, str) and (math.isnan(val) or math.isinf(val))))
              else val for val in result_list]),
              sep= DELIMITER)

teradataml/data/scripts/sklearn/sklearn_function.template

@@ -8,23 +8,16 @@ params = json.loads('<params>')
 
 DELIMITER = '\t'
 
-def get_value(value):
-    ret_val = value
-    try:
-        ret_val = float(value.replace(' ', ''))
-    except Exception as ex:
-        # If the value can't be converted to float, then it is string.
-        pass
-    return ret_val
-
 def convert_to_type(val, typee):
     if typee == 'int':
-        return int(val)
+        return int(val) if val != "" else np.nan
     if typee == 'float':
-        return get_value(val)
+        if isinstance(val, str):
+            val = val.replace(' ', '')
+        return float(val) if val != "" else np.nan
     if typee == 'bool':
-        return bool(val)
-    return str(val)
+        return eval(val) if val != "" else None
+    return str(val) if val != "" else None
 
 def splitter(strr, delim=",", convert_to="str"):
     """
@@ -39,17 +32,19 @@ if len(sys.argv) != 4:
     # 4 arguments command line arguments should be passed to this file.
     # 1: file to be run
     # 2. Comma separated indices of partition columns.
-    # 3. Comma separated types of the partition columns.
+    # 3. Comma separated types of all the data columns.
     # 4. Data columns information separted by "--" where each data column information is in the form
     #    "<arg_name>-<comma separated data indices>-<comma separated data types>".
     sys.exit("4 arguments command line arguments should be passed: file to be run,"
-             " comma separated indices and types of partition columns, data columns information"
-             " separated by '--' where each data column information is in the form"
-             " '<arg_name>-<comma separated data indices>-<comma separated data types>'.")
+             " comma separated indices of partition columns, comma separated types of all columns,"
+             " data columns information separated by '--' where each data column information is"
+             " in the form '<arg_name>-<comma separated data indices>-<comma separated data types>'.")
 
 db = sys.argv[0].split("/")[1]
 data_partition_column_indices = splitter(sys.argv[1], convert_to="int") # indices are integers.
-data_partition_column_types = splitter(sys.argv[2])
+data_column_types = splitter(sys.argv[2], delim="--")
+
+data_partition_column_types = [data_column_types[idx] for idx in data_partition_column_indices]
 
 # Data related arguments information of indices and types.
 data_args_indices_types = OrderedDict()

teradataml/data/scripts/sklearn/sklearn_model_selection_split.py

@@ -6,35 +6,22 @@ import base64
 
 DELIMITER = '\t'
 
-
-def get_value(value):
-    ret_val = value
-    try:
-        ret_val = round(float("".join(value.split())), 2)
-    except Exception as ex:
-        # If the value can't be converted to float, then it is string.
-        pass
-    return ret_val
-
-
-def get_values_list(values, ignore_none=True):
+def get_values_list(values, types):
     ret_vals = []
-    for val in values:
-        if val == "" and ignore_none:
-            # Empty cell value in the database table.
-            continue
-        ret_vals.append(get_value(val))
-
+    for i, val in enumerate(values):
+        ret_vals.append(convert_to_type(val, types[i]))
     return ret_vals
 
 def convert_to_type(val, typee):
     if typee == 'int':
-        return int(val)
+        return int(val) if val != "" else np.nan
     if typee == 'float':
-        return float(val)
+        if isinstance(val, str):
+            val = val.replace(' ', '')
+        return float(val) if val != "" else np.nan
     if typee == 'bool':
-        return eval(val)
-    return str(val)
+        return eval(val) if val != "" else None
+    return str(val) if val != "" else None
 
 def splitter(strr, delim=",", convert_to="str"):
     """
@@ -54,13 +41,14 @@ if len(sys.argv) != 9:
     # 4. No of class labels.
     # 5. No of group columns.
     # 6. Comma separated indices of partition columns.
-    # 7. Comma separated types of the partition columns.
+    # 7. Comma separated types of all the data columns.
     # 8. Model file prefix to generated model file using partition columns.
     # 9. Flag to check the system type. True, means Lake, Enterprise otherwise.
     sys.exit("9 arguments command line arguments should be passed: file to be run,"
              " function name, no of feature columns, no of class labels, no of group columns,"
-             " comma separated indices and types of partition columns, model file prefix to"
-             " generated model file using partition columns and flag to check lake or enterprise.")
+             " comma separated indices of partition columns, comma separated types of all columns,"
+             " model file prefix to generated model file using partition columns and flag to check"
+             " lake or enterprise.")
 
 
 is_lake_system = eval(sys.argv[8])
@@ -70,10 +58,11 @@ function_name = sys.argv[1]
 n_f_cols = int(sys.argv[2])
 n_c_labels = int(sys.argv[3])
 n_g_cols = int(sys.argv[4])
-data_partition_column_types = splitter(sys.argv[6])
+data_column_types = splitter(sys.argv[6], delim="--")
 data_partition_column_indices = splitter(sys.argv[5], convert_to="int") # indices are integers.
 model_file_prefix = sys.argv[7]
 
+data_partition_column_types = [data_column_types[idx] for idx in data_partition_column_indices]
 
 model = None
 data_partition_column_values = []
@@ -93,6 +82,7 @@ while 1:
             break
         else:
             values = line.split(DELIMITER)
+            values = get_values_list(values, data_column_types)
             if not data_partition_column_values:
                 # Partition column values is same for all rows. Hence, only read once.
                 for i, val in enumerate(data_partition_column_indices):
@@ -117,13 +107,13 @@ while 1:
 
             start = 0
             if n_f_cols > 0:
-                features.append(get_values_list(values[:n_f_cols]))
+                features.append(values[:n_f_cols])
                 start = start + n_f_cols
             if n_c_labels > 0:
-                labels.append(get_values_list(values[start:(start+n_c_labels)]))
+                labels.append(values[start:(start+n_c_labels)])
                 start = start + n_c_labels
             if n_g_cols > 0:
-                groups.append(get_values_list(values[start:(start+n_g_cols)]))
+                groups.append(values[start:(start+n_g_cols)])
 
     except EOFError:  # Exit if reached EOF or CTRL-D
         break
@@ -144,14 +134,14 @@ if function_name == "split":
         y_train, y_test = labels[train_idx], labels[test_idx]
         for X, y in zip(X_train, y_train):
             print(*(data_partition_column_values + [split_id, "train"] +
-                    ['' if (val is None or math.isnan(val) or math.isinf(val)) else val 
+                    ['' if (val is None or (not isinstance(val, str) and (math.isnan(val) or math.isinf(val)))) else val 
                      for val in X] + [y]
-                    ),sep=DELIMITER)
+                    ), sep=DELIMITER)
         for X, y in zip(X_test, y_test):
             print(*(data_partition_column_values + [split_id, "test"] +
-                    ['' if (val is None or math.isnan(val) or math.isinf(val)) else val 
+                    ['' if (val is None or (not isinstance(val, str) and (math.isnan(val) or math.isinf(val)))) else val 
                      for val in X] + [y]
-                    ),sep=DELIMITER)
+                    ), sep=DELIMITER)
         split_id += 1
 else:
     val = getattr(model, function_name)(features, labels, groups)

teradataml/data/scripts/sklearn/sklearn_neighbors.py

@@ -12,34 +12,22 @@ from scipy.sparse.csr import csr_matrix
 DELIMITER = '\t'
 
 
-def get_value(value):
-    ret_val = value
-    try:
-        ret_val = float(value.replace(' ', ''))
-    except Exception as ex:
-        # If the value can't be converted to float, then it is string.
-        pass
-    return ret_val
-
-
-def get_values_list(values, ignore_none=True):
+def get_values_list(values, types):
     ret_vals = []
-    for val in values:
-        if val == "" and ignore_none:
-            # Empty cell value in the database table.
-            continue
-        ret_vals.append(get_value(val))
-
+    for i, val in enumerate(values):
+        ret_vals.append(convert_to_type(val, types[i]))
     return ret_vals
 
 def convert_to_type(val, typee):
     if typee == 'int':
-        return int(val)
+        return int(val) if val != "" else np.nan
     if typee == 'float':
-        return float(val)
+        if isinstance(val, str):
+            val = val.replace(' ', '')
+        return float(val) if val != "" else np.nan
     if typee == 'bool':
-        return eval(val)
-    return str(val)
+        return eval(val) if val != "" else None
+    return str(val) if val != "" else None
 
 def splitter(strr, delim=",", convert_to="str"):
     """
@@ -57,15 +45,15 @@ if len(sys.argv) < 7:
     # 2. function name.
     # 3. No of feature columns.
     # 4. Comma separated indices of partition columns.
-    # 5. Comma separated types of the partition columns.
+    # 5. Comma separated types of all the data columns.
     # 6. Model file prefix to generate model file using partition columns.
     # 7. Flag to check the system type. True, means Lake, Enterprise otherwise.
     # 8. OPTIONAL - Arguments in string format like "return_distance True-bool",
     #    "n_neighbors 3-int", "radius 3.4-float" etc.
     sys.exit("At least 7 arguments should be passed to this file - file to be run, function name, "\
-             "no of feature columns, comma separated indices and types of partition columns, "\
-             "model file prefix to generate model file using partition columns, flag to check "\
-             "lake or enterprise and optional arguments in string format.")
+             "no of feature columns, comma separated indices of partition columns, comma "\
+             "separated types of all columns, model file prefix to generate model file using "\
+             "partition columns, flag to check lake or enterprise and optional arguments in string format.")
 
 convert_to_int = lambda x: int(x) if x != "None" else None
 
@@ -74,7 +62,7 @@ if not is_lake_system:
     db = sys.argv[0].split("/")[1]
 func_name = sys.argv[1]
 n_f_cols = convert_to_int(sys.argv[2])
-data_partition_column_types = splitter(sys.argv[4])
+data_column_types = splitter(sys.argv[4], delim="--")
 data_partition_column_indices = splitter(sys.argv[3], convert_to="int") # indices are integers.
 model_file_prefix = sys.argv[5]
 # Extract arguments from string.
@@ -83,6 +71,8 @@ for i in range(7, len(sys.argv), 2):
     value = sys.argv[i + 1].split("-", 1)
     arguments[sys.argv[i]] = convert_to_type(value[0], value[1])
 
+data_partition_column_types = [data_column_types[idx] for idx in data_partition_column_indices]
+
 model = None
 data_partition_column_values = []
 
@@ -101,6 +91,7 @@ while 1:
             break
         else:
             values = line.split(DELIMITER)
+            values = get_values_list(values, data_column_types)
             if not data_partition_column_values:
                 # Partition column values is same for all rows. Hence, only read once.
                 for i, val in enumerate(data_partition_column_indices):
@@ -123,7 +114,7 @@ while 1:
                 if not model:
                     sys.exit("Model file is not installed in Vantage.")
 
-            f_ = get_values_list(values[:n_f_cols])
+            f_ = values[:n_f_cols]
             if f_:
                 output = getattr(model, func_name)(np.array([f_]), **arguments)
             else:

teradataml/data/scripts/sklearn/sklearn_score.py

@@ -6,34 +6,22 @@ import numpy as np
 DELIMITER = '\t'
 
 
-def get_value(value):
-    ret_val = value
-    try:
-        ret_val = float("".join(value.split()))
-    except Exception as ex:
-        # If the value can't be converted to float, then it is string.
-        pass
-    return ret_val
-
-
-def get_values_list(values, ignore_none=True):
+def get_values_list(values, types):
     ret_vals = []
-    for val in values:
-        if val == "" and ignore_none:
-            # Empty cell value in the database table.
-            continue
-        ret_vals.append(get_value(val))
-
+    for i, val in enumerate(values):
+        ret_vals.append(convert_to_type(val, types[i]))
     return ret_vals
 
 def convert_to_type(val, typee):
     if typee == 'int':
-        return int(val)
+        return int(val) if val != "" else np.nan
     if typee == 'float':
-        return float(val)
+        if isinstance(val, str):
+            val = val.replace(' ', '')
+        return float(val) if val != "" else np.nan
     if typee == 'bool':
-        return bool(val)
-    return str(val)
+        return eval(val) if val != "" else None
+    return str(val) if val != "" else None
 
 def splitter(strr, delim=",", convert_to="str"):
     """
@@ -51,13 +39,13 @@ if len(sys.argv) != 8:
     # 3. No of feature columns.
     # 4. No of class labels.
     # 5. Comma separated indices of partition columns.
-    # 6. Comma separated types of the partition columns.
+    # 6. Comma separated types of all the data columns.
     # 7. Model file prefix to generated model file using partition columns.
     # 8. Flag to check the system type. True, means Lake, Enterprise otherwise.
     sys.exit("8 arguments should be passed to this file - file to be run, function name, "\
-             "no of feature columns, no of class labels, comma separated indices and types of "\
-             "partition columns, model file prefix to generate model file using partition "\
-             "columns and flag to check lake or enterprise.")
+             "no of feature columns, no of class labels, comma separated indices "
+             "of partition columns, comma separated types of all columns, model file prefix to "\
+             "generate model file using partition columns and flag to check lake or enterprise.")
 
 is_lake_system = eval(sys.argv[7])
 if not is_lake_system:
@@ -65,10 +53,12 @@ if not is_lake_system:
 func_name = sys.argv[1]
 n_f_cols = int(sys.argv[2])
 n_c_labels = int(sys.argv[3])
-data_partition_column_types = splitter(sys.argv[5])
+data_column_types = splitter(sys.argv[5], delim="--")
 data_partition_column_indices = splitter(sys.argv[4], convert_to="int") # indices are integers.
 model_file_prefix = sys.argv[6]
 
+data_partition_column_types = [data_column_types[idx] for idx in data_partition_column_indices]
+
 model = None
 
 # Data Format (n_features, k_labels, one data_partition_column):
@@ -87,9 +77,10 @@ while 1:
             break
         else:
             values = line.split(DELIMITER)
-            features.append(get_values_list(values[:n_f_cols]))
+            values = get_values_list(values, data_column_types)
+            features.append(values[:n_f_cols])
             if n_c_labels > 0:
-                labels.append(get_values_list(values[n_f_cols:(n_f_cols+n_c_labels)]))
+                labels.append(values[n_f_cols:(n_f_cols+n_c_labels)])
 
             if not data_partition_column_values:
                 # Partition column values is same for all rows. Hence, only read once.
@@ -124,5 +115,5 @@ if labels:
 else:
     val = getattr(model, func_name)(np.array(features))
 
-result_val = ['' if (val is None or math.isnan(val) or math.isinf(val)) else val]
+result_val = ['' if (val is None or (not isinstance(val, str) and (math.isnan(val) or math.isinf(val)))) else val]
 print(*(data_partition_column_values + result_val), sep=DELIMITER)