teradataml 20.0.0.0__py3-none-any.whl → 20.0.0.2__py3-none-any.whl

teradataml/data/openml_example.json

@@ -59,5 +59,34 @@
         "group_column" : "integer",
         "partition_column_1" : "integer",
         "partition_column_2" : "integer"
+    },
+    "onehot_encoder_train": {
+        "gender" : "varchar(20)",
+        "numb" : "integer"
+    },
+    "customer_segmentation_train": {
+        "ID" : "integer",
+        "Gender" : "varchar(10)",
+        "Ever_Married" : "varchar(10)",
+        "Age" : "integer",
+        "Graduated" : "varchar(10)",
+        "Profession" : "varchar(30)",
+        "Work_Experience" : "integer",
+        "Spending_Score" : "varchar(10)",
+        "Family_Size": "integer",
+        "Var_1": "varchar(10)",
+        "Segmentation": "varchar(2)"
+    },
+    "customer_segmentation_test": {
+        "ID" : "integer",
+        "Gender" : "varchar(10)",
+        "Ever_Married" : "varchar(10)",
+        "Age" : "integer",
+        "Graduated" : "varchar(10)",
+        "Profession" : "varchar(30)",
+        "Work_Experience" : "integer",
+        "Spending_Score" : "varchar(10)",
+        "Family_Size": "integer",
+        "Var_1": "varchar(10)"
     }
 }

teradataml/data/real_values.csv

@@ -0,0 +1,14 @@
+"TD_TIMECODE","id","val"
+2020-01-01 08:00:00,33,1.2e+02
+2020-02-01 08:00:00,33,1.95e+02
+2020-03-01 08:00:00,33,8e+02
+2020-04-01 08:00:00,33,6.6e+01
+2020-05-01 08:00:00,33,1.44e+02
+2020-06-01 08:00:00,33,2.1e+04
+2020-07-01 08:00:00,33,3.2e+02
+2020-08-01 08:00:00,33,1.44e+02
+2020-09-01 08:00:00,33,2.2e+02
+2020-10-01 08:00:00,33,2.1e+02
+2020-11-01 08:00:00,33,1.34e+02
+2020-12-01 08:00:00,33,1.84e+02
+2020-12-02 08:00:00,33,1.98e+02

teradataml/data/sax_example.json

@@ -5,5 +5,13 @@
     "expenditure": "integer",
     "income": "integer",
     "investment": "integer"
+  },
+  "finance_data4":{
+    "id": "integer",
+    "period": "integer",
+    "expenditure": "float",
+    "income": "float",
+    "investment": "float"
+
   }
 }

teradataml/data/scale_attributes.csv

@@ -0,0 +1,3 @@
+pid,attribute_column
+1,fare
+2,age

teradataml/data/scale_example.json

@@ -19,5 +19,56 @@
                   "bathrms" : "real",
                   "stories" : "real"
                   
-        }
+        },
+"scale_attributes":{
+                   
+                   "pid" : "integer",
+                   "attribute_column" : "varchar(150)"
+                   
+           },
+"scale_parameters":{
+                   
+                   "pid" : "integer",
+                   "parameter_column" : "varchar(150)",
+                   "value_column" : "varchar(150)"
+                   
+           },
+"scale_input_partitioned":{
+                        "passenger" : "integer",
+                        "pid" : "integer",
+                        "survived" : "integer",
+                        "pclass" : "integer",
+                        "name" : "varchar(90)",
+                        "gender" : "varchar(10)",
+                        "age" : "integer",
+                        "sibsp" : "integer",
+                        "parch" : "integer",
+                        "ticket" : "varchar(20)",
+                        "fare" : "integer",
+                        "cabin" : "varchar(20)",
+                        "embarked" : "varchar(10)"
+
+
+},
+
+"scale_input_sparse":
+{
+                   
+        "passenger" : "integer",
+        "attribute_column" : "varchar(20)",
+        "attribute_value" : "real"
+                   
+},
+
+"scale_input_part_sparse":
+{
+                   
+        "pid" : "integer",
+        "passenger" : "integer",
+        "attribute_column" : "varchar(20)",
+        "attribute_value" : "real"
+                   
 }
+
+        
+}

teradataml/data/scale_input_part_sparse.csv

@@ -0,0 +1,31 @@
+pid,passenger,attribute_column,attribute_value
+3,56,age,
+3,56,fare,35.5
+3,63,age,45.0
+3,63,fare,83.475
+3,67,age,29.0
+3,67,fare,10.5
+3,76,age,25.0
+3,76,fare,7.65
+3,93,age,46.0
+3,93,fare,61.175
+1,2,age,38.0
+1,2,fare,71.2833
+1,4,age,35.0
+1,4,fare,53.1
+1,7,age,54.0
+1,7,fare,51.8625
+1,11,age,4.0
+1,11,fare,16.7
+1,12,age,58.0
+1,12,fare,26.55
+2,22,age,34.0
+2,22,fare,13.0
+2,24,age,28.0
+2,24,fare,35.5
+2,32,age,
+2,32,fare,146.5208
+2,53,age,49.0
+2,53,fare,76.7292
+2,55,age,65.0
+2,55,fare,61.9792

teradataml/data/scale_input_partitioned.csv

@@ -0,0 +1,16 @@
+passenger,pid,survived,pclass,name,gender,age,sibsp,parch,ticket,fare,cabin,embarked
+76,3,0,3,Moen; Mr. Sigurd Hansen,male,25.0,0,0,348123,7.65,F G73,S
+32,2,1,1,Spencer; Mrs. William Augustus (Marie Eugenie),female,,1,0,PC 17569,146.5208,B78,C
+55,2,0,1,Ostby; Mr. Engelhart Cornelius,male,65.0,0,1,113509,61.9792,B30,C
+53,2,1,1,Harper; Mrs. Henry Sleeper (Myna Haxtun),female,49.0,1,0,PC 17572,76.7292,D33,C
+93,3,0,1,Chaffee; Mr. Herbert Fuller,male,46.0,1,0,W.E.P. 5734,61.175,E31,S
+11,1,1,3,Sandstrom; Miss. Marguerite Rut,female,4.0,1,1,PP 9549,16.7,G6,S
+7,1,0,1,McCarthy; Mr. Timothy J,male,54.0,0,0,17463,51.8625,E46,S
+24,2,1,1,Sloper; Mr. William Thompson,male,28.0,0,0,113788,35.5,A6,S
+63,3,0,1,Harris; Mr. Henry Birkhardt,male,45.0,1,0,36973,83.475,C83,S
+22,2,1,2,Beesley; Mr. Lawrence,male,34.0,0,0,248698,13.0,D56,S
+56,3,1,1,Woolner; Mr. Hugh,male,,0,0,19947,35.5,C52,S
+12,1,1,1,Bonnell; Miss. Elizabeth,female,58.0,0,0,113783,26.55,C103,S
+2,1,1,1,Cumings; Mrs. John Bradley (Florence Briggs Thayer),female,38.0,1,0,PC 17599,71.2833,C85,C
+67,3,1,2,Nye; Mrs. (Elizabeth Ramell),female,29.0,0,0,C.A. 29395,10.5,F33,S
+4,1,1,1,Futrelle; Mrs. Jacques Heath (Lily May Peel),female,35.0,1,0,113803,53.1,C123,S

teradataml/data/scale_input_sparse.csv

@@ -0,0 +1,11 @@
+passenger,attribute_column,attribute_value
+873,age,33.0
+631,age,80.0
+97,age,71.0
+873,fare,5.0
+631,fare,30.0
+97,fare,34.6542
+488,age,58.0
+488,fare,29.7
+505,age,16.0
+505,fare,86.5

teradataml/data/scale_parameters.csv

@@ -0,0 +1,3 @@
+pid,parameter_column,value_column
+1,scalemethod,midrange
+2,scalemethod,range

teradataml/data/scripts/deploy_script.py

@@ -27,6 +27,11 @@ def get_values_list(values, ignore_none=True):
 
     return ret_vals
 
+if len(sys.argv) != 2:
+    sys.exit("Script command format: python deploy_script.py <enterprise/lake>")
+
+vantage_type = sys.argv[1]
+
 data_partition_column_values = []
 data_partition_column_indices = [5, 6]
 
@@ -55,11 +60,25 @@ if not len(features):
     sys.exit(0)
 
 X = np.array(features)
-y = np.array(labels)
+y = np.array(labels).ravel()
 
 clf = make_pipeline(StandardScaler(), SVC(gamma='auto'))
 clf.fit(X, y)
 
-model = base64.b64encode(pickle.dumps(clf))
+model_str = pickle.dumps(clf)
+
+# Prepare the corresponding model file name and extract model.
+partition_join = "_".join([str(x) for x in data_partition_column_values])
+# Replace '-' with '_' as '-' because partition_columns can be negative.
+partition_join = partition_join.replace("-", "_")
+
+if vantage_type == "lake":
+    model = f"/tmp/sklearn_model_{partition_join}.pickle"
+    with open(model, "wb") as fp:
+        fp.write(model_str)
+elif vantage_type == "enterprise":
+    model = base64.b64encode(model_str)
+else:
+    sys.exit("Invalid vantage type. Use either 'lake' or 'enterprise'.")
 
 print(*(data_partition_column_values + [model]), sep=DELIMITER)

teradataml/data/scripts/sklearn/sklearn_fit.py

@@ -5,35 +5,24 @@ import base64
 
 DELIMITER = '\t'
 
-def get_value(value):
-    ret_val = value
-    try:
-        ret_val = float(value.replace(' ', ''))
-    except Exception as ex:
-        # If the value can't be converted to float, then it is string.
-        pass
-    return ret_val
-
-
-def get_values_list(values, ignore_none=True):
+def get_values_list(values, types, model_obj):
     ret_vals = []
-    for val in values:
-        if val == "" and ignore_none:
-            # Empty cell value in the database table.
+    for i, val in enumerate(values):
+        if type(model_obj).__name__ == "MultiLabelBinarizer" and val == "":
             continue
-        ret_vals.append(get_value(val))
-
+        ret_vals.append(convert_to_type(val, types[i]))
     return ret_vals
 
 def convert_to_type(val, typee):
     if typee == 'int':
-        return int(val)
+        return int(val) if val != "" else np.nan
     if typee == 'float':
-        val = get_value(val)
-        return float(val)
+        if isinstance(val, str):
+            val = val.replace(' ', '')
+        return float(val) if val != "" else np.nan
     if typee == 'bool':
-        return bool(val)
-    return str(val)
+        return eval(val) if val != "" else None
+    return str(val) if val != "" else None
 
 def get_classes_as_list(classes, actual_type):
     if classes == "None":
@@ -66,14 +55,14 @@ if len(sys.argv) != 10:
     # 3. No of feature columns.
     # 4. No of class labels.
     # 5. Comma separated indices of partition columns.
-    # 6. Comma separated types of the partition columns.
+    # 6. Comma separated types of all the data columns.
     # 7. Model file prefix to generated model file using partition columns.
     # 8. classes (separated by '--') - should be converted to list. "None" if no classes exists.
     # 9. type of elements in passed in classes. "None" if no classes exists.
     # 10. Flag to check the system type. True, means Lake, Enterprise otherwise
     sys.exit("10 arguments command line arguments should be passed: file to be run,"
              " function name, no of feature columns, no of class labels, comma separated indices"
-             " and types of partition columns, model file prefix ,"
+             " of partition columns, comma separated types of all columns, model file prefix ,"
              " classes, type of elements in classes and flag to check lake or enterprise.")
 
 is_lake_system = eval(sys.argv[9])
@@ -82,12 +71,14 @@ if not is_lake_system:
 function_name = sys.argv[1]
 n_f_cols = int(sys.argv[2])
 n_c_labels = int(sys.argv[3])
-data_partition_column_types = splitter(sys.argv[5])
+data_column_types = splitter(sys.argv[5], delim="--")
 data_partition_column_indices = splitter(sys.argv[4], convert_to="int") # indices are integers.
 model_file_prefix = sys.argv[6]
 class_type = sys.argv[8]
 classes = get_classes_as_list(sys.argv[7], class_type)
 
+data_partition_column_types = [data_column_types[idx] for idx in data_partition_column_indices]
+
 model = None
 
 # Data Format (n_features, k_labels, one data_partition_column):
@@ -108,9 +99,7 @@ while 1:
             break
         else:
             values = line.split(DELIMITER)
-            features.append(get_values_list(values[:n_f_cols]))
-            if n_c_labels > 0:
-                labels.append(get_values_list(values[n_f_cols:(n_f_cols+n_c_labels)]))
+
             if not data_partition_column_values:
                 # Partition column values is same for all rows. Hence, only read once.
                 for i, val in enumerate(data_partition_column_indices):
@@ -133,6 +122,13 @@ while 1:
                 if model is None:
                     sys.exit("Model file is not installed in Vantage.")
 
+            values = get_values_list(values, data_column_types, model)
+            values = values[:-len(data_partition_column_indices)] # Already processed partition columns.
+            features.append(values[:n_f_cols])
+            if n_c_labels > 0:
+                labels.append(values[n_f_cols:(n_f_cols+n_c_labels)])
+
+
     except EOFError:  # Exit if reached EOF or CTRL-D
         break
 
@@ -142,22 +138,29 @@ if not len(features):
 # Fit/partial_fit the model to the data.
 if function_name == "partial_fit":
     if labels and classes:
-        model.partial_fit(np.array(features), np.array(labels), classes=classes)
+        model.partial_fit(features, labels, classes=classes)
     elif labels:
-        model.partial_fit(np.array(features), np.array(labels))
+        model.partial_fit(features, labels)
     elif classes:
-        model.partial_fit(np.array(features), classes=classes)
+        model.partial_fit(features, classes=classes)
     else:
-        model.partial_fit(np.array(features))
+        model.partial_fit(features)
 elif function_name == "fit":
-    # For IsotonicRegression, fit() accepts training target as
-    # y: array-like of shape (n_samples,).
+    model_name = model.__class__.__name__
+    np_func_list = ["OneVsRestClassifier", "LabelBinarizer", "TSNE"]
     if labels:
-        labels = np.array(labels).reshape(-1) \
-            if model.__class__.__name__ == "IsotonicRegression" else np.array(labels)
-        model.fit(np.array(features), labels)
+        # For IsotonicRegression, fit() accepts training target as
+        # y: array-like of shape (n_samples,).
+        if model_name in ["IsotonicRegression", "LinearSVC"]:
+            labels = np.array(labels).reshape(-1)
+        if model_name in np_func_list:
+            labels = np.array(labels)
+            features = np.array(features)
+        model.fit(features, labels)
     else:
-        model.fit(np.array(features))
+        if model_name in np_func_list:
+            features = np.array(features)
+        model.fit(features)
 
 model_str = pickle.dumps(model)
 

teradataml/data/scripts/sklearn/sklearn_fit_predict.py

@@ -5,33 +5,22 @@ import math
 
 DELIMITER = '\t'
 
-def get_value(value):
-    ret_val = value
-    try:
-        ret_val = float(value.replace(' ', ''))
-    except Exception as ex:
-        # If the value can't be converted to float, then it is string.
-        pass
-    return ret_val
-
-def get_values_list(values, ignore_none=True):
+def get_values_list(values, types):
     ret_vals = []
-    for val in values:
-        if val == "" and ignore_none:
-            # Empty cell value in the database table.
-            continue
-        ret_vals.append(get_value(val))
-
+    for i, val in enumerate(values):
+        ret_vals.append(convert_to_type(val, types[i]))
     return ret_vals
 
 def convert_to_type(val, typee):
     if typee == 'int':
-        return int(val)
+        return int(val) if val != "" else np.nan
     if typee == 'float':
-        return float(val)
+        if isinstance(val, str):
+            val = val.replace(' ', '')
+        return float(val) if val != "" else np.nan
     if typee == 'bool':
-        return bool(val)
-    return str(val)
+        return eval(val) if val != "" else None
+    return str(val) if val != "" else None
 
 def splitter(strr, delim=",", convert_to="str"):
     """
@@ -48,13 +37,13 @@ if len(sys.argv) != 7:
     # 2. No of feature columns.
     # 3. No of class labels.
     # 4. Comma separated indices of partition columns.
-    # 5. Comma separated types of the partition columns.
+    # 5. Comma separated types of all the data columns.
     # 6. Model file prefix to generated model file using partition columns.
     # 7. Flag to check the system type. True, means Lake, Enterprise otherwise.
     sys.exit("7 arguments should be passed to this file - file to be run, "\
-             "no of feature columns, no of class labels, comma separated indices and types of "\
-             "partition columns, model file prefix to generate model file using partition "\
-             "columns and flag to check lake or enterprise.")
+             "no of feature columns, no of class labels, comma separated indices of partition "
+             "columns, comma separated types of all columns, model file prefix to generate model "
+             "file using partition columns and flag to check lake or enterprise.")
 
 is_lake_system = eval(sys.argv[6])
 if not is_lake_system:
@@ -62,9 +51,11 @@ if not is_lake_system:
 n_f_cols = int(sys.argv[1])
 n_c_labels = int(sys.argv[2])
 model_file_prefix = sys.argv[5]
-data_partition_column_types = splitter(sys.argv[4])
+data_column_types = splitter(sys.argv[4], delim="--")
 data_partition_column_indices = splitter(sys.argv[3], convert_to="int") # indices are integers.
 
+data_partition_column_types = [data_column_types[idx] for idx in data_partition_column_indices]
+
 model = None
 
 # Data Format (n_features, k_labels, one data_partition_columns):
@@ -85,9 +76,10 @@ while 1:
             break
         else:
             values = line.split(DELIMITER)
-            features.append(get_values_list(values[:n_f_cols]))
+            values = get_values_list(values, data_column_types)
+            features.append(values[:n_f_cols])
             if n_c_labels > 0:
-                labels.append(get_values_list(values[n_f_cols:(n_f_cols+n_c_labels)]))
+                labels.append(values[n_f_cols:(n_f_cols+n_c_labels)])
             if not data_partition_column_values:
                 # Partition column values is same for all rows. Hence, only read once.
                 for i, val in enumerate(data_partition_column_indices):
@@ -118,9 +110,9 @@ if not len(features):
 
 # write code to call fit_predict with features and labels when n_c_labels > 0
 if n_c_labels > 0:
-    predictions = model.fit_predict(np.array(features), np.array(labels))
+    predictions = model.fit_predict(features, labels)
 else:
-    predictions = model.fit_predict(np.array(features))
+    predictions = model.fit_predict(features)
 
 # Export results to to the Databse through standard output
 for i in range(len(predictions)):
@@ -130,6 +122,6 @@ for i in range(len(predictions)):
     else:
         result_list = features[i] + [predictions[i]]
     print(*(data_partition_column_values +
-            ['' if (val is None or math.isnan(val) or math.isinf(val))
+            ['' if (val is None or (not isinstance(val, str) and (math.isnan(val) or math.isinf(val))))
              else val for val in result_list]),
              sep= DELIMITER)

teradataml/data/scripts/sklearn/sklearn_function.template

@@ -8,23 +8,16 @@ params = json.loads('<params>')
 
 DELIMITER = '\t'
 
-def get_value(value):
-    ret_val = value
-    try:
-        ret_val = float(value.replace(' ', ''))
-    except Exception as ex:
-        # If the value can't be converted to float, then it is string.
-        pass
-    return ret_val
-
 def convert_to_type(val, typee):
     if typee == 'int':
-        return int(val)
+        return int(val) if val != "" else np.nan
     if typee == 'float':
-        return get_value(val)
+        if isinstance(val, str):
+            val = val.replace(' ', '')
+        return float(val) if val != "" else np.nan
     if typee == 'bool':
-        return bool(val)
-    return str(val)
+        return eval(val) if val != "" else None
+    return str(val) if val != "" else None
 
 def splitter(strr, delim=",", convert_to="str"):
     """
@@ -35,21 +28,30 @@ def splitter(strr, delim=",", convert_to="str"):
     return [convert_to_type(i, convert_to) for i in strr.split(delim)]
 
 # Arguments to the Script.
-if len(sys.argv) != 4:
-    # 4 arguments command line arguments should be passed to this file.
+if len(sys.argv) != 6:
+    # 5 arguments command line arguments should be passed to this file.
     # 1: file to be run
     # 2. Comma separated indices of partition columns.
-    # 3. Comma separated types of the partition columns.
+    # 3. Comma separated types of all the data columns.
     # 4. Data columns information separted by "--" where each data column information is in the form
     #    "<arg_name>-<comma separated data indices>-<comma separated data types>".
-    sys.exit("4 arguments command line arguments should be passed: file to be run,"
-             " comma separated indices and types of partition columns, data columns information"
-             " separated by '--' where each data column information is in the form"
-             " '<arg_name>-<comma separated data indices>-<comma separated data types>'.")
-
-db = sys.argv[0].split("/")[1]
+    # 5. Flag to check the system type. True, means Lake, Enterprise otherwise.
+    # 6. Model file prefix for lake system, None otherwise.
+    sys.exit("5 arguments command line arguments should be passed: file to be run,"
+             " comma separated indices of partition columns, comma separated types of all columns,"
+             " data columns information separated by '--' where each data column information is"
+             " in the form '<arg_name>-<comma separated data indices>-<comma separated data types>',"
+             " flag to check lake or enterprise and model file prefix used only for lake system.")
+
+is_lake_system = eval(sys.argv[4])
+if not is_lake_system:
+    db = sys.argv[0].split("/")[1]
+else:
+    model_file_prefix = sys.argv[5]
 data_partition_column_indices = splitter(sys.argv[1], convert_to="int") # indices are integers.
-data_partition_column_types = splitter(sys.argv[2])
+data_column_types = splitter(sys.argv[2], delim="--")
+
+data_partition_column_types = [data_column_types[idx] for idx in data_partition_column_indices]
 
 # Data related arguments information of indices and types.
 data_args_indices_types = OrderedDict()
@@ -84,6 +86,11 @@ while 1:
                     data_partition_column_values.append(
                         convert_to_type(values[val], typee=data_partition_column_types[i])
                         )
+
+                # Prepare the corresponding model file name and extract model.
+                partition_join = "_".join([str(x) for x in data_partition_column_values])
+                # Replace '-' with '_' as '-' because partition_columns can be negative.
+                partition_join = partition_join.replace("-", "_")
             
             # Prepare data dictionary containing only arguments related to data.
             for arg_name in data_args_values:
@@ -110,4 +117,15 @@ all_args = {**data_args_values, **params}
 module_ = importlib.import_module(module_name)
 sklearn_model = getattr(module_, func_name)(**all_args)
 
-print(*(data_partition_column_values + [base64.b64encode(pickle.dumps(sklearn_model))]), sep=DELIMITER)
+model_str = pickle.dumps(sklearn_model)
+
+if is_lake_system:
+    model_file_path = f"/tmp/{model_file_prefix}_{partition_join}.pickle"
+
+    # Write to file in Vantage, to be used in predict/scoring.
+    with open(model_file_path, "wb") as fp:
+        fp.write(model_str)
+
+model_data = model_file_path if is_lake_system else base64.b64encode(model_str)
+
+print(*(data_partition_column_values + [model_data]), sep=DELIMITER)