teradataml 20.0.0.8__py3-none-any.whl

teradataml/data/scripts/lightgbm/lightgbm_function.template

@@ -0,0 +1,234 @@
+import sys, json, io
+import pickle, base64, importlib, numpy as np
+from collections import OrderedDict
+import os
+from contextlib import contextmanager
+
+func_name = "<func_name>"
+module_name = "<module_name>"
+is_lake_system = <is_lake_system>
+params = json.loads('<params>')
+data_partition_column_indices = <partition_cols_indices>
+data_partition_column_types = <partition_cols_types>
+model_file_prefix = "<model_file_prefix>" # Needed in case of lake system for writing model to /tmp
+
+DELIMITER = '\t'
+
+@contextmanager
+def suppress_stderr():
+    """
+    Function to suppress the warnings(lake systems treats warnings as errors).
+    """
+    with open(os.devnull, "w") as devnull:
+        old_stderr = sys.stderr
+        sys.stderr = devnull
+        try:
+            yield
+        finally:
+            sys.stderr = old_stderr
+
+## On Lake system warnings raised by script are treated as a errors.
+## Hence, to suppress it putting the under suppress_stderr().
+with suppress_stderr():
+    def convert_to_type(val, typee):
+        if typee == 'int':
+            return int(val) if val != "" else np.nan
+        if typee == 'float':
+            if isinstance(val, str):
+                val = val.replace(' ', '')
+            return float(val) if val != "" else np.nan
+        if typee == 'bool':
+            return eval(val) if val != "" else None
+        return str(val) if val != "" else None
+
+    if not is_lake_system:
+        db = sys.argv[0].split("/")[1]
+
+    data_present = False
+    data_partition_column_values = []
+
+    while 1:
+        try:
+            line = input()
+            if line == '':  # Exit if user provides blank line
+                break
+            else:
+                data_present = True
+                values = line.split(DELIMITER)
+                if not data_partition_column_values:
+                    # Partition column values is same for all rows. Hence, only read once.
+                    for i, val in enumerate(data_partition_column_indices): # Only partition columns are
+                        data_partition_column_values.append(
+                            convert_to_type(values[val], typee=data_partition_column_types[i])
+                            )
+
+                    # Prepare the corresponding model file name and extract model.
+                    partition_join = "_".join([str(x) for x in data_partition_column_values])
+                    # Replace '-' with '_' because partition_columns can be negative containing '-'.
+                    partition_join = partition_join.replace("-", "_")
+
+                    train_set = params.get("train_set") # Gets file name prefix.
+                    model_file_path = f"{train_set}_{partition_join}"\
+                        if is_lake_system else \
+                        f"./{db}/{train_set}_{partition_join}"
+
+                    with open(model_file_path, "rb") as fp:
+                        params["train_set"] = pickle.loads(fp.read())
+
+                    valid_sets = params.get("valid_sets", None) # Gets file names prefix.
+                    if valid_sets:
+                        params["valid_sets"] = []
+                        for valid_set in valid_sets:
+                            model_file_path = f"{valid_set}_{partition_join}"\
+                                if is_lake_system else \
+                                f"./{db}/{valid_set}_{partition_join}"
+                            with open(model_file_path, "rb") as fp:
+                                params["valid_sets"].append(pickle.load(fp))
+
+        except EOFError:  # Exit if reached EOF or CTRL-D
+            break
+
+    if not data_present:
+        sys.exit(0)
+
+    # Handle callbacks.
+    rec_eval = None
+    if "callbacks" in params and params["callbacks"] is not None:
+        callbacks = params["callbacks"]
+        callbacks = [callbacks] if not isinstance(callbacks, list) else callbacks
+        for i, callback in enumerate(callbacks):
+            c_module_name = callback["module"]
+            c_func_name = callback["func_name"]
+            c_kwargs = callback["kwargs"]
+            c_module = importlib.import_module(c_module_name)
+            if c_func_name == "record_evaluation":
+                # record_evaluation function takes empty dict. If the argument has elements in the
+                # dict, they will be deleted as per the documentation from lightgbm as described below:
+                # eval_result (dict) -
+                #   Dictionary used to store all evaluation results of all validation sets. This should
+                #   be initialized outside of your call to record_evaluation() and should be empty. Any
+                #   initial contents of the dictionary will be deleted.
+                rec_eval = {}
+                callbacks[i] = getattr(c_module, c_func_name)(rec_eval)
+            else:
+                callbacks[i] = getattr(c_module, c_func_name)(**c_kwargs)
+
+        params["callbacks"] = callbacks
+
+    module_ = importlib.import_module(module_name)
+
+    ### LightGBM training is giving some meaningful console output like this:
+    ### Hence, capturing it to show to the user.
+
+    # [LightGBM] [Warning] Auto-choosing row-wise multi-threading, the overhead of testing was 0.000190 seconds.
+    # You can set `force_row_wise=true` to remove the overhead.
+    # And if memory is not enough, you can set `force_col_wise=true`.
+    # [LightGBM] [Info] Total Bins 136
+    # [LightGBM] [Info] Number of data points in the train set: 97, number of used features: 4
+    # [LightGBM] [Info] Start training from score 0.556701
+    # [LightGBM] [Warning] No further splits with positive gain, best gain: -inf
+    # [1]	valid_0's l2: 0.219637	valid_1's l2: 0.219637
+    # [LightGBM] [Warning] No further splits with positive gain, best gain: -inf
+    # [2]	valid_0's l2: 0.196525	valid_1's l2: 0.196525
+    # [LightGBM] [Warning] No further splits with positive gain, best gain: -inf
+    # [3]	valid_0's l2: 0.178462	valid_1's l2: 0.178462
+    # [LightGBM] [Warning] No further splits with positive gain, best gain: -inf
+    # [4]	valid_0's l2: 0.162887	valid_1's l2: 0.162887
+    # [LightGBM] [Warning] No further splits with positive gain, best gain: -inf
+    # [5]	valid_0's l2: 0.150271	valid_1's l2: 0.150271
+    # [LightGBM] [Warning] No further splits with positive gain, best gain: -inf
+    # [6]	valid_0's l2: 0.140219	valid_1's l2: 0.140219
+    # [LightGBM] [Warning] No further splits with positive gain, best gain: -inf
+    # [7]	valid_0's l2: 0.131697	valid_1's l2: 0.131697
+    # [LightGBM] [Warning] No further splits with positive gain, best gain: -inf
+    # [8]	valid_0's l2: 0.124056	valid_1's l2: 0.124056
+    # [LightGBM] [Warning] No further splits with positive gain, best gain: -inf
+    # [9]	valid_0's l2: 0.117944	valid_1's l2: 0.117944
+    # [LightGBM] [Warning] No further splits with positive gain, best gain: -inf
+    # [10]	valid_0's l2: 0.11263	valid_1's l2: 0.11263
+    # [LightGBM] [Warning] No further splits with positive gain, best gain: -inf
+    # [11]	valid_0's l2: 0.105228	valid_1's l2: 0.105228
+    # [LightGBM] [Warning] No further splits with positive gain, best gain: -inf
+    # [12]	valid_0's l2: 0.0981571	valid_1's l2: 0.0981571
+    # [LightGBM] [Warning] No further splits with positive gain, best gain: -inf
+    # [13]	valid_0's l2: 0.0924294	valid_1's l2: 0.0924294
+    # [LightGBM] [Warning] No further splits with positive gain, best gain: -inf
+    # [14]	valid_0's l2: 0.0877899	valid_1's l2: 0.0877899
+    # [LightGBM] [Warning] No further splits with positive gain, best gain: -inf
+    # [15]	valid_0's l2: 0.084032	valid_1's l2: 0.084032
+    # [LightGBM] [Warning] No further splits with positive gain, best gain: -inf
+    # [16]	valid_0's l2: 0.080988	valid_1's l2: 0.080988
+    # [LightGBM] [Warning] No further splits with positive gain, best gain: -inf
+    # [17]	valid_0's l2: 0.0785224	valid_1's l2: 0.0785224
+    # [LightGBM] [Warning] No further splits with positive gain, best gain: -inf
+    # [18]	valid_0's l2: 0.0765253	valid_1's l2: 0.0765253
+    # [LightGBM] [Warning] No further splits with positive gain, best gain: -inf
+    # [19]	valid_0's l2: 0.0750803	valid_1's l2: 0.0750803
+    # [LightGBM] [Warning] No further splits with positive gain, best gain: -inf
+    # [20]	valid_0's l2: 0.0738915	valid_1's l2: 0.0738915
+    # [LightGBM] [Warning] No further splits with positive gain, best gain: -inf
+    # [21]	valid_0's l2: 0.07288	valid_1's l2: 0.07288
+    # [LightGBM] [Warning] No further splits with positive gain, best gain: -inf
+    # [22]	valid_0's l2: 0.0718676	valid_1's l2: 0.0718676
+    # [LightGBM] [Warning] No further splits with positive gain, best gain: -inf
+    # [23]	valid_0's l2: 0.0706037	valid_1's l2: 0.0706037
+    # [LightGBM] [Warning] No further splits with positive gain, best gain: -inf
+    # [24]	valid_0's l2: 0.0695799	valid_1's l2: 0.0695799
+    # [LightGBM] [Warning] No further splits with positive gain, best gain: -inf
+    # [25]	valid_0's l2: 0.0687507	valid_1's l2: 0.0687507
+    # [LightGBM] [Warning] No further splits with positive gain, best gain: -inf
+    # [26]	valid_0's l2: 0.0680819	valid_1's l2: 0.0680819
+    # [LightGBM] [Warning] No further splits with positive gain, best gain: -inf
+    # [27]	valid_0's l2: 0.0674077	valid_1's l2: 0.0674077
+    # [LightGBM] [Warning] No further splits with positive gain, best gain: -inf
+    # [28]	valid_0's l2: 0.0665111	valid_1's l2: 0.0665111
+    # [LightGBM] [Warning] No further splits with positive gain, best gain: -inf
+    # [29]	valid_0's l2: 0.0659656	valid_1's l2: 0.0659656
+    # [LightGBM] [Warning] No further splits with positive gain, best gain: -inf
+    # [30]	valid_0's l2: 0.0652665	valid_1's l2: 0.0652665
+    result = ""
+    stdout = None
+    try:
+        stdout = sys.stdout
+        new_stdout = io.StringIO()
+        sys.stdout = new_stdout
+        trained_model = getattr(module_, func_name)(**params)
+        result = new_stdout.getvalue()
+    except Exception:
+        raise
+    finally:
+        sys.stdout = stdout
+
+    model_str = pickle.dumps(trained_model)
+    console_output_str = result.encode()
+
+    if is_lake_system:
+        model_file_path = f"/tmp/{model_file_prefix}_{partition_join}.pickle"
+        model_console_output_path = f"/tmp/{model_file_prefix}_{partition_join}_console_output.pickle"
+
+        # Write to file in Vantage, to be used in predict/scoring.
+        with open(model_file_path, "wb") as fp:
+            fp.write(model_str)
+
+        with open(model_console_output_path, "wb") as fpc:
+            fpc.write(console_output_str)
+
+
+    model_data = model_file_path if is_lake_system else base64.b64encode(model_str)
+    console_output = model_console_output_path if is_lake_system else base64.b64encode(console_output_str)
+
+    output_data = [model_data, console_output]
+
+    if rec_eval is not None:
+        rec_eval = pickle.dumps(rec_eval)
+        if is_lake_system:
+            rec_eval_file_path = f"/tmp/{model_file_prefix}_{partition_join}_rec_eval.pickle"
+
+            with open(rec_eval_file_path, "wb") as fp:
+                fp.write(rec_eval)
+
+        rec_eval = rec_eval_file_path if is_lake_system else base64.b64encode(rec_eval)
+
+        output_data.append(rec_eval)
+
+    print(*(data_partition_column_values + output_data), sep=DELIMITER)

teradataml/data/scripts/lightgbm/lightgbm_sklearn.template

@@ -0,0 +1,177 @@
+import sys, json
+import pickle, base64, importlib, numpy as np
+from collections import OrderedDict
+import os
+from contextlib import contextmanager
+
+DELIMITER = '\t'
+
+func_name = <func_name>
+params = json.loads('<params>')
+is_lake_system = <is_lake_system>
+model_file_prefix = <model_file_prefix>
+
+@contextmanager
+def suppress_stderr():
+    """
+    Function to suppress the warnings(lake systems treats warnings as errors).
+    """
+    with open(os.devnull, "w") as devnull:
+        old_stderr = sys.stderr
+        sys.stderr = devnull
+        try:
+            yield
+        finally:
+            sys.stderr = old_stderr
+
+## On Lake system warnings raised by script are treated as a errors.
+## Hence, to suppress it putting the under suppress_stderr().
+with suppress_stderr():
+    def convert_to_type(val, typee):
+        if typee == 'int':
+            return int(val) if val != "" else np.nan
+        if typee == 'float':
+            if isinstance(val, str):
+                val = val.replace(' ', '')
+            return float(val) if val != "" else np.nan
+        if typee == 'bool':
+            return eval(val) if val != "" else None
+        return str(val) if val != "" else None
+
+    def splitter(strr, delim=",", convert_to="str"):
+        """
+        Split the string based on delimiter and convert to the type specified.
+        """
+        if strr == "None":
+            return []
+        return [convert_to_type(i, convert_to) for i in strr.split(delim)]
+
+
+    if not is_lake_system:
+        db = sys.argv[0].split("/")[1]
+
+    data_partition_column_indices = <partition_cols_indices>
+    data_column_types = <types_of_data_cols>
+
+    data_partition_column_types = [data_column_types[idx] for idx in data_partition_column_indices]
+
+    # Data related arguments information of indices and types.
+    data_args_indices_types = OrderedDict()
+
+    # Data related arguments values - prepare dictionary and populate data later.
+    data_args_values = {}
+
+    data_args_info_str = <data_args_info_str>
+    for data_arg in data_args_info_str.split("--"):
+        arg_name, indices, types = data_arg.split("-")
+        indices = splitter(indices, convert_to="int")
+        types = splitter(types)
+
+        data_args_indices_types[arg_name] = {"indices": indices, "types": types}
+        data_args_values[arg_name] = [] # Keeping empty for each data arg name and populate data later.
+
+    data_partition_column_values = []
+    data_present = False
+
+    model = None
+
+    # Read data - columns information is passed as command line argument and stored in
+    # data_args_indices_types dictionary.
+    while 1:
+        try:
+            line = input()
+            if line == '':  # Exit if user provides blank line
+                break
+            else:
+                data_present = True
+                values = line.split(DELIMITER)
+                if not data_partition_column_values:
+                    # Partition column values is same for all rows. Hence, only read once.
+                    for i, val in enumerate(data_partition_column_indices):
+                        data_partition_column_values.append(
+                            convert_to_type(values[val], typee=data_partition_column_types[i])
+                            )
+
+                    # Prepare the corresponding model file name and extract model.
+                    partition_join = "_".join([str(x) for x in data_partition_column_values])
+                    # Replace '-' with '_' as '-' because partition_columns can be negative.
+                    partition_join = partition_join.replace("-", "_")
+
+
+                    model_file_path = f"{model_file_prefix}_{partition_join}"\
+                        if is_lake_system else \
+                        f"./{db}/{model_file_prefix}_{partition_join}"
+
+                    with open(model_file_path, "rb") as fp:
+                        model = pickle.loads(fp.read())
+
+                    if model is None:
+                        sys.exit("Model file is not installed in Vantage.")
+
+                # Prepare data dictionary containing only arguments related to data.
+                for arg_name in data_args_values:
+                    data_indices = data_args_indices_types[arg_name]["indices"]
+                    types = data_args_indices_types[arg_name]["types"]
+                    cur_row = []
+                    for idx, data_idx in enumerate(data_indices):
+                        cur_row.append(convert_to_type(values[data_idx], types[idx]))
+                    data_args_values[arg_name].append(cur_row)
+        except EOFError:  # Exit if reached EOF or CTRL-D
+            break
+
+    if not data_present:
+        sys.exit(0)
+
+    # Handle callbacks.
+    rec_eval = None
+    if "callbacks" in params and params["callbacks"] is not None:
+        callbacks = params["callbacks"]
+        callbacks = [callbacks] if not isinstance(callbacks, list) else callbacks
+        for i, callback in enumerate(callbacks):
+            c_module_name = callback["module"]
+            c_func_name = callback["func_name"]
+            c_kwargs = callback["kwargs"]
+            c_module = importlib.import_module(c_module_name)
+            if c_func_name == "record_evaluation":
+                # record_evaluation function takes empty dict. If the argument has elements in the
+                # dict, they will be deleted as per the documentation from lightgbm as described below:
+                # eval_result (dict) -
+                #   Dictionary used to store all evaluation results of all validation sets. This should
+                #   be initialized outside of your call to record_evaluation() and should be empty. Any
+                #   initial contents of the dictionary will be deleted.
+                rec_eval = {}
+                callbacks[i] = getattr(c_module, c_func_name)(rec_eval)
+            else:
+                callbacks[i] = getattr(c_module, c_func_name)(**c_kwargs)
+
+        params["callbacks"] = callbacks
+
+    # Update data as numpy arrays.
+    for arg_name in data_args_values:
+        np_values = np.array(data_args_values[arg_name])
+        data_args_values[arg_name] = np_values
+        if arg_name == "sample_weight":
+            data_args_values[arg_name] = np_values.ravel()
+
+    # Combine all arguments.
+    all_args = {**data_args_values, **params}
+
+    trained_model = getattr(model, func_name)(**all_args)
+
+    model_data = 0
+    if func_name == "fit":
+        model_str = pickle.dumps(trained_model)
+
+        if is_lake_system:
+            model_file_path = f"/tmp/{model_file_prefix}_{partition_join}.pickle"
+
+        # Write to file in Vantage, to be used in predict/scoring.
+        with open(model_file_path, "wb") as fp:
+            fp.write(model_str)
+
+        model_data = model_file_path if is_lake_system else base64.b64encode(model_str)
+
+    elif func_name == "score":
+        model_data = trained_model
+
+    print(*(data_partition_column_values + [model_data]), sep=DELIMITER)

teradataml/data/scripts/mapper.R

@@ -0,0 +1,20 @@
+#!/usr/bin/Rscript
+
+# Read input from STDIN (standard input)
+input <- file("stdin", open = "r")
+while (length(line <- readLines(input, n = 1)) > 0) {
+  # Remove leading and trailing whitespace
+  line <- trimws(line)
+  # Split the line into words
+  words <- unlist(strsplit(line, "\\s+"))
+  # Increase counters
+  for (word in words) {
+    # Write the results to STDOUT (standard output);
+    # what we output here will be the input for the
+    # Reduce step, i.e., the input for reducer.py
+    #
+    # tab-delimited; the trivial word count is 1
+    cat(paste(word,"\t","1\n"))
+  }
+}
+close(input)

teradataml/data/scripts/mapper.py

@@ -0,0 +1,16 @@
+#!/usr/bin/python
+import sys
+# input comes from STDIN (standard input)
+for line in sys.stdin:
+    # remove leading and trailing whitespace
+    line = line.strip()
+    # split the line into words
+    words = line.split()
+    # increase counters
+    for word in words:
+        # write the results to STDOUT (standard output);
+        # what we output here will be the input for the
+        # Reduce step, i.e. the input for reducer.py
+        #
+        # tab-delimited; the trivial word count is 1
+        print ('%s\t%s' % (word, 1))

teradataml/data/scripts/mapper_replace.py

@@ -0,0 +1,16 @@
+#!/usr/bin/python
+import sys
+# input comes from STDIN (standard input)
+for line in sys.stdin:
+    # remove leading and trailing whitespace
+    line = line.strip()
+    # split the line into words
+    words = line.split()
+    # increase counters
+    for newword in words:
+        # write the results to STDOUT (standard output);
+        # what we output here will be the input for the
+        # Reduce step, i.e. the input for reducer.py
+        #
+        # tab-delimited; the trivial word count is 1
+        print ('%s,%s' % (newword, 1))

teradataml/data/scripts/sklearn/sklearn_fit.py

@@ -0,0 +1,205 @@
+import sys
+import numpy as np
+import pickle
+import base64
+import os
+from contextlib import contextmanager
+
+DELIMITER = '\t'
+
+
+@contextmanager
+def suppress_stderr():
+    """
+    Function to suppress the warnings(lake systems treats warnings as errors).
+    """
+    with open(os.devnull, "w") as devnull:
+        old_stderr = sys.stderr
+        sys.stderr = devnull
+        try:
+            yield
+        finally:
+            sys.stderr = old_stderr
+
+## On Lake system warnings raised by script are treated as a errors. 
+## Hence, to suppress it putting the under suppress_stderr().
+with suppress_stderr():
+    def get_values_list(values, types, model_obj):
+        ret_vals = []
+        for i, val in enumerate(values):
+            if type(model_obj).__name__ == "MultiLabelBinarizer" and val == "":
+                continue
+            ret_vals.append(convert_to_type(val, types[i]))
+        return ret_vals
+
+    def convert_to_type(val, typee):
+        if typee == 'int':
+            return int(val) if val != "" else np.nan
+        if typee == 'float':
+            if isinstance(val, str):
+                val = val.replace(' ', '')
+            return float(val) if val != "" else np.nan
+        if typee == 'bool':
+            return eval(val) if val != "" else None
+        return str(val) if val != "" else None
+
+    def get_classes_as_list(classes, actual_type):
+        if classes == "None":
+            return None
+        if actual_type == "None":
+            sys.exit("type of class elements is None where class elements exists.")
+
+        # separated by '--'
+        classes = classes.split("--")
+
+        for idx, cls in enumerate(classes):
+            classes[idx] = convert_to_type(cls, actual_type)
+
+        return classes
+
+
+    def splitter(strr, delim=",", convert_to="str"):
+        """
+        Split the string based on delimiter and convert to the type specified.
+        """
+        if strr == "None":
+            return []
+        return [convert_to_type(i, convert_to) for i in strr.split(delim)]
+
+    # Arguments to the Script
+    if len(sys.argv) != 10:
+        # 10 arguments command line arguments should be passed to this file.
+        # 1: file to be run
+        # 2. function name
+        # 3. No of feature columns.
+        # 4. No of class labels.
+        # 5. Comma separated indices of partition columns.
+        # 6. Comma separated types of all the data columns.
+        # 7. Model file prefix to generated model file using partition columns.
+        # 8. classes (separated by '--') - should be converted to list. "None" if no classes exists.
+        # 9. type of elements in passed in classes. "None" if no classes exists.
+        # 10. Flag to check the system type. True, means Lake, Enterprise otherwise
+        sys.exit("10 arguments command line arguments should be passed: file to be run,"
+                 " function name, no of feature columns, no of class labels, comma separated indices"
+                 " of partition columns, comma separated types of all columns, model file prefix ,"
+                 " classes, type of elements in classes and flag to check lake or enterprise.")
+
+    is_lake_system = eval(sys.argv[9])
+    if not is_lake_system:
+        db = sys.argv[0].split("/")[1]
+    function_name = sys.argv[1]
+    n_f_cols = int(sys.argv[2])
+    n_c_labels = int(sys.argv[3])
+    data_column_types = splitter(sys.argv[5], delim="--")
+    data_partition_column_indices = splitter(sys.argv[4], convert_to="int") # indices are integers.
+    model_file_prefix = sys.argv[6]
+    class_type = sys.argv[8]
+    classes = get_classes_as_list(sys.argv[7], class_type)
+
+    data_partition_column_types = [data_column_types[idx] for idx in data_partition_column_indices]
+
+    model = None
+
+    # Data Format (n_features, k_labels, one data_partition_column):
+    # feature1, feature2, ..., featuren, label1, label2, ... labelk, data_partition_column1, ...,
+    # data_partition_columnn
+    # There can be no labels also.
+
+    # Read data from table through STO and build features and labels.
+    features = []
+    labels = []
+    data_partition_column_values = []
+
+
+    while 1:
+        try:
+            line = input()
+            if line == '':  # Exit if user provides blank line
+                break
+            else:
+                values = line.split(DELIMITER)
+
+                if not data_partition_column_values:
+                    # Partition column values is same for all rows. Hence, only read once.
+                    for i, val in enumerate(data_partition_column_indices):
+                        data_partition_column_values.append(
+                            convert_to_type(values[val], typee=data_partition_column_types[i])
+                            )
+
+                    # Prepare the corresponding model file name and extract model.
+                    partition_join = "_".join([str(x) for x in data_partition_column_values])
+                    # Replace '-' with '_' as '-' because partition_columns can be negative.
+                    partition_join = partition_join.replace("-", "_")
+
+                    model_file_path = f"{model_file_prefix}_{partition_join}"\
+                        if is_lake_system else \
+                        f"./{db}/{model_file_prefix}_{partition_join}"
+
+                    with open(model_file_path, "rb") as fp:
+                        model = pickle.loads(fp.read())
+
+                    if model is None:
+                        sys.exit("Model file is not installed in Vantage.")
+
+                values = get_values_list(values, data_column_types, model)
+                values = values[:-len(data_partition_column_indices)] # Already processed partition columns.
+                features.append(values[:n_f_cols])
+                if n_c_labels > 0:
+                    labels.append(values[n_f_cols:(n_f_cols+n_c_labels)])
+
+
+        except EOFError:  # Exit if reached EOF or CTRL-D
+            break
+
+    if not len(features):
+        sys.exit(0)
+
+    # Fit/partial_fit the model to the data.
+    model_name = model.__class__.__name__
+    if function_name == "partial_fit":
+        if labels and classes:
+            if model_name == "SelectFromModel":
+               features = np.array(features)
+               classes = np.array(classes)
+               labels = np.array(labels).ravel()
+            model.partial_fit(features, labels, classes=classes)
+        elif labels:
+            model.partial_fit(features, labels)
+        elif classes:
+            model.partial_fit(features, classes=classes)
+        else:
+            model.partial_fit(features)
+    elif function_name == "fit":
+        np_func_list = ["OneVsRestClassifier", "LabelBinarizer", "TSNE"]
+        if labels:
+            # For IsotonicRegression, fit() accepts training target as
+            # y: array-like of shape (n_samples,).
+            if model_name in ["CalibratedClassifierCV", "GaussianProcessClassifier", "GenericUnivariateSelect",
+                              "IsotonicRegression", "LinearSVC", "GridSearchCV", "LinearDiscriminantAnalysis", "RFECV",
+                              "RFE", "RandomizedSearchCV", "SelectFdr", "SelectFpr", "SelectFromModel", "SelectFwe",
+                              "SelectKBest", "SelectPercentile", "SequentialFeatureSelector", "GaussianNB",
+                              "QuadraticDiscriminantAnalysis"]:
+                labels = np.array(labels).reshape(-1)
+            if model_name in np_func_list:
+                labels = np.array(labels)
+                features = np.array(features)
+            model.fit(features, labels)
+        else:
+            if model_name in np_func_list:
+                features = np.array(features)
+            model.fit(features)
+
+    model_str = pickle.dumps(model)
+
+    if is_lake_system:
+        model_file_path = f"/tmp/{model_file_prefix}_{partition_join}.pickle"
+
+    # Write to file in Vantage, to be used in predict/scoring.
+    with open(model_file_path, "wb") as fp:
+        fp.write(model_str)
+
+    model_data = model_file_path if is_lake_system \
+        else base64.b64encode(model_str)
+
+    # Print the model to be read from script.
+    print(*(data_partition_column_values + [model_data]), sep=DELIMITER)