teradataml 20.0.0.0__py3-none-any.whl → 20.0.0.1__py3-none-any.whl

teradataml/data/scripts/sklearn/sklearn_transform.py

@@ -4,41 +4,24 @@ import os
 import sys
 import numpy as np
 
-# The below import is needed to convert sparse matrix to dense array as sparse matrices are NOT
-# supported in Vantage.
-# This is in scipy 1.10.0. Might vary based on scipy version.
-from scipy.sparse import csr_matrix
-
 DELIMITER = '\t'
 
-def get_value(value):
-    ret_val = value
-    try:
-        ret_val = float(value.replace(' ', ''))
-    except Exception as ex:
-        # If the value can't be converted to float, then it is string.
-        pass
-    return ret_val
-
-
-def get_values_list(values, ignore_none=True):
+def get_values_list(values, types):
     ret_vals = []
-    for val in values:
-        if val == "" and ignore_none:
-            # Empty cell value in the database table.
-            continue
-        ret_vals.append(get_value(val))
-
+    for i, val in enumerate(values):
+        ret_vals.append(convert_to_type(val, types[i]))
     return ret_vals
 
 def convert_to_type(val, typee):
     if typee == 'int':
-        return int(val)
+        return int(val) if val != "" else np.nan
     if typee == 'float':
-        return float(val)
+        if isinstance(val, str):
+            val = val.replace(' ', '')
+        return float(val) if val != "" else np.nan
     if typee == 'bool':
-        return eval(val)
-    return str(val)
+        return eval(val) if val != "" else None
+    return str(val) if val != "" else None
 
 def splitter(strr, delim=",", convert_to="str"):
     """
@@ -54,7 +37,7 @@ def get_output_data(trans_values, func_name, model_obj, n_c_labels):
     # supported in Vantage.
     module_name = model_obj.__module__.split("._")[0]
 
-    if isinstance(trans_values, csr_matrix):
+    if type(trans_values).__name__ in ["csr_matrix", "csc_matrix"]:
         trans_values = trans_values.toarray()
 
     if module_name == "sklearn.cross_decomposition" and n_c_labels > 0 and func_name == "transform":
@@ -86,13 +69,13 @@ if len(sys.argv) != 8:
     # 3. No of feature columns.
     # 4. No of class labels.
     # 5. Comma separated indices of partition columns.
-    # 6. Comma separated types of the partition columns.
+    # 6. Comma separated types of all the data columns.
     # 7. Model file prefix to generated model file using partition columns.
     # 8. Flag to check the system type. True, means Lake, Enterprise otherwise.
     sys.exit("8 arguments should be passed to this file - file to be run, function name, "\
-             "no of feature columns, no of class labels, comma separated indices and types of "\
-             "partition columns, model file prefix to generate model file using partition "\
-             "columns and flag to check lake or enterprise.")
+             "no of feature columns, no of class labels, comma separated indices of partition "\
+             "columns, comma separated types of all columns, model file prefix to generate model "\
+             "file using partition columns and flag to check lake or enterprise.")
 
 is_lake_system = eval(sys.argv[7])
 if not is_lake_system:
@@ -100,10 +83,12 @@ if not is_lake_system:
 func_name = sys.argv[1]
 n_f_cols = int(sys.argv[2])
 n_c_labels = int(sys.argv[3])
-data_partition_column_types = splitter(sys.argv[5])
+data_column_types = splitter(sys.argv[5], delim="--")
 data_partition_column_indices = splitter(sys.argv[4], convert_to="int") # indices are integers.
 model_file_prefix = sys.argv[6]
 
+data_partition_column_types = [data_column_types[idx] for idx in data_partition_column_indices]
+
 model = None
 data_partition_column_values = []
 
@@ -119,6 +104,7 @@ while 1:
             break
         else:
             values = line.split(DELIMITER)
+            values = get_values_list(values, data_column_types)
             if not data_partition_column_values:
                 # Partition column values is same for all rows. Hence, only read once.
                 for i, val in enumerate(data_partition_column_indices):
@@ -141,10 +127,10 @@ while 1:
                 if not model:
                     sys.exit("Model file is not installed in Vantage.")
 
-            f_ = get_values_list(values[:n_f_cols])
+            f_ = values[:n_f_cols]
             if n_c_labels > 0:
                 # Labels are present in last column.
-                l_ = get_values_list(values[n_f_cols:n_f_cols+n_c_labels])
+                l_ = values[n_f_cols:n_f_cols+n_c_labels]
                 # predict() now takes 'y' also for it to return the labels from script. Skipping 'y'
                 # in function call. Generally, 'y' is passed to return y along with actual output.
                 try:
@@ -170,10 +156,16 @@ while 1:
             result_list += get_output_data(trans_values=trans_values, func_name=func_name,
                                            model_obj=model, n_c_labels=n_c_labels)
 
-            print(*(data_partition_column_values +
-                    ['' if (val is None or math.isnan(val) or math.isinf(val))
-                     else val for val in result_list]),
-                     sep=DELIMITER)
+            for i, val in enumerate(result_list):
+                if (val is None or (not isinstance(val, str) and (math.isnan(val) or math.isinf(val)))):
+                    result_list[i] = ""
+                # MissingIndicator returns boolean values. Convert them to 0/1.
+                elif val == False:
+                    result_list[i] = 0
+                elif val == True:
+                    result_list[i] = 1
+
+            print(*(data_partition_column_values + result_list), sep=DELIMITER)
 
     except EOFError:  # Exit if reached EOF or CTRL-D
         break

teradataml/data/teradataml_example.json

@@ -1271,6 +1271,83 @@
     "height":"INTEGER",
     "weight":"INTEGER",
     "bmi":"INTEGER"
+  },
+  "breast_cancer":{
+    "id":"BIGINT",  
+    "diagnosis":"VARCHAR(20)", 
+    "radius_mean":"FLOAT",
+    "texture_mean":"FLOAT",
+    "perimeter_mean":"FLOAT",
+    "area_mean":"FLOAT",
+    "smoothness_mean":"FLOAT",
+    "compactness_mean":"FLOAT",
+    "concavity_mean":"FLOAT",
+    "concave_points_mean":"FLOAT",
+    "symmetry_mean":"FLOAT",
+    "fractal_dimension_mean":"FLOAT",
+    "radius_se":"FLOAT",
+    "texture_se":"FLOAT",
+    "perimeter_se":"FLOAT",
+    "area_se":"FLOAT",
+    "smoothness_se":"FLOAT",
+    "compactness_se":"FLOAT",
+    "concavity_se":"FLOAT",
+    "concave_points_se":"FLOAT",
+    "symmetry_se":"FLOAT",
+    "fractal_dimension_se":"FLOAT",
+    "radius_worst":"FLOAT",
+    "texture_worst":"FLOAT",
+    "perimeter_worst":"FLOAT",
+    "area_worst":"FLOAT",
+    "smoothness_worst":"FLOAT",
+    "compactness_worst":"FLOAT",
+    "concavity_worst":"FLOAT",
+    "concave_points_worst":"FLOAT",
+    "symmetry_worst":"FLOAT",
+    "fractal_dimension_worst":"FLOAT"
+  },
+  "bike_sharing" :{
+    "instant":"BIGINT", 
+    "dteday":"DATE FORMAT 'dd-mm-yyyy'", 
+    "season":"BIGINT", 
+    "yr":"BIGINT", 
+    "mnth":"BIGINT", 
+    "holiday":"BIGINT", 
+    "weekday":"BIGINT", 
+    "workingday":"BIGINT", 
+    "weathersit":"BIGINT", 
+    "temp":"FLOAT",
+    "atemp":"FLOAT",
+    "hum":"FLOAT",
+    "windspeed":"FLOAT",
+    "casual":"BIGINT", 
+    "registered":"BIGINT", 
+    "cnt":"BIGINT"
+  },
+  "bank_marketing":{
+    "age":"BIGINT",
+    "job":"VARCHAR(20)",
+    "marital":"VARCHAR(20)",
+    "education":"VARCHAR(20)",
+    "default_value":"VARCHAR(20)",
+    "balance":"BIGINT",
+    "housing":"VARCHAR(20)",
+    "loan":"VARCHAR(20)",
+    "contact":"VARCHAR(20)",
+    "day_of_month":"BIGINT",
+    "month_of_year":"VARCHAR(20)",
+    "duration":"BIGINT",
+    "campaign":"BIGINT",
+    "pdays":"BIGINT",
+    "previous":"BIGINT",
+    "poutcome":"VARCHAR(20)",
+    "deposit":"VARCHAR(20)"
+  },
+  "advertising":{
+    "TV":"FLOAT",
+    "radio":"FLOAT",
+    "newspaper":"FLOAT",
+    "sales":"FLOAT"
   }
 
 }

teradataml/data/ztest_example.json

@@ -0,0 +1,16 @@
+{
+    "roc_input" : {
+      "model_id" : "integer",
+      "id" : "integer",
+      "observation" : "integer",
+      "probability" : "real"
+    },
+    "boston2cols":{
+        "groupName":"VARCHAR(40)",
+        "groupValue":"REAL"
+    },
+    "insect2Cols":{
+        "groupName":"VARCHAR(40)",
+        "groupValue":"INTEGER"
+    }
+  }

teradataml/dataframe/copy_to.py

@@ -569,7 +569,6 @@ def copy_to_sql(df, table_name,
                                        chunksize, is_pti, timecode_column,
                                        sequence_column, match_column_order)
 
-
         # df is a teradataml DataFrame object (to_sql wrapper used)
         elif isinstance(df, tdmldf.DataFrame):
             df_column_list = [col.name for col in df._metaexpr.c]
@@ -578,7 +577,13 @@ def copy_to_sql(df, table_name,
                 # Reorder the column list to reposition the timecode and sequence columns
                 df_column_list = _reorder_insert_list_for_pti(df_column_list, timecode_column, sequence_column)
 
-            df_utils._insert_all_from_table(table_name, df._table_name, df_column_list, schema_name, temporary)
+            source_tbl_name = UtilFuncs._extract_table_name(df._table_name)
+            from_schema_name = UtilFuncs._extract_db_name(df._table_name)
+
+            df_utils._insert_all_from_table(table_name, source_tbl_name, df_column_list,
+                                            to_schema_name=schema_name,
+                                            from_schema_name=from_schema_name,
+                                            temporary=temporary)
 
         # While table name conflict is present, Delete the source table after creation of temporary table.
         # Rename the temporary table to destination table name. 
@@ -1656,7 +1661,7 @@ def _get_all_sqlalchemy_mappings():
     EXAMPLES:
         _get_all_sqlalchemy_mappings()
     """
-    teradata_types_map = {'int32':INTEGER(), 'int64':BIGINT(),
+    teradata_types_map = {'int32':INTEGER(), 'int64':BIGINT(), "Int64": INTEGER(),
                           'object':VARCHAR(configure.default_varchar_size,charset='UNICODE'),
                           'O':VARCHAR(configure.default_varchar_size,charset='UNICODE'),
                           'float64':FLOAT(), 'float32':FLOAT(), 'bool':BYTEINT(),