teradataml 20.0.0.1__py3-none-any.whl → 20.0.0.3__py3-none-any.whl

teradataml/data/scripts/sklearn/sklearn_transform.py

@@ -31,39 +31,83 @@ def splitter(strr, delim=",", convert_to="str"):
         return []
     return [convert_to_type(i, convert_to) for i in strr.split(delim)]
 
+def should_convert(t_val, py_type):
+    """
+    Function to check type of value and whether value is nan and infinity.
+    """
+    return not isinstance(t_val, eval(py_type)) and not math.isinf(t_val) and not math.isnan(t_val)
+
+def convert_value(t_val, py_type):
+    """
+    Function to convert value to specified python type.
+    """
+    return convert_to_type(t_val, py_type) if should_convert(t_val, py_type) else t_val
+
 # Process output returned by sklearn function.
-def get_output_data(trans_values, func_name, model_obj, n_c_labels):
-    # Converting sparse matrix to dense array as sparse matrices are NOT
+def get_output_data(trans_values, func_name, model_obj, n_c_labels, n_out_columns):
+    # Converting    sparse matrix to dense array as sparse matrices are NOT
     # supported in Vantage.
     module_name = model_obj.__module__.split("._")[0]
 
-    if type(trans_values).__name__ in ["csr_matrix", "csc_matrix"]:
-        trans_values = trans_values.toarray()
+    # Converting the translated values into corresponding the return column's 
+    # python type.
+    if (func_name == "decision_path" or return_columns_python_types is None \
+            or not isinstance(trans_values, np.ndarray)):
+        trans_values_list = trans_values
+    else:
+        # Conversion.....
+        trans_values_list = []
+        for trans_value in trans_values.tolist():
+            if not isinstance(trans_value, list):
+                trans_value = [trans_value]
+
+            converted_list = []
+            if len(return_columns_python_types) == len(trans_value):
+                for t_val, py_type in zip(trans_value, return_columns_python_types):
+                    converted_list.append(convert_value(t_val, py_type))
+            ## transform() is having only 1 python return type, But it actually returns more than 1 column  
+            else:
+                for t_val in trans_value:
+                    converted_list.append(convert_value(t_val, "".join(return_columns_python_types)))
+
+            trans_values_list.append(converted_list)
+
+    if type(trans_values_list).__name__ in ["csr_matrix", "csc_matrix"]:
+        trans_values_list = trans_values_list.toarray()
 
     if module_name == "sklearn.cross_decomposition" and n_c_labels > 0 and func_name == "transform":
         # For cross_decomposition, output is a tuple of arrays when label columns are provided
         # along with feature columns for transform function. In this case, concatenate the
         # arrays and return the combined values.
-        if isinstance(trans_values, tuple):
-            return np.concatenate(trans_values, axis=1).tolist()[0]
+        if isinstance(trans_values_list, tuple):
+            return np.concatenate(trans_values_list, axis=1).tolist()[0]
 
-    if isinstance(trans_values[0], np.ndarray) \
-            or isinstance(trans_values[0], list) \
-            or isinstance(trans_values[0], tuple):
+    if isinstance(trans_values_list[0], np.ndarray) \
+            or isinstance(trans_values_list[0], list) \
+            or isinstance(trans_values_list[0], tuple):
         # Here, the value returned by sklearn function is list type.
-        opt_list = list(trans_values[0])
+        opt_list = list(trans_values_list[0])
+
+        if len(opt_list) < n_out_columns:
+            # If the output list is less than the required number of columns, append
+            # empty strings to the list.
+            opt_list += [""] * (n_out_columns - len(opt_list))
+
         if func_name == "inverse_transform" and type(model_obj).__name__ == "MultiLabelBinarizer":
             # output array "trans_values[0]" may not be of same size. It should be of
             # maximum size of `model.classes_`
             # Append None to last elements.
             if len(opt_list) < len(model_obj.classes_):
                 opt_list += [""] * (len(model_obj.classes_) - len(opt_list))
+
         return opt_list
-    return [trans_values[0]]
+
+    # Only one element is returned by the function.
+    return [trans_values_list[0]]
 
 # Arguments to the Script
-if len(sys.argv) != 8:
-    # 8 arguments command line arguments should be passed to this file.
+if len(sys.argv) != 10:
+    # 10 arguments command line arguments should be passed to this file.
     # 1: file to be run
     # 2. function name (Eg. predict, fit etc)
     # 3. No of feature columns.
@@ -71,13 +115,17 @@ if len(sys.argv) != 8:
     # 5. Comma separated indices of partition columns.
     # 6. Comma separated types of all the data columns.
     # 7. Model file prefix to generated model file using partition columns.
-    # 8. Flag to check the system type. True, means Lake, Enterprise otherwise.
-    sys.exit("8 arguments should be passed to this file - file to be run, function name, "\
-             "no of feature columns, no of class labels, comma separated indices of partition "\
-             "columns, comma separated types of all columns, model file prefix to generate model "\
-             "file using partition columns and flag to check lake or enterprise.")
+    # 8. Number of columns to be returned by the sklearn's transform function.
+    # 9. Flag to check the system type. True, means Lake, Enterprise otherwise.
+    # 10. Python types of returned/transfromed columns.
+    sys.exit("10 arguments should be passed to this file - file to be run, function name, "\
+                 "no of feature columns, no of class labels, comma separated indices of partition "\
+                 "columns, comma separated types of all columns, model file prefix to generate model "\
+                 "file using partition columns, number of columns to be returnd by sklearn's "\
+                 "transform function, flag to check lake or enterprise and Python types of "\
+                 "returned/transfromed columns.")
 
-is_lake_system = eval(sys.argv[7])
+is_lake_system = eval(sys.argv[8])
 if not is_lake_system:
     db = sys.argv[0].split("/")[1]
 func_name = sys.argv[1]
@@ -86,17 +134,29 @@ n_c_labels = int(sys.argv[3])
 data_column_types = splitter(sys.argv[5], delim="--")
 data_partition_column_indices = splitter(sys.argv[4], convert_to="int") # indices are integers.
 model_file_prefix = sys.argv[6]
+# sys.argv[9] will contain a string of python datatypes with '--'
+# separator OR a single datatype OR None in string format.
+ret_col_argv = sys.argv[9]
+if ret_col_argv == "None":
+    return_columns_python_types = eval(ret_col_argv)
+else:
+    return_columns_python_types = splitter(ret_col_argv, delim="--")
+
+no_of_output_columns = int(sys.argv[7])
 
 data_partition_column_types = [data_column_types[idx] for idx in data_partition_column_indices]
 
 model = None
 data_partition_column_values = []
 
+all_rows_input = []
+
 # Data Format:
 # feature1, feature2, ..., featuren, label1, label2, ... labelk, data_partition_column1, ..., 
 # data_partition_columnn.
 # label is optional (it is present when label_exists is not "None")
 
+model_name = ""
 while 1:
     try:
         line = input()
@@ -128,9 +188,48 @@ while 1:
                     sys.exit("Model file is not installed in Vantage.")
 
             f_ = values[:n_f_cols]
+
+            model_name = model.__class__.__name__
+            np_func_list = ["ClassifierChain", "EllipticEnvelope", "MinCovDet",  
+                            "FeatureAgglomeration", "LabelBinarizer", "MultiLabelBinarizer",
+                            "BernoulliRBM"]
+
+            # MissingIndicator's transform() and SimpleImputer's inverse_transform() requires processing
+            # the entire dataset simultaneously, rather than on a row-by-row basis.
+
+            # Error getting during row-by-row processing of MissingIndicator - 
+            # "ValueError: MissingIndicator does not support data with dtype <U13. 
+            # Please provide either a numeric array (with a floating point or 
+            # integer dtype) or categorical data represented ei
+
+            # Error getting during row-by-row processing of SimpleImputer -
+            # "IndexError: index 3 is out of bounds for axis 1 with size 3".
+            if ((model_name == "MissingIndicator" and func_name == "transform") or \
+                (model_name == "SimpleImputer" and func_name == "inverse_transform") or \
+                    (model_name in ["EllipticEnvelope", "MinCovDet"]
+                        and func_name == "correct_covariance")):
+                all_rows_input.append(f_)
+                continue
+
+            f__ = np.array([f_]) if model_name in np_func_list else [f_]
+
+            # transform() function in these functions generate different number of output columns and
+            # NULLS/NaNs are appended to the end of the output.
+            # If we run inverse_transform() on these models, it will take same number of input columns
+            # with NULLs/NaNs but those NULLs/NaNs should be ignored while reading the input to 
+            # inverse_transform() function.
+            models_with_all_null_in_last_cols = ["SelectFpr", "SelectFdr", "SelectFwe", "SelectFromModel", "RFECV"]
+            if model_name in models_with_all_null_in_last_cols and func_name == "inverse_transform":
+                # Remove NULLs/NaNs from the end of one input row.
+                _f  = np.array([f_])
+                _f = _f[~np.isnan(_f)]
+                f__ = [_f.tolist()]
+
             if n_c_labels > 0:
                 # Labels are present in last column.
                 l_ = values[n_f_cols:n_f_cols+n_c_labels]
+
+                l__ = np.array([l_]) if model_name in np_func_list else [l_]
                 # predict() now takes 'y' also for it to return the labels from script. Skipping 'y'
                 # in function call. Generally, 'y' is passed to return y along with actual output.
                 try:
@@ -138,23 +237,24 @@ while 1:
                     # used 'in' in if constion, as model.__module__ is giving 
                     # 'sklearn.cross_decomposition._pls'.  
                     if "cross_decomposition" in model.__module__:
-                        trans_values = getattr(model, func_name)(X=np.array([f_]), Y=np.array([l_]))
+                        trans_values = getattr(model, func_name)(X=f__, Y=l__)
                     else:
-                        trans_values = getattr(model, func_name)(X=np.array([f_]), y=np.array([l_]))
+                        trans_values = getattr(model, func_name)(X=f__, y=l__)
 
                 except TypeError as ex:
                     # Function which does not accept 'y' like predict_proba() raises error like
                     # "TypeError: predict_proba() takes 2 positional arguments but 3 were given".
-                    trans_values = getattr(model, func_name)(np.array([f_]))
+                    trans_values = getattr(model, func_name)(f__)
             else:
                 # If class labels do not exist in data, don't read labels, read just features.
-                trans_values = getattr(model, func_name)(np.array([f_]))
+                trans_values = getattr(model, func_name)(f__)
 
             result_list = f_
             if n_c_labels > 0 and func_name in ["predict", "decision_function"]:
                 result_list += l_
             result_list += get_output_data(trans_values=trans_values, func_name=func_name,
-                                           model_obj=model, n_c_labels=n_c_labels)
+                                           model_obj=model, n_c_labels=n_c_labels,
+                                           n_out_columns=no_of_output_columns)
 
             for i, val in enumerate(result_list):
                 if (val is None or (not isinstance(val, str) and (math.isnan(val) or math.isinf(val)))):
@@ -169,3 +269,41 @@ while 1:
 
     except EOFError:  # Exit if reached EOF or CTRL-D
         break
+
+
+# MissingIndicator and SimpleImputer needs processing of all the dataset at the same time, instead of row by row. 
+# Hence, handling it outside of the while loop
+if model_name == "MissingIndicator" and func_name == "transform" or \
+    (model_name == "SimpleImputer" and func_name == "inverse_transform"):
+    if model_name == "SimpleImputer":
+        all_rows_input = np.array(all_rows_input)
+    m_out = getattr(model, func_name)(all_rows_input)
+
+    if type(m_out).__name__ in ["csr_matrix", "csc_matrix"]:
+        m_out = m_out.toarray()
+
+    for j in range(len(all_rows_input)):
+        m_out_list = get_output_data(trans_values=[m_out[j]], func_name=func_name,
+                                     model_obj=model, n_c_labels=n_c_labels,
+                                     n_out_columns=no_of_output_columns)
+
+        result_list = list(all_rows_input[j]) + list(m_out_list)
+
+        for i, val in enumerate(result_list):
+            if (val is None or (not isinstance(val, str) and (math.isnan(val) or math.isinf(val)))):
+                result_list[i] = ""
+            # MissingIndicator returns boolean values. Convert them to 0/1.
+            elif val == False:
+                result_list[i] = 0
+            elif val == True:
+                result_list[i] = 1
+
+        print(*(data_partition_column_values + result_list), sep=DELIMITER)
+
+## correct_covariance() requires processing of all the input rows at the same time.
+## It returns the output dataset  in (n_features, n_features) shape, i.e., based on
+## no. of columns.
+if model_name in ["EllipticEnvelope", "MinCovDet"] and func_name == "correct_covariance":
+    result_list = model.correct_covariance(np.array(all_rows_input))
+    for l, vals in enumerate(result_list):
+        print(*(data_partition_column_values + vals.tolist()), sep=DELIMITER)

teradataml/data/star_pivot.csv

@@ -0,0 +1,8 @@
+country,state,yr,qtr,sales,cogs,rating
+USA,CA,2001,Q1,30,15,A
+USA,NY,2001,Q1,45,25,D
+USA,CA,2001,Q2,50,20,A
+USA,CA,2001,Q2,5,5,B
+Canada,ON,2001,Q2,10,0,B
+Canada,BC,2001,Q3,15,0,A
+Canada,BC,2001,Q3,10,0,A

teradataml/data/target_udt_data.csv

@@ -0,0 +1,8 @@
+id,array_col
+1,"3.33e-05,0.2,0.1"
+2,"0.5,0.4,0.42"
+3,"1,0.8,0.9"
+4,"0.01,0.4,0.2"
+5,"0.93,0.4,0.7"
+6,"0.83,0.3,0.6"
+7,"0.73,0.5,0.7"

teradataml/data/templates/open_source_ml.json

@@ -2,7 +2,9 @@
   "env_specs": [
     {
       "env_name": "openml_env",
-      "libs": "scikit-learn",
+      "libs": ["scikit-learn==1.5.1", "joblib==1.4.2", "numpy==1.23.5",
+               "scipy==1.14.0", "threadpoolctl==3.5.0", "lightgbm==3.3.3",
+               "pandas==2.2.3"],
       "desc": "DONT DELETE: OpenML environment"
     }
   ]

teradataml/data/teradataml_example.json

@@ -1348,6 +1348,25 @@
     "radio":"FLOAT",
     "newspaper":"FLOAT",
     "sales":"FLOAT"
+  },
+  "timestamp_data":{
+    "id": "INTEGER",
+    "timestamp_col": "VARCHAR(50)",
+    "timestamp_col1": "BIGINT",
+    "format_col": "VARCHAR(50)",
+    "timezone_col": "VARCHAR(50)"
+  },
+  "interval_data":{
+    "id": "INTEGER",
+    "int_col": "BIGINT",
+    "value_col": "VARCHAR(30)",
+    "value_col1": "VARCHAR(30)",
+    "str_col1": "VARCHAR(30)",
+    "str_col2": "VARCHAR(30)"
+  },
+  "url_data": {
+    "id": "INTEGER",
+    "urls": "VARCHAR(60)",
+    "part": "VARCHAR(20)"
   }
-
 }

teradataml/data/timestamp_data.csv

@@ -0,0 +1,4 @@
+id,timestamp_col,timestamp_col1,format_col,timezone_col
+0,"2015-01-08 00:00:12.2",123456,"YYYY-MM-DD HH24:MI:SS.FF6","GMT"
+1,"2015-01-08 13:00",878986,"YYYY-MM-DD HH24:MI","America Pacific"
+2,"2015-01-08 00:00:12.2+10:00",45678910234,"YYYY-MM-DD HH24:MI:SS.FF6 TZH:TZM","GMT+10"

teradataml/data/titanic_dataset_unpivoted.csv

@@ -0,0 +1,19 @@
+passenger,AttributeName,AttributeValue,survived
+61,pclass,3,0
+1000,pclass,3,1
+40,pclass,3,1
+21,pclass,2,0
+61,gender,male,0
+1000,gender,,1
+40,gender,female,1
+21,gender,male,0
+2,pclass,1,1
+16,pclass,2,1
+7,pclass,1,0
+2,gender,female,1
+16,gender,female,1
+7,gender,male,0
+10,pclass,2,1
+4,pclass,1,1
+10,gender,female,1
+4,gender,female,1

teradataml/data/uaf_example.json

@@ -471,5 +471,59 @@
     "CONF_OFF_v": "FLOAT",
     "CONF_LOW_v": "FLOAT",
     "CONF_HI_v": "FLOAT"
+  },
+  "dwt_dataTable":{
+	  "id": "INTEGER",
+	  "rowi": "INTEGER",
+	  "v": "FLOAT"
+  },
+  "dwt_filterTable":{
+	  "id": "INTEGER",
+	  "seq": "INTEGER",
+	  "lo": "FLOAT",
+	  "hi": "FLOAT"
+  },
+  "idwt_dataTable":{
+	  "id": "INTEGER",
+	  "rowi": "INTEGER",
+	  "approx": "FLOAT",
+	  "detail": "FLOAT"
+  },
+  "idwt_filterTable":{
+	  "id": "INTEGER",
+	  "seq": "INTEGER",
+	  "lo": "FLOAT",
+	  "hi": "FLOAT"
+  },
+   "dwt2d_dataTable":{
+	  "id": "INTEGER",
+	  "x": "INTEGER",
+	  "y": "INTEGER",
+	  "v": "FLOAT"
+  },
+  "idwt2d_dataTable":{
+	  "id": "INTEGER",
+	  "x": "INTEGER",
+	  "y": "INTEGER",
+	  "v": "FLOAT"
+  },
+  "covid_confirm_sd":{
+	  "city": "VARCHAR(15)",
+      "row_axis": "INTEGER",
+      "cnumber": "INTEGER"
+  },
+  "real_values":{
+	  "TD_TIMECODE": "TIMESTAMP(0)",
+      "id": "INTEGER",
+      "val": "FLOAT",
+      "<PTI_CLAUSE>": "(TIMESTAMP(0), DATE '2020-01-01', HOURS(1), COLUMNS(id), nonsequenced)"
+  },
+  "windowdfft":{
+    "id": "INTEGER",
+    "row_i": "INTEGER",
+    "v1": "FLOAT",
+    "v2": "FLOAT",
+    "v3": "FLOAT",
+    "v4": "FLOAT"
   }
-}
+}

teradataml/data/unpivot_example.json

@@ -6,5 +6,20 @@
     "temp": "integer",
     "pressure": "real",
     "dewpoint": "varchar(30)"
+  },
+  "titanic_dataset_unpivoted":{
+    "passenger": "integer",
+    "AttributeName": "varchar(30)",
+    "AttributeValue": "varchar(30)",
+    "survived": "integer"
+  },
+  "star_pivot":{
+    "country": "varchar(30)",
+    "state": "varchar(30)",
+    "yr": "integer",
+    "qtr": "varchar(30)",
+    "sales": "integer",
+    "cogs": "integer",
+    "rating": "varchar(30)"
   }
 }

teradataml/data/url_data.csv

@@ -0,0 +1,9 @@
+"id","urls","part"
+0,"http://example.com:8080/path","FILE"
+1,"ftp://example.net:21/path","PATH"
+2,"https://example.net/path4/path5/path6?query4=value4#fragment3","REF"
+3,"https://www.facebook.com","HOST"
+4,"https://teracloud-pod-services-pod-account-service.dummyvalue.production.pods.teracloud.ninja/v1/accounts/acc-dummyvalue/user-environment-service/api/v1/","QUERY"
+5,"http://pg.example.ml/path150#fragment90","AUTHORITY"
+6,"smtp://user:password@smtp.example.com:21/file.txt","USERINFO"
+7,"https://www.google.com","PROTOCOL"

teradataml/data/vectordistance_example.json

@@ -22,5 +22,9 @@
     "CallDuration": "REAL",
     "DataCounter": "REAL",
     "SMS": "REAL"
+  },
+  "target_udt_data":{
+    "id": "INTEGER", 
+    "array_col":"AIVector"
   }
 }

teradataml/data/windowdfft.csv

@@ -0,0 +1,16 @@
+id,row_i,v1,v2,v3,v4
+3,1,0.0,1.4,1.0,1.0
+3,2,1.0,2.4,2.0,2.0
+3,3,2.0,3.4,3.0,3.0
+3,4,3.0,4.6,4.0,4.0
+3,5,0.0,5.9,5.0,5.0
+3,6,1.0,6.7,6.0,6.0
+3,7,2.0,7.7,7.0,7.0
+3,8,3.0,8.7,8.0,8.0
+3,9,0.0,9.9,9.0,9.0
+3,10,1.0,10.2,10.0,10.0
+3,11,2.0,11.2,11.0,11.0
+3,12,3.0,12.2,12.0,12.0
+3,13,1.0,10.2,13.0,13.0
+3,14,2.0,11.2,14.0,14.0
+3,15,3.0,12.2,15.0,15.0

teradataml/dataframe/copy_to.py

@@ -30,7 +30,7 @@ from teradatasql import OperationalError
 from teradataml.common.wrapper_utils import AnalyticsWrapperUtils
 from teradataml.utils.utils import execute_sql
 from teradataml.utils.validators import _Validators
-from teradatasqlalchemy.telemetry.queryband import collect_queryband
+from teradataml.telemetry_utils.queryband import collect_queryband
 
 
 @collect_queryband(queryband="CpToSql")

teradataml/dataframe/data_transfer.py

@@ -27,7 +27,7 @@ from teradataml.dataframe.copy_to import copy_to_sql, _create_table_object, \
 from teradataml.dataframe.dataframe_utils import DataFrameUtils as df_utils
 from teradataml.dbutils.dbutils import _create_table, _execute_query_and_generate_pandas_df
 from teradataml.utils.validators import _Validators
-from teradatasqlalchemy.telemetry.queryband import collect_queryband
+from teradataml.telemetry_utils.queryband import collect_queryband
 
 
 @collect_queryband(queryband="fstExprt")
@@ -1958,7 +1958,8 @@ class _DataTransferUtils():
             dt_obj = _DataTransferUtils(df)
             ins_query = dt_obj._table_exists()
         """
-        return con.dialect.has_table(get_connection(), self.table_name, self.schema_name)
+        return con.dialect.has_table(get_connection(), self.table_name, self.schema_name,
+                                     table_only=True)
 
     def _get_fully_qualified_table_name(self, table_name=None, schema_name=None):
         """
@@ -2144,7 +2145,8 @@ class _DataTransferUtils():
             # drop the tables created by FastloadCSV.
             if not self.save_errors:
                 for table in fastloadcsv_err_tables:
-                    if conn.dialect.has_table(conn, table_name=table, schema=self.schema_name):
+                    if conn.dialect.has_table(conn, table_name=table, schema=self.schema_name,
+                                              table_only=True):
                         UtilFuncs._drop_table(self._get_fully_qualified_table_name(table))
                     err_warn_dict.update({"fastloadcsv_error_tables": []})
                 return err_warn_dict