teradataml 20.0.0.1__py3-none-any.whl → 20.0.0.2__py3-none-any.whl

teradataml/data/scripts/deploy_script.py

@@ -60,7 +60,7 @@ if not len(features):
     sys.exit(0)
 
 X = np.array(features)
-y = np.array(labels)
+y = np.array(labels).ravel()
 
 clf = make_pipeline(StandardScaler(), SVC(gamma='auto'))
 clf.fit(X, y)

teradataml/data/scripts/sklearn/sklearn_fit.py

@@ -138,22 +138,29 @@ if not len(features):
 # Fit/partial_fit the model to the data.
 if function_name == "partial_fit":
     if labels and classes:
-        model.partial_fit(np.array(features), np.array(labels), classes=classes)
+        model.partial_fit(features, labels, classes=classes)
     elif labels:
-        model.partial_fit(np.array(features), np.array(labels))
+        model.partial_fit(features, labels)
     elif classes:
-        model.partial_fit(np.array(features), classes=classes)
+        model.partial_fit(features, classes=classes)
     else:
-        model.partial_fit(np.array(features))
+        model.partial_fit(features)
 elif function_name == "fit":
-    # For IsotonicRegression, fit() accepts training target as
-    # y: array-like of shape (n_samples,).
+    model_name = model.__class__.__name__
+    np_func_list = ["OneVsRestClassifier", "LabelBinarizer", "TSNE"]
     if labels:
-        labels = np.array(labels).reshape(-1) \
-            if model.__class__.__name__ == "IsotonicRegression" else np.array(labels)
-        model.fit(np.array(features), labels)
+        # For IsotonicRegression, fit() accepts training target as
+        # y: array-like of shape (n_samples,).
+        if model_name in ["IsotonicRegression", "LinearSVC"]:
+            labels = np.array(labels).reshape(-1)
+        if model_name in np_func_list:
+            labels = np.array(labels)
+            features = np.array(features)
+        model.fit(features, labels)
     else:
-        model.fit(np.array(features))
+        if model_name in np_func_list:
+            features = np.array(features)
+        model.fit(features)
 
 model_str = pickle.dumps(model)
 

teradataml/data/scripts/sklearn/sklearn_fit_predict.py

@@ -110,9 +110,9 @@ if not len(features):
 
 # write code to call fit_predict with features and labels when n_c_labels > 0
 if n_c_labels > 0:
-    predictions = model.fit_predict(np.array(features), np.array(labels))
+    predictions = model.fit_predict(features, labels)
 else:
-    predictions = model.fit_predict(np.array(features))
+    predictions = model.fit_predict(features)
 
 # Export results to to the Databse through standard output
 for i in range(len(predictions)):

teradataml/data/scripts/sklearn/sklearn_function.template

@@ -28,19 +28,26 @@ def splitter(strr, delim=",", convert_to="str"):
     return [convert_to_type(i, convert_to) for i in strr.split(delim)]
 
 # Arguments to the Script.
-if len(sys.argv) != 4:
-    # 4 arguments command line arguments should be passed to this file.
+if len(sys.argv) != 6:
+    # 5 arguments command line arguments should be passed to this file.
     # 1: file to be run
     # 2. Comma separated indices of partition columns.
     # 3. Comma separated types of all the data columns.
     # 4. Data columns information separted by "--" where each data column information is in the form
     #    "<arg_name>-<comma separated data indices>-<comma separated data types>".
-    sys.exit("4 arguments command line arguments should be passed: file to be run,"
+    # 5. Flag to check the system type. True, means Lake, Enterprise otherwise.
+    # 6. Model file prefix for lake system, None otherwise.
+    sys.exit("5 arguments command line arguments should be passed: file to be run,"
              " comma separated indices of partition columns, comma separated types of all columns,"
              " data columns information separated by '--' where each data column information is"
-             " in the form '<arg_name>-<comma separated data indices>-<comma separated data types>'.")
-
-db = sys.argv[0].split("/")[1]
+             " in the form '<arg_name>-<comma separated data indices>-<comma separated data types>',"
+             " flag to check lake or enterprise and model file prefix used only for lake system.")
+
+is_lake_system = eval(sys.argv[4])
+if not is_lake_system:
+    db = sys.argv[0].split("/")[1]
+else:
+    model_file_prefix = sys.argv[5]
 data_partition_column_indices = splitter(sys.argv[1], convert_to="int") # indices are integers.
 data_column_types = splitter(sys.argv[2], delim="--")
 
@@ -79,6 +86,11 @@ while 1:
                     data_partition_column_values.append(
                         convert_to_type(values[val], typee=data_partition_column_types[i])
                         )
+
+                # Prepare the corresponding model file name and extract model.
+                partition_join = "_".join([str(x) for x in data_partition_column_values])
+                # Replace '-' with '_' as '-' because partition_columns can be negative.
+                partition_join = partition_join.replace("-", "_")
             
             # Prepare data dictionary containing only arguments related to data.
             for arg_name in data_args_values:
@@ -105,4 +117,15 @@ all_args = {**data_args_values, **params}
 module_ = importlib.import_module(module_name)
 sklearn_model = getattr(module_, func_name)(**all_args)
 
-print(*(data_partition_column_values + [base64.b64encode(pickle.dumps(sklearn_model))]), sep=DELIMITER)
+model_str = pickle.dumps(sklearn_model)
+
+if is_lake_system:
+    model_file_path = f"/tmp/{model_file_prefix}_{partition_join}.pickle"
+
+    # Write to file in Vantage, to be used in predict/scoring.
+    with open(model_file_path, "wb") as fp:
+        fp.write(model_str)
+
+model_data = model_file_path if is_lake_system else base64.b64encode(model_str)
+
+print(*(data_partition_column_values + [model_data]), sep=DELIMITER)

teradataml/data/scripts/sklearn/sklearn_neighbors.py

@@ -116,7 +116,7 @@ while 1:
 
             f_ = values[:n_f_cols]
             if f_:
-                output = getattr(model, func_name)(np.array([f_]), **arguments)
+                output = getattr(model, func_name)([f_], **arguments)
             else:
                 output = getattr(model, func_name)(**arguments)
             result_list = f_

teradataml/data/scripts/sklearn/sklearn_score.py

@@ -110,10 +110,19 @@ while 1:
 if len(features) == 0:
     sys.exit(0)
 
+
+model_name = model.__class__.__name__
+np_func_list = ["MultiOutputClassifier", "GaussianMixture"]
+
+if model_name in np_func_list:
+    features = np.array(features)
+
 if labels:
-    val = getattr(model, func_name)(np.array(features), np.array(labels))
+    if model_name in np_func_list:
+        labels = np.array(labels)
+    val = getattr(model, func_name)(features, labels)
 else:
-    val = getattr(model, func_name)(np.array(features))
+    val = getattr(model, func_name)(features)
 
 result_val = ['' if (val is None or (not isinstance(val, str) and (math.isnan(val) or math.isinf(val)))) else val]
-print(*(data_partition_column_values + result_val), sep=DELIMITER)
+print(*(data_partition_column_values + result_val), sep=DELIMITER)

teradataml/data/scripts/sklearn/sklearn_transform.py

@@ -92,11 +92,14 @@ data_partition_column_types = [data_column_types[idx] for idx in data_partition_
 model = None
 data_partition_column_values = []
 
+missing_indicator_input = []
+
 # Data Format:
 # feature1, feature2, ..., featuren, label1, label2, ... labelk, data_partition_column1, ..., 
 # data_partition_columnn.
 # label is optional (it is present when label_exists is not "None")
 
+model_name = ""
 while 1:
     try:
         line = input()
@@ -128,9 +131,33 @@ while 1:
                     sys.exit("Model file is not installed in Vantage.")
 
             f_ = values[:n_f_cols]
+
+            model_name = model.__class__.__name__
+            np_func_list = ["ClassifierChain", "EllipticEnvelope", "MinCovDet",  
+                            "FeatureAgglomeration", "LabelBinarizer", "MultiLabelBinarizer"]
+
+            # MissingIndicator requires processing the entire dataset simultaneously, 
+            # rather than on a row-by-row basis.
+
+            # Error getting during row-by-row processing - 
+            # "ValueError: MissingIndicator does not support data with dtype <U13. 
+            # Please provide either a numeric array (with a floating point or 
+            i# integer dtype) or categorical data represented ei
+            if model_name == "MissingIndicator" and func_name == "transform":
+                missing_indicator_input.append(f_)
+                continue
+
+            f__ = np.array([f_]) if model_name in np_func_list or \
+                                    (model_name == "SimpleImputer" and func_name == "inverse_transform")\
+                else [f_]
+
             if n_c_labels > 0:
                 # Labels are present in last column.
                 l_ = values[n_f_cols:n_f_cols+n_c_labels]
+
+                l__ = np.array([l_]) if model_name in np_func_list or \
+                                        (model_name == "SimpleImputer" and func_name == "inverse_transform")\
+                    else [l_]
                 # predict() now takes 'y' also for it to return the labels from script. Skipping 'y'
                 # in function call. Generally, 'y' is passed to return y along with actual output.
                 try:
@@ -138,17 +165,17 @@ while 1:
                     # used 'in' in if constion, as model.__module__ is giving 
                     # 'sklearn.cross_decomposition._pls'.  
                     if "cross_decomposition" in model.__module__:
-                        trans_values = getattr(model, func_name)(X=np.array([f_]), Y=np.array([l_]))
+                        trans_values = getattr(model, func_name)(X=f__, Y=l__)
                     else:
-                        trans_values = getattr(model, func_name)(X=np.array([f_]), y=np.array([l_]))
+                        trans_values = getattr(model, func_name)(X=f__, y=l__)
 
                 except TypeError as ex:
                     # Function which does not accept 'y' like predict_proba() raises error like
                     # "TypeError: predict_proba() takes 2 positional arguments but 3 were given".
-                    trans_values = getattr(model, func_name)(np.array([f_]))
+                    trans_values = getattr(model, func_name)(f__)
             else:
                 # If class labels do not exist in data, don't read labels, read just features.
-                trans_values = getattr(model, func_name)(np.array([f_]))
+                trans_values = getattr(model, func_name)(f__)
 
             result_list = f_
             if n_c_labels > 0 and func_name in ["predict", "decision_function"]:
@@ -169,3 +196,27 @@ while 1:
 
     except EOFError:  # Exit if reached EOF or CTRL-D
         break
+
+
+# MissingIndicator needs processing of all the dataset at the same time, instead of row by row. 
+# Hence, handling it outside of the while loop
+if model_name == "MissingIndicator" and func_name == "transform":
+    m_out = model.transform(missing_indicator_input)
+
+    for j, vals in enumerate(missing_indicator_input):
+
+        m_out_list = get_output_data(trans_values=m_out[j], func_name=func_name,
+                                     model_obj=model, n_c_labels=n_c_labels)
+
+        result_list = missing_indicator_input[j] + m_out_list
+
+        for i, val in enumerate(result_list):
+            if (val is None or (not isinstance(val, str) and (math.isnan(val) or math.isinf(val)))):
+                result_list[i] = ""
+            # MissingIndicator returns boolean values. Convert them to 0/1.
+            elif val == False:
+                result_list[i] = 0
+            elif val == True:
+                result_list[i] = 1
+
+        print(*(data_partition_column_values + result_list), sep=DELIMITER)

teradataml/data/star_pivot.csv

@@ -0,0 +1,8 @@
+country,state,yr,qtr,sales,cogs,rating
+USA,CA,2001,Q1,30,15,A
+USA,NY,2001,Q1,45,25,D
+USA,CA,2001,Q2,50,20,A
+USA,CA,2001,Q2,5,5,B
+Canada,ON,2001,Q2,10,0,B
+Canada,BC,2001,Q3,15,0,A
+Canada,BC,2001,Q3,10,0,A

teradataml/data/templates/open_source_ml.json

@@ -2,7 +2,8 @@
   "env_specs": [
     {
       "env_name": "openml_env",
-      "libs": "scikit-learn",
+      "libs": ["scikit-learn==1.5.1", "joblib==1.4.2", "numpy==2.0.0",
+               "scipy==1.14.0", "threadpoolctl==3.5.0"],
       "desc": "DONT DELETE: OpenML environment"
     }
   ]

teradataml/data/teradataml_example.json

@@ -1348,6 +1348,25 @@
     "radio":"FLOAT",
     "newspaper":"FLOAT",
     "sales":"FLOAT"
+  },
+  "timestamp_data":{
+    "id": "INTEGER",
+    "timestamp_col": "VARCHAR(50)",
+    "timestamp_col1": "BIGINT",
+    "format_col": "VARCHAR(50)",
+    "timezone_col": "VARCHAR(50)"
+  },
+  "interval_data":{
+    "id": "INTEGER",
+    "int_col": "BIGINT",
+    "value_col": "VARCHAR(30)",
+    "value_col1": "VARCHAR(30)",
+    "str_col1": "VARCHAR(30)",
+    "str_col2": "VARCHAR(30)"
+  },
+  "url_data": {
+    "id": "INTEGER",
+    "urls": "VARCHAR(60)",
+    "part": "VARCHAR(20)"
   }
-
 }

teradataml/data/timestamp_data.csv

@@ -0,0 +1,4 @@
+id,timestamp_col,timestamp_col1,format_col,timezone_col
+0,"2015-01-08 00:00:12.2",123456,"YYYY-MM-DD HH24:MI:SS.FF6","GMT"
+1,"2015-01-08 13:00",878986,"YYYY-MM-DD HH24:MI","America Pacific"
+2,"2015-01-08 00:00:12.2+10:00",45678910234,"YYYY-MM-DD HH24:MI:SS.FF6 TZH:TZM","GMT+10"

teradataml/data/titanic_dataset_unpivoted.csv

@@ -0,0 +1,19 @@
+passenger,AttributeName,AttributeValue,survived
+61,pclass,3,0
+1000,pclass,3,1
+40,pclass,3,1
+21,pclass,2,0
+61,gender,male,0
+1000,gender,,1
+40,gender,female,1
+21,gender,male,0
+2,pclass,1,1
+16,pclass,2,1
+7,pclass,1,0
+2,gender,female,1
+16,gender,female,1
+7,gender,male,0
+10,pclass,2,1
+4,pclass,1,1
+10,gender,female,1
+4,gender,female,1

teradataml/data/uaf_example.json

@@ -471,5 +471,59 @@
     "CONF_OFF_v": "FLOAT",
     "CONF_LOW_v": "FLOAT",
     "CONF_HI_v": "FLOAT"
+  },
+  "dwt_dataTable":{
+	  "id": "INTEGER",
+	  "rowi": "INTEGER",
+	  "v": "FLOAT"
+  },
+  "dwt_filterTable":{
+	  "id": "INTEGER",
+	  "seq": "INTEGER",
+	  "lo": "FLOAT",
+	  "hi": "FLOAT"
+  },
+  "idwt_dataTable":{
+	  "id": "INTEGER",
+	  "rowi": "INTEGER",
+	  "approx": "FLOAT",
+	  "detail": "FLOAT"
+  },
+  "idwt_filterTable":{
+	  "id": "INTEGER",
+	  "seq": "INTEGER",
+	  "lo": "FLOAT",
+	  "hi": "FLOAT"
+  },
+   "dwt2d_dataTable":{
+	  "id": "INTEGER",
+	  "x": "INTEGER",
+	  "y": "INTEGER",
+	  "v": "FLOAT"
+  },
+  "idwt2d_dataTable":{
+	  "id": "INTEGER",
+	  "x": "INTEGER",
+	  "y": "INTEGER",
+	  "v": "FLOAT"
+  },
+  "covid_confirm_sd":{
+	  "city": "VARCHAR(15)",
+      "row_axis": "INTEGER",
+      "cnumber": "INTEGER"
+  },
+  "real_values":{
+	  "TD_TIMECODE": "TIMESTAMP(0)",
+      "id": "INTEGER",
+      "val": "FLOAT",
+      "<PTI_CLAUSE>": "(TIMESTAMP(0), DATE '2020-01-01', HOURS(1), COLUMNS(id), nonsequenced)"
+  },
+  "windowdfft":{
+    "id": "INTEGER",
+    "row_i": "INTEGER",
+    "v1": "FLOAT",
+    "v2": "FLOAT",
+    "v3": "FLOAT",
+    "v4": "FLOAT"
   }
-}
+}

teradataml/data/unpivot_example.json

@@ -6,5 +6,20 @@
     "temp": "integer",
     "pressure": "real",
     "dewpoint": "varchar(30)"
+  },
+  "titanic_dataset_unpivoted":{
+    "passenger": "integer",
+    "AttributeName": "varchar(30)",
+    "AttributeValue": "varchar(30)",
+    "survived": "integer"
+  },
+  "star_pivot":{
+    "country": "varchar(30)",
+    "state": "varchar(30)",
+    "yr": "integer",
+    "qtr": "varchar(30)",
+    "sales": "integer",
+    "cogs": "integer",
+    "rating": "varchar(30)"
   }
 }

teradataml/data/url_data.csv

@@ -0,0 +1,9 @@
+"id","urls","part"
+0,"http://example.com:8080/path","FILE"
+1,"ftp://example.net:21/path","PATH"
+2,"https://example.net/path4/path5/path6?query4=value4#fragment3","REF"
+3,"https://www.facebook.com","HOST"
+4,"https://teracloud-pod-services-pod-account-service.dummyvalue.production.pods.teracloud.ninja/v1/accounts/acc-dummyvalue/user-environment-service/api/v1/","QUERY"
+5,"http://pg.example.ml/path150#fragment90","AUTHORITY"
+6,"smtp://user:password@smtp.example.com:21/file.txt","USERINFO"
+7,"https://www.google.com","PROTOCOL"

teradataml/data/windowdfft.csv

@@ -0,0 +1,16 @@
+id,row_i,v1,v2,v3,v4
+3,1,0.0,1.4,1.0,1.0
+3,2,1.0,2.4,2.0,2.0
+3,3,2.0,3.4,3.0,3.0
+3,4,3.0,4.6,4.0,4.0
+3,5,0.0,5.9,5.0,5.0
+3,6,1.0,6.7,6.0,6.0
+3,7,2.0,7.7,7.0,7.0
+3,8,3.0,8.7,8.0,8.0
+3,9,0.0,9.9,9.0,9.0
+3,10,1.0,10.2,10.0,10.0
+3,11,2.0,11.2,11.0,11.0
+3,12,3.0,12.2,12.0,12.0
+3,13,1.0,10.2,13.0,13.0
+3,14,2.0,11.2,14.0,14.0
+3,15,3.0,12.2,15.0,15.0

teradataml/dataframe/copy_to.py

@@ -30,7 +30,7 @@ from teradatasql import OperationalError
 from teradataml.common.wrapper_utils import AnalyticsWrapperUtils
 from teradataml.utils.utils import execute_sql
 from teradataml.utils.validators import _Validators
-from teradatasqlalchemy.telemetry.queryband import collect_queryband
+from teradataml.telemetry_utils.queryband import collect_queryband
 
 
 @collect_queryband(queryband="CpToSql")

teradataml/dataframe/data_transfer.py

@@ -27,7 +27,7 @@ from teradataml.dataframe.copy_to import copy_to_sql, _create_table_object, \
 from teradataml.dataframe.dataframe_utils import DataFrameUtils as df_utils
 from teradataml.dbutils.dbutils import _create_table, _execute_query_and_generate_pandas_df
 from teradataml.utils.validators import _Validators
-from teradatasqlalchemy.telemetry.queryband import collect_queryband
+from teradataml.telemetry_utils.queryband import collect_queryband
 
 
 @collect_queryband(queryband="fstExprt")
@@ -1958,7 +1958,8 @@ class _DataTransferUtils():
             dt_obj = _DataTransferUtils(df)
             ins_query = dt_obj._table_exists()
         """
-        return con.dialect.has_table(get_connection(), self.table_name, self.schema_name)
+        return con.dialect.has_table(get_connection(), self.table_name, self.schema_name,
+                                     table_only=True)
 
     def _get_fully_qualified_table_name(self, table_name=None, schema_name=None):
         """
@@ -2144,7 +2145,8 @@ class _DataTransferUtils():
             # drop the tables created by FastloadCSV.
             if not self.save_errors:
                 for table in fastloadcsv_err_tables:
-                    if conn.dialect.has_table(conn, table_name=table, schema=self.schema_name):
+                    if conn.dialect.has_table(conn, table_name=table, schema=self.schema_name,
+                                              table_only=True):
                         UtilFuncs._drop_table(self._get_fully_qualified_table_name(table))
                     err_warn_dict.update({"fastloadcsv_error_tables": []})
                 return err_warn_dict