teradataml 20.0.0.3__py3-none-any.whl → 20.0.0.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of teradataml might be problematic. Click here for more details.
- teradataml/LICENSE-3RD-PARTY.pdf +0 -0
- teradataml/README.md +119 -0
- teradataml/_version.py +1 -1
- teradataml/analytics/analytic_function_executor.py +18 -6
- teradataml/analytics/byom/__init__.py +1 -1
- teradataml/analytics/sqle/__init__.py +4 -1
- teradataml/analytics/valib.py +18 -4
- teradataml/automl/__init__.py +51 -6
- teradataml/automl/data_preparation.py +56 -33
- teradataml/automl/data_transformation.py +58 -33
- teradataml/automl/feature_engineering.py +12 -5
- teradataml/automl/model_training.py +34 -13
- teradataml/common/__init__.py +1 -2
- teradataml/common/constants.py +64 -40
- teradataml/common/messagecodes.py +13 -3
- teradataml/common/messages.py +4 -1
- teradataml/common/sqlbundle.py +40 -10
- teradataml/common/utils.py +113 -39
- teradataml/common/warnings.py +11 -0
- teradataml/context/context.py +141 -17
- teradataml/data/amazon_reviews_25.csv +26 -0
- teradataml/data/byom_example.json +11 -0
- teradataml/data/docs/byom/docs/DataRobotPredict.py +2 -2
- teradataml/data/docs/byom/docs/DataikuPredict.py +40 -1
- teradataml/data/docs/byom/docs/H2OPredict.py +2 -2
- teradataml/data/docs/byom/docs/ONNXEmbeddings.py +242 -0
- teradataml/data/docs/byom/docs/ONNXPredict.py +2 -2
- teradataml/data/docs/byom/docs/PMMLPredict.py +2 -2
- teradataml/data/docs/sqle/docs_17_20/Shap.py +28 -6
- teradataml/data/docs/uaf/docs_17_20/DWT2D.py +4 -1
- teradataml/data/hnsw_alter_data.csv +5 -0
- teradataml/data/hnsw_data.csv +10 -0
- teradataml/data/jsons/byom/h2opredict.json +1 -1
- teradataml/data/jsons/byom/onnxembeddings.json +266 -0
- teradataml/data/jsons/sqle/17.20/TD_Shap.json +0 -1
- teradataml/data/jsons/sqle/20.00/TD_HNSW.json +296 -0
- teradataml/data/jsons/sqle/20.00/TD_HNSWPredict.json +206 -0
- teradataml/data/jsons/sqle/20.00/TD_HNSWSummary.json +32 -0
- teradataml/data/jsons/sqle/20.00/TD_KMeans.json +2 -2
- teradataml/data/jsons/sqle/20.00/TD_SMOTE.json +1 -1
- teradataml/data/jsons/sqle/20.00/TD_VectorDistance.json +5 -5
- teradataml/data/teradataml_example.json +8 -0
- teradataml/data/vectordistance_example.json +1 -1
- teradataml/dataframe/copy_to.py +8 -3
- teradataml/dataframe/data_transfer.py +11 -1
- teradataml/dataframe/dataframe.py +517 -121
- teradataml/dataframe/dataframe_utils.py +152 -20
- teradataml/dataframe/functions.py +26 -11
- teradataml/dataframe/setop.py +11 -6
- teradataml/dataframe/sql.py +2 -2
- teradataml/dbutils/dbutils.py +525 -129
- teradataml/hyperparameter_tuner/optimizer.py +12 -1
- teradataml/opensource/{sklearn/_sklearn_wrapper.py → _base.py} +317 -1011
- teradataml/opensource/_class.py +141 -17
- teradataml/opensource/{constants.py → _constants.py} +7 -3
- teradataml/opensource/_lightgbm.py +52 -53
- teradataml/opensource/_sklearn.py +1008 -0
- teradataml/opensource/_wrapper_utils.py +5 -5
- teradataml/options/__init__.py +47 -15
- teradataml/options/configure.py +103 -25
- teradataml/options/display.py +13 -2
- teradataml/plot/axis.py +47 -8
- teradataml/plot/figure.py +33 -0
- teradataml/plot/plot.py +63 -13
- teradataml/scriptmgmt/UserEnv.py +2 -2
- teradataml/scriptmgmt/lls_utils.py +63 -26
- teradataml/store/__init__.py +1 -2
- teradataml/store/feature_store/feature_store.py +102 -7
- teradataml/table_operators/Apply.py +32 -18
- teradataml/table_operators/Script.py +3 -1
- teradataml/table_operators/TableOperator.py +3 -1
- teradataml/utils/dtypes.py +47 -0
- teradataml/utils/internal_buffer.py +18 -0
- teradataml/utils/validators.py +68 -9
- {teradataml-20.0.0.3.dist-info → teradataml-20.0.0.4.dist-info}/METADATA +123 -2
- {teradataml-20.0.0.3.dist-info → teradataml-20.0.0.4.dist-info}/RECORD +79 -75
- teradataml/data/SQL_Fundamentals.pdf +0 -0
- teradataml/libaed_0_1.dylib +0 -0
- teradataml/libaed_0_1.so +0 -0
- teradataml/opensource/sklearn/__init__.py +0 -0
- teradataml/store/vector_store/__init__.py +0 -1586
- {teradataml-20.0.0.3.dist-info → teradataml-20.0.0.4.dist-info}/WHEEL +0 -0
- {teradataml-20.0.0.3.dist-info → teradataml-20.0.0.4.dist-info}/top_level.txt +0 -0
- {teradataml-20.0.0.3.dist-info → teradataml-20.0.0.4.dist-info}/zip-safe +0 -0
teradataml/common/messages.py
CHANGED
|
@@ -168,6 +168,8 @@ class Messages():
|
|
|
168
168
|
[ErrorInfoCodes.UNSUPPORTED_FILE_EXTENSION, MessageCodes.UNSUPPORTED_FILE_EXTENSION],
|
|
169
169
|
[ErrorInfoCodes.FILE_EMPTY, MessageCodes.FILE_EMPTY],
|
|
170
170
|
[ErrorInfoCodes.PYTHON_NOT_INSTALLED, MessageCodes.PYTHON_NOT_INSTALLED],
|
|
171
|
+
[ErrorInfoCodes.PYTHON_VERSION_MISMATCH, MessageCodes.PYTHON_VERSION_MISMATCH],
|
|
172
|
+
[ErrorInfoCodes.PYTHON_VERSION_MISMATCH_OAF, MessageCodes.PYTHON_VERSION_MISMATCH_OAF],
|
|
171
173
|
[ErrorInfoCodes.INT_ARGUMENT_COMPARISON, MessageCodes.INT_ARGUMENT_COMPARISON],
|
|
172
174
|
[ErrorInfoCodes.EXECUTION_FAILED, MessageCodes.EXECUTION_FAILED],
|
|
173
175
|
[ErrorInfoCodes.INVALID_COLUMN_DATATYPE, MessageCodes.INVALID_COLUMN_DATATYPE],
|
|
@@ -192,7 +194,8 @@ class Messages():
|
|
|
192
194
|
[ErrorInfoCodes.INVALID_PARTITIONING_COLS, MessageCodes.INVALID_PARTITIONING_COLS],
|
|
193
195
|
[ErrorInfoCodes.PATH_NOT_FOUND, MessageCodes.PATH_NOT_FOUND],
|
|
194
196
|
[ErrorInfoCodes.TARGET_COL_NOT_FOUND_FOR_EVALUATE, MessageCodes.TARGET_COL_NOT_FOUND_FOR_EVALUATE],
|
|
195
|
-
[ErrorInfoCodes.
|
|
197
|
+
[ErrorInfoCodes.SET_REQUIRED_PARAMS, MessageCodes.SET_REQUIRED_PARAMS],
|
|
198
|
+
[ErrorInfoCodes.MISSING_ARGS, MessageCodes.CONNECTION_PARAMS]
|
|
196
199
|
]
|
|
197
200
|
|
|
198
201
|
@staticmethod
|
teradataml/common/sqlbundle.py
CHANGED
|
@@ -40,9 +40,9 @@ class SQLBundle:
|
|
|
40
40
|
[SQLConstants.SQL_BASE_QUERY, "SELECT * FROM {0}"],
|
|
41
41
|
[SQLConstants.SQL_SAMPLE_QUERY, " {0} SAMPLE {1}"],
|
|
42
42
|
[SQLConstants.SQL_SAMPLE_WITH_WHERE_QUERY, " {0} WHERE {1} SAMPLE {2}"],
|
|
43
|
-
[SQLConstants.SQL_CREATE_VOLATILE_TABLE_FROM_QUERY_WITH_DATA, "CREATE MULTISET VOLATILE TABLE {0} AS ({1}) WITH DATA ON COMMIT PRESERVE ROWS"],
|
|
44
|
-
[SQLConstants.SQL_CREATE_VOLATILE_TABLE_FROM_QUERY_WITHOUT_DATA, "CREATE MULTISET VOLATILE TABLE {0} AS ({1}) WITH NO DATA"],
|
|
45
|
-
[SQLConstants.SQL_CREATE_VOLATILE_TABLE_USING_COLUMNS, "CREATE MULTISET VOLATILE TABLE {0}( {1} )"],
|
|
43
|
+
[SQLConstants.SQL_CREATE_VOLATILE_TABLE_FROM_QUERY_WITH_DATA, "CREATE MULTISET VOLATILE TABLE {0} AS ({1}) WITH DATA NO PRIMARY INDEX ON COMMIT PRESERVE ROWS"],
|
|
44
|
+
[SQLConstants.SQL_CREATE_VOLATILE_TABLE_FROM_QUERY_WITHOUT_DATA, "CREATE MULTISET VOLATILE TABLE {0} AS ({1}) WITH NO DATA NO PRIMARY INDEX"],
|
|
45
|
+
[SQLConstants.SQL_CREATE_VOLATILE_TABLE_USING_COLUMNS, "CREATE MULTISET VOLATILE TABLE {0}( {1} ) NO PRIMARY INDEX"],
|
|
46
46
|
[SQLConstants.SQL_CREATE_TABLE_FROM_QUERY_WITH_DATA, "CREATE MULTISET TABLE {0} AS ({1}) WITH DATA"],
|
|
47
47
|
[SQLConstants.SQL_HELP_COLUMNS, "help column {0}.*"],
|
|
48
48
|
[SQLConstants.SQL_DROP_TABLE, "DROP TABLE {0}"],
|
|
@@ -69,6 +69,8 @@ class SQLBundle:
|
|
|
69
69
|
[SQLConstants.SQL_CREATE_TABLE_USING_COLUMNS, "CREATE MULTISET TABLE {0}( {1} )"],
|
|
70
70
|
[SQLConstants.SQL_EXEC_STORED_PROCEDURE, "call {0}"],
|
|
71
71
|
[SQLConstants.SQL_SELECT_COLUMNNAMES_WITH_WHERE, "sel {0} from {1} where {2}"],
|
|
72
|
+
[SQLConstants.SQL_HELP_DATABASE, "HELP DATABASE {0}"],
|
|
73
|
+
[SQLConstants.SQL_HELP_DATALAKE, "HELP DATALAKE {0}"]
|
|
72
74
|
|
|
73
75
|
]
|
|
74
76
|
self._add_sql_version()
|
|
@@ -257,14 +259,15 @@ class SQLBundle:
|
|
|
257
259
|
return query.format(tablename, select_query)
|
|
258
260
|
|
|
259
261
|
@staticmethod
|
|
260
|
-
def _build_drop_table(tablename):
|
|
262
|
+
def _build_drop_table(tablename, purge_clause=None):
|
|
261
263
|
"""
|
|
262
264
|
Returns a drop table DDL statement for a table.
|
|
263
265
|
Example:
|
|
264
266
|
drop table mytab
|
|
265
267
|
|
|
266
268
|
PARAMETERS:
|
|
267
|
-
tablename - The table to drop
|
|
269
|
+
tablename - The table to drop.
|
|
270
|
+
purge_clause - String representing purge clause.
|
|
268
271
|
|
|
269
272
|
RETURNS:
|
|
270
273
|
Returns a drop table DDL statement for the table.
|
|
@@ -273,12 +276,17 @@ class SQLBundle:
|
|
|
273
276
|
None
|
|
274
277
|
|
|
275
278
|
EXAMPLES:
|
|
276
|
-
|
|
277
|
-
|
|
279
|
+
drop_stmt = SQLBundle._build_drop_table('my_tab')
|
|
280
|
+
drop_stmt = SQLBundle._build_drop_table("my_lake"."my_db"."my_tab",
|
|
281
|
+
purge_clause='PURGE ALL')
|
|
282
|
+
drop_stmt = SQLBundle._build_drop_table("my_lake"."my_db"."my_tab",
|
|
283
|
+
purge_clause='NO PURGE')
|
|
278
284
|
"""
|
|
279
285
|
sqlbundle = SQLBundle()
|
|
280
|
-
|
|
281
|
-
|
|
286
|
+
drop_stmt = sqlbundle._get_sql_query(SQLConstants.SQL_DROP_TABLE).format(tablename)
|
|
287
|
+
if purge_clause:
|
|
288
|
+
drop_stmt = "{} {}".format(drop_stmt, purge_clause)
|
|
289
|
+
return drop_stmt
|
|
282
290
|
|
|
283
291
|
@staticmethod
|
|
284
292
|
def _build_drop_view(viewname):
|
|
@@ -400,6 +408,28 @@ class SQLBundle:
|
|
|
400
408
|
return sqlbundle._get_sql_query(SQLConstants.SQL_HELP_VOLATILE_TABLE)
|
|
401
409
|
|
|
402
410
|
@staticmethod
|
|
411
|
+
def _build_help_datalake(datalake_name):
|
|
412
|
+
"""
|
|
413
|
+
Builds a query to get help on datalake.
|
|
414
|
+
|
|
415
|
+
Example:
|
|
416
|
+
"HELP DATALAKE datalake_name;"
|
|
417
|
+
|
|
418
|
+
PARAMETERS:
|
|
419
|
+
|
|
420
|
+
RETURNS:
|
|
421
|
+
returns a HELP DATALAKE <datalake_name> command.
|
|
422
|
+
|
|
423
|
+
RAISES:
|
|
424
|
+
None
|
|
425
|
+
|
|
426
|
+
EXAMPLES:
|
|
427
|
+
query = SQLBundle._build_help_datalake(datalake_name)
|
|
428
|
+
"""
|
|
429
|
+
sqlbundle = SQLBundle()
|
|
430
|
+
help_datalake_sql = sqlbundle._get_sql_query(SQLConstants.SQL_HELP_DATALAKE)
|
|
431
|
+
return help_datalake_sql.format(datalake_name)
|
|
432
|
+
|
|
403
433
|
def _build_select_table_name(tab_name):
|
|
404
434
|
"""
|
|
405
435
|
Builds a query to get table name from DBC.TABLESV
|
|
@@ -463,7 +493,7 @@ class SQLBundle:
|
|
|
463
493
|
# Extracting table name without '%' character
|
|
464
494
|
table_name_str = table_name.replace('%', '')
|
|
465
495
|
|
|
466
|
-
# Adding condition to check if table name contains the string using POSITION function
|
|
496
|
+
# Adding condition to check if table name contains the string using POSITION function.
|
|
467
497
|
# POSITION function returns the position index of the substring in the string if found,
|
|
468
498
|
# else returns 0
|
|
469
499
|
query = "{0}{1}{2}".format(query, sqlbundle._get_sql_query(SQLConstants.SQL_AND_TABLE_NAME_LIKE).format(table_name), \
|
teradataml/common/utils.py
CHANGED
|
@@ -311,6 +311,10 @@ class UtilFuncs():
|
|
|
311
311
|
tabname = "\"{}\".\"{}\"".format(_get_user(), tabname)
|
|
312
312
|
return tabname
|
|
313
313
|
|
|
314
|
+
if configure.temp_object_type == TeradataConstants.TERADATA_VOLATILE_TABLE:
|
|
315
|
+
from teradataml.context.context import _get_user
|
|
316
|
+
return "\"{}\".\"{}_{}\"".format(_get_user(), "vt", tabname)
|
|
317
|
+
|
|
314
318
|
if use_default_database and databasename is None:
|
|
315
319
|
tabname = "\"{}\".\"{}\"".format(tdmlctx._get_context_temp_databasename(
|
|
316
320
|
table_type=table_type), tabname)
|
|
@@ -694,6 +698,7 @@ class UtilFuncs():
|
|
|
694
698
|
EXAMPLES:
|
|
695
699
|
UtilFuncs._create_table('"dbname"."table_name"', "select * from table_name")
|
|
696
700
|
"""
|
|
701
|
+
|
|
697
702
|
crt_table = SQLBundle._build_create_table_with_data(table_name, query)
|
|
698
703
|
if volatile:
|
|
699
704
|
crt_table = SQLBundle._build_create_volatile_table_with_data(table_name, query)
|
|
@@ -757,13 +762,14 @@ class UtilFuncs():
|
|
|
757
762
|
return vtab_name
|
|
758
763
|
|
|
759
764
|
@staticmethod
|
|
760
|
-
def _drop_table(table_name, check_table_exist=True):
|
|
765
|
+
def _drop_table(table_name, check_table_exist=True, purge_clause=None):
|
|
761
766
|
"""
|
|
762
767
|
Drops a table.
|
|
763
768
|
|
|
764
769
|
PARAMETERS:
|
|
765
770
|
table_name - The table to drop.
|
|
766
771
|
check_table_exist - Checks if the table exist before dropping the table.
|
|
772
|
+
purge_clause - Specifies string representing purge clause to be appended to drop table query.
|
|
767
773
|
|
|
768
774
|
RETURNS:
|
|
769
775
|
True - if the table is dropped.
|
|
@@ -776,9 +782,11 @@ class UtilFuncs():
|
|
|
776
782
|
UtilFuncs._drop_table('mytab', check_table_exist = False)
|
|
777
783
|
UtilFuncs._drop_table('mydb.mytab', check_table_exist = False)
|
|
778
784
|
UtilFuncs._drop_table("mydb"."mytab", check_table_exist = True)
|
|
785
|
+
UtilFuncs._drop_table("my_lake"."my_db"."my_tab", purge_clause='PURGE ALL')
|
|
786
|
+
UtilFuncs._drop_table("my_lake"."my_db"."my_tab", purge_clause='NO PURGE')
|
|
779
787
|
|
|
780
788
|
"""
|
|
781
|
-
drop_tab = SQLBundle._build_drop_table(table_name)
|
|
789
|
+
drop_tab = SQLBundle._build_drop_table(table_name, purge_clause)
|
|
782
790
|
if check_table_exist is True:
|
|
783
791
|
helptable = UtilFuncs._get_help_tablename(table_name)
|
|
784
792
|
if helptable:
|
|
@@ -958,6 +966,22 @@ class UtilFuncs():
|
|
|
958
966
|
"""
|
|
959
967
|
return UtilFuncs._execute_query(SQLBundle._build_help_table(table_name))
|
|
960
968
|
|
|
969
|
+
@staticmethod
|
|
970
|
+
def _get_help_datalakename(datalake_name):
|
|
971
|
+
"""
|
|
972
|
+
Function to get help of the datalake.
|
|
973
|
+
|
|
974
|
+
PARAMETERS:
|
|
975
|
+
datalake_name - The name of the datalake.
|
|
976
|
+
|
|
977
|
+
RETURNS:
|
|
978
|
+
The help information of the datalake specified by datalake_name.
|
|
979
|
+
|
|
980
|
+
EXAMPLES:
|
|
981
|
+
UtilFuncs._get_help_datalakename(mydatalake)
|
|
982
|
+
"""
|
|
983
|
+
return UtilFuncs._execute_query(SQLBundle._build_help_datalake(datalake_name))
|
|
984
|
+
|
|
961
985
|
@staticmethod
|
|
962
986
|
def _get_select_table(table_name):
|
|
963
987
|
"""
|
|
@@ -1286,43 +1310,71 @@ class UtilFuncs():
|
|
|
1286
1310
|
return any(UtilFuncs._contains_space(col) for col in item)
|
|
1287
1311
|
|
|
1288
1312
|
return False
|
|
1289
|
-
|
|
1290
|
-
|
|
1291
|
-
def _in_schema(schema_name, table_name):
|
|
1313
|
+
|
|
1314
|
+
def _is_ascii(col_lst):
|
|
1292
1315
|
"""
|
|
1293
|
-
|
|
1294
|
-
|
|
1295
|
-
Note:
|
|
1296
|
-
teradataml recommends to use this function to access table(s)/view(s),
|
|
1297
|
-
from the database other than the default database.
|
|
1316
|
+
Description:
|
|
1317
|
+
Check if the specified string in col_lst has non-ASCII characters in it.
|
|
1298
1318
|
|
|
1299
1319
|
PARAMETERS:
|
|
1300
|
-
|
|
1301
|
-
Required Argument
|
|
1302
|
-
Specifies
|
|
1303
|
-
Types:
|
|
1320
|
+
col_lst:
|
|
1321
|
+
Required Argument.
|
|
1322
|
+
Specifies a list of strings to check for non-ASCII characters.
|
|
1323
|
+
Types: list
|
|
1304
1324
|
|
|
1305
|
-
|
|
1306
|
-
|
|
1307
|
-
Specifies the table name or view name in Vantage.
|
|
1308
|
-
Types: str
|
|
1325
|
+
RETURNS:
|
|
1326
|
+
True, if the specified string has non-ASCII characters in it, else False.
|
|
1309
1327
|
|
|
1328
|
+
RAISES:
|
|
1329
|
+
None.
|
|
1330
|
+
|
|
1310
1331
|
EXAMPLES:
|
|
1311
|
-
|
|
1332
|
+
# Passing column name with non-ASCII characters returns True.
|
|
1333
|
+
>>> is_non_ascii = UtilFuncs._is_ascii(["col name", "がく片の長さ@name"])
|
|
1334
|
+
>>> print(is_non_ascii)
|
|
1335
|
+
>>> True
|
|
1336
|
+
# Passing column name without non-ASCII characters returns False.
|
|
1337
|
+
>>> is_non_ascii = UtilFuncs._is_ascii(["colname", "col_name"])
|
|
1338
|
+
>>> print(is_non_ascii)
|
|
1339
|
+
>>> False
|
|
1340
|
+
"""
|
|
1341
|
+
if isinstance(col_lst, str):
|
|
1342
|
+
# Check if the input is a string and look for non-ASCII characters
|
|
1343
|
+
return not col_lst.isascii()
|
|
1344
|
+
|
|
1345
|
+
if isinstance(col_lst, list):
|
|
1346
|
+
for item in col_lst:
|
|
1347
|
+
if isinstance(item, str) and not item.isascii():
|
|
1348
|
+
return True
|
|
1349
|
+
return False
|
|
1350
|
+
|
|
1351
|
+
@staticmethod
|
|
1352
|
+
def __get_dot_separated_names(full_qualified_name):
|
|
1353
|
+
"""
|
|
1354
|
+
Takes in fully qualified name of the table/view (db.table), and returns
|
|
1355
|
+
a dot separated name from the same.
|
|
1356
|
+
|
|
1357
|
+
PARAMETERS:
|
|
1358
|
+
full_qualified_name - Name of the table/view
|
|
1312
1359
|
|
|
1313
|
-
|
|
1314
|
-
|
|
1315
|
-
|
|
1316
|
-
df = DataFrame(in_schema("dbc", "dbcinfo"))
|
|
1360
|
+
EXAMPLES:
|
|
1361
|
+
UtilFuncs._extract_db_name('"db1"."tablename"')
|
|
1362
|
+
UtilFuncs._extract_db_name('"Icebergs"."db1"."tablename"')
|
|
1317
1363
|
|
|
1318
|
-
|
|
1319
|
-
|
|
1320
|
-
# create a teradataml DataFrame.
|
|
1321
|
-
df = DataFrame.from_table(in_schema("dbc","dbcinfo"))
|
|
1364
|
+
RETURNS:
|
|
1365
|
+
List of dot separated name from the provided name.
|
|
1322
1366
|
|
|
1323
1367
|
"""
|
|
1324
|
-
|
|
1325
|
-
|
|
1368
|
+
# If condition to handle the quoted name.
|
|
1369
|
+
if '"' in full_qualified_name:
|
|
1370
|
+
# Extract the double quoted strings.
|
|
1371
|
+
names = re.findall(r'["](.*?)["]', full_qualified_name)
|
|
1372
|
+
# Remove quotes around the string.
|
|
1373
|
+
names = [i.replace('"', '') for i in names]
|
|
1374
|
+
# Handle non-quoted string with dot separated name.
|
|
1375
|
+
else:
|
|
1376
|
+
names = full_qualified_name.split(".")
|
|
1377
|
+
return names
|
|
1326
1378
|
|
|
1327
1379
|
@staticmethod
|
|
1328
1380
|
def _extract_db_name(full_qualified_name):
|
|
@@ -1334,19 +1386,19 @@ class UtilFuncs():
|
|
|
1334
1386
|
full_qualified_name - Name of the table/view
|
|
1335
1387
|
|
|
1336
1388
|
EXAMPLES:
|
|
1337
|
-
UtilFuncs._extract_db_name("db1"."tablename")
|
|
1389
|
+
UtilFuncs._extract_db_name('"db1"."tablename"')
|
|
1390
|
+
UtilFuncs._extract_db_name('"Icebergs"."db1"."tablename"')
|
|
1338
1391
|
|
|
1339
1392
|
RETURNS:
|
|
1340
1393
|
Database name from the provided name.
|
|
1341
1394
|
|
|
1342
1395
|
"""
|
|
1343
|
-
|
|
1344
|
-
names = re.findall(r'["](.*?)["]', full_qualified_name)
|
|
1345
|
-
# Remove quotes around the string.
|
|
1346
|
-
names = [i.replace('"', '') for i in names]
|
|
1396
|
+
names = UtilFuncs.__get_dot_separated_names(full_qualified_name)
|
|
1347
1397
|
if names:
|
|
1348
1398
|
if len(names) == 2:
|
|
1349
1399
|
return names[0]
|
|
1400
|
+
elif len(names) == 3:
|
|
1401
|
+
return names[1]
|
|
1350
1402
|
else:
|
|
1351
1403
|
return None
|
|
1352
1404
|
|
|
@@ -1360,23 +1412,45 @@ class UtilFuncs():
|
|
|
1360
1412
|
full_qualified_name - Name of the table/view
|
|
1361
1413
|
|
|
1362
1414
|
EXAMPLES:
|
|
1363
|
-
UtilFuncs.
|
|
1415
|
+
UtilFuncs._extract_table_name('"db1"."tablename"')
|
|
1416
|
+
UtilFuncs._extract_table_name('"Icebergs"."db1"."tablename"')
|
|
1364
1417
|
|
|
1365
1418
|
RETURNS:
|
|
1366
1419
|
Table/View name from the provided name.
|
|
1367
1420
|
|
|
1368
1421
|
"""
|
|
1369
|
-
|
|
1370
|
-
names = re.findall(r'["](.*?)["]', full_qualified_name)
|
|
1371
|
-
# Remove quotes around the string.
|
|
1372
|
-
names = [i.replace('"', '') for i in names]
|
|
1422
|
+
names = UtilFuncs.__get_dot_separated_names(full_qualified_name)
|
|
1373
1423
|
if names:
|
|
1374
1424
|
if len(names) == 2:
|
|
1375
1425
|
return names[1]
|
|
1426
|
+
elif len(names) == 3:
|
|
1427
|
+
return names[2]
|
|
1376
1428
|
else:
|
|
1377
1429
|
return names[0]
|
|
1378
1430
|
return full_qualified_name
|
|
1379
1431
|
|
|
1432
|
+
@staticmethod
|
|
1433
|
+
def _extract_datalake_name(full_qualified_name):
|
|
1434
|
+
"""
|
|
1435
|
+
Takes in fully qualified name of the table/view (db.table), and returns
|
|
1436
|
+
a datalake name from the same.
|
|
1437
|
+
|
|
1438
|
+
PARAMETERS:
|
|
1439
|
+
full_qualified_name - Name of the table/view
|
|
1440
|
+
|
|
1441
|
+
EXAMPLES:
|
|
1442
|
+
UtilFuncs._extract_datalake_name('"db1"."tablename"')
|
|
1443
|
+
UtilFuncs._extract_datalake_name('"Icebergs"."db1"."tablename"')
|
|
1444
|
+
|
|
1445
|
+
RETURNS:
|
|
1446
|
+
Database name from the provided name.
|
|
1447
|
+
|
|
1448
|
+
"""
|
|
1449
|
+
names = UtilFuncs.__get_dot_separated_names(full_qualified_name)
|
|
1450
|
+
if names and len(names) == 3:
|
|
1451
|
+
return names[0]
|
|
1452
|
+
return None
|
|
1453
|
+
|
|
1380
1454
|
@staticmethod
|
|
1381
1455
|
def _teradata_quote_arg(args, quote="'", call_from_wrapper=True):
|
|
1382
1456
|
"""
|
teradataml/common/warnings.py
CHANGED
|
@@ -17,9 +17,20 @@ class VantageRuntimeWarning(RuntimeWarning):
|
|
|
17
17
|
"""
|
|
18
18
|
pass
|
|
19
19
|
|
|
20
|
+
|
|
20
21
|
class TeradataMlRuntimeWarning(RuntimeWarning):
|
|
21
22
|
"""
|
|
22
23
|
The TeradataMlRuntimeWarning is thrown whenever there are Warnings in runtime execution of objects in teradataml.
|
|
23
24
|
"""
|
|
24
25
|
# For future purpose
|
|
25
26
|
pass
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class OneTimeUserWarning(UserWarning):
|
|
31
|
+
"""
|
|
32
|
+
The OneTimeUserWarning is thrown when the warning should only be displayed once to the user.
|
|
33
|
+
This can be useful for deprecation warnings, configuration issues, or other
|
|
34
|
+
non-critical warnings that do not need to be repeated multiple times.
|
|
35
|
+
"""
|
|
36
|
+
pass
|
teradataml/context/context.py
CHANGED
|
@@ -13,32 +13,35 @@ A teradataml context functions provide interface to Teradata Vantage. Provides f
|
|
|
13
13
|
context which can be used by other analytical functions to get the Teradata Vantage connection.
|
|
14
14
|
|
|
15
15
|
"""
|
|
16
|
+
import atexit
|
|
16
17
|
import ipaddress
|
|
18
|
+
import os
|
|
19
|
+
import socket
|
|
20
|
+
import sys
|
|
21
|
+
import threading
|
|
22
|
+
import urllib.parse
|
|
23
|
+
import warnings
|
|
17
24
|
from pathlib import Path
|
|
25
|
+
from dotenv import dotenv_values
|
|
26
|
+
|
|
18
27
|
from sqlalchemy import create_engine
|
|
19
|
-
from
|
|
28
|
+
from sqlalchemy.engine.base import Engine
|
|
29
|
+
from sqlalchemy.engine.url import URL
|
|
30
|
+
|
|
31
|
+
from teradataml.common.constants import Query, SQLConstants, TeradataConstants
|
|
20
32
|
from teradataml.common.deprecations import argument_deprecation
|
|
21
33
|
from teradataml.common.exceptions import TeradataMlException
|
|
22
|
-
from teradataml.common.
|
|
23
|
-
from teradataml.common.messages import Messages
|
|
34
|
+
from teradataml.common.garbagecollector import GarbageCollector
|
|
24
35
|
from teradataml.common.messagecodes import MessageCodes
|
|
36
|
+
from teradataml.common.messages import Messages
|
|
25
37
|
from teradataml.common.sqlbundle import SQLBundle
|
|
26
|
-
from teradataml.common.
|
|
27
|
-
from teradataml.common.garbagecollector import GarbageCollector
|
|
38
|
+
from teradataml.common.warnings import TeradataMlRuntimeWarning
|
|
28
39
|
from teradataml.context.aed_context import AEDContext
|
|
29
|
-
from teradataml.common.constants import TeradataConstants, Query
|
|
30
40
|
from teradataml.options.configure import configure
|
|
41
|
+
from teradataml.telemetry_utils.queryband import collect_queryband
|
|
42
|
+
from teradataml.utils.internal_buffer import _InternalBuffer
|
|
31
43
|
from teradataml.utils.utils import execute_sql
|
|
32
44
|
from teradataml.utils.validators import _Validators
|
|
33
|
-
from teradataml.utils.internal_buffer import _InternalBuffer
|
|
34
|
-
from sqlalchemy.engine.base import Engine
|
|
35
|
-
from sqlalchemy.engine.url import URL
|
|
36
|
-
import os
|
|
37
|
-
import warnings
|
|
38
|
-
import atexit
|
|
39
|
-
import socket
|
|
40
|
-
import threading
|
|
41
|
-
import urllib.parse
|
|
42
45
|
|
|
43
46
|
# Store a global Teradata Vantage Connection.
|
|
44
47
|
# Right now user can only provide a single Vantage connection at any point of time.
|
|
@@ -47,6 +50,8 @@ td_sqlalchemy_engine = None
|
|
|
47
50
|
temporary_database_name = None
|
|
48
51
|
user_specified_connection = False
|
|
49
52
|
python_packages_installed = False
|
|
53
|
+
python_version_vantage = None
|
|
54
|
+
python_version_local = None
|
|
50
55
|
td_user = None
|
|
51
56
|
|
|
52
57
|
function_alias_mappings = {}
|
|
@@ -201,9 +206,16 @@ def create_context(host = None, username = None, password = None, tdsqlengine =
|
|
|
201
206
|
"""
|
|
202
207
|
DESCRIPTION:
|
|
203
208
|
Creates a connection to the Teradata Vantage using the teradatasql + teradatasqlalchemy DBAPI and dialect
|
|
204
|
-
combination. Users can pass all required parameters (host, username, password) for establishing a connection to
|
|
205
|
-
Vantage, or pass a sqlalchemy engine to the tdsqlengine parameter to override the default DBAPI and dialect
|
|
206
209
|
combination.
|
|
210
|
+
Users can create a connection by passing the connection parameters using any of the following methods:
|
|
211
|
+
1. Pass all required parameters (host, username, password) directly to the function, OR
|
|
212
|
+
2. Set the connection parameters in a configuration file (.cfg or .env) and
|
|
213
|
+
pass the configuration file OR
|
|
214
|
+
3. Set the connection parameters in environment variables and create_context() reads from
|
|
215
|
+
environment variables.
|
|
216
|
+
|
|
217
|
+
Alternatively, users can pass a SQLAlchemy engine to the `tdsqlengine` parameter to override the default DBAPI
|
|
218
|
+
and dialect combination.
|
|
207
219
|
|
|
208
220
|
Note:
|
|
209
221
|
1. teradataml requires that the user has certain permissions on the user's default database or the initial
|
|
@@ -241,6 +253,17 @@ def create_context(host = None, username = None, password = None, tdsqlengine =
|
|
|
241
253
|
| | default database. |
|
|
242
254
|
+------------------------------------------------------+---------------------------------------------+
|
|
243
255
|
|
|
256
|
+
3. The function prioritizes parameters in the following order:
|
|
257
|
+
1. Explicitly passed arguments (host, username, password).
|
|
258
|
+
2. Environment variables (TD_HOST, TD_USERNAME, TD_PASSWORD, etc.).
|
|
259
|
+
Note:
|
|
260
|
+
* The environment variables should start with 'TD_' and all must be in upper case.
|
|
261
|
+
Example:
|
|
262
|
+
os.environ['TD_HOST'] = 'tdhost'
|
|
263
|
+
os.environ['TD_USERNAME'] = 'tduser'
|
|
264
|
+
os.environ['TD_PASSWORD'] = 'tdpassword'
|
|
265
|
+
3. A configuration file if provided (such as a .env file).
|
|
266
|
+
|
|
244
267
|
PARAMETERS:
|
|
245
268
|
host:
|
|
246
269
|
Optional Argument.
|
|
@@ -331,6 +354,25 @@ def create_context(host = None, username = None, password = None, tdsqlengine =
|
|
|
331
354
|
integer or boolean value, instead of quoted integer or quoted boolean as suggested in the
|
|
332
355
|
documentation. Please check the examples for usage.
|
|
333
356
|
|
|
357
|
+
config_file:
|
|
358
|
+
Optional Argument.
|
|
359
|
+
Specifies the name of the configuration file to read the connection parameters.
|
|
360
|
+
Notes:
|
|
361
|
+
* If user do not specify full path of file,then file look up is done at current working directory.
|
|
362
|
+
* The content of the file must be in '.env' format.
|
|
363
|
+
* Use parameters of create_context() as key in the configuration file.
|
|
364
|
+
Example:
|
|
365
|
+
host=tdhost
|
|
366
|
+
username=tduser
|
|
367
|
+
password=tdpassword
|
|
368
|
+
temp_database_name=tdtemp_database_name
|
|
369
|
+
logmech=tdlogmech
|
|
370
|
+
logdata=tdlogdata
|
|
371
|
+
database=tddatabase
|
|
372
|
+
* For more information please refer examples section.
|
|
373
|
+
Default Value : td_properties.cfg
|
|
374
|
+
Types: str
|
|
375
|
+
|
|
334
376
|
RETURNS:
|
|
335
377
|
A Teradata sqlalchemy engine object.
|
|
336
378
|
|
|
@@ -400,13 +442,56 @@ def create_context(host = None, username = None, password = None, tdsqlengine =
|
|
|
400
442
|
>>> td_context = create_context(host = 'tdhost', username='tduser', password = 'tdpassword', tmode = 'tera',
|
|
401
443
|
log = 8, lob_support = False)
|
|
402
444
|
|
|
445
|
+
# Example 14: Create context from config file with name 'td_properties.cfg'
|
|
446
|
+
# available under current working directory.
|
|
447
|
+
# td_properties.cfg content:
|
|
448
|
+
# host=tdhost
|
|
449
|
+
# username=tduser
|
|
450
|
+
# password=tdpassword
|
|
451
|
+
# temp_database_name=tdtemp_database_name
|
|
452
|
+
# logmech=tdlogmech
|
|
453
|
+
# logdata=tdlogdata
|
|
454
|
+
# database=tddatabase
|
|
455
|
+
>>> td_context = create_context()
|
|
456
|
+
|
|
457
|
+
# Example 15: Create context using the file specified in user's home directory
|
|
458
|
+
# with name user_td_properties.cfg.
|
|
459
|
+
# user_td_properties.cfg content:
|
|
460
|
+
# host=tdhost
|
|
461
|
+
# username=tduser
|
|
462
|
+
# password=tdpassword
|
|
463
|
+
# temp_database_name=tdtemp_database_name
|
|
464
|
+
# logmech=tdlogmech
|
|
465
|
+
# logdata=tdlogdata
|
|
466
|
+
# database=tddatabase
|
|
467
|
+
>>> td_context = create_context(config_file="user_td_properties.cfg")
|
|
468
|
+
|
|
469
|
+
# Example 16: Create context using environment variables.
|
|
470
|
+
# Set these using os.environ and then run the example:
|
|
471
|
+
# os.environ['TD_HOST'] = 'tdhost'
|
|
472
|
+
# os.environ['TD_USERNAME'] = 'tduser'
|
|
473
|
+
# os.environ['TD_PASSWORD'] = 'tdpassword'
|
|
474
|
+
# os.environ['TD_TEMP_DATABASE_NAME'] = 'tdtemp_database_name'
|
|
475
|
+
# os.environ['TD_LOGMECH'] = 'tdlogmech'
|
|
476
|
+
# os.environ['TD_LOGDATA'] = 'tdlogdata'
|
|
477
|
+
# os.environ['TD_DATABASE'] = 'tddatabase'
|
|
478
|
+
>>> td_context = create_context()
|
|
403
479
|
"""
|
|
404
480
|
global td_connection
|
|
405
481
|
global td_sqlalchemy_engine
|
|
406
482
|
global temporary_database_name
|
|
407
483
|
global user_specified_connection
|
|
408
484
|
global python_packages_installed
|
|
485
|
+
global python_version_vantage
|
|
486
|
+
global python_version_local
|
|
409
487
|
global td_user
|
|
488
|
+
|
|
489
|
+
# Check if the user has provided the connection parameters or tdsqlengine.
|
|
490
|
+
# If not, check if the user has provided the connection parameters in the environment variables.
|
|
491
|
+
# If not, check if the user has provided the connection parameters in the config file.
|
|
492
|
+
if not (host or tdsqlengine) and host!="":
|
|
493
|
+
return _load_context_from_env_config(kwargs.pop('config_file', 'td_properties.cfg'))
|
|
494
|
+
|
|
410
495
|
awu_matrix = []
|
|
411
496
|
awu_matrix.append(["host", host, True, (str), True])
|
|
412
497
|
awu_matrix.append(["username", username, True, (str), True])
|
|
@@ -497,6 +582,8 @@ def create_context(host = None, username = None, password = None, tdsqlengine =
|
|
|
497
582
|
_load_function_aliases()
|
|
498
583
|
|
|
499
584
|
python_packages_installed = False
|
|
585
|
+
python_version_vantage = None
|
|
586
|
+
python_version_local = sys.version.split(" ")[0].strip()
|
|
500
587
|
|
|
501
588
|
# Assign the tempdatabase name to global
|
|
502
589
|
if temp_database_name is None:
|
|
@@ -527,6 +614,34 @@ def create_context(host = None, username = None, password = None, tdsqlengine =
|
|
|
527
614
|
# Return the connection by default
|
|
528
615
|
return td_sqlalchemy_engine
|
|
529
616
|
|
|
617
|
+
def _load_context_from_env_config(config_file):
|
|
618
|
+
"""
|
|
619
|
+
DESCRIPTION:
|
|
620
|
+
Reads the connection parameters from the configuration file or environment variables.
|
|
621
|
+
|
|
622
|
+
PARAMETERS:
|
|
623
|
+
config_file:
|
|
624
|
+
Required Argument.
|
|
625
|
+
Specifies the name of the configuration file to read the connection parameters.
|
|
626
|
+
Types: str
|
|
627
|
+
|
|
628
|
+
RETURNS:
|
|
629
|
+
A Teradata sqlalchemy engine object.
|
|
630
|
+
|
|
631
|
+
RAISES:
|
|
632
|
+
TeradataMlException
|
|
633
|
+
"""
|
|
634
|
+
host = os.environ.get('TD_HOST')
|
|
635
|
+
connection_params_from_file = dotenv_values(config_file)
|
|
636
|
+
if host:
|
|
637
|
+
connection_params_from_env = {key[3:].lower(): value for key, value in os.environ.items()
|
|
638
|
+
if key.startswith('TD_')}
|
|
639
|
+
return create_context(**connection_params_from_env)
|
|
640
|
+
elif connection_params_from_file.get('host'):
|
|
641
|
+
return create_context(**connection_params_from_file)
|
|
642
|
+
else:
|
|
643
|
+
raise TeradataMlException(Messages.get_message(MessageCodes.CONNECTION_PARAMS),
|
|
644
|
+
MessageCodes.MISSING_ARGS)
|
|
530
645
|
|
|
531
646
|
def _mask_logmech_logdata():
|
|
532
647
|
"""
|
|
@@ -625,6 +740,8 @@ def set_context(tdsqlengine, temp_database_name=None):
|
|
|
625
740
|
global temporary_database_name
|
|
626
741
|
global user_specified_connection
|
|
627
742
|
global python_packages_installed
|
|
743
|
+
global python_version_local
|
|
744
|
+
global python_version_vantage
|
|
628
745
|
if td_connection is not None:
|
|
629
746
|
# Clearing the internal buffer.
|
|
630
747
|
_InternalBuffer.clean()
|
|
@@ -654,6 +771,8 @@ def set_context(tdsqlengine, temp_database_name=None):
|
|
|
654
771
|
_load_function_aliases()
|
|
655
772
|
|
|
656
773
|
python_packages_installed = False
|
|
774
|
+
python_version_vantage = None
|
|
775
|
+
python_version_local = sys.version.split(" ")[0].strip()
|
|
657
776
|
|
|
658
777
|
# Initialise Dag
|
|
659
778
|
__initalise_dag()
|
|
@@ -687,6 +806,8 @@ def remove_context():
|
|
|
687
806
|
global td_sqlalchemy_engine
|
|
688
807
|
global user_specified_connection
|
|
689
808
|
global python_packages_installed
|
|
809
|
+
global python_version_vantage
|
|
810
|
+
global python_version_local
|
|
690
811
|
global td_user
|
|
691
812
|
|
|
692
813
|
# Initiate the garbage collection
|
|
@@ -706,10 +827,13 @@ def remove_context():
|
|
|
706
827
|
td_connection = None
|
|
707
828
|
td_sqlalchemy_engine = None
|
|
708
829
|
python_packages_installed = False
|
|
830
|
+
python_version_local = None
|
|
831
|
+
python_version_vantage = None
|
|
709
832
|
td_user = None
|
|
710
833
|
configure._current_database_name = None
|
|
711
834
|
configure._database_username = None
|
|
712
835
|
configure.database_version = None
|
|
836
|
+
configure.indb_install_location = ''
|
|
713
837
|
|
|
714
838
|
# Closing Dag
|
|
715
839
|
__close_dag()
|