teradataml 20.0.0.6__py3-none-any.whl → 20.0.0.7__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of teradataml might be problematic. Click here for more details.
- teradataml/README.md +210 -0
- teradataml/__init__.py +1 -1
- teradataml/_version.py +1 -1
- teradataml/analytics/analytic_function_executor.py +162 -76
- teradataml/analytics/byom/__init__.py +1 -1
- teradataml/analytics/json_parser/__init__.py +2 -0
- teradataml/analytics/json_parser/analytic_functions_argument.py +95 -2
- teradataml/analytics/json_parser/metadata.py +22 -4
- teradataml/analytics/sqle/DecisionTreePredict.py +3 -2
- teradataml/analytics/sqle/NaiveBayesPredict.py +3 -2
- teradataml/analytics/sqle/__init__.py +3 -0
- teradataml/analytics/utils.py +4 -1
- teradataml/automl/__init__.py +2369 -464
- teradataml/automl/autodataprep/__init__.py +15 -0
- teradataml/automl/custom_json_utils.py +184 -112
- teradataml/automl/data_preparation.py +113 -58
- teradataml/automl/data_transformation.py +154 -53
- teradataml/automl/feature_engineering.py +113 -53
- teradataml/automl/feature_exploration.py +548 -25
- teradataml/automl/model_evaluation.py +260 -32
- teradataml/automl/model_training.py +399 -206
- teradataml/clients/auth_client.py +2 -2
- teradataml/common/aed_utils.py +11 -2
- teradataml/common/bulk_exposed_utils.py +4 -2
- teradataml/common/constants.py +62 -2
- teradataml/common/garbagecollector.py +50 -21
- teradataml/common/messagecodes.py +47 -2
- teradataml/common/messages.py +19 -1
- teradataml/common/sqlbundle.py +23 -6
- teradataml/common/utils.py +116 -10
- teradataml/context/aed_context.py +16 -10
- teradataml/data/Employee.csv +5 -0
- teradataml/data/Employee_Address.csv +4 -0
- teradataml/data/Employee_roles.csv +5 -0
- teradataml/data/JulesBelvezeDummyData.csv +100 -0
- teradataml/data/byom_example.json +5 -0
- teradataml/data/creditcard_data.csv +284618 -0
- teradataml/data/docs/byom/docs/ONNXSeq2Seq.py +255 -0
- teradataml/data/docs/sqle/docs_17_10/NGramSplitter.py +1 -1
- teradataml/data/docs/sqle/docs_17_20/NGramSplitter.py +1 -1
- teradataml/data/docs/sqle/docs_17_20/TextParser.py +1 -1
- teradataml/data/jsons/byom/ONNXSeq2Seq.json +287 -0
- teradataml/data/jsons/sqle/20.00/AI_AnalyzeSentiment.json +3 -7
- teradataml/data/jsons/sqle/20.00/AI_AskLLM.json +3 -7
- teradataml/data/jsons/sqle/20.00/AI_DetectLanguage.json +3 -7
- teradataml/data/jsons/sqle/20.00/AI_ExtractKeyPhrases.json +3 -7
- teradataml/data/jsons/sqle/20.00/AI_MaskPII.json +3 -7
- teradataml/data/jsons/sqle/20.00/AI_RecognizeEntities.json +3 -7
- teradataml/data/jsons/sqle/20.00/AI_RecognizePIIEntities.json +3 -7
- teradataml/data/jsons/sqle/20.00/AI_TextClassifier.json +3 -7
- teradataml/data/jsons/sqle/20.00/AI_TextEmbeddings.json +3 -7
- teradataml/data/jsons/sqle/20.00/AI_TextSummarize.json +3 -7
- teradataml/data/jsons/sqle/20.00/AI_TextTranslate.json +3 -7
- teradataml/data/jsons/sqle/20.00/TD_API_AzureML.json +151 -0
- teradataml/data/jsons/sqle/20.00/TD_API_Sagemaker.json +182 -0
- teradataml/data/jsons/sqle/20.00/TD_API_VertexAI.json +183 -0
- teradataml/data/load_example_data.py +29 -11
- teradataml/data/payment_fraud_dataset.csv +10001 -0
- teradataml/data/teradataml_example.json +67 -0
- teradataml/dataframe/copy_to.py +714 -54
- teradataml/dataframe/dataframe.py +1153 -33
- teradataml/dataframe/dataframe_utils.py +8 -3
- teradataml/dataframe/functions.py +168 -1
- teradataml/dataframe/setop.py +4 -1
- teradataml/dataframe/sql.py +141 -9
- teradataml/dbutils/dbutils.py +470 -35
- teradataml/dbutils/filemgr.py +1 -1
- teradataml/hyperparameter_tuner/optimizer.py +456 -142
- teradataml/lib/aed_0_1.dll +0 -0
- teradataml/lib/libaed_0_1.dylib +0 -0
- teradataml/lib/libaed_0_1.so +0 -0
- teradataml/lib/libaed_0_1_aarch64.so +0 -0
- teradataml/scriptmgmt/UserEnv.py +234 -34
- teradataml/scriptmgmt/lls_utils.py +43 -17
- teradataml/sdk/_json_parser.py +1 -1
- teradataml/sdk/api_client.py +9 -6
- teradataml/sdk/modelops/_client.py +3 -0
- teradataml/series/series.py +12 -7
- teradataml/store/feature_store/constants.py +601 -234
- teradataml/store/feature_store/feature_store.py +2886 -616
- teradataml/store/feature_store/mind_map.py +639 -0
- teradataml/store/feature_store/models.py +5831 -214
- teradataml/store/feature_store/utils.py +390 -0
- teradataml/table_operators/table_operator_util.py +1 -1
- teradataml/table_operators/templates/dataframe_register.template +6 -2
- teradataml/table_operators/templates/dataframe_udf.template +6 -2
- teradataml/utils/docstring.py +527 -0
- teradataml/utils/dtypes.py +93 -0
- teradataml/utils/internal_buffer.py +2 -2
- teradataml/utils/utils.py +41 -2
- teradataml/utils/validators.py +694 -17
- {teradataml-20.0.0.6.dist-info → teradataml-20.0.0.7.dist-info}/METADATA +213 -2
- {teradataml-20.0.0.6.dist-info → teradataml-20.0.0.7.dist-info}/RECORD +96 -81
- {teradataml-20.0.0.6.dist-info → teradataml-20.0.0.7.dist-info}/WHEEL +0 -0
- {teradataml-20.0.0.6.dist-info → teradataml-20.0.0.7.dist-info}/top_level.txt +0 -0
- {teradataml-20.0.0.6.dist-info → teradataml-20.0.0.7.dist-info}/zip-safe +0 -0
teradataml/dbutils/dbutils.py
CHANGED
|
@@ -10,12 +10,14 @@ teradataml db utilities
|
|
|
10
10
|
A teradataml database utility functions provide interface to Teradata Vantage common tasks such as drop_table, drop_view, create_table etc.
|
|
11
11
|
"""
|
|
12
12
|
import concurrent.futures
|
|
13
|
+
import enum
|
|
13
14
|
import json
|
|
14
15
|
import os
|
|
15
16
|
import re
|
|
16
17
|
import shutil
|
|
17
18
|
import tempfile
|
|
18
19
|
from datetime import datetime
|
|
20
|
+
import functools
|
|
19
21
|
|
|
20
22
|
import pandas as pd
|
|
21
23
|
from sqlalchemy import (CheckConstraint, Column, ForeignKeyConstraint,
|
|
@@ -69,7 +71,7 @@ def db_drop_table(table_name, schema_name=None, suppress_error=False,
|
|
|
69
71
|
Optional Argument
|
|
70
72
|
Specifies whether to raise error or not.
|
|
71
73
|
Default Value: False
|
|
72
|
-
Types:
|
|
74
|
+
Types: bool
|
|
73
75
|
|
|
74
76
|
datalake_name:
|
|
75
77
|
Optional Argument
|
|
@@ -113,6 +115,7 @@ def db_drop_table(table_name, schema_name=None, suppress_error=False,
|
|
|
113
115
|
awu_matrix.append(["table_name", table_name, False, (str), True])
|
|
114
116
|
awu_matrix.append(["datalake_name", datalake_name, True, (str), True])
|
|
115
117
|
awu_matrix.append(["purge", purge, True, (bool, type(None)), True])
|
|
118
|
+
awu_matrix.append(["suppress_error", suppress_error, True, (bool)])
|
|
116
119
|
# Validate argument types
|
|
117
120
|
_Validators._validate_function_arguments(awu_matrix)
|
|
118
121
|
|
|
@@ -149,7 +152,7 @@ def db_drop_table(table_name, schema_name=None, suppress_error=False,
|
|
|
149
152
|
|
|
150
153
|
|
|
151
154
|
@collect_queryband(queryband='DrpVw')
|
|
152
|
-
def db_drop_view(view_name, schema_name=None):
|
|
155
|
+
def db_drop_view(view_name, schema_name=None, suppress_error=False):
|
|
153
156
|
"""
|
|
154
157
|
DESCRIPTION:
|
|
155
158
|
Drops the view from the given schema.
|
|
@@ -167,6 +170,12 @@ def db_drop_view(view_name, schema_name=None):
|
|
|
167
170
|
Default Value: None
|
|
168
171
|
Types: str
|
|
169
172
|
|
|
173
|
+
suppress_error:
|
|
174
|
+
Optional Argument
|
|
175
|
+
Specifies whether to raise error or not.
|
|
176
|
+
Default Value: False
|
|
177
|
+
Types: bool
|
|
178
|
+
|
|
170
179
|
RETURNS:
|
|
171
180
|
True - if the operation is successful.
|
|
172
181
|
|
|
@@ -174,19 +183,23 @@ def db_drop_view(view_name, schema_name=None):
|
|
|
174
183
|
TeradataMlException - If the view doesn't exist.
|
|
175
184
|
|
|
176
185
|
EXAMPLES:
|
|
177
|
-
# Create a view
|
|
186
|
+
# Create a view.
|
|
178
187
|
>>> execute_sql("create view temporary_view as (select 1 as dummy_col1, 2 as dummy_col2);")
|
|
179
188
|
|
|
180
|
-
# Drop view in current schema
|
|
189
|
+
# Drop view in current schema.
|
|
181
190
|
>>> db_drop_view(view_name = 'temporary_view')
|
|
182
191
|
|
|
183
|
-
# Drop view from the given schema
|
|
192
|
+
# Drop view from the given schema.
|
|
184
193
|
>>> db_drop_view(view_name = 'temporary_view', schema_name = 'alice')
|
|
194
|
+
|
|
195
|
+
# Drop view by suppressing errors.
|
|
196
|
+
>>> db_drop_view(view_name = 'temporary_view', suppress_error = True)
|
|
185
197
|
"""
|
|
186
198
|
# Argument validations
|
|
187
199
|
awu_matrix = []
|
|
188
200
|
awu_matrix.append(["schema_name", schema_name, True, (str), True])
|
|
189
201
|
awu_matrix.append(["view_name", view_name, False, (str), True])
|
|
202
|
+
awu_matrix.append(["suppress_error", suppress_error, True, (bool)])
|
|
190
203
|
|
|
191
204
|
# Validate argument types
|
|
192
205
|
_Validators._validate_function_arguments(awu_matrix)
|
|
@@ -196,14 +209,18 @@ def db_drop_view(view_name, schema_name=None):
|
|
|
196
209
|
|
|
197
210
|
try:
|
|
198
211
|
return UtilFuncs._drop_view(view_name)
|
|
199
|
-
except TeradataMlException:
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
212
|
+
except (TeradataMlException, OperationalError):
|
|
213
|
+
if suppress_error:
|
|
214
|
+
pass
|
|
215
|
+
else:
|
|
216
|
+
raise
|
|
203
217
|
except Exception as err:
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
218
|
+
if suppress_error:
|
|
219
|
+
pass
|
|
220
|
+
else:
|
|
221
|
+
raise TeradataMlException(Messages.get_message(MessageCodes.DROP_FAILED, "view",
|
|
222
|
+
view_name),
|
|
223
|
+
MessageCodes.DROP_FAILED) from err
|
|
207
224
|
|
|
208
225
|
|
|
209
226
|
@collect_queryband(queryband='LstTbls')
|
|
@@ -422,7 +439,7 @@ def _get_select_table_kind(schema_name, table_name, table_kind, datalake_name):
|
|
|
422
439
|
if object_table_kind is not None:
|
|
423
440
|
object_table_kind = ', '.join([f"'{value}'" for value in object_table_kind])
|
|
424
441
|
query = SQLBundle._build_select_table_kind(schema_name, object_name_str, object_table_kind)
|
|
425
|
-
pddf = pd.read_sql(query, tdmlctx.
|
|
442
|
+
pddf = pd.read_sql(query, tdmlctx.get_connection())
|
|
426
443
|
|
|
427
444
|
# Check if all table kind or volatile table kind is requested.
|
|
428
445
|
# If so,add volatile tables to the pddf.
|
|
@@ -431,7 +448,7 @@ def _get_select_table_kind(schema_name, table_name, table_kind, datalake_name):
|
|
|
431
448
|
# Create list of volatile tables.
|
|
432
449
|
try:
|
|
433
450
|
vtquery = SQLBundle._build_help_volatile_table()
|
|
434
|
-
vtdf = pd.read_sql(vtquery, tdmlctx.
|
|
451
|
+
vtdf = pd.read_sql(vtquery, tdmlctx.get_connection())
|
|
435
452
|
if not vtdf.empty:
|
|
436
453
|
# Volatile table query returns different column names.
|
|
437
454
|
# So, rename its column names to match with normal
|
|
@@ -579,7 +596,7 @@ def db_transaction(func):
|
|
|
579
596
|
True
|
|
580
597
|
>>>
|
|
581
598
|
"""
|
|
582
|
-
|
|
599
|
+
@functools.wraps(func)
|
|
583
600
|
def execute_transaction(*args, **kwargs):
|
|
584
601
|
auto_commit_off = "{fn teradata_nativesql}{fn teradata_autocommit_off}"
|
|
585
602
|
auto_commit_on = "{fn teradata_nativesql}{fn teradata_autocommit_on}"
|
|
@@ -1215,7 +1232,7 @@ def db_python_version_diff():
|
|
|
1215
1232
|
raise TeradataMlException(Messages.get_message(MessageCodes.PYTHON_NOT_INSTALLED),
|
|
1216
1233
|
MessageCodes.PYTHON_NOT_INSTALLED)
|
|
1217
1234
|
|
|
1218
|
-
# Get major version of
|
|
1235
|
+
# Get major version of Python installed on Vantage and the current environment.
|
|
1219
1236
|
python_local = tdmlctx.python_version_local.rsplit(".", 1)[0]
|
|
1220
1237
|
python_vantage = tdmlctx.python_version_vantage.rsplit(".", 1)[0]
|
|
1221
1238
|
|
|
@@ -1304,7 +1321,7 @@ def _create_table(table_name,
|
|
|
1304
1321
|
|
|
1305
1322
|
columns:
|
|
1306
1323
|
Required Argument.
|
|
1307
|
-
Specifies a
|
|
1324
|
+
Specifies a Python dictionary with column-name(key) to column-type(value) mapping
|
|
1308
1325
|
to create table.
|
|
1309
1326
|
Types: dict
|
|
1310
1327
|
|
|
@@ -1457,6 +1474,116 @@ def _create_table(table_name,
|
|
|
1457
1474
|
raise TeradataMlException(Messages.get_message(msg_code, "create table", str(err)), msg_code)
|
|
1458
1475
|
|
|
1459
1476
|
|
|
1477
|
+
def _create_temporal_table(table_name,
|
|
1478
|
+
columns,
|
|
1479
|
+
validtime_columns,
|
|
1480
|
+
primary_index=None,
|
|
1481
|
+
partition_by_range=None,
|
|
1482
|
+
schema_name=None,
|
|
1483
|
+
skip_if_exists=False):
|
|
1484
|
+
"""
|
|
1485
|
+
DESCRIPTION:
|
|
1486
|
+
Internal function used to create validTime dimension temporal table.
|
|
1487
|
+
|
|
1488
|
+
PARAMETERS:
|
|
1489
|
+
table_name:
|
|
1490
|
+
Required Argument.
|
|
1491
|
+
Specifies the name of SQL table.
|
|
1492
|
+
Types: str
|
|
1493
|
+
|
|
1494
|
+
columns:
|
|
1495
|
+
Required Argument.
|
|
1496
|
+
Specifies a Python dictionary with column-name(key) to column-type(value) mapping
|
|
1497
|
+
to create table. Column-type can be of type string or teradatasqlalchemy type.
|
|
1498
|
+
Types: dict
|
|
1499
|
+
|
|
1500
|
+
validtime_columns:
|
|
1501
|
+
Required Argument.
|
|
1502
|
+
Specifies the validTime columns to be created in the table.
|
|
1503
|
+
Note:
|
|
1504
|
+
The columns specified in "validtime_columns" should be present in
|
|
1505
|
+
"columns" argument.
|
|
1506
|
+
Types: tuple of str
|
|
1507
|
+
|
|
1508
|
+
primary_index:
|
|
1509
|
+
Optional Argument.
|
|
1510
|
+
Specifies the column name(s) on which primary index needs to be created.
|
|
1511
|
+
Types: str OR list of Strings (str)
|
|
1512
|
+
|
|
1513
|
+
partition_by_range:
|
|
1514
|
+
Optional Argument.
|
|
1515
|
+
Specifies the column name(s) on which partition by range needs to be created.
|
|
1516
|
+
Types: str OR ColumnExpression
|
|
1517
|
+
|
|
1518
|
+
schema_name:
|
|
1519
|
+
Optional Argument.
|
|
1520
|
+
Specifies the name of the SQL schema in the database to write to.
|
|
1521
|
+
If not specified, table is created in default schema.
|
|
1522
|
+
Types: str
|
|
1523
|
+
|
|
1524
|
+
RETURNS:
|
|
1525
|
+
None
|
|
1526
|
+
|
|
1527
|
+
RAISES:
|
|
1528
|
+
None
|
|
1529
|
+
|
|
1530
|
+
EXAMPLES:
|
|
1531
|
+
>>> from teradataml.dbutils.dbutils import _create_temporal_table
|
|
1532
|
+
>>> from teradatasqlalchemy.types import *
|
|
1533
|
+
# Example: Create a temporal table "Table1" with primary key constraint, partition it by range.
|
|
1534
|
+
# Make sure to specify column validTime temporal column from columns 'start_time'
|
|
1535
|
+
# and 'end_time'.
|
|
1536
|
+
>>> _create_temporal_table(table_name="Table1",
|
|
1537
|
+
... columns={"column1": "VARCHAR(100)",
|
|
1538
|
+
... "column2": INTEGER,
|
|
1539
|
+
... "start_time": "TIMESTAMP(6)",
|
|
1540
|
+
... "end_time": TIMESTAMP(6)},
|
|
1541
|
+
... schema_name = "vfs_test",
|
|
1542
|
+
... primary_index='column_name',
|
|
1543
|
+
... partition_by_range='column_name',
|
|
1544
|
+
... validtime_columns=('start_time', 'end_time'))
|
|
1545
|
+
"""
|
|
1546
|
+
# Prepare column clause first.
|
|
1547
|
+
columns_clause_ = ['{} {}'.format(k, v if isinstance(v, str)
|
|
1548
|
+
else v.compile(td_dialect())) for k, v in columns.items()]
|
|
1549
|
+
if validtime_columns:
|
|
1550
|
+
period_for_clause = ['PERIOD FOR ValidPeriod ({}, {}) AS VALIDTIME'.format(
|
|
1551
|
+
validtime_columns[0], validtime_columns[1])
|
|
1552
|
+
]
|
|
1553
|
+
else:
|
|
1554
|
+
period_for_clause = []
|
|
1555
|
+
columns_clause = ",\n ".join(columns_clause_+period_for_clause)
|
|
1556
|
+
|
|
1557
|
+
# Prepare primary index clause.
|
|
1558
|
+
if primary_index:
|
|
1559
|
+
primary_index_clause = "PRIMARY INDEX ({})".format(
|
|
1560
|
+
", ".join(UtilFuncs._as_list(primary_index)))
|
|
1561
|
+
else:
|
|
1562
|
+
primary_index_clause = ""
|
|
1563
|
+
|
|
1564
|
+
# Prepare partition by range clause.
|
|
1565
|
+
if partition_by_range:
|
|
1566
|
+
partition_by_range_clause = "PARTITION BY RANGE_N({})".format(
|
|
1567
|
+
partition_by_range if isinstance(partition_by_range, str) else partition_by_range.compile())
|
|
1568
|
+
else:
|
|
1569
|
+
partition_by_range_clause = ""
|
|
1570
|
+
|
|
1571
|
+
# Prepare create table statement.
|
|
1572
|
+
table_name = UtilFuncs._get_qualified_table_name(schema_name, table_name) if\
|
|
1573
|
+
schema_name else table_name
|
|
1574
|
+
sql = """
|
|
1575
|
+
CREATE MULTISET TABLE {}
|
|
1576
|
+
(\n{}\n)\n{}\n{}
|
|
1577
|
+
""".format(table_name, columns_clause, primary_index_clause, partition_by_range_clause)
|
|
1578
|
+
|
|
1579
|
+
if skip_if_exists:
|
|
1580
|
+
execute_sql(sql, ignore_errors=3803)
|
|
1581
|
+
else:
|
|
1582
|
+
execute_sql(sql)
|
|
1583
|
+
|
|
1584
|
+
return True
|
|
1585
|
+
|
|
1586
|
+
|
|
1460
1587
|
def _create_database(schema_name, size='10e6', spool_size=None,
|
|
1461
1588
|
datalake=None, **kwargs):
|
|
1462
1589
|
"""
|
|
@@ -1612,7 +1739,13 @@ def _update_data(update_columns_values, table_name, schema_name, datalake_name=N
|
|
|
1612
1739
|
return True
|
|
1613
1740
|
|
|
1614
1741
|
|
|
1615
|
-
def _insert_data(table_name,
|
|
1742
|
+
def _insert_data(table_name,
|
|
1743
|
+
values,
|
|
1744
|
+
columns=None,
|
|
1745
|
+
schema_name=None,
|
|
1746
|
+
datalake_name=None,
|
|
1747
|
+
return_uid=False,
|
|
1748
|
+
ignore_errors=None):
|
|
1616
1749
|
"""
|
|
1617
1750
|
DESCRIPTION:
|
|
1618
1751
|
Internal function to insert the data in a table.
|
|
@@ -1644,8 +1777,30 @@ def _insert_data(table_name, values, columns=None, schema_name=None, datalake_na
|
|
|
1644
1777
|
Specifies the name of the datalake to look for "schema_name".
|
|
1645
1778
|
Types: str
|
|
1646
1779
|
|
|
1780
|
+
return_uid:
|
|
1781
|
+
Optional Argument.
|
|
1782
|
+
Specifies whether the function should return the unique identifier
|
|
1783
|
+
of the inserted row or not. When set to True, function returns the
|
|
1784
|
+
unique ID generated by Teradata Vantage for the inserted row. Otherwise,
|
|
1785
|
+
it returns True if the insert operation is successful.
|
|
1786
|
+
Note:
|
|
1787
|
+
This argument is only applicable when the table is created
|
|
1788
|
+
in such a way it generates unique ID automatically.
|
|
1789
|
+
Default Value: False
|
|
1790
|
+
Types: bool
|
|
1791
|
+
|
|
1792
|
+
ignore_errors:
|
|
1793
|
+
Optional Argument.
|
|
1794
|
+
Specifies the error code(s) to ignore while inserting data.
|
|
1795
|
+
If this argument is not specified, no errors are ignored.
|
|
1796
|
+
Note:
|
|
1797
|
+
Error codes are Teradata Vantage error codes and not
|
|
1798
|
+
teradataml error codes.
|
|
1799
|
+
Default Value: None
|
|
1800
|
+
Types: int or list of int
|
|
1801
|
+
|
|
1647
1802
|
RETURNS:
|
|
1648
|
-
bool
|
|
1803
|
+
bool or int
|
|
1649
1804
|
|
|
1650
1805
|
RAISES:
|
|
1651
1806
|
TeradataMlException.
|
|
@@ -1668,10 +1823,14 @@ def _insert_data(table_name, values, columns=None, schema_name=None, datalake_na
|
|
|
1668
1823
|
columns = ""
|
|
1669
1824
|
_q_marks = ["?"] * (len(values[0]))
|
|
1670
1825
|
|
|
1671
|
-
|
|
1672
|
-
|
|
1826
|
+
if not return_uid:
|
|
1827
|
+
sql = "insert into {} {} values ({});".format(qualified_table_name, columns, ", ".join(_q_marks))
|
|
1828
|
+
execute_sql(sql, values, ignore_errors)
|
|
1829
|
+
return True
|
|
1673
1830
|
|
|
1674
|
-
|
|
1831
|
+
sql = "{{fn teradata_agkr(C)}}insert into {} {} values ({});".format(qualified_table_name, columns, ", ".join(_q_marks))
|
|
1832
|
+
c = execute_sql(sql, values, ignore_errors)
|
|
1833
|
+
return c.fetchone()[0]
|
|
1675
1834
|
|
|
1676
1835
|
|
|
1677
1836
|
def _upsert_data(update_columns_values,
|
|
@@ -1757,7 +1916,164 @@ def _upsert_data(update_columns_values,
|
|
|
1757
1916
|
execute_sql(sql, (*update_values, *where_values, *insert_values))
|
|
1758
1917
|
|
|
1759
1918
|
|
|
1760
|
-
def
|
|
1919
|
+
def _merge_data(target_table,
|
|
1920
|
+
target_table_alias_name,
|
|
1921
|
+
source,
|
|
1922
|
+
source_alias_name,
|
|
1923
|
+
condition,
|
|
1924
|
+
matched_details=None,
|
|
1925
|
+
non_matched_clause=None,
|
|
1926
|
+
temporal_clause=None,
|
|
1927
|
+
target_table_schema=None,
|
|
1928
|
+
source_table_schema=None):
|
|
1929
|
+
"""
|
|
1930
|
+
DESCRIPTION:
|
|
1931
|
+
Internal function to merge the data in a table.
|
|
1932
|
+
|
|
1933
|
+
PARAMETERS:
|
|
1934
|
+
target_table:
|
|
1935
|
+
Required Argument.
|
|
1936
|
+
Specifies the name of the target table to merge.
|
|
1937
|
+
Types: str
|
|
1938
|
+
|
|
1939
|
+
target_table_alias_name:
|
|
1940
|
+
Required Argument.
|
|
1941
|
+
Specifies the alias name of the target table to merge.
|
|
1942
|
+
Types: str
|
|
1943
|
+
|
|
1944
|
+
source:
|
|
1945
|
+
Required Argument.
|
|
1946
|
+
Specifies the name of the source table to merge.
|
|
1947
|
+
Can be a table name or a teradataml DataFrame.
|
|
1948
|
+
Note:
|
|
1949
|
+
Source can be a SELECT statement also. In this case,
|
|
1950
|
+
one should add paranthesis for the query. For example,
|
|
1951
|
+
value of source should be '(SELECT * FROM TABLE)' if
|
|
1952
|
+
source is a query.
|
|
1953
|
+
Types: str OR teradataml DataFrame
|
|
1954
|
+
|
|
1955
|
+
source_alias_name:
|
|
1956
|
+
Required Argument.
|
|
1957
|
+
Specifies the alias name of the source table to merge.
|
|
1958
|
+
Types: str
|
|
1959
|
+
|
|
1960
|
+
condition:
|
|
1961
|
+
Required Argument.
|
|
1962
|
+
Specifies the condition to merge the data.
|
|
1963
|
+
Types: str OR ColumnExpression
|
|
1964
|
+
|
|
1965
|
+
matched_details:
|
|
1966
|
+
Optional Argument.
|
|
1967
|
+
Specifies what to do when the condition is matched.
|
|
1968
|
+
Teradata allows either UPDATE or DELETE when the condition is matched.
|
|
1969
|
+
Note:
|
|
1970
|
+
ColumnExpressions are not allowed for key 'set' since the aliases
|
|
1971
|
+
should be with the alias name and setting alias name is not straight forward.
|
|
1972
|
+
Hence, not allowing it for now.
|
|
1973
|
+
Types: dict
|
|
1974
|
+
Example: {"action": "UPDATE", "set": {"col1": "src.col1", "col2": "src.col2"}}
|
|
1975
|
+
|
|
1976
|
+
non_matched_clause:
|
|
1977
|
+
Optional Argument.
|
|
1978
|
+
Specifies what to do when the condition is not matched.
|
|
1979
|
+
Teradata allows INSERT when the condition is not matched.
|
|
1980
|
+
Note:
|
|
1981
|
+
ColumnExpressions are not allowed in 'values' since the aliases
|
|
1982
|
+
should be with the alias name and setting alias name is not straight forward.
|
|
1983
|
+
Hence, not allowing it for now.
|
|
1984
|
+
Types: dict
|
|
1985
|
+
Example: {"action": "INSERT", "columns": ["col1", "col2"], "values": ["src.col1", "src.col2"]}
|
|
1986
|
+
|
|
1987
|
+
temporal_clause:
|
|
1988
|
+
Optional Argument.
|
|
1989
|
+
Specifies the temporal clause to be added to the MERGE statement.
|
|
1990
|
+
Types: str
|
|
1991
|
+
|
|
1992
|
+
target_table_schema:
|
|
1993
|
+
Optional Argument.
|
|
1994
|
+
Specifies the schema name of the target table.
|
|
1995
|
+
Types: str
|
|
1996
|
+
|
|
1997
|
+
source_table_schema:
|
|
1998
|
+
Optional Argument.
|
|
1999
|
+
Specifies the schema name of the source table.
|
|
2000
|
+
Note:
|
|
2001
|
+
If source is a DataFrame, this argument is ignored.
|
|
2002
|
+
Types: str
|
|
2003
|
+
|
|
2004
|
+
RETURNS:
|
|
2005
|
+
None
|
|
2006
|
+
|
|
2007
|
+
RAISES:
|
|
2008
|
+
ValueError: If required parameters are missing or invalid.
|
|
2009
|
+
|
|
2010
|
+
EXAMPLES:
|
|
2011
|
+
>>> _merge_data(
|
|
2012
|
+
... target_table="target_table",
|
|
2013
|
+
... target_table_alias_name="tgt",
|
|
2014
|
+
... source="source_table",
|
|
2015
|
+
... source_alias_name="src",
|
|
2016
|
+
... condition="tgt.id = src.id",
|
|
2017
|
+
... matched_details={"action": "UPDATE", "set": {"col1": "src.col1", "col2": "src.col2"}},
|
|
2018
|
+
... non_matched_clause={"action": "INSERT", "columns": ["id", "col1"], "values": ["src.id", "src.col1"]}
|
|
2019
|
+
... )
|
|
2020
|
+
"""
|
|
2021
|
+
# Note: Table names are not quoted because source can be a query also.
|
|
2022
|
+
# To keep it intact, both target tables and source tables are not
|
|
2023
|
+
# quoted. Hence it is caller function responsibility to add quote
|
|
2024
|
+
# if either source table or target table has special characters or
|
|
2025
|
+
# is from the user.
|
|
2026
|
+
quote = UtilFuncs._get_dialect_quoted_name
|
|
2027
|
+
if target_table_schema:
|
|
2028
|
+
target_table = "{}.{}".format(quote(target_table_schema), target_table)
|
|
2029
|
+
else:
|
|
2030
|
+
target_table = target_table
|
|
2031
|
+
|
|
2032
|
+
# If source is DataFrame, extract the query from it.
|
|
2033
|
+
if isinstance(source, str):
|
|
2034
|
+
source = "{}.{}".format(quote(source_table_schema), source) \
|
|
2035
|
+
if source_table_schema else source
|
|
2036
|
+
else:
|
|
2037
|
+
source = "({})".format(source.show_query())
|
|
2038
|
+
|
|
2039
|
+
# If condition is not a string, then prepare from it.
|
|
2040
|
+
condition = condition if isinstance(condition, str) else condition.compile()
|
|
2041
|
+
|
|
2042
|
+
# Start building the MERGE statement
|
|
2043
|
+
merge_sql = (f"MERGE INTO {target_table} AS {target_table_alias_name} \n\tUSING "
|
|
2044
|
+
f"{source} AS {source_alias_name} \n\tON {condition}")
|
|
2045
|
+
|
|
2046
|
+
# Handle matched clause
|
|
2047
|
+
if matched_details:
|
|
2048
|
+
action = matched_details.get("action", "").upper()
|
|
2049
|
+
if action == "UPDATE":
|
|
2050
|
+
set_clause = ", ".join([f"{col} = {val}"
|
|
2051
|
+
for col, val in matched_details.get("set", {}).items()])
|
|
2052
|
+
merge_sql += f"\n\tWHEN MATCHED THEN \n\t\tUPDATE \n\t\tSET \n\t\t{set_clause}"
|
|
2053
|
+
elif action == "DELETE":
|
|
2054
|
+
merge_sql += "\n\tWHEN MATCHED THEN \n\tDELETE\n\t\t"
|
|
2055
|
+
else:
|
|
2056
|
+
raise ValueError("Invalid action in matched_details. Supported actions are 'UPDATE' and 'DELETE'.")
|
|
2057
|
+
|
|
2058
|
+
# Handle non-matched clause
|
|
2059
|
+
if non_matched_clause:
|
|
2060
|
+
action = non_matched_clause.get("action", "").upper()
|
|
2061
|
+
if action == "INSERT":
|
|
2062
|
+
columns = ", ".join(non_matched_clause.get("columns", []))
|
|
2063
|
+
values = ", ".join(non_matched_clause.get("values", []))
|
|
2064
|
+
merge_sql += f"\n\tWHEN NOT MATCHED THEN \n\t\tINSERT ({columns}) \n\t\tVALUES \n\t\t({values})"
|
|
2065
|
+
else:
|
|
2066
|
+
raise ValueError("Invalid action in non_matched_clause. Supported action is 'INSERT'.")
|
|
2067
|
+
|
|
2068
|
+
# Finalize the statement
|
|
2069
|
+
merge_sql += ";"
|
|
2070
|
+
if temporal_clause:
|
|
2071
|
+
merge_sql = "{} {}".format(temporal_clause, merge_sql)
|
|
2072
|
+
|
|
2073
|
+
execute_sql(merge_sql)
|
|
2074
|
+
|
|
2075
|
+
|
|
2076
|
+
def _delete_data(table_name, schema_name=None, datalake_name=None, delete_conditions=None, temporal_clause=None):
|
|
1761
2077
|
"""
|
|
1762
2078
|
DESCRIPTION:
|
|
1763
2079
|
Internal function to delete the data in a table.
|
|
@@ -1784,6 +2100,11 @@ def _delete_data(table_name, schema_name=None, datalake_name=None, delete_condit
|
|
|
1784
2100
|
Specifies the ColumnExpression or dictionary containing key values
|
|
1785
2101
|
pairs to use for removing the data.
|
|
1786
2102
|
Types: ColumnExpression, dict
|
|
2103
|
+
|
|
2104
|
+
temporal_clause:
|
|
2105
|
+
Optional Argument.
|
|
2106
|
+
Specifies the temporal clause to be added to the DELETE statement.
|
|
2107
|
+
Types: str
|
|
1787
2108
|
|
|
1788
2109
|
RETURNS:
|
|
1789
2110
|
int, specifies the number of records those are deleted.
|
|
@@ -1819,6 +2140,9 @@ def _delete_data(table_name, schema_name=None, datalake_name=None, delete_condit
|
|
|
1819
2140
|
|
|
1820
2141
|
sql = sqlbundle._get_sql_query(SQLConstants.SQL_DELETE_SPECIFIC_ROW).format(qualified_table_name, where_clause)
|
|
1821
2142
|
|
|
2143
|
+
if temporal_clause:
|
|
2144
|
+
sql = "{} {}".format(temporal_clause, sql)
|
|
2145
|
+
|
|
1822
2146
|
res = execute_sql(sql)
|
|
1823
2147
|
return res.rowcount
|
|
1824
2148
|
|
|
@@ -1998,6 +2322,59 @@ def _execute_query_and_generate_pandas_df(query, index=None, **kwargs):
|
|
|
1998
2322
|
|
|
1999
2323
|
return pandas_df
|
|
2000
2324
|
|
|
2325
|
+
def _is_trigger_exist(schema_name, trigger_names):
|
|
2326
|
+
"""
|
|
2327
|
+
DESCRIPTION:
|
|
2328
|
+
Checks if all given triggers exist in the specified schema.
|
|
2329
|
+
|
|
2330
|
+
PARAMETERS:
|
|
2331
|
+
schema_name:
|
|
2332
|
+
Required Argument.
|
|
2333
|
+
Specifies the schema/database name.
|
|
2334
|
+
Types: str
|
|
2335
|
+
|
|
2336
|
+
trigger_names:
|
|
2337
|
+
Required Argument.
|
|
2338
|
+
Specifies the trigger name(s) to check.
|
|
2339
|
+
Types: str or list of str
|
|
2340
|
+
|
|
2341
|
+
RETURNS:
|
|
2342
|
+
Tuple - first element specifies whether all provided triggers exist or not.
|
|
2343
|
+
second element specifies total number of triggers found from "trigger_names".
|
|
2344
|
+
|
|
2345
|
+
RAISES:
|
|
2346
|
+
TeradataMlException
|
|
2347
|
+
|
|
2348
|
+
EXAMPLES:
|
|
2349
|
+
>>> is_trigger_exist("mydb", ["trg1", "trg2"])
|
|
2350
|
+
(True, 2)
|
|
2351
|
+
>>> is_trigger_exist("mydb", ["trg1", "missing_trg"])
|
|
2352
|
+
(False, 1)
|
|
2353
|
+
"""
|
|
2354
|
+
# Normalize trigger_names to list
|
|
2355
|
+
triggers = UtilFuncs._as_list(trigger_names)
|
|
2356
|
+
if not triggers:
|
|
2357
|
+
return False
|
|
2358
|
+
|
|
2359
|
+
# Prepare SQL to check all triggers in one call
|
|
2360
|
+
triggers_str = ", ".join("'{}'".format(t) for t in triggers)
|
|
2361
|
+
sql = f"""
|
|
2362
|
+
SELECT TriggerName
|
|
2363
|
+
FROM DBC.TriggersV
|
|
2364
|
+
WHERE DatabaseName = '{schema_name}'
|
|
2365
|
+
AND TriggerName IN ({triggers_str})
|
|
2366
|
+
"""
|
|
2367
|
+
|
|
2368
|
+
try:
|
|
2369
|
+
result = execute_sql(sql)
|
|
2370
|
+
found = {row[0] for row in result.fetchall()}
|
|
2371
|
+
num_found = len(found)
|
|
2372
|
+
all_exist = all(t in found for t in triggers)
|
|
2373
|
+
return all_exist, num_found
|
|
2374
|
+
except Exception as e:
|
|
2375
|
+
raise TeradataMlException(
|
|
2376
|
+
Messages.get_message(MessageCodes.EXECUTION_FAILED, "is_triggers_exist", str(e)),
|
|
2377
|
+
MessageCodes.EXECUTION_FAILED)
|
|
2001
2378
|
|
|
2002
2379
|
class _TDSessionParams:
|
|
2003
2380
|
"""
|
|
@@ -2277,6 +2654,14 @@ def unset_session_param(name):
|
|
|
2277
2654
|
msg_code = MessageCodes.FUNC_EXECUTION_FAILED
|
|
2278
2655
|
error_msg = Messages.get_message(msg_code, "unset_session_param", "Set the parameter before unsetting it.")
|
|
2279
2656
|
raise TeradataMlException(error_msg, msg_code)
|
|
2657
|
+
if name.upper() == "DEBUG_FUNCTION":
|
|
2658
|
+
# If unset param is debug_function, then check if any function name is available to unset.
|
|
2659
|
+
if _InternalBuffer.get('function_name') in ('', None):
|
|
2660
|
+
raise TeradataMlException(
|
|
2661
|
+
Messages.get_message(MessageCodes.FUNC_EXECUTION_FAILED,
|
|
2662
|
+
"unset_session_param",
|
|
2663
|
+
"Set the parameter before unsetting it."),
|
|
2664
|
+
MessageCodes.FUNC_EXECUTION_FAILED)
|
|
2280
2665
|
# unset_values stores params which are not available in _InternalBuffer, to unset create a dictionary
|
|
2281
2666
|
# with param as key and unset param as value
|
|
2282
2667
|
# TODO: Unset for ISOLATED_LOADING should revert to previous behaviour, but we are setting it to NO.
|
|
@@ -2307,10 +2692,10 @@ def unset_session_param(name):
|
|
|
2307
2692
|
|
|
2308
2693
|
|
|
2309
2694
|
class _Authorize:
|
|
2310
|
-
""" Parent class to either provide or revoke access on
|
|
2695
|
+
""" Parent class to either provide or revoke access on database objects. """
|
|
2311
2696
|
_property = None
|
|
2312
2697
|
|
|
2313
|
-
def __init__(self, objects):
|
|
2698
|
+
def __init__(self, objects, database=None):
|
|
2314
2699
|
"""
|
|
2315
2700
|
DESCRIPTION:
|
|
2316
2701
|
Constructor for creating Authorize object.
|
|
@@ -2319,7 +2704,12 @@ class _Authorize:
|
|
|
2319
2704
|
objects:
|
|
2320
2705
|
Required Argument.
|
|
2321
2706
|
Specifies the name(s) of the database objects to be authorized.
|
|
2322
|
-
Types: str OR list of str
|
|
2707
|
+
Types: str OR list of str OR AccessType Enum
|
|
2708
|
+
|
|
2709
|
+
database:
|
|
2710
|
+
Optional Argument.
|
|
2711
|
+
Specifies the name of the database to grant or revoke access.
|
|
2712
|
+
Types: str
|
|
2323
2713
|
|
|
2324
2714
|
RETURNS:
|
|
2325
2715
|
Object of _Authorize.
|
|
@@ -2331,8 +2721,10 @@ class _Authorize:
|
|
|
2331
2721
|
>>> auth = _Authorize('vfs_v1')
|
|
2332
2722
|
"""
|
|
2333
2723
|
# Store the objects here. Then use this where ever required.
|
|
2724
|
+
self._is_enum = issubclass(objects, enum.Enum)
|
|
2334
2725
|
self._objects = objects
|
|
2335
2726
|
self._access_method = self.__class__.__name__.upper()
|
|
2727
|
+
self.database = database
|
|
2336
2728
|
|
|
2337
2729
|
def read(self, user):
|
|
2338
2730
|
"""
|
|
@@ -2356,11 +2748,7 @@ class _Authorize:
|
|
|
2356
2748
|
EXAMPLES:
|
|
2357
2749
|
>>> _Authorize('repo').read('BoB')
|
|
2358
2750
|
"""
|
|
2359
|
-
|
|
2360
|
-
sql = "{} SELECT ON {} {} {}".format(self._access_method, object, self._property, user)
|
|
2361
|
-
execute_sql(sql)
|
|
2362
|
-
|
|
2363
|
-
return True
|
|
2751
|
+
return self._apply_access(user, 'read', 'SELECT')
|
|
2364
2752
|
|
|
2365
2753
|
def write(self, user):
|
|
2366
2754
|
"""
|
|
@@ -2384,11 +2772,58 @@ class _Authorize:
|
|
|
2384
2772
|
EXAMPLES:
|
|
2385
2773
|
>>> _Authorize('repo').write('BoB')
|
|
2386
2774
|
"""
|
|
2387
|
-
|
|
2388
|
-
|
|
2389
|
-
|
|
2390
|
-
|
|
2775
|
+
return self._apply_access(user, 'write', 'INSERT, UPDATE, DELETE')
|
|
2776
|
+
|
|
2777
|
+
def _apply_access(self, user, operation, access_type):
|
|
2778
|
+
"""
|
|
2779
|
+
DESCRIPTION:
|
|
2780
|
+
Internal function to grant or revoke access.
|
|
2781
|
+
|
|
2782
|
+
PARAMETERS:
|
|
2783
|
+
user:
|
|
2784
|
+
Required Argument.
|
|
2785
|
+
Specifies the name of the user to have access.
|
|
2786
|
+
Types: str
|
|
2787
|
+
|
|
2788
|
+
operation:
|
|
2789
|
+
Required Argument.
|
|
2790
|
+
Specifies the operation to perform.
|
|
2791
|
+
Permitted Values: 'read', 'write'
|
|
2792
|
+
Types: str
|
|
2391
2793
|
|
|
2794
|
+
access_type:
|
|
2795
|
+
Required Argument.
|
|
2796
|
+
Specifies the type of access to grant or revoke.
|
|
2797
|
+
Permitted Values:
|
|
2798
|
+
* 'SELECT' for read
|
|
2799
|
+
* 'INSERT, UPDATE, DELETE' for write
|
|
2800
|
+
Types: str
|
|
2801
|
+
|
|
2802
|
+
RETURNS:
|
|
2803
|
+
bool, True if access is granted or revoked successfully.
|
|
2804
|
+
|
|
2805
|
+
RAISES:
|
|
2806
|
+
TeradataMlException, OperationalError
|
|
2807
|
+
|
|
2808
|
+
EXAMPLES:
|
|
2809
|
+
>>> _Authorize('repo')._apply_access('BoB', 'read', 'SELECT')
|
|
2810
|
+
"""
|
|
2811
|
+
sql_objects = UtilFuncs._as_list(self._objects) if not self._is_enum else \
|
|
2812
|
+
getattr(self._objects, operation).value
|
|
2813
|
+
|
|
2814
|
+
for obj in sql_objects:
|
|
2815
|
+
if self._is_enum:
|
|
2816
|
+
sql = obj.format(
|
|
2817
|
+
grant_revoke_=self._access_method,
|
|
2818
|
+
database_=self.database,
|
|
2819
|
+
to_from_=self._property,
|
|
2820
|
+
user_=user
|
|
2821
|
+
)
|
|
2822
|
+
else:
|
|
2823
|
+
sql = "{} {} ON {} {} {}".format(
|
|
2824
|
+
self._access_method, access_type, obj, self._property, user
|
|
2825
|
+
)
|
|
2826
|
+
execute_sql(sql)
|
|
2392
2827
|
return True
|
|
2393
2828
|
|
|
2394
2829
|
def read_write(self, user):
|