teradataml 20.0.0.4__py3-none-any.whl → 20.0.0.5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of teradataml might be problematic. Click here for more details.
- teradataml/LICENSE-3RD-PARTY.pdf +0 -0
- teradataml/README.md +86 -13
- teradataml/__init__.py +2 -1
- teradataml/_version.py +2 -2
- teradataml/analytics/analytic_function_executor.py +7 -12
- teradataml/analytics/json_parser/analytic_functions_argument.py +4 -0
- teradataml/analytics/sqle/__init__.py +16 -1
- teradataml/analytics/utils.py +15 -1
- teradataml/automl/__init__.py +290 -106
- teradataml/automl/autodataprep/__init__.py +471 -0
- teradataml/automl/data_preparation.py +29 -10
- teradataml/automl/data_transformation.py +11 -0
- teradataml/automl/feature_engineering.py +64 -4
- teradataml/automl/feature_exploration.py +639 -25
- teradataml/automl/model_training.py +1 -1
- teradataml/clients/auth_client.py +2 -2
- teradataml/common/constants.py +61 -26
- teradataml/common/messagecodes.py +2 -1
- teradataml/common/messages.py +5 -4
- teradataml/common/utils.py +255 -37
- teradataml/context/context.py +225 -87
- teradataml/data/apriori_example.json +22 -0
- teradataml/data/docs/sqle/docs_17_20/Apriori.py +138 -0
- teradataml/data/docs/sqle/docs_17_20/NERExtractor.py +121 -0
- teradataml/data/docs/sqle/docs_17_20/NGramSplitter.py +3 -3
- teradataml/data/docs/sqle/docs_17_20/SMOTE.py +212 -0
- teradataml/data/docs/sqle/docs_17_20/TextMorph.py +119 -0
- teradataml/data/docs/sqle/docs_17_20/TextParser.py +54 -3
- teradataml/data/docs/uaf/docs_17_20/ACF.py +1 -1
- teradataml/data/docs/uaf/docs_17_20/ArimaEstimate.py +2 -2
- teradataml/data/docs/uaf/docs_17_20/ArimaXEstimate.py +2 -2
- teradataml/data/docs/uaf/docs_17_20/DFFT.py +1 -1
- teradataml/data/docs/uaf/docs_17_20/DFFT2.py +1 -1
- teradataml/data/docs/uaf/docs_17_20/DFFT2Conv.py +1 -1
- teradataml/data/docs/uaf/docs_17_20/DFFTConv.py +1 -1
- teradataml/data/docs/uaf/docs_17_20/FilterFactory1d.py +4 -4
- teradataml/data/docs/uaf/docs_17_20/GenseriesSinusoids.py +2 -2
- teradataml/data/docs/uaf/docs_17_20/GoldfeldQuandt.py +2 -2
- teradataml/data/docs/uaf/docs_17_20/HoltWintersForecaster.py +6 -6
- teradataml/data/docs/uaf/docs_17_20/LineSpec.py +1 -1
- teradataml/data/docs/uaf/docs_17_20/LinearRegr.py +1 -1
- teradataml/data/docs/uaf/docs_17_20/Matrix2Image.py +4 -4
- teradataml/data/docs/uaf/docs_17_20/MultivarRegr.py +1 -1
- teradataml/data/docs/uaf/docs_17_20/PACF.py +1 -1
- teradataml/data/docs/uaf/docs_17_20/PowerSpec.py +2 -2
- teradataml/data/docs/uaf/docs_17_20/PowerTransform.py +3 -3
- teradataml/data/docs/uaf/docs_17_20/Resample.py +5 -5
- teradataml/data/docs/uaf/docs_17_20/SAX.py +3 -3
- teradataml/data/docs/uaf/docs_17_20/SignifPeriodicities.py +1 -1
- teradataml/data/docs/uaf/docs_17_20/SimpleExp.py +1 -1
- teradataml/data/docs/uaf/docs_17_20/Smoothma.py +3 -3
- teradataml/data/docs/uaf/docs_17_20/UNDIFF.py +1 -1
- teradataml/data/jsons/sqle/17.20/NGramSplitter.json +6 -6
- teradataml/data/jsons/sqle/17.20/TD_Apriori.json +181 -0
- teradataml/data/jsons/sqle/17.20/TD_NERExtractor.json +145 -0
- teradataml/data/jsons/sqle/17.20/TD_SMOTE.json +267 -0
- teradataml/data/jsons/sqle/17.20/TD_TextMorph.json +134 -0
- teradataml/data/jsons/sqle/17.20/TD_TextParser.json +114 -9
- teradataml/data/jsons/sqle/20.00/AI_AnalyzeSentiment.json +328 -0
- teradataml/data/jsons/sqle/20.00/AI_AskLLM.json +420 -0
- teradataml/data/jsons/sqle/20.00/AI_DetectLanguage.json +343 -0
- teradataml/data/jsons/sqle/20.00/AI_ExtractKeyPhrases.json +328 -0
- teradataml/data/jsons/sqle/20.00/AI_MaskPII.json +328 -0
- teradataml/data/jsons/sqle/20.00/AI_RecognizeEntities.json +328 -0
- teradataml/data/jsons/sqle/20.00/AI_RecognizePIIEntities.json +328 -0
- teradataml/data/jsons/sqle/20.00/AI_TextClassifier.json +359 -0
- teradataml/data/jsons/sqle/20.00/AI_TextEmbeddings.json +360 -0
- teradataml/data/jsons/sqle/20.00/AI_TextSummarize.json +343 -0
- teradataml/data/jsons/sqle/20.00/AI_TextTranslate.json +343 -0
- teradataml/data/jsons/sqle/20.00/TD_SMOTE.json +2 -2
- teradataml/data/jsons/sqle/20.00/TD_VectorDistance.json +1 -1
- teradataml/data/ner_dict.csv +8 -0
- teradataml/data/ner_input_eng.csv +7 -0
- teradataml/data/ner_rule.csv +5 -0
- teradataml/data/pos_input.csv +40 -0
- teradataml/data/tdnerextractor_example.json +14 -0
- teradataml/data/teradataml_example.json +13 -0
- teradataml/data/textmorph_example.json +5 -0
- teradataml/data/to_num_data.csv +4 -0
- teradataml/data/tochar_data.csv +5 -0
- teradataml/data/trans_dense.csv +16 -0
- teradataml/data/trans_sparse.csv +55 -0
- teradataml/dataframe/copy_to.py +37 -26
- teradataml/dataframe/data_transfer.py +61 -45
- teradataml/dataframe/dataframe.py +130 -50
- teradataml/dataframe/dataframe_utils.py +15 -2
- teradataml/dataframe/functions.py +109 -9
- teradataml/dataframe/sql.py +328 -76
- teradataml/dbutils/dbutils.py +33 -13
- teradataml/dbutils/filemgr.py +14 -10
- teradataml/lib/aed_0_1.dll +0 -0
- teradataml/opensource/_base.py +6 -157
- teradataml/options/configure.py +4 -5
- teradataml/scriptmgmt/UserEnv.py +305 -38
- teradataml/scriptmgmt/lls_utils.py +376 -130
- teradataml/store/__init__.py +1 -1
- teradataml/table_operators/Apply.py +16 -1
- teradataml/table_operators/Script.py +20 -1
- teradataml/table_operators/table_operator_util.py +58 -9
- teradataml/utils/dtypes.py +2 -1
- teradataml/utils/internal_buffer.py +22 -2
- teradataml/utils/validators.py +313 -57
- {teradataml-20.0.0.4.dist-info → teradataml-20.0.0.5.dist-info}/METADATA +89 -14
- {teradataml-20.0.0.4.dist-info → teradataml-20.0.0.5.dist-info}/RECORD +107 -77
- {teradataml-20.0.0.4.dist-info → teradataml-20.0.0.5.dist-info}/WHEEL +0 -0
- {teradataml-20.0.0.4.dist-info → teradataml-20.0.0.5.dist-info}/top_level.txt +0 -0
- {teradataml-20.0.0.4.dist-info → teradataml-20.0.0.5.dist-info}/zip-safe +0 -0
teradataml/context/context.py
CHANGED
|
@@ -19,17 +19,15 @@ import os
|
|
|
19
19
|
import socket
|
|
20
20
|
import sys
|
|
21
21
|
import threading
|
|
22
|
-
import urllib.parse
|
|
23
22
|
import warnings
|
|
24
23
|
from pathlib import Path
|
|
25
|
-
from dotenv import dotenv_values
|
|
26
24
|
|
|
25
|
+
from dotenv import dotenv_values
|
|
27
26
|
from sqlalchemy import create_engine
|
|
28
27
|
from sqlalchemy.engine.base import Engine
|
|
29
28
|
from sqlalchemy.engine.url import URL
|
|
30
29
|
|
|
31
30
|
from teradataml.common.constants import Query, SQLConstants, TeradataConstants
|
|
32
|
-
from teradataml.common.deprecations import argument_deprecation
|
|
33
31
|
from teradataml.common.exceptions import TeradataMlException
|
|
34
32
|
from teradataml.common.garbagecollector import GarbageCollector
|
|
35
33
|
from teradataml.common.messagecodes import MessageCodes
|
|
@@ -138,7 +136,7 @@ def __cleanup_garbage_collection():
|
|
|
138
136
|
GarbageCollector._cleanup_garbage_collector()
|
|
139
137
|
|
|
140
138
|
|
|
141
|
-
def _get_other_connection_parameters(logmech
|
|
139
|
+
def _get_other_connection_parameters(logmech=None, logdata=None, database=None, **kwargs):
|
|
142
140
|
"""
|
|
143
141
|
DESCRIPTION:
|
|
144
142
|
Internal function to return the connection parameters.
|
|
@@ -199,24 +197,27 @@ def _get_other_connection_parameters(logmech = None, logdata = None, database =
|
|
|
199
197
|
|
|
200
198
|
return result
|
|
201
199
|
|
|
202
|
-
|
|
200
|
+
|
|
203
201
|
@collect_queryband(queryband='CrtCxt')
|
|
204
|
-
def create_context(host
|
|
205
|
-
logmech
|
|
202
|
+
def create_context(host=None, username=None, password=None, tdsqlengine=None, temp_database_name=None,
|
|
203
|
+
logmech=None, logdata=None, database=None, **kwargs):
|
|
206
204
|
"""
|
|
207
205
|
DESCRIPTION:
|
|
208
206
|
Creates a connection to the Teradata Vantage using the teradatasql + teradatasqlalchemy DBAPI and dialect
|
|
209
207
|
combination.
|
|
210
|
-
Users can create a connection by passing the connection parameters using any of the following methods:
|
|
211
|
-
1. Pass all required parameters (host, username, password) directly to the function
|
|
208
|
+
Users can create a connection by passing the connection parameters using any one of the following methods:
|
|
209
|
+
1. Pass all required parameters (host, username, password) directly to the function.
|
|
212
210
|
2. Set the connection parameters in a configuration file (.cfg or .env) and
|
|
213
|
-
pass the configuration file
|
|
211
|
+
pass the configuration file.
|
|
214
212
|
3. Set the connection parameters in environment variables and create_context() reads from
|
|
215
213
|
environment variables.
|
|
216
214
|
|
|
217
|
-
Alternatively, users can pass a SQLAlchemy engine to the `tdsqlengine` parameter to override the default DBAPI
|
|
215
|
+
Alternatively, users can pass a SQLAlchemy engine object to the `tdsqlengine` parameter to override the default DBAPI
|
|
218
216
|
and dialect combination.
|
|
219
217
|
|
|
218
|
+
Function also enables user to set the authentication token which is required to access services running
|
|
219
|
+
on Teradata Vantage.
|
|
220
|
+
|
|
220
221
|
Note:
|
|
221
222
|
1. teradataml requires that the user has certain permissions on the user's default database or the initial
|
|
222
223
|
default database specified using the database argument, or the temporary database when specified using
|
|
@@ -264,6 +265,28 @@ def create_context(host = None, username = None, password = None, tdsqlengine =
|
|
|
264
265
|
os.environ['TD_PASSWORD'] = 'tdpassword'
|
|
265
266
|
3. A configuration file if provided (such as a .env file).
|
|
266
267
|
|
|
268
|
+
4. Points to note when user sets authentication token with create_context():
|
|
269
|
+
* The username provided in create_context() is not case-sensitive. For example,
|
|
270
|
+
if a user is created with the username xyz, create_context() still establishes
|
|
271
|
+
a connection if user passes the username as XyZ. However, authentication token
|
|
272
|
+
generation requires the username to be in the same case as when it was created.
|
|
273
|
+
Therefore, Teradata recommends to pass the username with the same case as when
|
|
274
|
+
it was created.
|
|
275
|
+
* User must have a privilege to login with a NULL password to use set_auth_token().
|
|
276
|
+
Refer to GRANT LOGON section in Teradata Documentation for more details.
|
|
277
|
+
* When "auth_mech" is not specified, arguments are used in the following combination
|
|
278
|
+
to derive authentication mechanism.
|
|
279
|
+
* If "base_url" and "client_id" are specified then token generation is done through OAuth.
|
|
280
|
+
* If "base_url", "pat_token", "pem_file" are specified then token generation is done using PAT.
|
|
281
|
+
* If "base_url" and "auth_token" are specified then value provided for "auth_token" is used.
|
|
282
|
+
* If only "base_url" is specified then token generation is done through OAuth.
|
|
283
|
+
* If Basic authentication mechanism is to be used then user must specify argument
|
|
284
|
+
"auth_mech" as "BASIC" along with "username" and "password".
|
|
285
|
+
* Refresh token works only for OAuth authentication.
|
|
286
|
+
* Use the argument "kid" only when key used during the pem file generation is different
|
|
287
|
+
from pem file name. For example, if you use the key as 'key1' while generating pem file
|
|
288
|
+
and the name of the pem file is `key1(1).pem`, then pass value 'key1' to the argument "kid".
|
|
289
|
+
|
|
267
290
|
PARAMETERS:
|
|
268
291
|
host:
|
|
269
292
|
Optional Argument.
|
|
@@ -277,39 +300,15 @@ def create_context(host = None, username = None, password = None, tdsqlengine =
|
|
|
277
300
|
|
|
278
301
|
password:
|
|
279
302
|
Optional Argument.
|
|
280
|
-
Specifies the password required for the username.
|
|
303
|
+
Specifies the password required for the "username".
|
|
281
304
|
Types: str
|
|
282
305
|
Note:
|
|
283
306
|
* Encrypted passwords can also be passed to this argument, using Stored Password Protection feature.
|
|
284
307
|
Examples section below demonstrates passing encrypted password to 'create_context'.
|
|
285
308
|
More details on Stored Password Protection and how to generate key and encrypted password file
|
|
286
309
|
can be found at https://pypi.org/project/teradatasql/#StoredPasswordProtection
|
|
287
|
-
* Special characters
|
|
288
|
-
|
|
289
|
-
disabled by setting "url_encode" = False.
|
|
290
|
-
For example, if the password is: "kx%jj5/g", then this password is encoded as below:
|
|
291
|
-
"kx%25jj5%2Fg"
|
|
292
|
-
where,
|
|
293
|
-
%25 represents the '%' character and
|
|
294
|
-
%2F represents the '/' character
|
|
295
|
-
The details of how the special characters are replaced can be found in the below link:
|
|
296
|
-
"https://docs.microfocus.com/OMi/10.62/Content/OMi/ExtGuide/ExtApps/URL_encoding.htm".
|
|
297
|
-
Note:
|
|
298
|
-
When password contains a space:
|
|
299
|
-
* "url_encode" must be set to False.
|
|
300
|
-
* In addition to space, other special characters in this password should be manually encoded,
|
|
301
|
-
following the link: https://docs.microfocus.com/OMi/10.62/Content/OMi/ExtGuide/ExtApps/URL_encoding.htm
|
|
302
|
-
Refer Example 16 in examples section for a detailed demonstration.
|
|
303
|
-
|
|
304
|
-
When password contains unreserved characters like tilde("~"), dot("."), underscore("_"), hyphen("-"):
|
|
305
|
-
* Character is not URL encoded by default.
|
|
306
|
-
* If unreserved character in passed needs to be encoded then,
|
|
307
|
-
it must be encoded manually and encoded password must be passed, along with "url_encode" set to False.
|
|
308
|
-
* The unreserved characters differ from one Python version to another Python version.
|
|
309
|
-
For example, The encoding standards for Python version 3.7 found in the below link:
|
|
310
|
-
"https://docs.python.org/3/library/urllib.parse.html#url-quoting".
|
|
311
|
-
|
|
312
|
-
|
|
310
|
+
* Special characters used in the password are encoded by default.
|
|
311
|
+
|
|
313
312
|
tdsqlengine:
|
|
314
313
|
Optional Argument.
|
|
315
314
|
Specifies Teradata Vantage sqlalchemy engine object that should be used to establish a Teradata Vantage
|
|
@@ -347,18 +346,23 @@ def create_context(host = None, username = None, password = None, tdsqlengine =
|
|
|
347
346
|
Types: str
|
|
348
347
|
|
|
349
348
|
kwargs:
|
|
350
|
-
Specifies
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
349
|
+
Specifies optional keyword arguments accepted by create_context().
|
|
350
|
+
Below are the supported keyword arguments:
|
|
351
|
+
|
|
352
|
+
Connection parameters for Teradata SQL Driver:
|
|
353
|
+
Specifies the keyword-value pairs of connection parameters that are passed to Teradata SQL Driver for
|
|
354
|
+
Python. Please refer to https://github.com/Teradata/python-driver#ConnectionParameters to get information
|
|
355
|
+
on connection parameters of the driver.
|
|
356
|
+
Note:
|
|
357
|
+
* When type of a connection parameter is integer or boolean (eg: log, lob_support etc,.), pass
|
|
358
|
+
integer or boolean value, instead of quoted integer or quoted boolean as suggested in the
|
|
359
|
+
documentation. Please check the examples for usage.
|
|
360
|
+
* "sql_timeout" represents "request_timeout" connection parameter.
|
|
356
361
|
|
|
357
362
|
config_file:
|
|
358
|
-
Optional Argument.
|
|
359
363
|
Specifies the name of the configuration file to read the connection parameters.
|
|
360
364
|
Notes:
|
|
361
|
-
* If user
|
|
365
|
+
* If user does not specify full path of file, then file look up is done at current working directory.
|
|
362
366
|
* The content of the file must be in '.env' format.
|
|
363
367
|
* Use parameters of create_context() as key in the configuration file.
|
|
364
368
|
Example:
|
|
@@ -373,6 +377,60 @@ def create_context(host = None, username = None, password = None, tdsqlengine =
|
|
|
373
377
|
Default Value : td_properties.cfg
|
|
374
378
|
Types: str
|
|
375
379
|
|
|
380
|
+
base_url:
|
|
381
|
+
Specifies the endpoint URL for a given environment on Teradata Vantage.
|
|
382
|
+
Types: str
|
|
383
|
+
|
|
384
|
+
client_id:
|
|
385
|
+
Specifies the id of the application that requests the access token from
|
|
386
|
+
VantageCloud Lake.
|
|
387
|
+
Types: str
|
|
388
|
+
|
|
389
|
+
pat_token:
|
|
390
|
+
Specifies the PAT token generated from VantageCloud Lake Console.
|
|
391
|
+
Types: str
|
|
392
|
+
|
|
393
|
+
pem_file:
|
|
394
|
+
Specifies the path to private key file which is generated from VantageCloud Lake Console.
|
|
395
|
+
Types: str
|
|
396
|
+
|
|
397
|
+
auth_token:
|
|
398
|
+
Specifies the authentication token required to access services running
|
|
399
|
+
on Teradata Vantage.
|
|
400
|
+
|
|
401
|
+
expiration_time:
|
|
402
|
+
Specifies the expiration time of the token in seconds. After expiry time JWT token expires and
|
|
403
|
+
UserEnv methods does not work, user should regenerate the token.
|
|
404
|
+
Note:
|
|
405
|
+
This option is used only for PAT and not for OAuth.
|
|
406
|
+
Default Value: 31536000
|
|
407
|
+
Types: int
|
|
408
|
+
|
|
409
|
+
kid:
|
|
410
|
+
Specifies the name of the key which is used while generating 'pem_file'.
|
|
411
|
+
Note:
|
|
412
|
+
* Use the argument "kid" only when key used during the pem file generation is different
|
|
413
|
+
from pem file name. For example, if you use the key as 'key1' while generating pem file
|
|
414
|
+
and the name of the pem file is `key1(1).pem`, then pass value 'key1' to the argument "kid".
|
|
415
|
+
Types: str
|
|
416
|
+
|
|
417
|
+
auth_mech:
|
|
418
|
+
Specifies the mechanism to be used for generating authentication token.
|
|
419
|
+
Notes:
|
|
420
|
+
* User must use this argument if Basic authentication is to be used.
|
|
421
|
+
* When "auth_mech" is provided, other arguments are used in the following
|
|
422
|
+
combination as per value of "auth_mech":
|
|
423
|
+
* OAuth: Token generation is done through OAuth by using client id
|
|
424
|
+
which can be sepcified by user in "client_id" argument or
|
|
425
|
+
can be derived internally from "base_url".
|
|
426
|
+
* PAT : Token generation is done using "pat_token" and "pem_file".
|
|
427
|
+
* BASIC: Authentication is done via Basic authentication mechanism
|
|
428
|
+
using user credentials passed in "username" and "password"
|
|
429
|
+
arguments.
|
|
430
|
+
* JWT : Readily available token in "auth_token" argument is used.
|
|
431
|
+
Permitted Values: "OAuth", "PAT", "BASIC", "JWT".
|
|
432
|
+
Types: str
|
|
433
|
+
|
|
376
434
|
RETURNS:
|
|
377
435
|
A Teradata sqlalchemy engine object.
|
|
378
436
|
|
|
@@ -383,7 +441,7 @@ def create_context(host = None, username = None, password = None, tdsqlengine =
|
|
|
383
441
|
>>> from teradataml.context.context import *
|
|
384
442
|
|
|
385
443
|
# Example 1: Create context using hostname, username and password
|
|
386
|
-
>>> td_context = create_context(host = 'tdhost', username='tduser', password = 'tdpassword')
|
|
444
|
+
>>> td_context = create_context(host = 'tdhost', username = 'tduser', password = 'tdpassword')
|
|
387
445
|
|
|
388
446
|
# Example 2: Create context using already created sqlalchemy engine
|
|
389
447
|
>>> from sqlalchemy import create_engine
|
|
@@ -391,56 +449,56 @@ def create_context(host = None, username = None, password = None, tdsqlengine =
|
|
|
391
449
|
>>> td_context = create_context(tdsqlengine = sqlalchemy_engine)
|
|
392
450
|
|
|
393
451
|
# Example 3: Creating context for Vantage with default logmech 'TD2'
|
|
394
|
-
>>> td_context = create_context(host = 'tdhost', username='tduser', password = 'tdpassword', logmech='TD2')
|
|
452
|
+
>>> td_context = create_context(host = 'tdhost', username = 'tduser', password = 'tdpassword', logmech = 'TD2')
|
|
395
453
|
|
|
396
454
|
# Example 4: Creating context for Vantage with logmech as 'TDNEGO'
|
|
397
|
-
>>> td_context = create_context(host = 'tdhost', username='tduser', password = 'tdpassword', logmech='TDNEGO')
|
|
455
|
+
>>> td_context = create_context(host = 'tdhost', username = 'tduser', password = 'tdpassword', logmech = 'TDNEGO')
|
|
398
456
|
|
|
399
457
|
# Example 5: Creating context for Vantage with logmech as 'LDAP'
|
|
400
|
-
>>> td_context = create_context(host = 'tdhost', username='tduser', password = 'tdpassword', logmech='LDAP')
|
|
458
|
+
>>> td_context = create_context(host = 'tdhost', username = 'tduser', password = 'tdpassword', logmech = 'LDAP')
|
|
401
459
|
|
|
402
460
|
# Example 6: Creating context for Vantage with logmech as 'KRB5'
|
|
403
|
-
>>> td_context = create_context(host = 'tdhost', logmech='KRB5')
|
|
461
|
+
>>> td_context = create_context(host = 'tdhost', logmech = 'KRB5')
|
|
404
462
|
|
|
405
463
|
# Example 7: Creating context for Vantage with logmech as 'JWT'
|
|
406
|
-
>>> td_context = create_context(host = 'tdhost', logmech='JWT', logdata='token=eyJpc...h8dA')
|
|
464
|
+
>>> td_context = create_context(host = 'tdhost', logmech = 'JWT', logdata = 'token=eyJpc...h8dA')
|
|
407
465
|
|
|
408
466
|
# Example 8: Create context using encrypted password and key passed to 'password' parameter.
|
|
409
|
-
#
|
|
410
|
-
#
|
|
411
|
-
#
|
|
412
|
-
#
|
|
413
|
-
#
|
|
414
|
-
#
|
|
415
|
-
#
|
|
416
|
-
#
|
|
467
|
+
# The password should be specified in the format mentioned below:
|
|
468
|
+
# ENCRYPTED_PASSWORD(file:<PasswordEncryptionKeyFileName>, file:<EncryptedPasswordFileName>)
|
|
469
|
+
# The PasswordEncryptionKeyFileName specifies the name of a file that contains the password encryption key
|
|
470
|
+
# and associated information.
|
|
471
|
+
# The EncryptedPasswordFileName specifies the name of a file that contains the encrypted password and
|
|
472
|
+
# associated information.
|
|
473
|
+
# Each filename must be preceded by the 'file:' prefix. The PasswordEncryptionKeyFileName must be separated
|
|
474
|
+
# from the EncryptedPasswordFileName by a single comma.
|
|
417
475
|
>>> encrypted_password = "ENCRYPTED_PASSWORD(file:PassKey.properties, file:EncPass.properties)"
|
|
418
|
-
>>> td_context = create_context(host = 'tdhost', username='tduser', password = encrypted_password)
|
|
476
|
+
>>> td_context = create_context(host = 'tdhost', username = 'tduser', password = encrypted_password)
|
|
419
477
|
|
|
420
478
|
# Example 9: Create context using encrypted password in LDAP logon mechanism.
|
|
421
|
-
>>> td_context = create_context(host = 'tdhost', username='tduser', password = encrypted_password,
|
|
422
|
-
|
|
479
|
+
>>> td_context = create_context(host = 'tdhost', username = 'tduser', password = encrypted_password,
|
|
480
|
+
... logmech = 'LDAP')
|
|
423
481
|
|
|
424
482
|
# Example 10: Create context using hostname, username, password and database parameters, and connect to a
|
|
425
483
|
# different initial database by setting the database parameter.
|
|
426
|
-
>>> td_context = create_context(host = 'tdhost', username='tduser', password = 'tdpassword', database =
|
|
427
|
-
|
|
484
|
+
>>> td_context = create_context(host = 'tdhost', username = 'tduser', password = 'tdpassword', database =
|
|
485
|
+
... 'database_name')
|
|
428
486
|
|
|
429
487
|
# Example 11: Create context using already created sqlalchemy engine, and connect to a different initial
|
|
430
488
|
# database by setting the database parameter.
|
|
431
489
|
>>> from sqlalchemy import create_engine
|
|
432
490
|
>>> sqlalchemy_engine = create_engine('teradatasql://'+ tduser +':' + tdpassword + '@'+tdhost +
|
|
433
|
-
|
|
491
|
+
... '/?DATABASE=database_name')
|
|
434
492
|
>>> td_context = create_context(tdsqlengine = sqlalchemy_engine)
|
|
435
493
|
|
|
436
494
|
# Example 12: Create context for Vantage with logmech as 'LDAP', and connect to a different initial
|
|
437
495
|
# database by setting the database parameter.
|
|
438
|
-
>>> td_context = create_context(host = 'tdhost', username='tduser', password = 'tdpassword', logmech='LDAP',
|
|
439
|
-
|
|
496
|
+
>>> td_context = create_context(host = 'tdhost', username = 'tduser', password = 'tdpassword', logmech = 'LDAP',
|
|
497
|
+
... database = 'database_name')
|
|
440
498
|
|
|
441
499
|
# Example 13: Create context using 'tera' mode with log value set to 8 and lob_support disabled.
|
|
442
|
-
>>> td_context = create_context(host = 'tdhost', username='tduser', password = 'tdpassword', tmode = 'tera',
|
|
443
|
-
|
|
500
|
+
>>> td_context = create_context(host = 'tdhost', username = 'tduser', password = 'tdpassword', tmode = 'tera',
|
|
501
|
+
... log = 8, lob_support = False)
|
|
444
502
|
|
|
445
503
|
# Example 14: Create context from config file with name 'td_properties.cfg'
|
|
446
504
|
# available under current working directory.
|
|
@@ -464,7 +522,7 @@ def create_context(host = None, username = None, password = None, tdsqlengine =
|
|
|
464
522
|
# logmech=tdlogmech
|
|
465
523
|
# logdata=tdlogdata
|
|
466
524
|
# database=tddatabase
|
|
467
|
-
>>> td_context = create_context(config_file="user_td_properties.cfg")
|
|
525
|
+
>>> td_context = create_context(config_file = "user_td_properties.cfg")
|
|
468
526
|
|
|
469
527
|
# Example 16: Create context using environment variables.
|
|
470
528
|
# Set these using os.environ and then run the example:
|
|
@@ -476,6 +534,57 @@ def create_context(host = None, username = None, password = None, tdsqlengine =
|
|
|
476
534
|
# os.environ['TD_LOGDATA'] = 'tdlogdata'
|
|
477
535
|
# os.environ['TD_DATABASE'] = 'tddatabase'
|
|
478
536
|
>>> td_context = create_context()
|
|
537
|
+
|
|
538
|
+
# Example 17: Create a context by providing username and password. Along with it,
|
|
539
|
+
# set authentication token by providing the pem file and pat token.
|
|
540
|
+
>>> td_context = create_context(host = 'tdhost', username = 'tduser', password = 'tdpassword',
|
|
541
|
+
... base_url = 'base_url', pat_token = 'pat_token', pem_file = 'pem_file')
|
|
542
|
+
|
|
543
|
+
# Example 18: Create a context by providing username and password. Along with it,
|
|
544
|
+
# generate authentication token by providing the client id.
|
|
545
|
+
>>> td_context = create_context(host = 'tdhost', username = 'tduser', password = 'tdpassword',
|
|
546
|
+
... base_url = 'base_url', client_id = 'client_id')
|
|
547
|
+
|
|
548
|
+
# Example 19: Create context and set authentication token by providing all the details in a config file.
|
|
549
|
+
# td_properties.cfg content:
|
|
550
|
+
# host=tdhost
|
|
551
|
+
# username=tduser
|
|
552
|
+
# password=tdpassword
|
|
553
|
+
# base_url=base_url
|
|
554
|
+
# pat_token=pat_token
|
|
555
|
+
# pem_file=pem_file
|
|
556
|
+
>>> td_context = create_context()
|
|
557
|
+
|
|
558
|
+
# Example 20: Create context and set authentication token by providing all the details in environment variables.
|
|
559
|
+
# Set these using os.environ and then run the example:
|
|
560
|
+
# os.environ['TD_HOST'] = 'tdhost'
|
|
561
|
+
# os.environ['TD_USERNAME'] = 'tduser'
|
|
562
|
+
# os.environ['TD_PASSWORD'] = 'tdpassword'
|
|
563
|
+
# os.environ['TD_BASE_URL'] = 'base_url'
|
|
564
|
+
# os.environ['TD_PAT_TOKEN'] = 'pat_token'
|
|
565
|
+
# os.environ['TD_PEM_FILE'] = 'pem_file'
|
|
566
|
+
>>> td_context = create_context()
|
|
567
|
+
|
|
568
|
+
# Example 21: Create context with sql_timeout set to 3.
|
|
569
|
+
>>> td_context = create_context(host = 'tdhost', username = 'tduser', password = 'tdpassword', sql_timeout = 3)
|
|
570
|
+
|
|
571
|
+
# Example 22: Create context and set authentication token via Basic authentication mechanism
|
|
572
|
+
# using username and password.
|
|
573
|
+
>>> import getpass
|
|
574
|
+
>>> create_context(host="host",
|
|
575
|
+
... username=getpass.getpass("username : "),
|
|
576
|
+
... password=getpass.getpass("password : "),
|
|
577
|
+
... base_url=getpass.getpass("base_url : "),
|
|
578
|
+
... auth_mech="BASIC")
|
|
579
|
+
|
|
580
|
+
# Example 23: Create context and set authentication token by providing auth_mech argument.
|
|
581
|
+
>>> import getpass
|
|
582
|
+
>>> create_context(host="vcl_host",
|
|
583
|
+
... username=getpass.getpass("username : "),
|
|
584
|
+
... password=getpass.getpass("password : "),
|
|
585
|
+
... base_url=getpass.getpass("base_url : "),
|
|
586
|
+
... auth_mech="OAuth")
|
|
587
|
+
|
|
479
588
|
"""
|
|
480
589
|
global td_connection
|
|
481
590
|
global td_sqlalchemy_engine
|
|
@@ -489,7 +598,7 @@ def create_context(host = None, username = None, password = None, tdsqlengine =
|
|
|
489
598
|
# Check if the user has provided the connection parameters or tdsqlengine.
|
|
490
599
|
# If not, check if the user has provided the connection parameters in the environment variables.
|
|
491
600
|
# If not, check if the user has provided the connection parameters in the config file.
|
|
492
|
-
if not (host or tdsqlengine) and host!="":
|
|
601
|
+
if not (host or tdsqlengine) and host != "":
|
|
493
602
|
return _load_context_from_env_config(kwargs.pop('config_file', 'td_properties.cfg'))
|
|
494
603
|
|
|
495
604
|
awu_matrix = []
|
|
@@ -500,13 +609,23 @@ def create_context(host = None, username = None, password = None, tdsqlengine =
|
|
|
500
609
|
awu_matrix.append(["logmech", logmech, True, (str), True])
|
|
501
610
|
awu_matrix.append(["logdata", logdata, True, (str), True])
|
|
502
611
|
awu_matrix.append(["database", database, True, (str), True])
|
|
612
|
+
# set_auth_token parameters
|
|
613
|
+
_set_auth_token_params = {}
|
|
614
|
+
auth_mech = kwargs.get('auth_mech', None)
|
|
615
|
+
for param in ['base_url', 'pat_token', 'pem_file', 'client_id', 'auth_token', 'expiration_time', 'kid', 'auth_mech']:
|
|
616
|
+
if kwargs.get(param):
|
|
617
|
+
_set_auth_token_params[param] = kwargs.pop(param)
|
|
618
|
+
|
|
619
|
+
# Set the "sql_timeout" parameter to "request_timeout" which is consumed by teradatasql.
|
|
620
|
+
if kwargs.get('sql_timeout'):
|
|
621
|
+
awu_matrix.append(["sql_timeout", kwargs.get('sql_timeout'), True, (int), True])
|
|
622
|
+
kwargs['request_timeout'] = kwargs.pop('sql_timeout')
|
|
503
623
|
|
|
504
624
|
awu = _Validators()
|
|
505
625
|
awu._validate_function_arguments(awu_matrix)
|
|
506
626
|
|
|
507
627
|
# Clearing the internal buffer.
|
|
508
628
|
_InternalBuffer.clean()
|
|
509
|
-
|
|
510
629
|
if logmech == "JWT" and not logdata:
|
|
511
630
|
raise TeradataMlException(Messages.get_message(MessageCodes.DEPENDENT_ARG_MISSING,
|
|
512
631
|
'logdata',
|
|
@@ -541,7 +660,6 @@ def create_context(host = None, username = None, password = None, tdsqlengine =
|
|
|
541
660
|
host_value = '{}:@{}'.format(username, host)
|
|
542
661
|
else:
|
|
543
662
|
host_value = '{}:{}@{}'.format(username, password, host)
|
|
544
|
-
|
|
545
663
|
url_object = URL.create(
|
|
546
664
|
"teradatasql",
|
|
547
665
|
username=username,
|
|
@@ -555,7 +673,7 @@ def create_context(host = None, username = None, password = None, tdsqlengine =
|
|
|
555
673
|
td_connection = td_sqlalchemy_engine.connect()
|
|
556
674
|
td_user = username.upper()
|
|
557
675
|
|
|
558
|
-
# Masking
|
|
676
|
+
# Masking sensitive information - password, logmech and logdata.
|
|
559
677
|
if password:
|
|
560
678
|
try:
|
|
561
679
|
# Below statement raises an AttributeError with SQLAlchemy
|
|
@@ -607,21 +725,34 @@ def create_context(host = None, username = None, password = None, tdsqlengine =
|
|
|
607
725
|
# Process Analytic functions.
|
|
608
726
|
from teradataml.analytics import _process_analytic_functions
|
|
609
727
|
_process_analytic_functions()
|
|
610
|
-
|
|
728
|
+
|
|
729
|
+
if _set_auth_token_params.get('base_url'):
|
|
730
|
+
from teradataml.scriptmgmt.lls_utils import set_auth_token
|
|
731
|
+
try:
|
|
732
|
+
if auth_mech and auth_mech.lower() == 'basic' and password:
|
|
733
|
+
_set_auth_token_params['password'] = password
|
|
734
|
+
set_auth_token(**_set_auth_token_params)
|
|
735
|
+
except Exception as err:
|
|
736
|
+
print("Connection to Vantage established successfully.")
|
|
737
|
+
mssg = f"Failed to set authentication token. Rerun \"set_auth_token()\" again to set the authentication token." \
|
|
738
|
+
f" Reason for failure: {err.args[0]}"
|
|
739
|
+
warnings.warn(mssg, stacklevel=2)
|
|
740
|
+
|
|
611
741
|
# Add global lock to internal buffer
|
|
612
742
|
_InternalBuffer.add(global_lock=threading.Lock())
|
|
613
743
|
|
|
614
744
|
# Return the connection by default
|
|
615
745
|
return td_sqlalchemy_engine
|
|
616
746
|
|
|
617
|
-
|
|
747
|
+
|
|
748
|
+
def _load_context_from_env_config(config_file=None):
|
|
618
749
|
"""
|
|
619
750
|
DESCRIPTION:
|
|
620
751
|
Reads the connection parameters from the configuration file or environment variables.
|
|
621
752
|
|
|
622
753
|
PARAMETERS:
|
|
623
754
|
config_file:
|
|
624
|
-
|
|
755
|
+
Optional Argument.
|
|
625
756
|
Specifies the name of the configuration file to read the connection parameters.
|
|
626
757
|
Types: str
|
|
627
758
|
|
|
@@ -632,16 +763,17 @@ def _load_context_from_env_config(config_file):
|
|
|
632
763
|
TeradataMlException
|
|
633
764
|
"""
|
|
634
765
|
host = os.environ.get('TD_HOST')
|
|
635
|
-
connection_params_from_file = dotenv_values(config_file)
|
|
636
766
|
if host:
|
|
637
767
|
connection_params_from_env = {key[3:].lower(): value for key, value in os.environ.items()
|
|
638
768
|
if key.startswith('TD_')}
|
|
639
769
|
return create_context(**connection_params_from_env)
|
|
640
|
-
elif
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
770
|
+
elif config_file is not None:
|
|
771
|
+
connection_params_from_file = dotenv_values(config_file)
|
|
772
|
+
if connection_params_from_file.get('host'):
|
|
773
|
+
return create_context(**connection_params_from_file)
|
|
774
|
+
raise TeradataMlException(Messages.get_message(MessageCodes.CONNECTION_PARAMS),
|
|
775
|
+
MessageCodes.MISSING_ARGS)
|
|
776
|
+
|
|
645
777
|
|
|
646
778
|
def _mask_logmech_logdata():
|
|
647
779
|
"""
|
|
@@ -776,7 +908,7 @@ def set_context(tdsqlengine, temp_database_name=None):
|
|
|
776
908
|
|
|
777
909
|
# Initialise Dag
|
|
778
910
|
__initalise_dag()
|
|
779
|
-
|
|
911
|
+
|
|
780
912
|
# Add global lock to internal buffer
|
|
781
913
|
_InternalBuffer.add(global_lock=threading.Lock())
|
|
782
914
|
|
|
@@ -813,6 +945,9 @@ def remove_context():
|
|
|
813
945
|
# Initiate the garbage collection
|
|
814
946
|
__cleanup_garbage_collection()
|
|
815
947
|
|
|
948
|
+
# Clearing the internal buffer.
|
|
949
|
+
_InternalBuffer.clean()
|
|
950
|
+
|
|
816
951
|
# Check if connection is established or not.
|
|
817
952
|
if user_specified_connection is not True:
|
|
818
953
|
try:
|
|
@@ -1117,6 +1252,7 @@ def _get_user():
|
|
|
1117
1252
|
td_user = _get_database_username()
|
|
1118
1253
|
return td_user
|
|
1119
1254
|
|
|
1255
|
+
|
|
1120
1256
|
def _get_host():
|
|
1121
1257
|
"""
|
|
1122
1258
|
DESCRIPTION:
|
|
@@ -1139,6 +1275,7 @@ def _get_host():
|
|
|
1139
1275
|
else:
|
|
1140
1276
|
return td_sqlalchemy_engine.url.host
|
|
1141
1277
|
|
|
1278
|
+
|
|
1142
1279
|
def _get_host_ip():
|
|
1143
1280
|
"""
|
|
1144
1281
|
DESCRIPTION:
|
|
@@ -1159,7 +1296,7 @@ def _get_host_ip():
|
|
|
1159
1296
|
# Return None if connection is not established.
|
|
1160
1297
|
if td_connection is None:
|
|
1161
1298
|
return None
|
|
1162
|
-
|
|
1299
|
+
|
|
1163
1300
|
host = _get_host()
|
|
1164
1301
|
try:
|
|
1165
1302
|
# Validate if host_ip is a valid IP address (IPv4 or IPv6)
|
|
@@ -1170,7 +1307,7 @@ def _get_host_ip():
|
|
|
1170
1307
|
dns_host_ip = _InternalBuffer.get('dns_host_ip')
|
|
1171
1308
|
if dns_host_ip:
|
|
1172
1309
|
return dns_host_ip
|
|
1173
|
-
|
|
1310
|
+
|
|
1174
1311
|
# If DNS host ip not found, resolve the host name to get the IP address.
|
|
1175
1312
|
# If there is issue in resolving the host name, it will proceed with DNS host as it is.
|
|
1176
1313
|
try:
|
|
@@ -1185,6 +1322,7 @@ def _get_host_ip():
|
|
|
1185
1322
|
host_ip = host
|
|
1186
1323
|
return host_ip
|
|
1187
1324
|
|
|
1325
|
+
|
|
1188
1326
|
class ContextUtilFuncs():
|
|
1189
1327
|
@staticmethod
|
|
1190
1328
|
def _check_alias_config_file_exists(vantage_version, alias_config_file):
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
{
|
|
2
|
+
"trans_dense": {
|
|
3
|
+
"location": "varchar(100)",
|
|
4
|
+
"tranid": "int",
|
|
5
|
+
"period": "varchar(20)",
|
|
6
|
+
"storeid": "int",
|
|
7
|
+
"region": "varchar(20)",
|
|
8
|
+
"item": "varchar(200)",
|
|
9
|
+
"sku": "int",
|
|
10
|
+
"category": "varchar(20)"
|
|
11
|
+
},
|
|
12
|
+
"trans_sparse": {
|
|
13
|
+
"location": "varchar(20)",
|
|
14
|
+
"tranid": "int",
|
|
15
|
+
"period": "varchar(20)",
|
|
16
|
+
"storeid": "int",
|
|
17
|
+
"region": "varchar(20)",
|
|
18
|
+
"item": "varchar(20)",
|
|
19
|
+
"sku": "int",
|
|
20
|
+
"category": "varchar(20)"
|
|
21
|
+
}
|
|
22
|
+
}
|