teradataml 20.0.0.4__py3-none-any.whl → 20.0.0.6__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of teradataml might be problematic. Click here for more details.
- teradataml/LICENSE-3RD-PARTY.pdf +0 -0
- teradataml/README.md +182 -13
- teradataml/__init__.py +2 -1
- teradataml/_version.py +2 -2
- teradataml/analytics/analytic_function_executor.py +8 -13
- teradataml/analytics/json_parser/analytic_functions_argument.py +4 -0
- teradataml/analytics/sqle/__init__.py +16 -1
- teradataml/analytics/utils.py +60 -1
- teradataml/automl/__init__.py +290 -106
- teradataml/automl/autodataprep/__init__.py +471 -0
- teradataml/automl/data_preparation.py +29 -10
- teradataml/automl/data_transformation.py +11 -0
- teradataml/automl/feature_engineering.py +64 -4
- teradataml/automl/feature_exploration.py +639 -25
- teradataml/automl/model_training.py +1 -1
- teradataml/clients/auth_client.py +12 -8
- teradataml/clients/keycloak_client.py +165 -0
- teradataml/common/constants.py +71 -26
- teradataml/common/exceptions.py +32 -0
- teradataml/common/messagecodes.py +28 -0
- teradataml/common/messages.py +13 -4
- teradataml/common/sqlbundle.py +3 -2
- teradataml/common/utils.py +345 -45
- teradataml/context/context.py +259 -93
- teradataml/data/apriori_example.json +22 -0
- teradataml/data/docs/sqle/docs_17_20/Apriori.py +138 -0
- teradataml/data/docs/sqle/docs_17_20/NERExtractor.py +121 -0
- teradataml/data/docs/sqle/docs_17_20/NGramSplitter.py +3 -3
- teradataml/data/docs/sqle/docs_17_20/SMOTE.py +212 -0
- teradataml/data/docs/sqle/docs_17_20/TextMorph.py +119 -0
- teradataml/data/docs/sqle/docs_17_20/TextParser.py +54 -3
- teradataml/data/docs/uaf/docs_17_20/ACF.py +1 -1
- teradataml/data/docs/uaf/docs_17_20/ArimaEstimate.py +2 -2
- teradataml/data/docs/uaf/docs_17_20/ArimaXEstimate.py +2 -2
- teradataml/data/docs/uaf/docs_17_20/DFFT.py +1 -1
- teradataml/data/docs/uaf/docs_17_20/DFFT2.py +1 -1
- teradataml/data/docs/uaf/docs_17_20/DFFT2Conv.py +1 -1
- teradataml/data/docs/uaf/docs_17_20/DFFTConv.py +1 -1
- teradataml/data/docs/uaf/docs_17_20/FilterFactory1d.py +4 -4
- teradataml/data/docs/uaf/docs_17_20/GenseriesSinusoids.py +2 -2
- teradataml/data/docs/uaf/docs_17_20/GoldfeldQuandt.py +2 -2
- teradataml/data/docs/uaf/docs_17_20/HoltWintersForecaster.py +6 -6
- teradataml/data/docs/uaf/docs_17_20/LineSpec.py +1 -1
- teradataml/data/docs/uaf/docs_17_20/LinearRegr.py +1 -1
- teradataml/data/docs/uaf/docs_17_20/Matrix2Image.py +4 -4
- teradataml/data/docs/uaf/docs_17_20/MultivarRegr.py +1 -1
- teradataml/data/docs/uaf/docs_17_20/PACF.py +1 -1
- teradataml/data/docs/uaf/docs_17_20/PowerSpec.py +2 -2
- teradataml/data/docs/uaf/docs_17_20/PowerTransform.py +3 -3
- teradataml/data/docs/uaf/docs_17_20/Resample.py +5 -5
- teradataml/data/docs/uaf/docs_17_20/SAX.py +3 -3
- teradataml/data/docs/uaf/docs_17_20/SignifPeriodicities.py +1 -1
- teradataml/data/docs/uaf/docs_17_20/SimpleExp.py +1 -1
- teradataml/data/docs/uaf/docs_17_20/Smoothma.py +3 -3
- teradataml/data/docs/uaf/docs_17_20/UNDIFF.py +1 -1
- teradataml/data/jsons/byom/onnxembeddings.json +1 -0
- teradataml/data/jsons/sqle/17.20/NGramSplitter.json +6 -6
- teradataml/data/jsons/sqle/17.20/TD_Apriori.json +181 -0
- teradataml/data/jsons/sqle/17.20/TD_NERExtractor.json +145 -0
- teradataml/data/jsons/sqle/17.20/TD_SMOTE.json +267 -0
- teradataml/data/jsons/sqle/17.20/TD_TextMorph.json +134 -0
- teradataml/data/jsons/sqle/17.20/TD_TextParser.json +114 -9
- teradataml/data/jsons/sqle/20.00/AI_AnalyzeSentiment.json +328 -0
- teradataml/data/jsons/sqle/20.00/AI_AskLLM.json +420 -0
- teradataml/data/jsons/sqle/20.00/AI_DetectLanguage.json +343 -0
- teradataml/data/jsons/sqle/20.00/AI_ExtractKeyPhrases.json +328 -0
- teradataml/data/jsons/sqle/20.00/AI_MaskPII.json +328 -0
- teradataml/data/jsons/sqle/20.00/AI_RecognizeEntities.json +328 -0
- teradataml/data/jsons/sqle/20.00/AI_RecognizePIIEntities.json +328 -0
- teradataml/data/jsons/sqle/20.00/AI_TextClassifier.json +359 -0
- teradataml/data/jsons/sqle/20.00/AI_TextEmbeddings.json +360 -0
- teradataml/data/jsons/sqle/20.00/AI_TextSummarize.json +343 -0
- teradataml/data/jsons/sqle/20.00/AI_TextTranslate.json +343 -0
- teradataml/data/jsons/sqle/20.00/TD_SMOTE.json +2 -2
- teradataml/data/jsons/sqle/20.00/TD_VectorDistance.json +1 -1
- teradataml/data/ner_dict.csv +8 -0
- teradataml/data/ner_input_eng.csv +7 -0
- teradataml/data/ner_rule.csv +5 -0
- teradataml/data/pattern_matching_data.csv +11 -0
- teradataml/data/pos_input.csv +40 -0
- teradataml/data/sdk/modelops/modelops_spec.json +101737 -0
- teradataml/data/tdnerextractor_example.json +14 -0
- teradataml/data/teradataml_example.json +21 -1
- teradataml/data/textmorph_example.json +5 -0
- teradataml/data/to_num_data.csv +4 -0
- teradataml/data/tochar_data.csv +5 -0
- teradataml/data/trans_dense.csv +16 -0
- teradataml/data/trans_sparse.csv +55 -0
- teradataml/data/url_data.csv +10 -9
- teradataml/dataframe/copy_to.py +38 -27
- teradataml/dataframe/data_transfer.py +61 -45
- teradataml/dataframe/dataframe.py +1110 -132
- teradataml/dataframe/dataframe_utils.py +73 -27
- teradataml/dataframe/functions.py +1070 -9
- teradataml/dataframe/sql.py +750 -959
- teradataml/dbutils/dbutils.py +33 -13
- teradataml/dbutils/filemgr.py +14 -10
- teradataml/hyperparameter_tuner/utils.py +4 -2
- teradataml/lib/aed_0_1.dll +0 -0
- teradataml/opensource/_base.py +12 -157
- teradataml/options/configure.py +24 -9
- teradataml/scriptmgmt/UserEnv.py +317 -39
- teradataml/scriptmgmt/lls_utils.py +456 -135
- teradataml/sdk/README.md +79 -0
- teradataml/sdk/__init__.py +4 -0
- teradataml/sdk/_auth_modes.py +422 -0
- teradataml/sdk/_func_params.py +487 -0
- teradataml/sdk/_json_parser.py +453 -0
- teradataml/sdk/_openapi_spec_constants.py +249 -0
- teradataml/sdk/_utils.py +236 -0
- teradataml/sdk/api_client.py +897 -0
- teradataml/sdk/constants.py +62 -0
- teradataml/sdk/modelops/__init__.py +98 -0
- teradataml/sdk/modelops/_client.py +406 -0
- teradataml/sdk/modelops/_constants.py +304 -0
- teradataml/sdk/modelops/models.py +2308 -0
- teradataml/sdk/spinner.py +107 -0
- teradataml/store/__init__.py +1 -1
- teradataml/table_operators/Apply.py +16 -1
- teradataml/table_operators/Script.py +20 -1
- teradataml/table_operators/query_generator.py +4 -21
- teradataml/table_operators/table_operator_util.py +58 -9
- teradataml/utils/dtypes.py +4 -2
- teradataml/utils/internal_buffer.py +22 -2
- teradataml/utils/utils.py +0 -1
- teradataml/utils/validators.py +318 -58
- {teradataml-20.0.0.4.dist-info → teradataml-20.0.0.6.dist-info}/METADATA +188 -14
- {teradataml-20.0.0.4.dist-info → teradataml-20.0.0.6.dist-info}/RECORD +131 -84
- {teradataml-20.0.0.4.dist-info → teradataml-20.0.0.6.dist-info}/WHEEL +0 -0
- {teradataml-20.0.0.4.dist-info → teradataml-20.0.0.6.dist-info}/top_level.txt +0 -0
- {teradataml-20.0.0.4.dist-info → teradataml-20.0.0.6.dist-info}/zip-safe +0 -0
teradataml/context/context.py
CHANGED
|
@@ -19,17 +19,15 @@ import os
|
|
|
19
19
|
import socket
|
|
20
20
|
import sys
|
|
21
21
|
import threading
|
|
22
|
-
import urllib.parse
|
|
23
22
|
import warnings
|
|
24
23
|
from pathlib import Path
|
|
25
|
-
from dotenv import dotenv_values
|
|
26
24
|
|
|
25
|
+
from dotenv import dotenv_values
|
|
27
26
|
from sqlalchemy import create_engine
|
|
28
27
|
from sqlalchemy.engine.base import Engine
|
|
29
28
|
from sqlalchemy.engine.url import URL
|
|
30
29
|
|
|
31
30
|
from teradataml.common.constants import Query, SQLConstants, TeradataConstants
|
|
32
|
-
from teradataml.common.deprecations import argument_deprecation
|
|
33
31
|
from teradataml.common.exceptions import TeradataMlException
|
|
34
32
|
from teradataml.common.garbagecollector import GarbageCollector
|
|
35
33
|
from teradataml.common.messagecodes import MessageCodes
|
|
@@ -138,7 +136,7 @@ def __cleanup_garbage_collection():
|
|
|
138
136
|
GarbageCollector._cleanup_garbage_collector()
|
|
139
137
|
|
|
140
138
|
|
|
141
|
-
def _get_other_connection_parameters(logmech
|
|
139
|
+
def _get_other_connection_parameters(logmech=None, logdata=None, database=None, **kwargs):
|
|
142
140
|
"""
|
|
143
141
|
DESCRIPTION:
|
|
144
142
|
Internal function to return the connection parameters.
|
|
@@ -199,24 +197,27 @@ def _get_other_connection_parameters(logmech = None, logdata = None, database =
|
|
|
199
197
|
|
|
200
198
|
return result
|
|
201
199
|
|
|
202
|
-
|
|
200
|
+
|
|
203
201
|
@collect_queryband(queryband='CrtCxt')
|
|
204
|
-
def create_context(host
|
|
205
|
-
logmech
|
|
202
|
+
def create_context(host=None, username=None, password=None, tdsqlengine=None, temp_database_name=None,
|
|
203
|
+
logmech=None, logdata=None, database=None, **kwargs):
|
|
206
204
|
"""
|
|
207
205
|
DESCRIPTION:
|
|
208
206
|
Creates a connection to the Teradata Vantage using the teradatasql + teradatasqlalchemy DBAPI and dialect
|
|
209
207
|
combination.
|
|
210
|
-
Users can create a connection by passing the connection parameters using any of the following methods:
|
|
211
|
-
1. Pass all required parameters (host, username, password) directly to the function
|
|
208
|
+
Users can create a connection by passing the connection parameters using any one of the following methods:
|
|
209
|
+
1. Pass all required parameters (host, username, password) directly to the function.
|
|
212
210
|
2. Set the connection parameters in a configuration file (.cfg or .env) and
|
|
213
|
-
pass the configuration file
|
|
211
|
+
pass the configuration file.
|
|
214
212
|
3. Set the connection parameters in environment variables and create_context() reads from
|
|
215
213
|
environment variables.
|
|
216
214
|
|
|
217
|
-
Alternatively, users can pass a SQLAlchemy engine to the `tdsqlengine` parameter to override the default DBAPI
|
|
215
|
+
Alternatively, users can pass a SQLAlchemy engine object to the `tdsqlengine` parameter to override the default DBAPI
|
|
218
216
|
and dialect combination.
|
|
219
217
|
|
|
218
|
+
Function also enables user to set the authentication token which is required to access services running
|
|
219
|
+
on Teradata Vantage.
|
|
220
|
+
|
|
220
221
|
Note:
|
|
221
222
|
1. teradataml requires that the user has certain permissions on the user's default database or the initial
|
|
222
223
|
default database specified using the database argument, or the temporary database when specified using
|
|
@@ -264,6 +265,28 @@ def create_context(host = None, username = None, password = None, tdsqlengine =
|
|
|
264
265
|
os.environ['TD_PASSWORD'] = 'tdpassword'
|
|
265
266
|
3. A configuration file if provided (such as a .env file).
|
|
266
267
|
|
|
268
|
+
4. Points to note when user sets authentication token with create_context():
|
|
269
|
+
* The username provided in create_context() is not case-sensitive. For example,
|
|
270
|
+
if a user is created with the username xyz, create_context() still establishes
|
|
271
|
+
a connection if user passes the username as XyZ. However, authentication token
|
|
272
|
+
generation requires the username to be in the same case as when it was created.
|
|
273
|
+
Therefore, Teradata recommends to pass the username with the same case as when
|
|
274
|
+
it was created.
|
|
275
|
+
* User must have a privilege to login with a NULL password to use set_auth_token().
|
|
276
|
+
Refer to GRANT LOGON section in Teradata Documentation for more details.
|
|
277
|
+
* When "auth_mech" is not specified, arguments are used in the following combination
|
|
278
|
+
to derive authentication mechanism.
|
|
279
|
+
* If "base_url" and "client_id" are specified then token generation is done through OAuth.
|
|
280
|
+
* If "base_url", "pat_token", "pem_file" are specified then token generation is done using PAT.
|
|
281
|
+
* If "base_url" and "auth_token" are specified then value provided for "auth_token" is used.
|
|
282
|
+
* If only "base_url" is specified then token generation is done through OAuth.
|
|
283
|
+
* If Basic authentication mechanism is to be used then user must specify argument
|
|
284
|
+
"auth_mech" as "BASIC" along with "username" and "password".
|
|
285
|
+
* Refresh token works only for OAuth authentication.
|
|
286
|
+
* Use the argument "kid" only when key used during the pem file generation is different
|
|
287
|
+
from pem file name. For example, if you use the key as 'key1' while generating pem file
|
|
288
|
+
and the name of the pem file is `key1(1).pem`, then pass value 'key1' to the argument "kid".
|
|
289
|
+
|
|
267
290
|
PARAMETERS:
|
|
268
291
|
host:
|
|
269
292
|
Optional Argument.
|
|
@@ -277,39 +300,15 @@ def create_context(host = None, username = None, password = None, tdsqlengine =
|
|
|
277
300
|
|
|
278
301
|
password:
|
|
279
302
|
Optional Argument.
|
|
280
|
-
Specifies the password required for the username.
|
|
303
|
+
Specifies the password required for the "username".
|
|
281
304
|
Types: str
|
|
282
305
|
Note:
|
|
283
306
|
* Encrypted passwords can also be passed to this argument, using Stored Password Protection feature.
|
|
284
307
|
Examples section below demonstrates passing encrypted password to 'create_context'.
|
|
285
308
|
More details on Stored Password Protection and how to generate key and encrypted password file
|
|
286
309
|
can be found at https://pypi.org/project/teradatasql/#StoredPasswordProtection
|
|
287
|
-
* Special characters
|
|
288
|
-
|
|
289
|
-
disabled by setting "url_encode" = False.
|
|
290
|
-
For example, if the password is: "kx%jj5/g", then this password is encoded as below:
|
|
291
|
-
"kx%25jj5%2Fg"
|
|
292
|
-
where,
|
|
293
|
-
%25 represents the '%' character and
|
|
294
|
-
%2F represents the '/' character
|
|
295
|
-
The details of how the special characters are replaced can be found in the below link:
|
|
296
|
-
"https://docs.microfocus.com/OMi/10.62/Content/OMi/ExtGuide/ExtApps/URL_encoding.htm".
|
|
297
|
-
Note:
|
|
298
|
-
When password contains a space:
|
|
299
|
-
* "url_encode" must be set to False.
|
|
300
|
-
* In addition to space, other special characters in this password should be manually encoded,
|
|
301
|
-
following the link: https://docs.microfocus.com/OMi/10.62/Content/OMi/ExtGuide/ExtApps/URL_encoding.htm
|
|
302
|
-
Refer Example 16 in examples section for a detailed demonstration.
|
|
303
|
-
|
|
304
|
-
When password contains unreserved characters like tilde("~"), dot("."), underscore("_"), hyphen("-"):
|
|
305
|
-
* Character is not URL encoded by default.
|
|
306
|
-
* If unreserved character in passed needs to be encoded then,
|
|
307
|
-
it must be encoded manually and encoded password must be passed, along with "url_encode" set to False.
|
|
308
|
-
* The unreserved characters differ from one Python version to another Python version.
|
|
309
|
-
For example, The encoding standards for Python version 3.7 found in the below link:
|
|
310
|
-
"https://docs.python.org/3/library/urllib.parse.html#url-quoting".
|
|
311
|
-
|
|
312
|
-
|
|
310
|
+
* Special characters used in the password are encoded by default.
|
|
311
|
+
|
|
313
312
|
tdsqlengine:
|
|
314
313
|
Optional Argument.
|
|
315
314
|
Specifies Teradata Vantage sqlalchemy engine object that should be used to establish a Teradata Vantage
|
|
@@ -347,18 +346,23 @@ def create_context(host = None, username = None, password = None, tdsqlengine =
|
|
|
347
346
|
Types: str
|
|
348
347
|
|
|
349
348
|
kwargs:
|
|
350
|
-
Specifies
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
349
|
+
Specifies optional keyword arguments accepted by create_context().
|
|
350
|
+
Below are the supported keyword arguments:
|
|
351
|
+
|
|
352
|
+
Connection parameters for Teradata SQL Driver:
|
|
353
|
+
Specifies the keyword-value pairs of connection parameters that are passed to Teradata SQL Driver for
|
|
354
|
+
Python. Please refer to https://github.com/Teradata/python-driver#ConnectionParameters to get information
|
|
355
|
+
on connection parameters of the driver.
|
|
356
|
+
Note:
|
|
357
|
+
* When type of a connection parameter is integer or boolean (eg: log, lob_support etc,.), pass
|
|
358
|
+
integer or boolean value, instead of quoted integer or quoted boolean as suggested in the
|
|
359
|
+
documentation. Please check the examples for usage.
|
|
360
|
+
* "sql_timeout" represents "request_timeout" connection parameter.
|
|
356
361
|
|
|
357
362
|
config_file:
|
|
358
|
-
Optional Argument.
|
|
359
363
|
Specifies the name of the configuration file to read the connection parameters.
|
|
360
364
|
Notes:
|
|
361
|
-
* If user
|
|
365
|
+
* If user does not specify full path of file, then file look up is done at current working directory.
|
|
362
366
|
* The content of the file must be in '.env' format.
|
|
363
367
|
* Use parameters of create_context() as key in the configuration file.
|
|
364
368
|
Example:
|
|
@@ -373,6 +377,73 @@ def create_context(host = None, username = None, password = None, tdsqlengine =
|
|
|
373
377
|
Default Value : td_properties.cfg
|
|
374
378
|
Types: str
|
|
375
379
|
|
|
380
|
+
base_url:
|
|
381
|
+
Specifies the endpoint URL for a given environment on Teradata Vantage.
|
|
382
|
+
Types: str
|
|
383
|
+
|
|
384
|
+
client_id:
|
|
385
|
+
Specifies the id of the application that requests the access token from
|
|
386
|
+
VantageCloud Lake.
|
|
387
|
+
Types: str
|
|
388
|
+
|
|
389
|
+
pat_token:
|
|
390
|
+
Specifies the PAT token generated from VantageCloud Lake Console.
|
|
391
|
+
Types: str
|
|
392
|
+
|
|
393
|
+
pem_file:
|
|
394
|
+
Specifies the path to private key file which is generated from VantageCloud Lake Console.
|
|
395
|
+
Types: str
|
|
396
|
+
|
|
397
|
+
auth_token:
|
|
398
|
+
Specifies the authentication token required to access services running
|
|
399
|
+
on Teradata Vantage.
|
|
400
|
+
|
|
401
|
+
expiration_time:
|
|
402
|
+
Specifies the expiration time of the token in seconds. After expiry time JWT token expires and
|
|
403
|
+
UserEnv methods does not work, user should regenerate the token.
|
|
404
|
+
Note:
|
|
405
|
+
This option is used only for PAT and not for OAuth.
|
|
406
|
+
Default Value: 31536000
|
|
407
|
+
Types: int
|
|
408
|
+
|
|
409
|
+
kid:
|
|
410
|
+
Specifies the name of the key which is used while generating 'pem_file'.
|
|
411
|
+
Note:
|
|
412
|
+
* Use the argument "kid" only when key used during the pem file generation is different
|
|
413
|
+
from pem file name. For example, if you use the key as 'key1' while generating pem file
|
|
414
|
+
and the name of the pem file is `key1(1).pem`, then pass value 'key1' to the argument "kid".
|
|
415
|
+
Types: str
|
|
416
|
+
|
|
417
|
+
auth_url:
|
|
418
|
+
Optional Argument.
|
|
419
|
+
Specifies the endpoint URL for a keycloak server.
|
|
420
|
+
Types: str
|
|
421
|
+
|
|
422
|
+
rest_client:
|
|
423
|
+
Optional Argument.
|
|
424
|
+
Specifies the service for which keycloak token is to be generated.
|
|
425
|
+
Permitted values: "VECTORSTORE"
|
|
426
|
+
Default value: "VECTORSTORE"
|
|
427
|
+
Types: str
|
|
428
|
+
|
|
429
|
+
auth_mech:
|
|
430
|
+
Specifies the mechanism to be used for generating authentication token.
|
|
431
|
+
Notes:
|
|
432
|
+
* User must use this argument if Basic authentication is to be used.
|
|
433
|
+
* When "auth_mech" is provided, other arguments are used in the following
|
|
434
|
+
combination as per value of "auth_mech":
|
|
435
|
+
* OAuth: Token generation is done through OAuth by using client id
|
|
436
|
+
which can be sepcified by user in "client_id" argument or
|
|
437
|
+
can be derived internally from "base_url".
|
|
438
|
+
* PAT : Token generation is done using "pat_token" and "pem_file".
|
|
439
|
+
* BASIC: Authentication is done via Basic authentication mechanism
|
|
440
|
+
using user credentials passed in "username" and "password"
|
|
441
|
+
arguments.
|
|
442
|
+
* JWT : Readily available token in "auth_token" argument is used.
|
|
443
|
+
* KEYCLOAK: Token generation is done using keycloak.
|
|
444
|
+
Permitted Values: "OAuth", "PAT", "BASIC", "JWT", "KEYCLOAK".
|
|
445
|
+
Types: str
|
|
446
|
+
|
|
376
447
|
RETURNS:
|
|
377
448
|
A Teradata sqlalchemy engine object.
|
|
378
449
|
|
|
@@ -383,7 +454,7 @@ def create_context(host = None, username = None, password = None, tdsqlengine =
|
|
|
383
454
|
>>> from teradataml.context.context import *
|
|
384
455
|
|
|
385
456
|
# Example 1: Create context using hostname, username and password
|
|
386
|
-
>>> td_context = create_context(host = 'tdhost', username='tduser', password = 'tdpassword')
|
|
457
|
+
>>> td_context = create_context(host = 'tdhost', username = 'tduser', password = 'tdpassword')
|
|
387
458
|
|
|
388
459
|
# Example 2: Create context using already created sqlalchemy engine
|
|
389
460
|
>>> from sqlalchemy import create_engine
|
|
@@ -391,56 +462,56 @@ def create_context(host = None, username = None, password = None, tdsqlengine =
|
|
|
391
462
|
>>> td_context = create_context(tdsqlengine = sqlalchemy_engine)
|
|
392
463
|
|
|
393
464
|
# Example 3: Creating context for Vantage with default logmech 'TD2'
|
|
394
|
-
>>> td_context = create_context(host = 'tdhost', username='tduser', password = 'tdpassword', logmech='TD2')
|
|
465
|
+
>>> td_context = create_context(host = 'tdhost', username = 'tduser', password = 'tdpassword', logmech = 'TD2')
|
|
395
466
|
|
|
396
467
|
# Example 4: Creating context for Vantage with logmech as 'TDNEGO'
|
|
397
|
-
>>> td_context = create_context(host = 'tdhost', username='tduser', password = 'tdpassword', logmech='TDNEGO')
|
|
468
|
+
>>> td_context = create_context(host = 'tdhost', username = 'tduser', password = 'tdpassword', logmech = 'TDNEGO')
|
|
398
469
|
|
|
399
470
|
# Example 5: Creating context for Vantage with logmech as 'LDAP'
|
|
400
|
-
>>> td_context = create_context(host = 'tdhost', username='tduser', password = 'tdpassword', logmech='LDAP')
|
|
471
|
+
>>> td_context = create_context(host = 'tdhost', username = 'tduser', password = 'tdpassword', logmech = 'LDAP')
|
|
401
472
|
|
|
402
473
|
# Example 6: Creating context for Vantage with logmech as 'KRB5'
|
|
403
|
-
>>> td_context = create_context(host = 'tdhost', logmech='KRB5')
|
|
474
|
+
>>> td_context = create_context(host = 'tdhost', logmech = 'KRB5')
|
|
404
475
|
|
|
405
476
|
# Example 7: Creating context for Vantage with logmech as 'JWT'
|
|
406
|
-
>>> td_context = create_context(host = 'tdhost', logmech='JWT', logdata='token=eyJpc...h8dA')
|
|
477
|
+
>>> td_context = create_context(host = 'tdhost', logmech = 'JWT', logdata = 'token=eyJpc...h8dA')
|
|
407
478
|
|
|
408
479
|
# Example 8: Create context using encrypted password and key passed to 'password' parameter.
|
|
409
|
-
#
|
|
410
|
-
#
|
|
411
|
-
#
|
|
412
|
-
#
|
|
413
|
-
#
|
|
414
|
-
#
|
|
415
|
-
#
|
|
416
|
-
#
|
|
480
|
+
# The password should be specified in the format mentioned below:
|
|
481
|
+
# ENCRYPTED_PASSWORD(file:<PasswordEncryptionKeyFileName>, file:<EncryptedPasswordFileName>)
|
|
482
|
+
# The PasswordEncryptionKeyFileName specifies the name of a file that contains the password encryption key
|
|
483
|
+
# and associated information.
|
|
484
|
+
# The EncryptedPasswordFileName specifies the name of a file that contains the encrypted password and
|
|
485
|
+
# associated information.
|
|
486
|
+
# Each filename must be preceded by the 'file:' prefix. The PasswordEncryptionKeyFileName must be separated
|
|
487
|
+
# from the EncryptedPasswordFileName by a single comma.
|
|
417
488
|
>>> encrypted_password = "ENCRYPTED_PASSWORD(file:PassKey.properties, file:EncPass.properties)"
|
|
418
|
-
>>> td_context = create_context(host = 'tdhost', username='tduser', password = encrypted_password)
|
|
489
|
+
>>> td_context = create_context(host = 'tdhost', username = 'tduser', password = encrypted_password)
|
|
419
490
|
|
|
420
491
|
# Example 9: Create context using encrypted password in LDAP logon mechanism.
|
|
421
|
-
>>> td_context = create_context(host = 'tdhost', username='tduser', password = encrypted_password,
|
|
422
|
-
|
|
492
|
+
>>> td_context = create_context(host = 'tdhost', username = 'tduser', password = encrypted_password,
|
|
493
|
+
... logmech = 'LDAP')
|
|
423
494
|
|
|
424
495
|
# Example 10: Create context using hostname, username, password and database parameters, and connect to a
|
|
425
496
|
# different initial database by setting the database parameter.
|
|
426
|
-
>>> td_context = create_context(host = 'tdhost', username='tduser', password = 'tdpassword', database =
|
|
427
|
-
|
|
497
|
+
>>> td_context = create_context(host = 'tdhost', username = 'tduser', password = 'tdpassword', database =
|
|
498
|
+
... 'database_name')
|
|
428
499
|
|
|
429
500
|
# Example 11: Create context using already created sqlalchemy engine, and connect to a different initial
|
|
430
501
|
# database by setting the database parameter.
|
|
431
502
|
>>> from sqlalchemy import create_engine
|
|
432
503
|
>>> sqlalchemy_engine = create_engine('teradatasql://'+ tduser +':' + tdpassword + '@'+tdhost +
|
|
433
|
-
|
|
504
|
+
... '/?DATABASE=database_name')
|
|
434
505
|
>>> td_context = create_context(tdsqlengine = sqlalchemy_engine)
|
|
435
506
|
|
|
436
507
|
# Example 12: Create context for Vantage with logmech as 'LDAP', and connect to a different initial
|
|
437
508
|
# database by setting the database parameter.
|
|
438
|
-
>>> td_context = create_context(host = 'tdhost', username='tduser', password = 'tdpassword', logmech='LDAP',
|
|
439
|
-
|
|
509
|
+
>>> td_context = create_context(host = 'tdhost', username = 'tduser', password = 'tdpassword', logmech = 'LDAP',
|
|
510
|
+
... database = 'database_name')
|
|
440
511
|
|
|
441
512
|
# Example 13: Create context using 'tera' mode with log value set to 8 and lob_support disabled.
|
|
442
|
-
>>> td_context = create_context(host = 'tdhost', username='tduser', password = 'tdpassword', tmode = 'tera',
|
|
443
|
-
|
|
513
|
+
>>> td_context = create_context(host = 'tdhost', username = 'tduser', password = 'tdpassword', tmode = 'tera',
|
|
514
|
+
... log = 8, lob_support = False)
|
|
444
515
|
|
|
445
516
|
# Example 14: Create context from config file with name 'td_properties.cfg'
|
|
446
517
|
# available under current working directory.
|
|
@@ -464,7 +535,7 @@ def create_context(host = None, username = None, password = None, tdsqlengine =
|
|
|
464
535
|
# logmech=tdlogmech
|
|
465
536
|
# logdata=tdlogdata
|
|
466
537
|
# database=tddatabase
|
|
467
|
-
>>> td_context = create_context(config_file="user_td_properties.cfg")
|
|
538
|
+
>>> td_context = create_context(config_file = "user_td_properties.cfg")
|
|
468
539
|
|
|
469
540
|
# Example 16: Create context using environment variables.
|
|
470
541
|
# Set these using os.environ and then run the example:
|
|
@@ -476,6 +547,66 @@ def create_context(host = None, username = None, password = None, tdsqlengine =
|
|
|
476
547
|
# os.environ['TD_LOGDATA'] = 'tdlogdata'
|
|
477
548
|
# os.environ['TD_DATABASE'] = 'tddatabase'
|
|
478
549
|
>>> td_context = create_context()
|
|
550
|
+
|
|
551
|
+
# Example 17: Create a context by providing username and password. Along with it,
|
|
552
|
+
# set authentication token by providing the pem file and pat token.
|
|
553
|
+
>>> td_context = create_context(host = 'tdhost', username = 'tduser', password = 'tdpassword',
|
|
554
|
+
... base_url = 'base_url', pat_token = 'pat_token', pem_file = 'pem_file')
|
|
555
|
+
|
|
556
|
+
# Example 18: Create a context by providing username and password. Along with it,
|
|
557
|
+
# generate authentication token by providing the client id.
|
|
558
|
+
>>> td_context = create_context(host = 'tdhost', username = 'tduser', password = 'tdpassword',
|
|
559
|
+
... base_url = 'base_url', client_id = 'client_id')
|
|
560
|
+
|
|
561
|
+
# Example 19: Create context and set authentication token by providing all the details in a config file.
|
|
562
|
+
# td_properties.cfg content:
|
|
563
|
+
# host=tdhost
|
|
564
|
+
# username=tduser
|
|
565
|
+
# password=tdpassword
|
|
566
|
+
# base_url=base_url
|
|
567
|
+
# pat_token=pat_token
|
|
568
|
+
# pem_file=pem_file
|
|
569
|
+
>>> td_context = create_context()
|
|
570
|
+
|
|
571
|
+
# Example 20: Create context and set authentication token by providing all the details in environment variables.
|
|
572
|
+
# Set these using os.environ and then run the example:
|
|
573
|
+
# os.environ['TD_HOST'] = 'tdhost'
|
|
574
|
+
# os.environ['TD_USERNAME'] = 'tduser'
|
|
575
|
+
# os.environ['TD_PASSWORD'] = 'tdpassword'
|
|
576
|
+
# os.environ['TD_BASE_URL'] = 'base_url'
|
|
577
|
+
# os.environ['TD_PAT_TOKEN'] = 'pat_token'
|
|
578
|
+
# os.environ['TD_PEM_FILE'] = 'pem_file'
|
|
579
|
+
>>> td_context = create_context()
|
|
580
|
+
|
|
581
|
+
# Example 21: Create context with sql_timeout set to 3.
|
|
582
|
+
>>> td_context = create_context(host = 'tdhost', username = 'tduser', password = 'tdpassword', sql_timeout = 3)
|
|
583
|
+
|
|
584
|
+
# Example 22: Create context and set authentication token via Basic authentication mechanism
|
|
585
|
+
# using username and password.
|
|
586
|
+
>>> import getpass
|
|
587
|
+
>>> create_context(host="host",
|
|
588
|
+
... username=getpass.getpass("username : "),
|
|
589
|
+
... password=getpass.getpass("password : "),
|
|
590
|
+
... base_url=getpass.getpass("base_url : "),
|
|
591
|
+
... auth_mech="BASIC")
|
|
592
|
+
|
|
593
|
+
# Example 23: Create context and set authentication token by providing auth_mech argument.
|
|
594
|
+
>>> import getpass
|
|
595
|
+
>>> create_context(host="vcl_host",
|
|
596
|
+
... username=getpass.getpass("username : "),
|
|
597
|
+
... password=getpass.getpass("password : "),
|
|
598
|
+
... base_url=getpass.getpass("base_url : "),
|
|
599
|
+
... auth_mech="OAuth")
|
|
600
|
+
|
|
601
|
+
# Example 24: Create context and set authentication token by providing auth_url and
|
|
602
|
+
# rest_client arguments.
|
|
603
|
+
>>> import getpass
|
|
604
|
+
>>> create_context(host="host",
|
|
605
|
+
... username=getpass.getpass("username : "),
|
|
606
|
+
... password=getpass.getpass("password : "),
|
|
607
|
+
... base_url=getpass.getpass("base_url : "),
|
|
608
|
+
... auth_url=getpass.getpass("auth_url : "),
|
|
609
|
+
... rest_client=getpass.getpass("rest_client : "))
|
|
479
610
|
"""
|
|
480
611
|
global td_connection
|
|
481
612
|
global td_sqlalchemy_engine
|
|
@@ -489,7 +620,7 @@ def create_context(host = None, username = None, password = None, tdsqlengine =
|
|
|
489
620
|
# Check if the user has provided the connection parameters or tdsqlengine.
|
|
490
621
|
# If not, check if the user has provided the connection parameters in the environment variables.
|
|
491
622
|
# If not, check if the user has provided the connection parameters in the config file.
|
|
492
|
-
if not (host or tdsqlengine) and host!="":
|
|
623
|
+
if not (host or tdsqlengine) and host != "":
|
|
493
624
|
return _load_context_from_env_config(kwargs.pop('config_file', 'td_properties.cfg'))
|
|
494
625
|
|
|
495
626
|
awu_matrix = []
|
|
@@ -500,13 +631,24 @@ def create_context(host = None, username = None, password = None, tdsqlengine =
|
|
|
500
631
|
awu_matrix.append(["logmech", logmech, True, (str), True])
|
|
501
632
|
awu_matrix.append(["logdata", logdata, True, (str), True])
|
|
502
633
|
awu_matrix.append(["database", database, True, (str), True])
|
|
634
|
+
# set_auth_token parameters
|
|
635
|
+
_set_auth_token_params = {}
|
|
636
|
+
auth_mech = kwargs.get('auth_mech', None)
|
|
637
|
+
for param in ['base_url', 'pat_token', 'pem_file', 'client_id', 'auth_token', 'expiration_time',
|
|
638
|
+
'kid', 'auth_mech', 'auth_url', 'rest_client']:
|
|
639
|
+
if kwargs.get(param):
|
|
640
|
+
_set_auth_token_params[param] = kwargs.pop(param)
|
|
641
|
+
|
|
642
|
+
# Set the "sql_timeout" parameter to "request_timeout" which is consumed by teradatasql.
|
|
643
|
+
if kwargs.get('sql_timeout'):
|
|
644
|
+
awu_matrix.append(["sql_timeout", kwargs.get('sql_timeout'), True, (int), True])
|
|
645
|
+
kwargs['request_timeout'] = kwargs.pop('sql_timeout')
|
|
503
646
|
|
|
504
647
|
awu = _Validators()
|
|
505
648
|
awu._validate_function_arguments(awu_matrix)
|
|
506
649
|
|
|
507
650
|
# Clearing the internal buffer.
|
|
508
651
|
_InternalBuffer.clean()
|
|
509
|
-
|
|
510
652
|
if logmech == "JWT" and not logdata:
|
|
511
653
|
raise TeradataMlException(Messages.get_message(MessageCodes.DEPENDENT_ARG_MISSING,
|
|
512
654
|
'logdata',
|
|
@@ -541,7 +683,6 @@ def create_context(host = None, username = None, password = None, tdsqlengine =
|
|
|
541
683
|
host_value = '{}:@{}'.format(username, host)
|
|
542
684
|
else:
|
|
543
685
|
host_value = '{}:{}@{}'.format(username, password, host)
|
|
544
|
-
|
|
545
686
|
url_object = URL.create(
|
|
546
687
|
"teradatasql",
|
|
547
688
|
username=username,
|
|
@@ -555,7 +696,7 @@ def create_context(host = None, username = None, password = None, tdsqlengine =
|
|
|
555
696
|
td_connection = td_sqlalchemy_engine.connect()
|
|
556
697
|
td_user = username.upper()
|
|
557
698
|
|
|
558
|
-
# Masking
|
|
699
|
+
# Masking sensitive information - password, logmech and logdata.
|
|
559
700
|
if password:
|
|
560
701
|
try:
|
|
561
702
|
# Below statement raises an AttributeError with SQLAlchemy
|
|
@@ -578,9 +719,6 @@ def create_context(host = None, username = None, password = None, tdsqlengine =
|
|
|
578
719
|
raise TeradataMlException(Messages.get_message(MessageCodes.CONNECTION_FAILURE),
|
|
579
720
|
MessageCodes.CONNECTION_FAILURE) from err
|
|
580
721
|
|
|
581
|
-
# Load function aliases from config.
|
|
582
|
-
_load_function_aliases()
|
|
583
|
-
|
|
584
722
|
python_packages_installed = False
|
|
585
723
|
python_version_vantage = None
|
|
586
724
|
python_version_local = sys.version.split(" ")[0].strip()
|
|
@@ -607,21 +745,42 @@ def create_context(host = None, username = None, password = None, tdsqlengine =
|
|
|
607
745
|
# Process Analytic functions.
|
|
608
746
|
from teradataml.analytics import _process_analytic_functions
|
|
609
747
|
_process_analytic_functions()
|
|
610
|
-
|
|
748
|
+
|
|
749
|
+
if _set_auth_token_params.get('base_url'):
|
|
750
|
+
from teradataml.scriptmgmt.lls_utils import set_auth_token
|
|
751
|
+
try:
|
|
752
|
+
# password needs to be passed to set_auth_token only when any of the following is True:
|
|
753
|
+
# 1. auth_mech is set to either 'basic' or 'keycloak'
|
|
754
|
+
# 2. 'auth_url' argument is passed which represents 'keycloak' authentication mechanism.
|
|
755
|
+
if ((auth_mech and auth_mech.lower() in ['basic', 'keycloak']) or
|
|
756
|
+
_set_auth_token_params.get('auth_url'))\
|
|
757
|
+
and password:
|
|
758
|
+
_set_auth_token_params['password'] = password
|
|
759
|
+
set_auth_token(**_set_auth_token_params)
|
|
760
|
+
except Exception as err:
|
|
761
|
+
print("Connection to Vantage established successfully.")
|
|
762
|
+
mssg = f"Failed to set authentication token. Rerun \"set_auth_token()\" again to set the authentication token." \
|
|
763
|
+
f" Reason for failure: {err.args[0]}"
|
|
764
|
+
warnings.warn(mssg, stacklevel=2)
|
|
765
|
+
|
|
611
766
|
# Add global lock to internal buffer
|
|
612
767
|
_InternalBuffer.add(global_lock=threading.Lock())
|
|
613
768
|
|
|
769
|
+
# Set _check_py_version to True to check the python version between local and Vantage.
|
|
770
|
+
_InternalBuffer.add(_check_py_version=True)
|
|
771
|
+
|
|
614
772
|
# Return the connection by default
|
|
615
773
|
return td_sqlalchemy_engine
|
|
616
774
|
|
|
617
|
-
|
|
775
|
+
|
|
776
|
+
def _load_context_from_env_config(config_file=None):
|
|
618
777
|
"""
|
|
619
778
|
DESCRIPTION:
|
|
620
779
|
Reads the connection parameters from the configuration file or environment variables.
|
|
621
780
|
|
|
622
781
|
PARAMETERS:
|
|
623
782
|
config_file:
|
|
624
|
-
|
|
783
|
+
Optional Argument.
|
|
625
784
|
Specifies the name of the configuration file to read the connection parameters.
|
|
626
785
|
Types: str
|
|
627
786
|
|
|
@@ -632,16 +791,17 @@ def _load_context_from_env_config(config_file):
|
|
|
632
791
|
TeradataMlException
|
|
633
792
|
"""
|
|
634
793
|
host = os.environ.get('TD_HOST')
|
|
635
|
-
connection_params_from_file = dotenv_values(config_file)
|
|
636
794
|
if host:
|
|
637
795
|
connection_params_from_env = {key[3:].lower(): value for key, value in os.environ.items()
|
|
638
796
|
if key.startswith('TD_')}
|
|
639
797
|
return create_context(**connection_params_from_env)
|
|
640
|
-
elif
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
798
|
+
elif config_file is not None:
|
|
799
|
+
connection_params_from_file = dotenv_values(config_file)
|
|
800
|
+
if connection_params_from_file.get('host'):
|
|
801
|
+
return create_context(**connection_params_from_file)
|
|
802
|
+
raise TeradataMlException(Messages.get_message(MessageCodes.CONNECTION_PARAMS),
|
|
803
|
+
MessageCodes.MISSING_ARGS)
|
|
804
|
+
|
|
645
805
|
|
|
646
806
|
def _mask_logmech_logdata():
|
|
647
807
|
"""
|
|
@@ -767,19 +927,19 @@ def set_context(tdsqlengine, temp_database_name=None):
|
|
|
767
927
|
else:
|
|
768
928
|
return None
|
|
769
929
|
|
|
770
|
-
# Load function aliases from config.
|
|
771
|
-
_load_function_aliases()
|
|
772
|
-
|
|
773
930
|
python_packages_installed = False
|
|
774
931
|
python_version_vantage = None
|
|
775
932
|
python_version_local = sys.version.split(" ")[0].strip()
|
|
776
933
|
|
|
777
934
|
# Initialise Dag
|
|
778
935
|
__initalise_dag()
|
|
779
|
-
|
|
936
|
+
|
|
780
937
|
# Add global lock to internal buffer
|
|
781
938
|
_InternalBuffer.add(global_lock=threading.Lock())
|
|
782
939
|
|
|
940
|
+
# Set _check_py_version to True to check the python version between local and Vantage.
|
|
941
|
+
_InternalBuffer.add(_check_py_version=True)
|
|
942
|
+
|
|
783
943
|
return td_connection
|
|
784
944
|
|
|
785
945
|
|
|
@@ -813,6 +973,9 @@ def remove_context():
|
|
|
813
973
|
# Initiate the garbage collection
|
|
814
974
|
__cleanup_garbage_collection()
|
|
815
975
|
|
|
976
|
+
# Clearing the internal buffer.
|
|
977
|
+
_InternalBuffer.clean()
|
|
978
|
+
|
|
816
979
|
# Check if connection is established or not.
|
|
817
980
|
if user_specified_connection is not True:
|
|
818
981
|
try:
|
|
@@ -1117,6 +1280,7 @@ def _get_user():
|
|
|
1117
1280
|
td_user = _get_database_username()
|
|
1118
1281
|
return td_user
|
|
1119
1282
|
|
|
1283
|
+
|
|
1120
1284
|
def _get_host():
|
|
1121
1285
|
"""
|
|
1122
1286
|
DESCRIPTION:
|
|
@@ -1139,6 +1303,7 @@ def _get_host():
|
|
|
1139
1303
|
else:
|
|
1140
1304
|
return td_sqlalchemy_engine.url.host
|
|
1141
1305
|
|
|
1306
|
+
|
|
1142
1307
|
def _get_host_ip():
|
|
1143
1308
|
"""
|
|
1144
1309
|
DESCRIPTION:
|
|
@@ -1159,7 +1324,7 @@ def _get_host_ip():
|
|
|
1159
1324
|
# Return None if connection is not established.
|
|
1160
1325
|
if td_connection is None:
|
|
1161
1326
|
return None
|
|
1162
|
-
|
|
1327
|
+
|
|
1163
1328
|
host = _get_host()
|
|
1164
1329
|
try:
|
|
1165
1330
|
# Validate if host_ip is a valid IP address (IPv4 or IPv6)
|
|
@@ -1170,7 +1335,7 @@ def _get_host_ip():
|
|
|
1170
1335
|
dns_host_ip = _InternalBuffer.get('dns_host_ip')
|
|
1171
1336
|
if dns_host_ip:
|
|
1172
1337
|
return dns_host_ip
|
|
1173
|
-
|
|
1338
|
+
|
|
1174
1339
|
# If DNS host ip not found, resolve the host name to get the IP address.
|
|
1175
1340
|
# If there is issue in resolving the host name, it will proceed with DNS host as it is.
|
|
1176
1341
|
try:
|
|
@@ -1185,6 +1350,7 @@ def _get_host_ip():
|
|
|
1185
1350
|
host_ip = host
|
|
1186
1351
|
return host_ip
|
|
1187
1352
|
|
|
1353
|
+
|
|
1188
1354
|
class ContextUtilFuncs():
|
|
1189
1355
|
@staticmethod
|
|
1190
1356
|
def _check_alias_config_file_exists(vantage_version, alias_config_file):
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
{
|
|
2
|
+
"trans_dense": {
|
|
3
|
+
"location": "varchar(100)",
|
|
4
|
+
"tranid": "int",
|
|
5
|
+
"period": "varchar(20)",
|
|
6
|
+
"storeid": "int",
|
|
7
|
+
"region": "varchar(20)",
|
|
8
|
+
"item": "varchar(200)",
|
|
9
|
+
"sku": "int",
|
|
10
|
+
"category": "varchar(20)"
|
|
11
|
+
},
|
|
12
|
+
"trans_sparse": {
|
|
13
|
+
"location": "varchar(20)",
|
|
14
|
+
"tranid": "int",
|
|
15
|
+
"period": "varchar(20)",
|
|
16
|
+
"storeid": "int",
|
|
17
|
+
"region": "varchar(20)",
|
|
18
|
+
"item": "varchar(20)",
|
|
19
|
+
"sku": "int",
|
|
20
|
+
"category": "varchar(20)"
|
|
21
|
+
}
|
|
22
|
+
}
|