teradataml 17.20.0.7__py3-none-any.whl → 20.0.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of teradataml might be problematic. Click here for more details.

Files changed (1285) hide show
  1. teradataml/LICENSE-3RD-PARTY.pdf +0 -0
  2. teradataml/LICENSE.pdf +0 -0
  3. teradataml/README.md +1864 -1640
  4. teradataml/__init__.py +70 -60
  5. teradataml/_version.py +11 -11
  6. teradataml/analytics/Transformations.py +2995 -2995
  7. teradataml/analytics/__init__.py +81 -83
  8. teradataml/analytics/analytic_function_executor.py +2013 -2010
  9. teradataml/analytics/analytic_query_generator.py +958 -958
  10. teradataml/analytics/byom/H2OPredict.py +514 -514
  11. teradataml/analytics/byom/PMMLPredict.py +437 -437
  12. teradataml/analytics/byom/__init__.py +14 -14
  13. teradataml/analytics/json_parser/__init__.py +130 -130
  14. teradataml/analytics/json_parser/analytic_functions_argument.py +1707 -1707
  15. teradataml/analytics/json_parser/json_store.py +191 -191
  16. teradataml/analytics/json_parser/metadata.py +1637 -1637
  17. teradataml/analytics/json_parser/utils.py +804 -803
  18. teradataml/analytics/meta_class.py +196 -196
  19. teradataml/analytics/sqle/DecisionTreePredict.py +455 -470
  20. teradataml/analytics/sqle/NaiveBayesPredict.py +419 -428
  21. teradataml/analytics/sqle/__init__.py +97 -110
  22. teradataml/analytics/sqle/json/decisiontreepredict_sqle.json +78 -78
  23. teradataml/analytics/sqle/json/naivebayespredict_sqle.json +62 -62
  24. teradataml/analytics/table_operator/__init__.py +10 -10
  25. teradataml/analytics/uaf/__init__.py +63 -63
  26. teradataml/analytics/utils.py +693 -692
  27. teradataml/analytics/valib.py +1603 -1600
  28. teradataml/automl/__init__.py +1628 -0
  29. teradataml/automl/custom_json_utils.py +1270 -0
  30. teradataml/automl/data_preparation.py +993 -0
  31. teradataml/automl/data_transformation.py +727 -0
  32. teradataml/automl/feature_engineering.py +1648 -0
  33. teradataml/automl/feature_exploration.py +547 -0
  34. teradataml/automl/model_evaluation.py +163 -0
  35. teradataml/automl/model_training.py +887 -0
  36. teradataml/catalog/__init__.py +1 -3
  37. teradataml/catalog/byom.py +1759 -1716
  38. teradataml/catalog/function_argument_mapper.py +859 -861
  39. teradataml/catalog/model_cataloging_utils.py +491 -1510
  40. teradataml/clients/pkce_client.py +481 -481
  41. teradataml/common/aed_utils.py +6 -2
  42. teradataml/common/bulk_exposed_utils.py +111 -111
  43. teradataml/common/constants.py +1433 -1441
  44. teradataml/common/deprecations.py +160 -0
  45. teradataml/common/exceptions.py +73 -73
  46. teradataml/common/formula.py +742 -742
  47. teradataml/common/garbagecollector.py +592 -635
  48. teradataml/common/messagecodes.py +422 -431
  49. teradataml/common/messages.py +227 -231
  50. teradataml/common/sqlbundle.py +693 -693
  51. teradataml/common/td_coltype_code_to_tdtype.py +48 -48
  52. teradataml/common/utils.py +2418 -2500
  53. teradataml/common/warnings.py +25 -25
  54. teradataml/common/wrapper_utils.py +1 -110
  55. teradataml/config/dummy_file1.cfg +4 -4
  56. teradataml/config/dummy_file2.cfg +2 -2
  57. teradataml/config/sqlengine_alias_definitions_v1.0 +13 -13
  58. teradataml/config/sqlengine_alias_definitions_v1.1 +19 -19
  59. teradataml/config/sqlengine_alias_definitions_v1.3 +18 -18
  60. teradataml/context/aed_context.py +217 -217
  61. teradataml/context/context.py +1071 -999
  62. teradataml/data/A_loan.csv +19 -19
  63. teradataml/data/BINARY_REALS_LEFT.csv +11 -11
  64. teradataml/data/BINARY_REALS_RIGHT.csv +11 -11
  65. teradataml/data/B_loan.csv +49 -49
  66. teradataml/data/BuoyData2.csv +17 -17
  67. teradataml/data/CONVOLVE2_COMPLEX_LEFT.csv +5 -5
  68. teradataml/data/CONVOLVE2_COMPLEX_RIGHT.csv +5 -5
  69. teradataml/data/Convolve2RealsLeft.csv +5 -5
  70. teradataml/data/Convolve2RealsRight.csv +5 -5
  71. teradataml/data/Convolve2ValidLeft.csv +11 -11
  72. teradataml/data/Convolve2ValidRight.csv +11 -11
  73. teradataml/data/DFFTConv_Real_8_8.csv +65 -65
  74. teradataml/data/Orders1_12mf.csv +24 -24
  75. teradataml/data/Pi_loan.csv +7 -7
  76. teradataml/data/SMOOTHED_DATA.csv +7 -7
  77. teradataml/data/TestDFFT8.csv +9 -9
  78. teradataml/data/TestRiver.csv +109 -109
  79. teradataml/data/Traindata.csv +28 -28
  80. teradataml/data/acf.csv +17 -17
  81. teradataml/data/adaboost_example.json +34 -34
  82. teradataml/data/adaboostpredict_example.json +24 -24
  83. teradataml/data/additional_table.csv +10 -10
  84. teradataml/data/admissions_test.csv +21 -21
  85. teradataml/data/admissions_train.csv +41 -41
  86. teradataml/data/admissions_train_nulls.csv +41 -41
  87. teradataml/data/ageandheight.csv +13 -13
  88. teradataml/data/ageandpressure.csv +31 -31
  89. teradataml/data/antiselect_example.json +36 -36
  90. teradataml/data/antiselect_input.csv +8 -8
  91. teradataml/data/antiselect_input_mixed_case.csv +8 -8
  92. teradataml/data/applicant_external.csv +6 -6
  93. teradataml/data/applicant_reference.csv +6 -6
  94. teradataml/data/arima_example.json +9 -9
  95. teradataml/data/assortedtext_input.csv +8 -8
  96. teradataml/data/attribution_example.json +33 -33
  97. teradataml/data/attribution_sample_table.csv +27 -27
  98. teradataml/data/attribution_sample_table1.csv +6 -6
  99. teradataml/data/attribution_sample_table2.csv +11 -11
  100. teradataml/data/bank_churn.csv +10001 -0
  101. teradataml/data/bank_web_clicks1.csv +42 -42
  102. teradataml/data/bank_web_clicks2.csv +91 -91
  103. teradataml/data/bank_web_url.csv +85 -85
  104. teradataml/data/barrier.csv +2 -2
  105. teradataml/data/barrier_new.csv +3 -3
  106. teradataml/data/betweenness_example.json +13 -13
  107. teradataml/data/bin_breaks.csv +8 -8
  108. teradataml/data/bin_fit_ip.csv +3 -3
  109. teradataml/data/binary_complex_left.csv +11 -11
  110. teradataml/data/binary_complex_right.csv +11 -11
  111. teradataml/data/binary_matrix_complex_left.csv +21 -21
  112. teradataml/data/binary_matrix_complex_right.csv +21 -21
  113. teradataml/data/binary_matrix_real_left.csv +21 -21
  114. teradataml/data/binary_matrix_real_right.csv +21 -21
  115. teradataml/data/blood2ageandweight.csv +26 -26
  116. teradataml/data/bmi.csv +501 -0
  117. teradataml/data/boston.csv +507 -507
  118. teradataml/data/buoydata_mix.csv +11 -11
  119. teradataml/data/burst_data.csv +5 -5
  120. teradataml/data/burst_example.json +20 -20
  121. teradataml/data/byom_example.json +17 -17
  122. teradataml/data/bytes_table.csv +3 -3
  123. teradataml/data/cal_housing_ex_raw.csv +70 -70
  124. teradataml/data/callers.csv +7 -7
  125. teradataml/data/calls.csv +10 -10
  126. teradataml/data/cars_hist.csv +33 -33
  127. teradataml/data/cat_table.csv +24 -24
  128. teradataml/data/ccm_example.json +31 -31
  129. teradataml/data/ccm_input.csv +91 -91
  130. teradataml/data/ccm_input2.csv +13 -13
  131. teradataml/data/ccmexample.csv +101 -101
  132. teradataml/data/ccmprepare_example.json +8 -8
  133. teradataml/data/ccmprepare_input.csv +91 -91
  134. teradataml/data/cfilter_example.json +12 -12
  135. teradataml/data/changepointdetection_example.json +18 -18
  136. teradataml/data/changepointdetectionrt_example.json +8 -8
  137. teradataml/data/chi_sq.csv +2 -2
  138. teradataml/data/churn_data.csv +14 -14
  139. teradataml/data/churn_emission.csv +35 -35
  140. teradataml/data/churn_initial.csv +3 -3
  141. teradataml/data/churn_state_transition.csv +5 -5
  142. teradataml/data/citedges_2.csv +745 -745
  143. teradataml/data/citvertices_2.csv +1210 -1210
  144. teradataml/data/clicks2.csv +16 -16
  145. teradataml/data/clickstream.csv +12 -12
  146. teradataml/data/clickstream1.csv +11 -11
  147. teradataml/data/closeness_example.json +15 -15
  148. teradataml/data/complaints.csv +21 -21
  149. teradataml/data/complaints_mini.csv +3 -3
  150. teradataml/data/complaints_testtoken.csv +224 -224
  151. teradataml/data/complaints_tokens_test.csv +353 -353
  152. teradataml/data/complaints_traintoken.csv +472 -472
  153. teradataml/data/computers_category.csv +1001 -1001
  154. teradataml/data/computers_test1.csv +1252 -1252
  155. teradataml/data/computers_train1.csv +5009 -5009
  156. teradataml/data/computers_train1_clustered.csv +5009 -5009
  157. teradataml/data/confusionmatrix_example.json +9 -9
  158. teradataml/data/conversion_event_table.csv +3 -3
  159. teradataml/data/corr_input.csv +17 -17
  160. teradataml/data/correlation_example.json +11 -11
  161. teradataml/data/coxhazardratio_example.json +39 -39
  162. teradataml/data/coxph_example.json +15 -15
  163. teradataml/data/coxsurvival_example.json +28 -28
  164. teradataml/data/cpt.csv +41 -41
  165. teradataml/data/credit_ex_merged.csv +45 -45
  166. teradataml/data/customer_loyalty.csv +301 -301
  167. teradataml/data/customer_loyalty_newseq.csv +31 -31
  168. teradataml/data/dataframe_example.json +146 -146
  169. teradataml/data/decisionforest_example.json +37 -37
  170. teradataml/data/decisionforestpredict_example.json +38 -38
  171. teradataml/data/decisiontree_example.json +21 -21
  172. teradataml/data/decisiontreepredict_example.json +45 -45
  173. teradataml/data/dfft2_size4_real.csv +17 -17
  174. teradataml/data/dfft2_test_matrix16.csv +17 -17
  175. teradataml/data/dfft2conv_real_4_4.csv +65 -65
  176. teradataml/data/diabetes.csv +443 -443
  177. teradataml/data/diabetes_test.csv +89 -89
  178. teradataml/data/dict_table.csv +5 -5
  179. teradataml/data/docperterm_table.csv +4 -4
  180. teradataml/data/docs/__init__.py +1 -1
  181. teradataml/data/docs/byom/docs/DataRobotPredict.py +180 -180
  182. teradataml/data/docs/byom/docs/DataikuPredict.py +177 -177
  183. teradataml/data/docs/byom/docs/H2OPredict.py +324 -324
  184. teradataml/data/docs/byom/docs/ONNXPredict.py +283 -283
  185. teradataml/data/docs/byom/docs/PMMLPredict.py +277 -277
  186. teradataml/data/docs/sqle/docs_17_10/Antiselect.py +82 -82
  187. teradataml/data/docs/sqle/docs_17_10/Attribution.py +199 -199
  188. teradataml/data/docs/sqle/docs_17_10/BincodeFit.py +171 -171
  189. teradataml/data/docs/sqle/docs_17_10/BincodeTransform.py +131 -130
  190. teradataml/data/docs/sqle/docs_17_10/CategoricalSummary.py +86 -86
  191. teradataml/data/docs/sqle/docs_17_10/ChiSq.py +90 -90
  192. teradataml/data/docs/sqle/docs_17_10/ColumnSummary.py +85 -85
  193. teradataml/data/docs/sqle/docs_17_10/ConvertTo.py +95 -95
  194. teradataml/data/docs/sqle/docs_17_10/DecisionForestPredict.py +139 -139
  195. teradataml/data/docs/sqle/docs_17_10/DecisionTreePredict.py +151 -151
  196. teradataml/data/docs/sqle/docs_17_10/FTest.py +160 -160
  197. teradataml/data/docs/sqle/docs_17_10/FillRowId.py +82 -82
  198. teradataml/data/docs/sqle/docs_17_10/Fit.py +87 -87
  199. teradataml/data/docs/sqle/docs_17_10/GLMPredict.py +144 -144
  200. teradataml/data/docs/sqle/docs_17_10/GetRowsWithMissingValues.py +84 -84
  201. teradataml/data/docs/sqle/docs_17_10/GetRowsWithoutMissingValues.py +81 -81
  202. teradataml/data/docs/sqle/docs_17_10/Histogram.py +164 -164
  203. teradataml/data/docs/sqle/docs_17_10/MovingAverage.py +134 -134
  204. teradataml/data/docs/sqle/docs_17_10/NGramSplitter.py +208 -208
  205. teradataml/data/docs/sqle/docs_17_10/NPath.py +265 -265
  206. teradataml/data/docs/sqle/docs_17_10/NaiveBayesPredict.py +116 -116
  207. teradataml/data/docs/sqle/docs_17_10/NaiveBayesTextClassifierPredict.py +176 -176
  208. teradataml/data/docs/sqle/docs_17_10/NumApply.py +147 -147
  209. teradataml/data/docs/sqle/docs_17_10/OneHotEncodingFit.py +132 -132
  210. teradataml/data/docs/sqle/docs_17_10/OneHotEncodingTransform.py +103 -103
  211. teradataml/data/docs/sqle/docs_17_10/OutlierFilterFit.py +165 -165
  212. teradataml/data/docs/sqle/docs_17_10/OutlierFilterTransform.py +101 -101
  213. teradataml/data/docs/sqle/docs_17_10/Pack.py +128 -128
  214. teradataml/data/docs/sqle/docs_17_10/PolynomialFeaturesFit.py +111 -111
  215. teradataml/data/docs/sqle/docs_17_10/PolynomialFeaturesTransform.py +102 -102
  216. teradataml/data/docs/sqle/docs_17_10/QQNorm.py +104 -104
  217. teradataml/data/docs/sqle/docs_17_10/RoundColumns.py +109 -109
  218. teradataml/data/docs/sqle/docs_17_10/RowNormalizeFit.py +117 -117
  219. teradataml/data/docs/sqle/docs_17_10/RowNormalizeTransform.py +99 -98
  220. teradataml/data/docs/sqle/docs_17_10/SVMSparsePredict.py +152 -152
  221. teradataml/data/docs/sqle/docs_17_10/ScaleFit.py +197 -197
  222. teradataml/data/docs/sqle/docs_17_10/ScaleTransform.py +99 -98
  223. teradataml/data/docs/sqle/docs_17_10/Sessionize.py +113 -113
  224. teradataml/data/docs/sqle/docs_17_10/SimpleImputeFit.py +116 -116
  225. teradataml/data/docs/sqle/docs_17_10/SimpleImputeTransform.py +98 -98
  226. teradataml/data/docs/sqle/docs_17_10/StrApply.py +187 -187
  227. teradataml/data/docs/sqle/docs_17_10/StringSimilarity.py +145 -145
  228. teradataml/data/docs/sqle/docs_17_10/Transform.py +105 -104
  229. teradataml/data/docs/sqle/docs_17_10/UnivariateStatistics.py +141 -141
  230. teradataml/data/docs/sqle/docs_17_10/Unpack.py +214 -214
  231. teradataml/data/docs/sqle/docs_17_10/WhichMax.py +83 -83
  232. teradataml/data/docs/sqle/docs_17_10/WhichMin.py +83 -83
  233. teradataml/data/docs/sqle/docs_17_10/ZTest.py +155 -155
  234. teradataml/data/docs/sqle/docs_17_20/ANOVA.py +126 -126
  235. teradataml/data/docs/sqle/docs_17_20/Antiselect.py +82 -82
  236. teradataml/data/docs/sqle/docs_17_20/Attribution.py +200 -200
  237. teradataml/data/docs/sqle/docs_17_20/BincodeFit.py +171 -171
  238. teradataml/data/docs/sqle/docs_17_20/BincodeTransform.py +139 -138
  239. teradataml/data/docs/sqle/docs_17_20/CategoricalSummary.py +86 -86
  240. teradataml/data/docs/sqle/docs_17_20/ChiSq.py +90 -90
  241. teradataml/data/docs/sqle/docs_17_20/ClassificationEvaluator.py +166 -166
  242. teradataml/data/docs/sqle/docs_17_20/ColumnSummary.py +85 -85
  243. teradataml/data/docs/sqle/docs_17_20/ColumnTransformer.py +243 -243
  244. teradataml/data/docs/sqle/docs_17_20/ConvertTo.py +113 -113
  245. teradataml/data/docs/sqle/docs_17_20/DecisionForest.py +279 -279
  246. teradataml/data/docs/sqle/docs_17_20/DecisionForestPredict.py +144 -144
  247. teradataml/data/docs/sqle/docs_17_20/DecisionTreePredict.py +135 -135
  248. teradataml/data/docs/sqle/docs_17_20/FTest.py +160 -160
  249. teradataml/data/docs/sqle/docs_17_20/FillRowId.py +82 -82
  250. teradataml/data/docs/sqle/docs_17_20/Fit.py +87 -87
  251. teradataml/data/docs/sqle/docs_17_20/GLM.py +380 -380
  252. teradataml/data/docs/sqle/docs_17_20/GLMPerSegment.py +414 -414
  253. teradataml/data/docs/sqle/docs_17_20/GLMPredict.py +144 -144
  254. teradataml/data/docs/sqle/docs_17_20/GLMPredictPerSegment.py +233 -234
  255. teradataml/data/docs/sqle/docs_17_20/GetFutileColumns.py +123 -123
  256. teradataml/data/docs/sqle/docs_17_20/GetRowsWithMissingValues.py +108 -108
  257. teradataml/data/docs/sqle/docs_17_20/GetRowsWithoutMissingValues.py +105 -105
  258. teradataml/data/docs/sqle/docs_17_20/Histogram.py +223 -223
  259. teradataml/data/docs/sqle/docs_17_20/KMeans.py +204 -204
  260. teradataml/data/docs/sqle/docs_17_20/KMeansPredict.py +144 -143
  261. teradataml/data/docs/sqle/docs_17_20/KNN.py +214 -214
  262. teradataml/data/docs/sqle/docs_17_20/MovingAverage.py +134 -134
  263. teradataml/data/docs/sqle/docs_17_20/NGramSplitter.py +208 -208
  264. teradataml/data/docs/sqle/docs_17_20/NPath.py +265 -265
  265. teradataml/data/docs/sqle/docs_17_20/NaiveBayesPredict.py +116 -116
  266. teradataml/data/docs/sqle/docs_17_20/NaiveBayesTextClassifierPredict.py +177 -176
  267. teradataml/data/docs/sqle/docs_17_20/NaiveBayesTextClassifierTrainer.py +126 -126
  268. teradataml/data/docs/sqle/docs_17_20/NonLinearCombineFit.py +117 -117
  269. teradataml/data/docs/sqle/docs_17_20/NonLinearCombineTransform.py +112 -112
  270. teradataml/data/docs/sqle/docs_17_20/NumApply.py +147 -147
  271. teradataml/data/docs/sqle/docs_17_20/OneClassSVM.py +307 -307
  272. teradataml/data/docs/sqle/docs_17_20/OneClassSVMPredict.py +185 -184
  273. teradataml/data/docs/sqle/docs_17_20/OneHotEncodingFit.py +225 -225
  274. teradataml/data/docs/sqle/docs_17_20/OneHotEncodingTransform.py +115 -115
  275. teradataml/data/docs/sqle/docs_17_20/OrdinalEncodingFit.py +219 -219
  276. teradataml/data/docs/sqle/docs_17_20/OrdinalEncodingTransform.py +127 -127
  277. teradataml/data/docs/sqle/docs_17_20/OutlierFilterFit.py +189 -189
  278. teradataml/data/docs/sqle/docs_17_20/OutlierFilterTransform.py +117 -112
  279. teradataml/data/docs/sqle/docs_17_20/Pack.py +128 -128
  280. teradataml/data/docs/sqle/docs_17_20/PolynomialFeaturesFit.py +111 -111
  281. teradataml/data/docs/sqle/docs_17_20/PolynomialFeaturesTransform.py +112 -111
  282. teradataml/data/docs/sqle/docs_17_20/QQNorm.py +104 -104
  283. teradataml/data/docs/sqle/docs_17_20/ROC.py +163 -163
  284. teradataml/data/docs/sqle/docs_17_20/RandomProjectionFit.py +154 -154
  285. teradataml/data/docs/sqle/docs_17_20/RandomProjectionMinComponents.py +106 -106
  286. teradataml/data/docs/sqle/docs_17_20/RandomProjectionTransform.py +120 -120
  287. teradataml/data/docs/sqle/docs_17_20/RegressionEvaluator.py +211 -211
  288. teradataml/data/docs/sqle/docs_17_20/RoundColumns.py +108 -108
  289. teradataml/data/docs/sqle/docs_17_20/RowNormalizeFit.py +117 -117
  290. teradataml/data/docs/sqle/docs_17_20/RowNormalizeTransform.py +111 -110
  291. teradataml/data/docs/sqle/docs_17_20/SVM.py +413 -413
  292. teradataml/data/docs/sqle/docs_17_20/SVMPredict.py +202 -202
  293. teradataml/data/docs/sqle/docs_17_20/SVMSparsePredict.py +152 -152
  294. teradataml/data/docs/sqle/docs_17_20/ScaleFit.py +197 -197
  295. teradataml/data/docs/sqle/docs_17_20/ScaleTransform.py +110 -109
  296. teradataml/data/docs/sqle/docs_17_20/SentimentExtractor.py +206 -206
  297. teradataml/data/docs/sqle/docs_17_20/Sessionize.py +113 -113
  298. teradataml/data/docs/sqle/docs_17_20/Silhouette.py +152 -152
  299. teradataml/data/docs/sqle/docs_17_20/SimpleImputeFit.py +116 -116
  300. teradataml/data/docs/sqle/docs_17_20/SimpleImputeTransform.py +109 -108
  301. teradataml/data/docs/sqle/docs_17_20/StrApply.py +187 -187
  302. teradataml/data/docs/sqle/docs_17_20/StringSimilarity.py +145 -145
  303. teradataml/data/docs/sqle/docs_17_20/TDDecisionForestPredict.py +207 -207
  304. teradataml/data/docs/sqle/docs_17_20/TDGLMPredict.py +171 -171
  305. teradataml/data/docs/sqle/docs_17_20/TargetEncodingFit.py +266 -266
  306. teradataml/data/docs/sqle/docs_17_20/TargetEncodingTransform.py +141 -140
  307. teradataml/data/docs/sqle/docs_17_20/TextParser.py +172 -172
  308. teradataml/data/docs/sqle/docs_17_20/TrainTestSplit.py +159 -159
  309. teradataml/data/docs/sqle/docs_17_20/Transform.py +123 -123
  310. teradataml/data/docs/sqle/docs_17_20/UnivariateStatistics.py +141 -141
  311. teradataml/data/docs/sqle/docs_17_20/Unpack.py +214 -214
  312. teradataml/data/docs/sqle/docs_17_20/VectorDistance.py +168 -168
  313. teradataml/data/docs/sqle/docs_17_20/WhichMax.py +83 -83
  314. teradataml/data/docs/sqle/docs_17_20/WhichMin.py +83 -83
  315. teradataml/data/docs/sqle/docs_17_20/WordEmbeddings.py +236 -236
  316. teradataml/data/docs/sqle/docs_17_20/XGBoost.py +353 -353
  317. teradataml/data/docs/sqle/docs_17_20/XGBoostPredict.py +275 -275
  318. teradataml/data/docs/sqle/docs_17_20/ZTest.py +155 -155
  319. teradataml/data/docs/tableoperator/docs_17_00/ReadNOS.py +429 -429
  320. teradataml/data/docs/tableoperator/docs_17_05/ReadNOS.py +429 -429
  321. teradataml/data/docs/tableoperator/docs_17_05/WriteNOS.py +347 -347
  322. teradataml/data/docs/tableoperator/docs_17_10/ReadNOS.py +428 -428
  323. teradataml/data/docs/tableoperator/docs_17_10/WriteNOS.py +347 -347
  324. teradataml/data/docs/tableoperator/docs_17_20/ReadNOS.py +439 -439
  325. teradataml/data/docs/tableoperator/docs_17_20/WriteNOS.py +386 -386
  326. teradataml/data/docs/uaf/docs_17_20/ACF.py +195 -195
  327. teradataml/data/docs/uaf/docs_17_20/ArimaEstimate.py +369 -369
  328. teradataml/data/docs/uaf/docs_17_20/ArimaForecast.py +142 -142
  329. teradataml/data/docs/uaf/docs_17_20/ArimaValidate.py +159 -159
  330. teradataml/data/docs/uaf/docs_17_20/BinaryMatrixOp.py +247 -247
  331. teradataml/data/docs/uaf/docs_17_20/BinarySeriesOp.py +252 -252
  332. teradataml/data/docs/uaf/docs_17_20/BreuschGodfrey.py +177 -177
  333. teradataml/data/docs/uaf/docs_17_20/BreuschPaganGodfrey.py +174 -174
  334. teradataml/data/docs/uaf/docs_17_20/Convolve.py +226 -226
  335. teradataml/data/docs/uaf/docs_17_20/Convolve2.py +214 -214
  336. teradataml/data/docs/uaf/docs_17_20/CumulPeriodogram.py +183 -183
  337. teradataml/data/docs/uaf/docs_17_20/DFFT.py +203 -203
  338. teradataml/data/docs/uaf/docs_17_20/DFFT2.py +216 -216
  339. teradataml/data/docs/uaf/docs_17_20/DFFT2Conv.py +215 -215
  340. teradataml/data/docs/uaf/docs_17_20/DFFTConv.py +191 -191
  341. teradataml/data/docs/uaf/docs_17_20/DTW.py +179 -179
  342. teradataml/data/docs/uaf/docs_17_20/DickeyFuller.py +144 -144
  343. teradataml/data/docs/uaf/docs_17_20/DurbinWatson.py +183 -183
  344. teradataml/data/docs/uaf/docs_17_20/ExtractResults.py +184 -184
  345. teradataml/data/docs/uaf/docs_17_20/FitMetrics.py +172 -172
  346. teradataml/data/docs/uaf/docs_17_20/GenseriesFormula.py +205 -205
  347. teradataml/data/docs/uaf/docs_17_20/GenseriesSinusoids.py +142 -142
  348. teradataml/data/docs/uaf/docs_17_20/HoltWintersForecaster.py +258 -258
  349. teradataml/data/docs/uaf/docs_17_20/IDFFT.py +164 -164
  350. teradataml/data/docs/uaf/docs_17_20/IDFFT2.py +198 -198
  351. teradataml/data/docs/uaf/docs_17_20/InputValidator.py +120 -120
  352. teradataml/data/docs/uaf/docs_17_20/LineSpec.py +155 -155
  353. teradataml/data/docs/uaf/docs_17_20/LinearRegr.py +214 -214
  354. teradataml/data/docs/uaf/docs_17_20/MAMean.py +173 -173
  355. teradataml/data/docs/uaf/docs_17_20/MInfo.py +133 -133
  356. teradataml/data/docs/uaf/docs_17_20/MatrixMultiply.py +135 -135
  357. teradataml/data/docs/uaf/docs_17_20/MultivarRegr.py +190 -190
  358. teradataml/data/docs/uaf/docs_17_20/PACF.py +158 -158
  359. teradataml/data/docs/uaf/docs_17_20/Portman.py +216 -216
  360. teradataml/data/docs/uaf/docs_17_20/PowerTransform.py +154 -154
  361. teradataml/data/docs/uaf/docs_17_20/Resample.py +228 -228
  362. teradataml/data/docs/uaf/docs_17_20/SInfo.py +122 -122
  363. teradataml/data/docs/uaf/docs_17_20/SeasonalNormalize.py +165 -165
  364. teradataml/data/docs/uaf/docs_17_20/SelectionCriteria.py +173 -173
  365. teradataml/data/docs/uaf/docs_17_20/SignifPeriodicities.py +170 -170
  366. teradataml/data/docs/uaf/docs_17_20/SignifResidmean.py +163 -163
  367. teradataml/data/docs/uaf/docs_17_20/SimpleExp.py +179 -179
  368. teradataml/data/docs/uaf/docs_17_20/Smoothma.py +207 -207
  369. teradataml/data/docs/uaf/docs_17_20/TrackingOp.py +150 -150
  370. teradataml/data/docs/uaf/docs_17_20/UNDIFF.py +171 -171
  371. teradataml/data/docs/uaf/docs_17_20/Unnormalize.py +201 -201
  372. teradataml/data/docs/uaf/docs_17_20/WhitesGeneral.py +169 -169
  373. teradataml/data/dtw_example.json +17 -17
  374. teradataml/data/dtw_t1.csv +11 -11
  375. teradataml/data/dtw_t2.csv +4 -4
  376. teradataml/data/dwt2d_example.json +15 -15
  377. teradataml/data/dwt_example.json +14 -14
  378. teradataml/data/dwt_filter_dim.csv +5 -5
  379. teradataml/data/emission.csv +9 -9
  380. teradataml/data/emp_table_by_dept.csv +19 -19
  381. teradataml/data/employee_info.csv +4 -4
  382. teradataml/data/employee_table.csv +6 -6
  383. teradataml/data/excluding_event_table.csv +2 -2
  384. teradataml/data/finance_data.csv +6 -6
  385. teradataml/data/finance_data2.csv +61 -61
  386. teradataml/data/finance_data3.csv +93 -93
  387. teradataml/data/fish.csv +160 -0
  388. teradataml/data/fm_blood2ageandweight.csv +26 -26
  389. teradataml/data/fmeasure_example.json +11 -11
  390. teradataml/data/followers_leaders.csv +10 -10
  391. teradataml/data/fpgrowth_example.json +12 -12
  392. teradataml/data/frequentpaths_example.json +29 -29
  393. teradataml/data/friends.csv +9 -9
  394. teradataml/data/fs_input.csv +33 -33
  395. teradataml/data/fs_input1.csv +33 -33
  396. teradataml/data/genData.csv +513 -513
  397. teradataml/data/geodataframe_example.json +39 -39
  398. teradataml/data/glass_types.csv +215 -0
  399. teradataml/data/glm_admissions_model.csv +12 -12
  400. teradataml/data/glm_example.json +29 -29
  401. teradataml/data/glml1l2_example.json +28 -28
  402. teradataml/data/glml1l2predict_example.json +54 -54
  403. teradataml/data/glmpredict_example.json +54 -54
  404. teradataml/data/gq_t1.csv +21 -21
  405. teradataml/data/hconvolve_complex_right.csv +5 -5
  406. teradataml/data/hconvolve_complex_rightmulti.csv +5 -5
  407. teradataml/data/histogram_example.json +11 -11
  408. teradataml/data/hmmdecoder_example.json +78 -78
  409. teradataml/data/hmmevaluator_example.json +24 -24
  410. teradataml/data/hmmsupervised_example.json +10 -10
  411. teradataml/data/hmmunsupervised_example.json +7 -7
  412. teradataml/data/house_values.csv +12 -12
  413. teradataml/data/house_values2.csv +13 -13
  414. teradataml/data/housing_cat.csv +7 -7
  415. teradataml/data/housing_data.csv +9 -9
  416. teradataml/data/housing_test.csv +47 -47
  417. teradataml/data/housing_test_binary.csv +47 -47
  418. teradataml/data/housing_train.csv +493 -493
  419. teradataml/data/housing_train_attribute.csv +4 -4
  420. teradataml/data/housing_train_binary.csv +437 -437
  421. teradataml/data/housing_train_parameter.csv +2 -2
  422. teradataml/data/housing_train_response.csv +493 -493
  423. teradataml/data/ibm_stock.csv +370 -370
  424. teradataml/data/ibm_stock1.csv +370 -370
  425. teradataml/data/identitymatch_example.json +21 -21
  426. teradataml/data/idf_table.csv +4 -4
  427. teradataml/data/impressions.csv +101 -101
  428. teradataml/data/inflation.csv +21 -21
  429. teradataml/data/initial.csv +3 -3
  430. teradataml/data/insect_sprays.csv +12 -12
  431. teradataml/data/insurance.csv +1339 -1339
  432. teradataml/data/interpolator_example.json +12 -12
  433. teradataml/data/iris_altinput.csv +481 -481
  434. teradataml/data/iris_attribute_output.csv +8 -8
  435. teradataml/data/iris_attribute_test.csv +121 -121
  436. teradataml/data/iris_attribute_train.csv +481 -481
  437. teradataml/data/iris_category_expect_predict.csv +31 -31
  438. teradataml/data/iris_data.csv +151 -0
  439. teradataml/data/iris_input.csv +151 -151
  440. teradataml/data/iris_response_train.csv +121 -121
  441. teradataml/data/iris_test.csv +31 -31
  442. teradataml/data/iris_train.csv +121 -121
  443. teradataml/data/join_table1.csv +4 -4
  444. teradataml/data/join_table2.csv +4 -4
  445. teradataml/data/jsons/anly_function_name.json +6 -6
  446. teradataml/data/jsons/byom/dataikupredict.json +147 -147
  447. teradataml/data/jsons/byom/datarobotpredict.json +146 -146
  448. teradataml/data/jsons/byom/h2opredict.json +194 -194
  449. teradataml/data/jsons/byom/onnxpredict.json +186 -186
  450. teradataml/data/jsons/byom/pmmlpredict.json +146 -146
  451. teradataml/data/jsons/paired_functions.json +435 -435
  452. teradataml/data/jsons/sqle/16.20/Antiselect.json +56 -56
  453. teradataml/data/jsons/sqle/16.20/Attribution.json +249 -249
  454. teradataml/data/jsons/sqle/16.20/DecisionForestPredict.json +156 -156
  455. teradataml/data/jsons/sqle/16.20/DecisionTreePredict.json +170 -170
  456. teradataml/data/jsons/sqle/16.20/GLMPredict.json +122 -122
  457. teradataml/data/jsons/sqle/16.20/MovingAverage.json +367 -367
  458. teradataml/data/jsons/sqle/16.20/NGramSplitter.json +239 -239
  459. teradataml/data/jsons/sqle/16.20/NaiveBayesPredict.json +136 -136
  460. teradataml/data/jsons/sqle/16.20/NaiveBayesTextClassifierPredict.json +235 -235
  461. teradataml/data/jsons/sqle/16.20/Pack.json +98 -98
  462. teradataml/data/jsons/sqle/16.20/SVMSparsePredict.json +162 -162
  463. teradataml/data/jsons/sqle/16.20/Sessionize.json +105 -105
  464. teradataml/data/jsons/sqle/16.20/StringSimilarity.json +86 -86
  465. teradataml/data/jsons/sqle/16.20/Unpack.json +166 -166
  466. teradataml/data/jsons/sqle/16.20/nPath.json +269 -269
  467. teradataml/data/jsons/sqle/17.00/Antiselect.json +56 -56
  468. teradataml/data/jsons/sqle/17.00/Attribution.json +249 -249
  469. teradataml/data/jsons/sqle/17.00/DecisionForestPredict.json +156 -156
  470. teradataml/data/jsons/sqle/17.00/DecisionTreePredict.json +170 -170
  471. teradataml/data/jsons/sqle/17.00/GLMPredict.json +122 -122
  472. teradataml/data/jsons/sqle/17.00/MovingAverage.json +367 -367
  473. teradataml/data/jsons/sqle/17.00/NGramSplitter.json +239 -239
  474. teradataml/data/jsons/sqle/17.00/NaiveBayesPredict.json +136 -136
  475. teradataml/data/jsons/sqle/17.00/NaiveBayesTextClassifierPredict.json +235 -235
  476. teradataml/data/jsons/sqle/17.00/Pack.json +98 -98
  477. teradataml/data/jsons/sqle/17.00/SVMSparsePredict.json +162 -162
  478. teradataml/data/jsons/sqle/17.00/Sessionize.json +105 -105
  479. teradataml/data/jsons/sqle/17.00/StringSimilarity.json +86 -86
  480. teradataml/data/jsons/sqle/17.00/Unpack.json +166 -166
  481. teradataml/data/jsons/sqle/17.00/nPath.json +269 -269
  482. teradataml/data/jsons/sqle/17.05/Antiselect.json +56 -56
  483. teradataml/data/jsons/sqle/17.05/Attribution.json +249 -249
  484. teradataml/data/jsons/sqle/17.05/DecisionForestPredict.json +156 -156
  485. teradataml/data/jsons/sqle/17.05/DecisionTreePredict.json +170 -170
  486. teradataml/data/jsons/sqle/17.05/GLMPredict.json +122 -122
  487. teradataml/data/jsons/sqle/17.05/MovingAverage.json +367 -367
  488. teradataml/data/jsons/sqle/17.05/NGramSplitter.json +239 -239
  489. teradataml/data/jsons/sqle/17.05/NaiveBayesPredict.json +136 -136
  490. teradataml/data/jsons/sqle/17.05/NaiveBayesTextClassifierPredict.json +235 -235
  491. teradataml/data/jsons/sqle/17.05/Pack.json +98 -98
  492. teradataml/data/jsons/sqle/17.05/SVMSparsePredict.json +162 -162
  493. teradataml/data/jsons/sqle/17.05/Sessionize.json +105 -105
  494. teradataml/data/jsons/sqle/17.05/StringSimilarity.json +86 -86
  495. teradataml/data/jsons/sqle/17.05/Unpack.json +166 -166
  496. teradataml/data/jsons/sqle/17.05/nPath.json +269 -269
  497. teradataml/data/jsons/sqle/17.10/Antiselect.json +56 -56
  498. teradataml/data/jsons/sqle/17.10/Attribution.json +249 -249
  499. teradataml/data/jsons/sqle/17.10/DecisionForestPredict.json +185 -185
  500. teradataml/data/jsons/sqle/17.10/DecisionTreePredict.json +171 -171
  501. teradataml/data/jsons/sqle/17.10/GLMPredict.json +151 -151
  502. teradataml/data/jsons/sqle/17.10/MovingAverage.json +368 -368
  503. teradataml/data/jsons/sqle/17.10/NGramSplitter.json +239 -239
  504. teradataml/data/jsons/sqle/17.10/NaiveBayesPredict.json +149 -149
  505. teradataml/data/jsons/sqle/17.10/NaiveBayesTextClassifierPredict.json +288 -288
  506. teradataml/data/jsons/sqle/17.10/Pack.json +133 -133
  507. teradataml/data/jsons/sqle/17.10/SVMSparsePredict.json +193 -193
  508. teradataml/data/jsons/sqle/17.10/Sessionize.json +105 -105
  509. teradataml/data/jsons/sqle/17.10/StringSimilarity.json +86 -86
  510. teradataml/data/jsons/sqle/17.10/TD_BinCodeFit.json +239 -239
  511. teradataml/data/jsons/sqle/17.10/TD_BinCodeTransform.json +70 -70
  512. teradataml/data/jsons/sqle/17.10/TD_CategoricalSummary.json +53 -53
  513. teradataml/data/jsons/sqle/17.10/TD_Chisq.json +67 -67
  514. teradataml/data/jsons/sqle/17.10/TD_ColumnSummary.json +53 -53
  515. teradataml/data/jsons/sqle/17.10/TD_ConvertTo.json +68 -68
  516. teradataml/data/jsons/sqle/17.10/TD_FTest.json +187 -187
  517. teradataml/data/jsons/sqle/17.10/TD_FillRowID.json +51 -51
  518. teradataml/data/jsons/sqle/17.10/TD_FunctionFit.json +46 -46
  519. teradataml/data/jsons/sqle/17.10/TD_FunctionTransform.json +72 -71
  520. teradataml/data/jsons/sqle/17.10/TD_GetRowsWithMissingValues.json +52 -52
  521. teradataml/data/jsons/sqle/17.10/TD_GetRowsWithoutMissingValues.json +52 -52
  522. teradataml/data/jsons/sqle/17.10/TD_Histogram.json +132 -132
  523. teradataml/data/jsons/sqle/17.10/TD_NumApply.json +147 -147
  524. teradataml/data/jsons/sqle/17.10/TD_OneHotEncodingFit.json +182 -182
  525. teradataml/data/jsons/sqle/17.10/TD_OneHotEncodingTransform.json +65 -64
  526. teradataml/data/jsons/sqle/17.10/TD_OutlierFilterFit.json +196 -196
  527. teradataml/data/jsons/sqle/17.10/TD_OutlierFilterTransform.json +48 -47
  528. teradataml/data/jsons/sqle/17.10/TD_PolynomialFeaturesFit.json +114 -114
  529. teradataml/data/jsons/sqle/17.10/TD_PolynomialFeaturesTransform.json +72 -71
  530. teradataml/data/jsons/sqle/17.10/TD_QQNorm.json +111 -111
  531. teradataml/data/jsons/sqle/17.10/TD_RoundColumns.json +93 -93
  532. teradataml/data/jsons/sqle/17.10/TD_RowNormalizeFit.json +127 -127
  533. teradataml/data/jsons/sqle/17.10/TD_RowNormalizeTransform.json +70 -69
  534. teradataml/data/jsons/sqle/17.10/TD_ScaleFit.json +156 -156
  535. teradataml/data/jsons/sqle/17.10/TD_ScaleTransform.json +70 -69
  536. teradataml/data/jsons/sqle/17.10/TD_SimpleImputeFit.json +147 -147
  537. teradataml/data/jsons/sqle/17.10/TD_SimpleImputeTransform.json +48 -47
  538. teradataml/data/jsons/sqle/17.10/TD_StrApply.json +240 -240
  539. teradataml/data/jsons/sqle/17.10/TD_UnivariateStatistics.json +118 -118
  540. teradataml/data/jsons/sqle/17.10/TD_WhichMax.json +52 -52
  541. teradataml/data/jsons/sqle/17.10/TD_WhichMin.json +52 -52
  542. teradataml/data/jsons/sqle/17.10/TD_ZTest.json +171 -171
  543. teradataml/data/jsons/sqle/17.10/Unpack.json +188 -188
  544. teradataml/data/jsons/sqle/17.10/nPath.json +269 -269
  545. teradataml/data/jsons/sqle/17.20/Antiselect.json +56 -56
  546. teradataml/data/jsons/sqle/17.20/Attribution.json +249 -249
  547. teradataml/data/jsons/sqle/17.20/DecisionForestPredict.json +185 -185
  548. teradataml/data/jsons/sqle/17.20/DecisionTreePredict.json +172 -172
  549. teradataml/data/jsons/sqle/17.20/GLMPredict.json +151 -151
  550. teradataml/data/jsons/sqle/17.20/MovingAverage.json +367 -367
  551. teradataml/data/jsons/sqle/17.20/NGramSplitter.json +239 -239
  552. teradataml/data/jsons/sqle/17.20/NaiveBayesPredict.json +149 -149
  553. teradataml/data/jsons/sqle/17.20/NaiveBayesTextClassifierPredict.json +287 -287
  554. teradataml/data/jsons/sqle/17.20/Pack.json +133 -133
  555. teradataml/data/jsons/sqle/17.20/SVMSparsePredict.json +192 -192
  556. teradataml/data/jsons/sqle/17.20/Sessionize.json +105 -105
  557. teradataml/data/jsons/sqle/17.20/StringSimilarity.json +86 -86
  558. teradataml/data/jsons/sqle/17.20/TD_ANOVA.json +76 -76
  559. teradataml/data/jsons/sqle/17.20/TD_BinCodeFit.json +239 -239
  560. teradataml/data/jsons/sqle/17.20/TD_BinCodeTransform.json +71 -71
  561. teradataml/data/jsons/sqle/17.20/TD_CategoricalSummary.json +53 -53
  562. teradataml/data/jsons/sqle/17.20/TD_Chisq.json +67 -67
  563. teradataml/data/jsons/sqle/17.20/TD_ClassificationEvaluator.json +145 -145
  564. teradataml/data/jsons/sqle/17.20/TD_ColumnSummary.json +53 -53
  565. teradataml/data/jsons/sqle/17.20/TD_ColumnTransformer.json +218 -218
  566. teradataml/data/jsons/sqle/17.20/TD_ConvertTo.json +92 -92
  567. teradataml/data/jsons/sqle/17.20/TD_DecisionForest.json +259 -259
  568. teradataml/data/jsons/sqle/17.20/TD_DecisionForestPredict.json +139 -139
  569. teradataml/data/jsons/sqle/17.20/TD_FTest.json +186 -186
  570. teradataml/data/jsons/sqle/17.20/TD_FillRowID.json +52 -52
  571. teradataml/data/jsons/sqle/17.20/TD_FunctionFit.json +46 -46
  572. teradataml/data/jsons/sqle/17.20/TD_FunctionTransform.json +72 -72
  573. teradataml/data/jsons/sqle/17.20/TD_GLM.json +431 -431
  574. teradataml/data/jsons/sqle/17.20/TD_GLMPREDICT.json +125 -125
  575. teradataml/data/jsons/sqle/17.20/TD_GLMPerSegment.json +411 -411
  576. teradataml/data/jsons/sqle/17.20/TD_GLMPredictPerSegment.json +146 -146
  577. teradataml/data/jsons/sqle/17.20/TD_GetFutileColumns.json +91 -91
  578. teradataml/data/jsons/sqle/17.20/TD_GetRowsWithMissingValues.json +76 -76
  579. teradataml/data/jsons/sqle/17.20/TD_GetRowsWithoutMissingValues.json +76 -76
  580. teradataml/data/jsons/sqle/17.20/TD_Histogram.json +152 -152
  581. teradataml/data/jsons/sqle/17.20/TD_KMeans.json +211 -211
  582. teradataml/data/jsons/sqle/17.20/TD_KMeansPredict.json +86 -86
  583. teradataml/data/jsons/sqle/17.20/TD_KNN.json +262 -262
  584. teradataml/data/jsons/sqle/17.20/TD_NaiveBayesTextClassifierTrainer.json +137 -137
  585. teradataml/data/jsons/sqle/17.20/TD_NonLinearCombineFit.json +101 -101
  586. teradataml/data/jsons/sqle/17.20/TD_NonLinearCombineTransform.json +71 -71
  587. teradataml/data/jsons/sqle/17.20/TD_NumApply.json +147 -147
  588. teradataml/data/jsons/sqle/17.20/TD_OneClassSVM.json +315 -315
  589. teradataml/data/jsons/sqle/17.20/TD_OneClassSVMPredict.json +123 -123
  590. teradataml/data/jsons/sqle/17.20/TD_OneHotEncodingFit.json +271 -271
  591. teradataml/data/jsons/sqle/17.20/TD_OneHotEncodingTransform.json +65 -65
  592. teradataml/data/jsons/sqle/17.20/TD_OrdinalEncodingFit.json +229 -229
  593. teradataml/data/jsons/sqle/17.20/TD_OrdinalEncodingTransform.json +75 -75
  594. teradataml/data/jsons/sqle/17.20/TD_OutlierFilterFit.json +217 -217
  595. teradataml/data/jsons/sqle/17.20/TD_OutlierFilterTransform.json +48 -48
  596. teradataml/data/jsons/sqle/17.20/TD_PolynomialFeaturesFit.json +114 -114
  597. teradataml/data/jsons/sqle/17.20/TD_PolynomialFeaturesTransform.json +72 -72
  598. teradataml/data/jsons/sqle/17.20/TD_QQNorm.json +111 -111
  599. teradataml/data/jsons/sqle/17.20/TD_ROC.json +177 -177
  600. teradataml/data/jsons/sqle/17.20/TD_RandomProjectionFit.json +178 -178
  601. teradataml/data/jsons/sqle/17.20/TD_RandomProjectionMinComponents.json +73 -73
  602. teradataml/data/jsons/sqle/17.20/TD_RandomProjectionTransform.json +74 -74
  603. teradataml/data/jsons/sqle/17.20/TD_RegressionEvaluator.json +137 -137
  604. teradataml/data/jsons/sqle/17.20/TD_RoundColumns.json +93 -93
  605. teradataml/data/jsons/sqle/17.20/TD_RowNormalizeFit.json +127 -127
  606. teradataml/data/jsons/sqle/17.20/TD_RowNormalizeTransform.json +70 -70
  607. teradataml/data/jsons/sqle/17.20/TD_SVM.json +389 -389
  608. teradataml/data/jsons/sqle/17.20/TD_SVMPredict.json +124 -124
  609. teradataml/data/jsons/sqle/17.20/TD_ScaleFit.json +156 -156
  610. teradataml/data/jsons/sqle/17.20/TD_ScaleTransform.json +70 -70
  611. teradataml/data/jsons/sqle/17.20/TD_SentimentExtractor.json +193 -193
  612. teradataml/data/jsons/sqle/17.20/TD_Silhouette.json +142 -142
  613. teradataml/data/jsons/sqle/17.20/TD_SimpleImputeFit.json +147 -147
  614. teradataml/data/jsons/sqle/17.20/TD_SimpleImputeTransform.json +48 -48
  615. teradataml/data/jsons/sqle/17.20/TD_StrApply.json +240 -240
  616. teradataml/data/jsons/sqle/17.20/TD_TargetEncodingFit.json +248 -248
  617. teradataml/data/jsons/sqle/17.20/TD_TargetEncodingTransform.json +75 -75
  618. teradataml/data/jsons/sqle/17.20/TD_TextParser.json +192 -192
  619. teradataml/data/jsons/sqle/17.20/TD_TrainTestSplit.json +142 -142
  620. teradataml/data/jsons/sqle/17.20/TD_UnivariateStatistics.json +117 -117
  621. teradataml/data/jsons/sqle/17.20/TD_VectorDistance.json +182 -182
  622. teradataml/data/jsons/sqle/17.20/TD_WhichMax.json +52 -52
  623. teradataml/data/jsons/sqle/17.20/TD_WhichMin.json +52 -52
  624. teradataml/data/jsons/sqle/17.20/TD_WordEmbeddings.json +241 -241
  625. teradataml/data/jsons/sqle/17.20/TD_XGBoost.json +312 -312
  626. teradataml/data/jsons/sqle/17.20/TD_XGBoostPredict.json +182 -182
  627. teradataml/data/jsons/sqle/17.20/TD_ZTest.json +170 -170
  628. teradataml/data/jsons/sqle/17.20/Unpack.json +188 -188
  629. teradataml/data/jsons/sqle/17.20/nPath.json +269 -269
  630. teradataml/data/jsons/tableoperator/17.00/read_nos.json +197 -197
  631. teradataml/data/jsons/tableoperator/17.05/read_nos.json +197 -197
  632. teradataml/data/jsons/tableoperator/17.05/write_nos.json +194 -194
  633. teradataml/data/jsons/tableoperator/17.10/read_nos.json +183 -183
  634. teradataml/data/jsons/tableoperator/17.10/write_nos.json +194 -194
  635. teradataml/data/jsons/tableoperator/17.20/read_nos.json +182 -182
  636. teradataml/data/jsons/tableoperator/17.20/write_nos.json +223 -223
  637. teradataml/data/jsons/uaf/17.20/TD_ACF.json +149 -149
  638. teradataml/data/jsons/uaf/17.20/TD_ARIMAESTIMATE.json +409 -409
  639. teradataml/data/jsons/uaf/17.20/TD_ARIMAFORECAST.json +79 -79
  640. teradataml/data/jsons/uaf/17.20/TD_ARIMAVALIDATE.json +151 -151
  641. teradataml/data/jsons/uaf/17.20/TD_BINARYMATRIXOP.json +109 -109
  642. teradataml/data/jsons/uaf/17.20/TD_BINARYSERIESOP.json +107 -107
  643. teradataml/data/jsons/uaf/17.20/TD_BREUSCH_GODFREY.json +87 -87
  644. teradataml/data/jsons/uaf/17.20/TD_BREUSCH_PAGAN_GODFREY.json +106 -106
  645. teradataml/data/jsons/uaf/17.20/TD_CONVOLVE.json +80 -80
  646. teradataml/data/jsons/uaf/17.20/TD_CONVOLVE2.json +67 -67
  647. teradataml/data/jsons/uaf/17.20/TD_CUMUL_PERIODOGRAM.json +91 -91
  648. teradataml/data/jsons/uaf/17.20/TD_DFFT.json +136 -136
  649. teradataml/data/jsons/uaf/17.20/TD_DFFT2.json +148 -148
  650. teradataml/data/jsons/uaf/17.20/TD_DFFT2CONV.json +108 -108
  651. teradataml/data/jsons/uaf/17.20/TD_DFFTCONV.json +109 -109
  652. teradataml/data/jsons/uaf/17.20/TD_DICKEY_FULLER.json +86 -86
  653. teradataml/data/jsons/uaf/17.20/TD_DIFF.json +91 -91
  654. teradataml/data/jsons/uaf/17.20/TD_DTW.json +116 -116
  655. teradataml/data/jsons/uaf/17.20/TD_DURBIN_WATSON.json +100 -100
  656. teradataml/data/jsons/uaf/17.20/TD_EXTRACT_RESULTS.json +38 -38
  657. teradataml/data/jsons/uaf/17.20/TD_FITMETRICS.json +100 -100
  658. teradataml/data/jsons/uaf/17.20/TD_GENSERIES4FORMULA.json +84 -84
  659. teradataml/data/jsons/uaf/17.20/TD_GENSERIES4SINUSOIDS.json +70 -70
  660. teradataml/data/jsons/uaf/17.20/TD_GOLDFELD_QUANDT.json +152 -152
  661. teradataml/data/jsons/uaf/17.20/TD_HOLT_WINTERS_FORECAST.json +313 -313
  662. teradataml/data/jsons/uaf/17.20/TD_IDFFT.json +57 -57
  663. teradataml/data/jsons/uaf/17.20/TD_IDFFT2.json +94 -94
  664. teradataml/data/jsons/uaf/17.20/TD_INPUTVALIDATOR.json +63 -63
  665. teradataml/data/jsons/uaf/17.20/TD_LINEAR_REGR.json +181 -181
  666. teradataml/data/jsons/uaf/17.20/TD_LINESPEC.json +102 -102
  667. teradataml/data/jsons/uaf/17.20/TD_MAMEAN.json +182 -182
  668. teradataml/data/jsons/uaf/17.20/TD_MATRIXMULTIPLY.json +67 -67
  669. teradataml/data/jsons/uaf/17.20/TD_MINFO.json +66 -66
  670. teradataml/data/jsons/uaf/17.20/TD_MULTIVAR_REGR.json +178 -178
  671. teradataml/data/jsons/uaf/17.20/TD_PACF.json +114 -114
  672. teradataml/data/jsons/uaf/17.20/TD_PORTMAN.json +118 -118
  673. teradataml/data/jsons/uaf/17.20/TD_POWERSPEC.json +175 -175
  674. teradataml/data/jsons/uaf/17.20/TD_POWERTRANSFORM.json +97 -97
  675. teradataml/data/jsons/uaf/17.20/TD_RESAMPLE.json +173 -173
  676. teradataml/data/jsons/uaf/17.20/TD_SEASONALNORMALIZE.json +136 -136
  677. teradataml/data/jsons/uaf/17.20/TD_SELECTION_CRITERIA.json +89 -89
  678. teradataml/data/jsons/uaf/17.20/TD_SIGNIF_PERIODICITIES.json +79 -79
  679. teradataml/data/jsons/uaf/17.20/TD_SIGNIF_RESIDMEAN.json +67 -67
  680. teradataml/data/jsons/uaf/17.20/TD_SIMPLEEXP.json +184 -184
  681. teradataml/data/jsons/uaf/17.20/TD_SINFO.json +57 -57
  682. teradataml/data/jsons/uaf/17.20/TD_SMOOTHMA.json +162 -162
  683. teradataml/data/jsons/uaf/17.20/TD_TRACKINGOP.json +100 -100
  684. teradataml/data/jsons/uaf/17.20/TD_UNDIFF.json +111 -111
  685. teradataml/data/jsons/uaf/17.20/TD_UNNORMALIZE.json +95 -95
  686. teradataml/data/jsons/uaf/17.20/TD_WHITES_GENERAL.json +77 -77
  687. teradataml/data/kmeans_example.json +17 -17
  688. teradataml/data/kmeans_us_arrests_data.csv +0 -0
  689. teradataml/data/knn_example.json +18 -18
  690. teradataml/data/knnrecommender_example.json +6 -6
  691. teradataml/data/knnrecommenderpredict_example.json +12 -12
  692. teradataml/data/lar_example.json +17 -17
  693. teradataml/data/larpredict_example.json +30 -30
  694. teradataml/data/lc_new_predictors.csv +5 -5
  695. teradataml/data/lc_new_reference.csv +9 -9
  696. teradataml/data/lda_example.json +8 -8
  697. teradataml/data/ldainference_example.json +14 -14
  698. teradataml/data/ldatopicsummary_example.json +8 -8
  699. teradataml/data/levendist_input.csv +13 -13
  700. teradataml/data/levenshteindistance_example.json +10 -10
  701. teradataml/data/linreg_example.json +9 -9
  702. teradataml/data/load_example_data.py +326 -323
  703. teradataml/data/loan_prediction.csv +295 -295
  704. teradataml/data/lungcancer.csv +138 -138
  705. teradataml/data/mappingdata.csv +12 -12
  706. teradataml/data/milk_timeseries.csv +157 -157
  707. teradataml/data/min_max_titanic.csv +4 -4
  708. teradataml/data/minhash_example.json +6 -6
  709. teradataml/data/ml_ratings.csv +7547 -7547
  710. teradataml/data/ml_ratings_10.csv +2445 -2445
  711. teradataml/data/model1_table.csv +5 -5
  712. teradataml/data/model2_table.csv +5 -5
  713. teradataml/data/models/iris_db_glm_model.pmml +56 -56
  714. teradataml/data/models/iris_db_xgb_model.pmml +4471 -4471
  715. teradataml/data/modularity_example.json +12 -12
  716. teradataml/data/movavg_example.json +7 -7
  717. teradataml/data/mtx1.csv +7 -7
  718. teradataml/data/mtx2.csv +13 -13
  719. teradataml/data/multi_model_classification.csv +401 -0
  720. teradataml/data/multi_model_regression.csv +401 -0
  721. teradataml/data/mvdfft8.csv +9 -9
  722. teradataml/data/naivebayes_example.json +9 -9
  723. teradataml/data/naivebayespredict_example.json +19 -19
  724. teradataml/data/naivebayestextclassifier2_example.json +6 -6
  725. teradataml/data/naivebayestextclassifier_example.json +8 -8
  726. teradataml/data/naivebayestextclassifierpredict_example.json +20 -20
  727. teradataml/data/name_Find_configure.csv +10 -10
  728. teradataml/data/namedentityfinder_example.json +14 -14
  729. teradataml/data/namedentityfinderevaluator_example.json +10 -10
  730. teradataml/data/namedentityfindertrainer_example.json +6 -6
  731. teradataml/data/nb_iris_input_test.csv +31 -31
  732. teradataml/data/nb_iris_input_train.csv +121 -121
  733. teradataml/data/nbp_iris_model.csv +13 -13
  734. teradataml/data/ner_extractor_text.csv +2 -2
  735. teradataml/data/ner_sports_test2.csv +29 -29
  736. teradataml/data/ner_sports_train.csv +501 -501
  737. teradataml/data/nerevaluator_example.json +5 -5
  738. teradataml/data/nerextractor_example.json +18 -18
  739. teradataml/data/nermem_sports_test.csv +17 -17
  740. teradataml/data/nermem_sports_train.csv +50 -50
  741. teradataml/data/nertrainer_example.json +6 -6
  742. teradataml/data/ngrams_example.json +6 -6
  743. teradataml/data/notebooks/sqlalchemy/Teradata Vantage Aggregate Functions using SQLAlchemy.ipynb +1455 -1455
  744. teradataml/data/notebooks/sqlalchemy/Teradata Vantage Arithmetic Functions Using SQLAlchemy.ipynb +1993 -1993
  745. teradataml/data/notebooks/sqlalchemy/Teradata Vantage Bit-Byte Manipulation Functions using SQLAlchemy.ipynb +1492 -1492
  746. teradataml/data/notebooks/sqlalchemy/Teradata Vantage Built-in functions using SQLAlchemy.ipynb +536 -536
  747. teradataml/data/notebooks/sqlalchemy/Teradata Vantage Regular Expressions Using SQLAlchemy.ipynb +570 -570
  748. teradataml/data/notebooks/sqlalchemy/Teradata Vantage String Functions Using SQLAlchemy.ipynb +2559 -2559
  749. teradataml/data/notebooks/sqlalchemy/Teradata Vantage Window Aggregate Functions using SQLAlchemy.ipynb +2911 -2911
  750. teradataml/data/notebooks/sqlalchemy/Using Generic SQLAlchemy ClauseElements teradataml DataFrame assign method.ipynb +698 -698
  751. teradataml/data/notebooks/sqlalchemy/teradataml filtering using SQLAlchemy ClauseElements.ipynb +784 -784
  752. teradataml/data/npath_example.json +23 -23
  753. teradataml/data/ntree_example.json +14 -14
  754. teradataml/data/numeric_strings.csv +4 -4
  755. teradataml/data/numerics.csv +4 -4
  756. teradataml/data/ocean_buoy.csv +17 -17
  757. teradataml/data/ocean_buoy2.csv +17 -17
  758. teradataml/data/ocean_buoys.csv +27 -27
  759. teradataml/data/ocean_buoys2.csv +10 -10
  760. teradataml/data/ocean_buoys_nonpti.csv +28 -28
  761. teradataml/data/ocean_buoys_seq.csv +29 -29
  762. teradataml/data/openml_example.json +63 -0
  763. teradataml/data/optional_event_table.csv +4 -4
  764. teradataml/data/orders1.csv +11 -11
  765. teradataml/data/orders1_12.csv +12 -12
  766. teradataml/data/orders_ex.csv +4 -4
  767. teradataml/data/pack_example.json +8 -8
  768. teradataml/data/package_tracking.csv +19 -19
  769. teradataml/data/package_tracking_pti.csv +18 -18
  770. teradataml/data/pagerank_example.json +13 -13
  771. teradataml/data/paragraphs_input.csv +6 -6
  772. teradataml/data/pathanalyzer_example.json +7 -7
  773. teradataml/data/pathgenerator_example.json +7 -7
  774. teradataml/data/phrases.csv +7 -7
  775. teradataml/data/pivot_example.json +8 -8
  776. teradataml/data/pivot_input.csv +22 -22
  777. teradataml/data/playerRating.csv +31 -31
  778. teradataml/data/postagger_example.json +6 -6
  779. teradataml/data/posttagger_output.csv +44 -44
  780. teradataml/data/production_data.csv +16 -16
  781. teradataml/data/production_data2.csv +7 -7
  782. teradataml/data/randomsample_example.json +31 -31
  783. teradataml/data/randomwalksample_example.json +8 -8
  784. teradataml/data/rank_table.csv +6 -6
  785. teradataml/data/ref_mobile_data.csv +4 -4
  786. teradataml/data/ref_mobile_data_dense.csv +2 -2
  787. teradataml/data/ref_url.csv +17 -17
  788. teradataml/data/restaurant_reviews.csv +7 -7
  789. teradataml/data/river_data.csv +145 -145
  790. teradataml/data/roc_example.json +7 -7
  791. teradataml/data/roc_input.csv +101 -101
  792. teradataml/data/rule_inputs.csv +6 -6
  793. teradataml/data/rule_table.csv +2 -2
  794. teradataml/data/sales.csv +7 -7
  795. teradataml/data/sales_transaction.csv +501 -501
  796. teradataml/data/salesdata.csv +342 -342
  797. teradataml/data/sample_cities.csv +2 -2
  798. teradataml/data/sample_shapes.csv +10 -10
  799. teradataml/data/sample_streets.csv +2 -2
  800. teradataml/data/sampling_example.json +15 -15
  801. teradataml/data/sax_example.json +8 -8
  802. teradataml/data/scale_example.json +23 -23
  803. teradataml/data/scale_housing.csv +11 -11
  804. teradataml/data/scale_housing_test.csv +6 -6
  805. teradataml/data/scale_stat.csv +11 -11
  806. teradataml/data/scalebypartition_example.json +13 -13
  807. teradataml/data/scalemap_example.json +13 -13
  808. teradataml/data/scalesummary_example.json +12 -12
  809. teradataml/data/score_category.csv +101 -101
  810. teradataml/data/score_summary.csv +4 -4
  811. teradataml/data/script_example.json +9 -9
  812. teradataml/data/scripts/deploy_script.py +65 -0
  813. teradataml/data/scripts/mapper.R +20 -0
  814. teradataml/data/scripts/mapper.py +15 -15
  815. teradataml/data/scripts/mapper_replace.py +15 -15
  816. teradataml/data/scripts/sklearn/__init__.py +0 -0
  817. teradataml/data/scripts/sklearn/sklearn_fit.py +175 -0
  818. teradataml/data/scripts/sklearn/sklearn_fit_predict.py +135 -0
  819. teradataml/data/scripts/sklearn/sklearn_function.template +113 -0
  820. teradataml/data/scripts/sklearn/sklearn_model_selection_split.py +158 -0
  821. teradataml/data/scripts/sklearn/sklearn_neighbors.py +152 -0
  822. teradataml/data/scripts/sklearn/sklearn_score.py +128 -0
  823. teradataml/data/scripts/sklearn/sklearn_transform.py +179 -0
  824. teradataml/data/seeds.csv +10 -10
  825. teradataml/data/sentenceextractor_example.json +6 -6
  826. teradataml/data/sentiment_extract_input.csv +11 -11
  827. teradataml/data/sentiment_train.csv +16 -16
  828. teradataml/data/sentiment_word.csv +20 -20
  829. teradataml/data/sentiment_word_input.csv +19 -19
  830. teradataml/data/sentimentextractor_example.json +24 -24
  831. teradataml/data/sentimenttrainer_example.json +8 -8
  832. teradataml/data/sequence_table.csv +10 -10
  833. teradataml/data/seriessplitter_example.json +7 -7
  834. teradataml/data/sessionize_example.json +17 -17
  835. teradataml/data/sessionize_table.csv +116 -116
  836. teradataml/data/setop_test1.csv +24 -24
  837. teradataml/data/setop_test2.csv +22 -22
  838. teradataml/data/soc_nw_edges.csv +10 -10
  839. teradataml/data/soc_nw_vertices.csv +7 -7
  840. teradataml/data/souvenir_timeseries.csv +167 -167
  841. teradataml/data/sparse_iris_attribute.csv +5 -5
  842. teradataml/data/sparse_iris_test.csv +121 -121
  843. teradataml/data/sparse_iris_train.csv +601 -601
  844. teradataml/data/star1.csv +6 -6
  845. teradataml/data/state_transition.csv +5 -5
  846. teradataml/data/stock_data.csv +53 -53
  847. teradataml/data/stock_movement.csv +11 -11
  848. teradataml/data/stock_vol.csv +76 -76
  849. teradataml/data/stop_words.csv +8 -8
  850. teradataml/data/store_sales.csv +37 -37
  851. teradataml/data/stringsimilarity_example.json +7 -7
  852. teradataml/data/strsimilarity_input.csv +13 -13
  853. teradataml/data/students.csv +101 -101
  854. teradataml/data/svm_iris_input_test.csv +121 -121
  855. teradataml/data/svm_iris_input_train.csv +481 -481
  856. teradataml/data/svm_iris_model.csv +7 -7
  857. teradataml/data/svmdense_example.json +9 -9
  858. teradataml/data/svmdensepredict_example.json +18 -18
  859. teradataml/data/svmsparse_example.json +7 -7
  860. teradataml/data/svmsparsepredict_example.json +13 -13
  861. teradataml/data/svmsparsesummary_example.json +7 -7
  862. teradataml/data/target_mobile_data.csv +13 -13
  863. teradataml/data/target_mobile_data_dense.csv +5 -5
  864. teradataml/data/templatedata.csv +1201 -1201
  865. teradataml/data/templates/open_source_ml.json +9 -0
  866. teradataml/data/teradataml_example.json +73 -1
  867. teradataml/data/test_classification.csv +101 -0
  868. teradataml/data/test_loan_prediction.csv +53 -53
  869. teradataml/data/test_pacf_12.csv +37 -37
  870. teradataml/data/test_prediction.csv +101 -0
  871. teradataml/data/test_regression.csv +101 -0
  872. teradataml/data/test_river2.csv +109 -109
  873. teradataml/data/text_inputs.csv +6 -6
  874. teradataml/data/textchunker_example.json +7 -7
  875. teradataml/data/textclassifier_example.json +6 -6
  876. teradataml/data/textclassifier_input.csv +7 -7
  877. teradataml/data/textclassifiertrainer_example.json +6 -6
  878. teradataml/data/textmorph_example.json +5 -5
  879. teradataml/data/textparser_example.json +15 -15
  880. teradataml/data/texttagger_example.json +11 -11
  881. teradataml/data/texttokenizer_example.json +6 -6
  882. teradataml/data/texttrainer_input.csv +11 -11
  883. teradataml/data/tf_example.json +6 -6
  884. teradataml/data/tfidf_example.json +13 -13
  885. teradataml/data/tfidf_input1.csv +201 -201
  886. teradataml/data/tfidf_train.csv +6 -6
  887. teradataml/data/time_table1.csv +535 -535
  888. teradataml/data/time_table2.csv +14 -14
  889. teradataml/data/timeseriesdata.csv +1601 -1601
  890. teradataml/data/timeseriesdatasetsd4.csv +105 -105
  891. teradataml/data/titanic.csv +892 -892
  892. teradataml/data/token_table.csv +696 -696
  893. teradataml/data/train_multiclass.csv +101 -0
  894. teradataml/data/train_regression.csv +101 -0
  895. teradataml/data/train_regression_multiple_labels.csv +101 -0
  896. teradataml/data/train_tracking.csv +27 -27
  897. teradataml/data/transformation_table.csv +5 -5
  898. teradataml/data/transformation_table_new.csv +1 -1
  899. teradataml/data/tv_spots.csv +16 -16
  900. teradataml/data/twod_climate_data.csv +117 -117
  901. teradataml/data/uaf_example.json +475 -475
  902. teradataml/data/univariatestatistics_example.json +8 -8
  903. teradataml/data/unpack_example.json +9 -9
  904. teradataml/data/unpivot_example.json +9 -9
  905. teradataml/data/unpivot_input.csv +8 -8
  906. teradataml/data/us_air_pass.csv +36 -36
  907. teradataml/data/us_population.csv +624 -624
  908. teradataml/data/us_states_shapes.csv +52 -52
  909. teradataml/data/varmax_example.json +17 -17
  910. teradataml/data/vectordistance_example.json +25 -25
  911. teradataml/data/ville_climatedata.csv +121 -121
  912. teradataml/data/ville_tempdata.csv +12 -12
  913. teradataml/data/ville_tempdata1.csv +12 -12
  914. teradataml/data/ville_temperature.csv +11 -11
  915. teradataml/data/waveletTable.csv +1605 -1605
  916. teradataml/data/waveletTable2.csv +1605 -1605
  917. teradataml/data/weightedmovavg_example.json +8 -8
  918. teradataml/data/wft_testing.csv +5 -5
  919. teradataml/data/wine_data.csv +1600 -0
  920. teradataml/data/word_embed_input_table1.csv +5 -5
  921. teradataml/data/word_embed_input_table2.csv +4 -4
  922. teradataml/data/word_embed_model.csv +22 -22
  923. teradataml/data/words_input.csv +13 -13
  924. teradataml/data/xconvolve_complex_left.csv +6 -6
  925. teradataml/data/xconvolve_complex_leftmulti.csv +6 -6
  926. teradataml/data/xgboost_example.json +35 -35
  927. teradataml/data/xgboostpredict_example.json +31 -31
  928. teradataml/dataframe/copy_to.py +1764 -1698
  929. teradataml/dataframe/data_transfer.py +2753 -2745
  930. teradataml/dataframe/dataframe.py +17545 -16946
  931. teradataml/dataframe/dataframe_utils.py +1837 -1740
  932. teradataml/dataframe/fastload.py +611 -603
  933. teradataml/dataframe/indexer.py +424 -424
  934. teradataml/dataframe/setop.py +1179 -1166
  935. teradataml/dataframe/sql.py +10090 -6432
  936. teradataml/dataframe/sql_function_parameters.py +439 -388
  937. teradataml/dataframe/sql_functions.py +652 -652
  938. teradataml/dataframe/sql_interfaces.py +220 -220
  939. teradataml/dataframe/vantage_function_types.py +674 -630
  940. teradataml/dataframe/window.py +693 -692
  941. teradataml/dbutils/__init__.py +3 -3
  942. teradataml/dbutils/dbutils.py +1167 -1150
  943. teradataml/dbutils/filemgr.py +267 -267
  944. teradataml/gen_ai/__init__.py +2 -2
  945. teradataml/gen_ai/convAI.py +472 -472
  946. teradataml/geospatial/__init__.py +3 -3
  947. teradataml/geospatial/geodataframe.py +1105 -1094
  948. teradataml/geospatial/geodataframecolumn.py +392 -387
  949. teradataml/geospatial/geometry_types.py +925 -925
  950. teradataml/hyperparameter_tuner/__init__.py +1 -1
  951. teradataml/hyperparameter_tuner/optimizer.py +3783 -2993
  952. teradataml/hyperparameter_tuner/utils.py +281 -187
  953. teradataml/lib/aed_0_1.dll +0 -0
  954. teradataml/lib/libaed_0_1.dylib +0 -0
  955. teradataml/lib/libaed_0_1.so +0 -0
  956. teradataml/libaed_0_1.dylib +0 -0
  957. teradataml/libaed_0_1.so +0 -0
  958. teradataml/opensource/__init__.py +1 -0
  959. teradataml/opensource/sklearn/__init__.py +1 -0
  960. teradataml/opensource/sklearn/_class.py +255 -0
  961. teradataml/opensource/sklearn/_sklearn_wrapper.py +1668 -0
  962. teradataml/opensource/sklearn/_wrapper_utils.py +268 -0
  963. teradataml/opensource/sklearn/constants.py +54 -0
  964. teradataml/options/__init__.py +121 -124
  965. teradataml/options/configure.py +337 -336
  966. teradataml/options/display.py +176 -176
  967. teradataml/plot/__init__.py +2 -2
  968. teradataml/plot/axis.py +1388 -1388
  969. teradataml/plot/constants.py +15 -15
  970. teradataml/plot/figure.py +398 -398
  971. teradataml/plot/plot.py +760 -760
  972. teradataml/plot/query_generator.py +83 -83
  973. teradataml/plot/subplot.py +216 -216
  974. teradataml/scriptmgmt/UserEnv.py +3788 -3761
  975. teradataml/scriptmgmt/__init__.py +3 -3
  976. teradataml/scriptmgmt/lls_utils.py +1616 -1604
  977. teradataml/series/series.py +532 -532
  978. teradataml/series/series_utils.py +71 -71
  979. teradataml/table_operators/Apply.py +949 -917
  980. teradataml/table_operators/Script.py +1719 -1982
  981. teradataml/table_operators/TableOperator.py +1207 -1616
  982. teradataml/table_operators/__init__.py +2 -3
  983. teradataml/table_operators/apply_query_generator.py +262 -262
  984. teradataml/table_operators/query_generator.py +507 -507
  985. teradataml/table_operators/table_operator_query_generator.py +460 -460
  986. teradataml/table_operators/table_operator_util.py +631 -639
  987. teradataml/table_operators/templates/dataframe_apply.template +184 -184
  988. teradataml/table_operators/templates/dataframe_map.template +176 -176
  989. teradataml/table_operators/templates/script_executor.template +170 -170
  990. teradataml/utils/dtypes.py +684 -684
  991. teradataml/utils/internal_buffer.py +84 -84
  992. teradataml/utils/print_versions.py +205 -205
  993. teradataml/utils/utils.py +410 -410
  994. teradataml/utils/validators.py +2239 -2115
  995. {teradataml-17.20.0.7.dist-info → teradataml-20.0.0.0.dist-info}/METADATA +270 -41
  996. teradataml-20.0.0.0.dist-info/RECORD +1038 -0
  997. {teradataml-17.20.0.7.dist-info → teradataml-20.0.0.0.dist-info}/WHEEL +1 -1
  998. {teradataml-17.20.0.7.dist-info → teradataml-20.0.0.0.dist-info}/zip-safe +1 -1
  999. teradataml/analytics/mle/AdaBoost.py +0 -651
  1000. teradataml/analytics/mle/AdaBoostPredict.py +0 -564
  1001. teradataml/analytics/mle/Antiselect.py +0 -342
  1002. teradataml/analytics/mle/Arima.py +0 -641
  1003. teradataml/analytics/mle/ArimaPredict.py +0 -477
  1004. teradataml/analytics/mle/Attribution.py +0 -1070
  1005. teradataml/analytics/mle/Betweenness.py +0 -658
  1006. teradataml/analytics/mle/Burst.py +0 -711
  1007. teradataml/analytics/mle/CCM.py +0 -600
  1008. teradataml/analytics/mle/CCMPrepare.py +0 -324
  1009. teradataml/analytics/mle/CFilter.py +0 -460
  1010. teradataml/analytics/mle/ChangePointDetection.py +0 -572
  1011. teradataml/analytics/mle/ChangePointDetectionRT.py +0 -477
  1012. teradataml/analytics/mle/Closeness.py +0 -737
  1013. teradataml/analytics/mle/ConfusionMatrix.py +0 -420
  1014. teradataml/analytics/mle/Correlation.py +0 -477
  1015. teradataml/analytics/mle/Correlation2.py +0 -573
  1016. teradataml/analytics/mle/CoxHazardRatio.py +0 -679
  1017. teradataml/analytics/mle/CoxPH.py +0 -556
  1018. teradataml/analytics/mle/CoxSurvival.py +0 -478
  1019. teradataml/analytics/mle/CumulativeMovAvg.py +0 -363
  1020. teradataml/analytics/mle/DTW.py +0 -623
  1021. teradataml/analytics/mle/DWT.py +0 -564
  1022. teradataml/analytics/mle/DWT2D.py +0 -599
  1023. teradataml/analytics/mle/DecisionForest.py +0 -716
  1024. teradataml/analytics/mle/DecisionForestEvaluator.py +0 -363
  1025. teradataml/analytics/mle/DecisionForestPredict.py +0 -561
  1026. teradataml/analytics/mle/DecisionTree.py +0 -830
  1027. teradataml/analytics/mle/DecisionTreePredict.py +0 -528
  1028. teradataml/analytics/mle/ExponentialMovAvg.py +0 -418
  1029. teradataml/analytics/mle/FMeasure.py +0 -402
  1030. teradataml/analytics/mle/FPGrowth.py +0 -734
  1031. teradataml/analytics/mle/FrequentPaths.py +0 -695
  1032. teradataml/analytics/mle/GLM.py +0 -558
  1033. teradataml/analytics/mle/GLML1L2.py +0 -547
  1034. teradataml/analytics/mle/GLML1L2Predict.py +0 -519
  1035. teradataml/analytics/mle/GLMPredict.py +0 -529
  1036. teradataml/analytics/mle/HMMDecoder.py +0 -945
  1037. teradataml/analytics/mle/HMMEvaluator.py +0 -901
  1038. teradataml/analytics/mle/HMMSupervised.py +0 -521
  1039. teradataml/analytics/mle/HMMUnsupervised.py +0 -572
  1040. teradataml/analytics/mle/Histogram.py +0 -561
  1041. teradataml/analytics/mle/IDWT.py +0 -476
  1042. teradataml/analytics/mle/IDWT2D.py +0 -493
  1043. teradataml/analytics/mle/IdentityMatch.py +0 -763
  1044. teradataml/analytics/mle/Interpolator.py +0 -918
  1045. teradataml/analytics/mle/KMeans.py +0 -485
  1046. teradataml/analytics/mle/KNN.py +0 -627
  1047. teradataml/analytics/mle/KNNRecommender.py +0 -488
  1048. teradataml/analytics/mle/KNNRecommenderPredict.py +0 -581
  1049. teradataml/analytics/mle/LAR.py +0 -439
  1050. teradataml/analytics/mle/LARPredict.py +0 -478
  1051. teradataml/analytics/mle/LDA.py +0 -548
  1052. teradataml/analytics/mle/LDAInference.py +0 -492
  1053. teradataml/analytics/mle/LDATopicSummary.py +0 -464
  1054. teradataml/analytics/mle/LevenshteinDistance.py +0 -450
  1055. teradataml/analytics/mle/LinReg.py +0 -433
  1056. teradataml/analytics/mle/LinRegPredict.py +0 -438
  1057. teradataml/analytics/mle/MinHash.py +0 -544
  1058. teradataml/analytics/mle/Modularity.py +0 -587
  1059. teradataml/analytics/mle/NEREvaluator.py +0 -410
  1060. teradataml/analytics/mle/NERExtractor.py +0 -595
  1061. teradataml/analytics/mle/NERTrainer.py +0 -458
  1062. teradataml/analytics/mle/NGrams.py +0 -570
  1063. teradataml/analytics/mle/NPath.py +0 -634
  1064. teradataml/analytics/mle/NTree.py +0 -549
  1065. teradataml/analytics/mle/NaiveBayes.py +0 -462
  1066. teradataml/analytics/mle/NaiveBayesPredict.py +0 -513
  1067. teradataml/analytics/mle/NaiveBayesTextClassifier.py +0 -607
  1068. teradataml/analytics/mle/NaiveBayesTextClassifier2.py +0 -531
  1069. teradataml/analytics/mle/NaiveBayesTextClassifierPredict.py +0 -799
  1070. teradataml/analytics/mle/NamedEntityFinder.py +0 -529
  1071. teradataml/analytics/mle/NamedEntityFinderEvaluator.py +0 -414
  1072. teradataml/analytics/mle/NamedEntityFinderTrainer.py +0 -396
  1073. teradataml/analytics/mle/POSTagger.py +0 -417
  1074. teradataml/analytics/mle/Pack.py +0 -411
  1075. teradataml/analytics/mle/PageRank.py +0 -535
  1076. teradataml/analytics/mle/PathAnalyzer.py +0 -426
  1077. teradataml/analytics/mle/PathGenerator.py +0 -367
  1078. teradataml/analytics/mle/PathStart.py +0 -464
  1079. teradataml/analytics/mle/PathSummarizer.py +0 -470
  1080. teradataml/analytics/mle/Pivot.py +0 -471
  1081. teradataml/analytics/mle/ROC.py +0 -425
  1082. teradataml/analytics/mle/RandomSample.py +0 -637
  1083. teradataml/analytics/mle/RandomWalkSample.py +0 -490
  1084. teradataml/analytics/mle/SAX.py +0 -779
  1085. teradataml/analytics/mle/SVMDense.py +0 -677
  1086. teradataml/analytics/mle/SVMDensePredict.py +0 -536
  1087. teradataml/analytics/mle/SVMDenseSummary.py +0 -437
  1088. teradataml/analytics/mle/SVMSparse.py +0 -557
  1089. teradataml/analytics/mle/SVMSparsePredict.py +0 -553
  1090. teradataml/analytics/mle/SVMSparseSummary.py +0 -435
  1091. teradataml/analytics/mle/Sampling.py +0 -549
  1092. teradataml/analytics/mle/Scale.py +0 -565
  1093. teradataml/analytics/mle/ScaleByPartition.py +0 -496
  1094. teradataml/analytics/mle/ScaleMap.py +0 -378
  1095. teradataml/analytics/mle/ScaleSummary.py +0 -320
  1096. teradataml/analytics/mle/SentenceExtractor.py +0 -363
  1097. teradataml/analytics/mle/SentimentEvaluator.py +0 -432
  1098. teradataml/analytics/mle/SentimentExtractor.py +0 -578
  1099. teradataml/analytics/mle/SentimentTrainer.py +0 -405
  1100. teradataml/analytics/mle/SeriesSplitter.py +0 -641
  1101. teradataml/analytics/mle/Sessionize.py +0 -475
  1102. teradataml/analytics/mle/SimpleMovAvg.py +0 -397
  1103. teradataml/analytics/mle/StringSimilarity.py +0 -425
  1104. teradataml/analytics/mle/TF.py +0 -389
  1105. teradataml/analytics/mle/TFIDF.py +0 -504
  1106. teradataml/analytics/mle/TextChunker.py +0 -414
  1107. teradataml/analytics/mle/TextClassifier.py +0 -399
  1108. teradataml/analytics/mle/TextClassifierEvaluator.py +0 -413
  1109. teradataml/analytics/mle/TextClassifierTrainer.py +0 -565
  1110. teradataml/analytics/mle/TextMorph.py +0 -494
  1111. teradataml/analytics/mle/TextParser.py +0 -623
  1112. teradataml/analytics/mle/TextTagger.py +0 -530
  1113. teradataml/analytics/mle/TextTokenizer.py +0 -502
  1114. teradataml/analytics/mle/UnivariateStatistics.py +0 -488
  1115. teradataml/analytics/mle/Unpack.py +0 -526
  1116. teradataml/analytics/mle/Unpivot.py +0 -438
  1117. teradataml/analytics/mle/VarMax.py +0 -776
  1118. teradataml/analytics/mle/VectorDistance.py +0 -762
  1119. teradataml/analytics/mle/WeightedMovAvg.py +0 -400
  1120. teradataml/analytics/mle/XGBoost.py +0 -842
  1121. teradataml/analytics/mle/XGBoostPredict.py +0 -627
  1122. teradataml/analytics/mle/__init__.py +0 -123
  1123. teradataml/analytics/mle/json/adaboost_mle.json +0 -135
  1124. teradataml/analytics/mle/json/adaboostpredict_mle.json +0 -85
  1125. teradataml/analytics/mle/json/antiselect_mle.json +0 -34
  1126. teradataml/analytics/mle/json/antiselect_mle_mle.json +0 -34
  1127. teradataml/analytics/mle/json/arima_mle.json +0 -172
  1128. teradataml/analytics/mle/json/arimapredict_mle.json +0 -52
  1129. teradataml/analytics/mle/json/attribution_mle_mle.json +0 -143
  1130. teradataml/analytics/mle/json/betweenness_mle.json +0 -97
  1131. teradataml/analytics/mle/json/burst_mle.json +0 -140
  1132. teradataml/analytics/mle/json/ccm_mle.json +0 -124
  1133. teradataml/analytics/mle/json/ccmprepare_mle.json +0 -14
  1134. teradataml/analytics/mle/json/cfilter_mle.json +0 -93
  1135. teradataml/analytics/mle/json/changepointdetection_mle.json +0 -92
  1136. teradataml/analytics/mle/json/changepointdetectionrt_mle.json +0 -78
  1137. teradataml/analytics/mle/json/closeness_mle.json +0 -104
  1138. teradataml/analytics/mle/json/confusionmatrix_mle.json +0 -79
  1139. teradataml/analytics/mle/json/correlation_mle.json +0 -86
  1140. teradataml/analytics/mle/json/correlationreduce_mle.json +0 -49
  1141. teradataml/analytics/mle/json/coxhazardratio_mle.json +0 -89
  1142. teradataml/analytics/mle/json/coxph_mle.json +0 -98
  1143. teradataml/analytics/mle/json/coxsurvival_mle.json +0 -79
  1144. teradataml/analytics/mle/json/cumulativemovavg_mle.json +0 -34
  1145. teradataml/analytics/mle/json/decisionforest_mle.json +0 -167
  1146. teradataml/analytics/mle/json/decisionforestevaluator_mle.json +0 -33
  1147. teradataml/analytics/mle/json/decisionforestpredict_mle_mle.json +0 -74
  1148. teradataml/analytics/mle/json/decisiontree_mle.json +0 -194
  1149. teradataml/analytics/mle/json/decisiontreepredict_mle_mle.json +0 -86
  1150. teradataml/analytics/mle/json/dtw_mle.json +0 -97
  1151. teradataml/analytics/mle/json/dwt2d_mle.json +0 -116
  1152. teradataml/analytics/mle/json/dwt_mle.json +0 -101
  1153. teradataml/analytics/mle/json/exponentialmovavg_mle.json +0 -55
  1154. teradataml/analytics/mle/json/fmeasure_mle.json +0 -58
  1155. teradataml/analytics/mle/json/fpgrowth_mle.json +0 -159
  1156. teradataml/analytics/mle/json/frequentpaths_mle.json +0 -129
  1157. teradataml/analytics/mle/json/glm_mle.json +0 -111
  1158. teradataml/analytics/mle/json/glml1l2_mle.json +0 -106
  1159. teradataml/analytics/mle/json/glml1l2predict_mle.json +0 -57
  1160. teradataml/analytics/mle/json/glmpredict_mle_mle.json +0 -74
  1161. teradataml/analytics/mle/json/histogram_mle.json +0 -100
  1162. teradataml/analytics/mle/json/hmmdecoder_mle.json +0 -192
  1163. teradataml/analytics/mle/json/hmmevaluator_mle.json +0 -206
  1164. teradataml/analytics/mle/json/hmmsupervised_mle.json +0 -91
  1165. teradataml/analytics/mle/json/hmmunsupervised_mle.json +0 -114
  1166. teradataml/analytics/mle/json/identitymatch_mle.json +0 -88
  1167. teradataml/analytics/mle/json/idwt2d_mle.json +0 -73
  1168. teradataml/analytics/mle/json/idwt_mle.json +0 -66
  1169. teradataml/analytics/mle/json/interpolator_mle.json +0 -151
  1170. teradataml/analytics/mle/json/kmeans_mle.json +0 -97
  1171. teradataml/analytics/mle/json/knn_mle.json +0 -141
  1172. teradataml/analytics/mle/json/knnrecommender_mle.json +0 -111
  1173. teradataml/analytics/mle/json/knnrecommenderpredict_mle.json +0 -75
  1174. teradataml/analytics/mle/json/lar_mle.json +0 -78
  1175. teradataml/analytics/mle/json/larpredict_mle.json +0 -69
  1176. teradataml/analytics/mle/json/lda_mle.json +0 -130
  1177. teradataml/analytics/mle/json/ldainference_mle.json +0 -78
  1178. teradataml/analytics/mle/json/ldatopicsummary_mle.json +0 -64
  1179. teradataml/analytics/mle/json/levenshteindistance_mle.json +0 -92
  1180. teradataml/analytics/mle/json/linreg_mle.json +0 -42
  1181. teradataml/analytics/mle/json/linregpredict_mle.json +0 -56
  1182. teradataml/analytics/mle/json/minhash_mle.json +0 -113
  1183. teradataml/analytics/mle/json/modularity_mle.json +0 -91
  1184. teradataml/analytics/mle/json/naivebayespredict_mle_mle.json +0 -85
  1185. teradataml/analytics/mle/json/naivebayesreduce_mle.json +0 -52
  1186. teradataml/analytics/mle/json/naivebayestextclassifierpredict_mle_mle.json +0 -147
  1187. teradataml/analytics/mle/json/naivebayestextclassifiertrainer2_mle.json +0 -108
  1188. teradataml/analytics/mle/json/naivebayestextclassifiertrainer_mle.json +0 -102
  1189. teradataml/analytics/mle/json/namedentityfinder_mle.json +0 -84
  1190. teradataml/analytics/mle/json/namedentityfinderevaluatorreduce_mle.json +0 -43
  1191. teradataml/analytics/mle/json/namedentityfindertrainer_mle.json +0 -64
  1192. teradataml/analytics/mle/json/nerevaluator_mle.json +0 -54
  1193. teradataml/analytics/mle/json/nerextractor_mle.json +0 -87
  1194. teradataml/analytics/mle/json/nertrainer_mle.json +0 -89
  1195. teradataml/analytics/mle/json/ngrams_mle.json +0 -137
  1196. teradataml/analytics/mle/json/ngramsplitter_mle_mle.json +0 -137
  1197. teradataml/analytics/mle/json/npath@coprocessor_mle.json +0 -73
  1198. teradataml/analytics/mle/json/ntree@coprocessor_mle.json +0 -123
  1199. teradataml/analytics/mle/json/pack_mle.json +0 -58
  1200. teradataml/analytics/mle/json/pack_mle_mle.json +0 -58
  1201. teradataml/analytics/mle/json/pagerank_mle.json +0 -81
  1202. teradataml/analytics/mle/json/pathanalyzer_mle.json +0 -63
  1203. teradataml/analytics/mle/json/pathgenerator_mle.json +0 -40
  1204. teradataml/analytics/mle/json/pathstart_mle.json +0 -62
  1205. teradataml/analytics/mle/json/pathsummarizer_mle.json +0 -72
  1206. teradataml/analytics/mle/json/pivoting_mle.json +0 -71
  1207. teradataml/analytics/mle/json/postagger_mle.json +0 -51
  1208. teradataml/analytics/mle/json/randomsample_mle.json +0 -131
  1209. teradataml/analytics/mle/json/randomwalksample_mle.json +0 -85
  1210. teradataml/analytics/mle/json/roc_mle.json +0 -73
  1211. teradataml/analytics/mle/json/sampling_mle.json +0 -75
  1212. teradataml/analytics/mle/json/sax_mle.json +0 -154
  1213. teradataml/analytics/mle/json/scale_mle.json +0 -93
  1214. teradataml/analytics/mle/json/scalebypartition_mle.json +0 -89
  1215. teradataml/analytics/mle/json/scalemap_mle.json +0 -44
  1216. teradataml/analytics/mle/json/scalesummary_mle.json +0 -14
  1217. teradataml/analytics/mle/json/sentenceextractor_mle.json +0 -41
  1218. teradataml/analytics/mle/json/sentimentevaluator_mle.json +0 -43
  1219. teradataml/analytics/mle/json/sentimentextractor_mle.json +0 -100
  1220. teradataml/analytics/mle/json/sentimenttrainer_mle.json +0 -68
  1221. teradataml/analytics/mle/json/seriessplitter_mle.json +0 -133
  1222. teradataml/analytics/mle/json/sessionize_mle_mle.json +0 -62
  1223. teradataml/analytics/mle/json/simplemovavg_mle.json +0 -48
  1224. teradataml/analytics/mle/json/stringsimilarity_mle.json +0 -50
  1225. teradataml/analytics/mle/json/stringsimilarity_mle_mle.json +0 -50
  1226. teradataml/analytics/mle/json/svmdense_mle.json +0 -165
  1227. teradataml/analytics/mle/json/svmdensepredict_mle.json +0 -95
  1228. teradataml/analytics/mle/json/svmdensesummary_mle.json +0 -58
  1229. teradataml/analytics/mle/json/svmsparse_mle.json +0 -148
  1230. teradataml/analytics/mle/json/svmsparsepredict_mle_mle.json +0 -103
  1231. teradataml/analytics/mle/json/svmsparsesummary_mle.json +0 -57
  1232. teradataml/analytics/mle/json/textchunker_mle.json +0 -40
  1233. teradataml/analytics/mle/json/textclassifier_mle.json +0 -51
  1234. teradataml/analytics/mle/json/textclassifierevaluator_mle.json +0 -43
  1235. teradataml/analytics/mle/json/textclassifiertrainer_mle.json +0 -103
  1236. teradataml/analytics/mle/json/textmorph_mle.json +0 -63
  1237. teradataml/analytics/mle/json/textparser_mle.json +0 -166
  1238. teradataml/analytics/mle/json/texttagger_mle.json +0 -81
  1239. teradataml/analytics/mle/json/texttokenizer_mle.json +0 -91
  1240. teradataml/analytics/mle/json/tf_mle.json +0 -33
  1241. teradataml/analytics/mle/json/tfidf_mle.json +0 -34
  1242. teradataml/analytics/mle/json/univariatestatistics_mle.json +0 -81
  1243. teradataml/analytics/mle/json/unpack_mle.json +0 -91
  1244. teradataml/analytics/mle/json/unpack_mle_mle.json +0 -91
  1245. teradataml/analytics/mle/json/unpivoting_mle.json +0 -63
  1246. teradataml/analytics/mle/json/varmax_mle.json +0 -176
  1247. teradataml/analytics/mle/json/vectordistance_mle.json +0 -179
  1248. teradataml/analytics/mle/json/weightedmovavg_mle.json +0 -48
  1249. teradataml/analytics/mle/json/xgboost_mle.json +0 -178
  1250. teradataml/analytics/mle/json/xgboostpredict_mle.json +0 -104
  1251. teradataml/analytics/sqle/Antiselect.py +0 -321
  1252. teradataml/analytics/sqle/Attribution.py +0 -603
  1253. teradataml/analytics/sqle/DecisionForestPredict.py +0 -408
  1254. teradataml/analytics/sqle/GLMPredict.py +0 -430
  1255. teradataml/analytics/sqle/MovingAverage.py +0 -543
  1256. teradataml/analytics/sqle/NGramSplitter.py +0 -548
  1257. teradataml/analytics/sqle/NPath.py +0 -632
  1258. teradataml/analytics/sqle/NaiveBayesTextClassifierPredict.py +0 -515
  1259. teradataml/analytics/sqle/Pack.py +0 -388
  1260. teradataml/analytics/sqle/SVMSparsePredict.py +0 -464
  1261. teradataml/analytics/sqle/Sessionize.py +0 -390
  1262. teradataml/analytics/sqle/StringSimilarity.py +0 -400
  1263. teradataml/analytics/sqle/Unpack.py +0 -503
  1264. teradataml/analytics/sqle/json/antiselect_sqle.json +0 -21
  1265. teradataml/analytics/sqle/json/attribution_sqle.json +0 -92
  1266. teradataml/analytics/sqle/json/decisionforestpredict_sqle.json +0 -48
  1267. teradataml/analytics/sqle/json/glmpredict_sqle.json +0 -48
  1268. teradataml/analytics/sqle/json/h2opredict_sqle.json +0 -63
  1269. teradataml/analytics/sqle/json/movingaverage_sqle.json +0 -58
  1270. teradataml/analytics/sqle/json/naivebayestextclassifierpredict_sqle.json +0 -76
  1271. teradataml/analytics/sqle/json/ngramsplitter_sqle.json +0 -126
  1272. teradataml/analytics/sqle/json/npath_sqle.json +0 -67
  1273. teradataml/analytics/sqle/json/pack_sqle.json +0 -47
  1274. teradataml/analytics/sqle/json/pmmlpredict_sqle.json +0 -55
  1275. teradataml/analytics/sqle/json/sessionize_sqle.json +0 -43
  1276. teradataml/analytics/sqle/json/stringsimilarity_sqle.json +0 -39
  1277. teradataml/analytics/sqle/json/svmsparsepredict_sqle.json +0 -74
  1278. teradataml/analytics/sqle/json/unpack_sqle.json +0 -80
  1279. teradataml/catalog/model_cataloging.py +0 -980
  1280. teradataml/config/mlengine_alias_definitions_v1.0 +0 -118
  1281. teradataml/config/mlengine_alias_definitions_v1.1 +0 -127
  1282. teradataml/config/mlengine_alias_definitions_v1.3 +0 -129
  1283. teradataml/table_operators/sandbox_container_util.py +0 -643
  1284. teradataml-17.20.0.7.dist-info/RECORD +0 -1280
  1285. {teradataml-17.20.0.7.dist-info → teradataml-20.0.0.0.dist-info}/top_level.txt +0 -0
@@ -1,1982 +1,1719 @@
1
- #!/usr/bin/python
2
- # ##################################################################
3
- #
4
- # Copyright 2020 Teradata. All rights reserved. #
5
- # TERADATA CONFIDENTIAL AND TRADE SECRET #
6
- #
7
- # Primary Owner: Gouri Patwardhan (gouri.patwardhan@teradata.com) #
8
- # Secondary Owner: Trupti Purohit (trupti.purohit@teradata.com) #
9
- #
10
- # Function Version: 1.0 #
11
- #
12
- # Description: Script is a TeradataML wrapper around Teradata's #
13
- # Script Table Operator #
14
- # ##################################################################
15
-
16
- import os
17
- import tarfile
18
- import teradataml.dataframe as tdmldf
19
- import subprocess
20
- import sys
21
- from pathlib import Path
22
- from datetime import datetime
23
- from io import StringIO
24
- from teradataml.common.constants import TableOperatorConstants
25
- from teradataml.common.wrapper_utils import AnalyticsWrapperUtils
26
- from teradataml.common.utils import UtilFuncs
27
- from teradataml.common.constants import OutputStyle, TeradataConstants
28
- from teradataml.context.context import _get_current_databasename
29
- from teradataml.context.context import get_context, get_connection
30
- from teradataml.dataframe.dataframe_utils import DataFrameUtils as df_utils
31
- from teradataml.common.garbagecollector import GarbageCollector
32
- from teradataml.dbutils.filemgr import install_file
33
- from teradataml.dbutils.filemgr import remove_file
34
- from teradataml.table_operators.table_operator_query_generator import \
35
- TableOperatorQueryGenerator
36
- from teradataml.common.exceptions import TeradataMlException
37
- from teradataml.common.messages import Messages
38
- from teradataml.common.messagecodes import MessageCodes
39
- from teradataml.utils.validators import _Validators
40
- from teradataml.options.display import display
41
- from teradataml.options.configure import configure
42
- from teradataml.table_operators.sandbox_container_util import setup_sandbox_env
43
- from teradataml.utils.utils import execute_sql
44
- from teradatasqlalchemy.dialect import dialect as td_dialect
45
- from teradatasqlalchemy import (BYTEINT, SMALLINT, INTEGER, BIGINT, DECIMAL, FLOAT,
46
- NUMBER)
47
- from teradatasqlalchemy import (TIMESTAMP, DATE, TIME)
48
- from teradatasqlalchemy import (CHAR, VARCHAR, CLOB)
49
- from teradatasqlalchemy import (BYTE, VARBYTE, BLOB)
50
- from teradatasqlalchemy import (PERIOD_DATE, PERIOD_TIME, PERIOD_TIMESTAMP)
51
- from teradatasqlalchemy import (INTERVAL_YEAR, INTERVAL_YEAR_TO_MONTH, INTERVAL_MONTH,
52
- INTERVAL_DAY,INTERVAL_DAY_TO_HOUR, INTERVAL_DAY_TO_MINUTE,
53
- INTERVAL_DAY_TO_SECOND, INTERVAL_HOUR,
54
- INTERVAL_HOUR_TO_MINUTE, INTERVAL_HOUR_TO_SECOND,
55
- INTERVAL_MINUTE, INTERVAL_MINUTE_TO_SECOND,
56
- INTERVAL_SECOND)
57
- from teradataml.table_operators.TableOperator import TableOperator
58
-
59
- class Script(TableOperator):
60
-
61
- def __init__(self,
62
- data=None,
63
- script_name=None,
64
- files_local_path=None,
65
- script_command=None,
66
- delimiter="\t",
67
- returns=None,
68
- auth=None,
69
- charset=None,
70
- quotechar=None,
71
- data_partition_column=None,
72
- data_hash_column=None,
73
- data_order_column=None,
74
- is_local_order=False,
75
- sort_ascending=True,
76
- nulls_first=True):
77
- """
78
- DESCRIPTION:
79
- The Script table operator function executes a user-installed script or
80
- any LINUX command inside database on Teradata Vantage.
81
-
82
- PARAMETERS:
83
- script_command:
84
- Required Argument.
85
- Specifies the command/script to run.
86
- Types: str
87
-
88
- script_name:
89
- Required Argument.
90
- Specifies the name of user script.
91
- User script should have at least permissions of mode 644.
92
- Types: str
93
-
94
- files_local_path:
95
- Required Argument.
96
- Specifies the absolute local path where user script and all supporting
97
- files like model files, input data file reside.
98
- Types: str
99
-
100
- returns:
101
- Required Argument.
102
- Specifies output column definition.
103
- Types: Dictionary specifying column name to teradatasqlalchemy type mapping.
104
- Default: None
105
- Note:
106
- User can pass a dictionary (dict or OrderedDict) to the "returns" argument,
107
- with the keys ordered to represent the order of the output columns.
108
- Preferred type is OrderedDict.
109
-
110
- data:
111
- Optional Argument.
112
- Specifies a teradataml DataFrame containing the input data for the
113
- script.
114
-
115
- data_hash_column:
116
- Optional Argument.
117
- Specifies the column to be used for hashing.
118
- The rows in the data are redistributed to AMPs based on the hash value of
119
- the column specified.
120
- The user-installed script file then runs once on each AMP.
121
- If there is no "data_partition_column", then the entire result set,
122
- delivered by the function, constitutes a single group or partition.
123
- Types: str
124
- Note:
125
- "data_hash_column" can not be specified along with
126
- "data_partition_column", "is_local_order" and "data_order_column".
127
-
128
- data_partition_column:
129
- Optional Argument.
130
- Specifies Partition By columns for "data".
131
- Values to this argument can be provided as a list, if multiple
132
- columns are used for partition.
133
- Default Value: ANY
134
- Types: str OR list of Strings (str)
135
- Note:
136
- 1) "data_partition_column" can not be specified along with
137
- "data_hash_column".
138
- 2) "data_partition_column" can not be specified along with
139
- "is_local_order = True".
140
-
141
- is_local_order:
142
- Optional Argument.
143
- Specifies a boolean value to determine whether the input data is to be
144
- ordered locally or not. Order by specifies the order in which the
145
- values in a group, or partition, are sorted. Local Order By specifies
146
- orders qualified rows on each AMP in preparation to be input to a table
147
- function. This argument is ignored, if "data_order_column" is None. When
148
- set to True, data is ordered locally.
149
- Default Value: False
150
- Types: bool
151
- Note:
152
- 1) "is_local_order" can not be specified along with "data_hash_column".
153
- 2) When "is_local_order" is set to True, "data_order_column" should be
154
- specified, and the columns specified in "data_order_column" are
155
- used for local ordering.
156
-
157
- data_order_column:
158
- Optional Argument.
159
- Specifies Order By columns for "data".
160
- Values to this argument can be provided as a list, if multiple
161
- columns are used for ordering. This argument is used with in both cases:
162
- "is_local_order = True" and "is_local_order = False".
163
- Types: str OR list of Strings (str)
164
- Note:
165
- "data_order_column" can not be specified along with "data_hash_column".
166
-
167
- sort_ascending:
168
- Optional Argument.
169
- Specifies a boolean value to determine if the result set is to be sorted
170
- on the "data_order_column" column in ascending or descending order.
171
- The sorting is ascending when this argument is set to True, and descending
172
- when set to False. This argument is ignored, if "data_order_column" is
173
- None.
174
- Default Value: True
175
- Types: bool
176
-
177
- nulls_first:
178
- Optional Argument.
179
- Specifies a boolean value to determine whether NULLS are listed first or
180
- last during ordering. This argument is ignored, if "data_order_column" is
181
- None. NULLS are listed first when this argument is set to True, and last
182
- when set to False.
183
- Default Value: True
184
- Types: bool
185
-
186
- delimiter:
187
- Optional Argument.
188
- Specifies a delimiter to use when reading columns from a row and
189
- writing result columns.
190
- Default Value: "\t" (tab)
191
- Types: str of length 1 character
192
- Notes:
193
- 1) This argument cannot be same as "quotechar" argument.
194
- 2) This argument cannot be a newline character i.e., '\\n'.
195
-
196
- auth:
197
- Optional Argument.
198
- Specifies an authorization to use when running the script.
199
- Types: str
200
-
201
- charset:
202
- Optional Argument.
203
- Specifies the character encoding for data.
204
- Permitted Values: utf-16, latin
205
- Types: str
206
-
207
- quotechar:
208
- Optional Argument.
209
- Specifies a character that forces all input and output of the script
210
- to be quoted using this specified character.
211
- Using this argument enables the Advanced SQL Engine to distinguish
212
- between NULL fields and empty strings. A string with length zero is
213
- quoted, while NULL fields are not.
214
- If this character is found in the data, it will be escaped by a second
215
- quote character.
216
- Types: character of length 1
217
- Notes:
218
- 1) This argument cannot be same as "delimiter" argument.
219
- 2) This argument cannot be a newline character i.e., '\\n'.
220
-
221
- RETURNS:
222
- Script Object
223
-
224
- RAISES:
225
- TeradataMlException
226
-
227
- EXAMPLES:
228
- # Note - Refer to User Guide for setting search path and required permissions.
229
- # Load example data.
230
- load_example_data("Script", ["barrier"])
231
-
232
- # Example - The script mapper.py reads in a line of text input
233
- # ("Old Macdonald Had A Farm") from csv and splits the line into individual
234
- # words, emitting a new row for each word.
235
-
236
- # Create teradataml DataFrame objects.
237
- >>> barrierdf = DataFrame.from_table("barrier")
238
-
239
- # Set SEARCHUIFDBPATH.
240
- >>> execute_sql("SET SESSION SEARCHUIFDBPATH = alice;")
241
-
242
- # Create a Script object that allows us to execute script on Vantage.
243
- >>> import teradataml, os
244
- >>> from teradatasqlalchemy import VARCHAR
245
- >>> td_path = os.path.dirname(teradataml.__file__)
246
- >>> sto = Script(data=barrierdf,
247
- ... script_name='mapper.py',
248
- ... files_local_path= os.path.join(td_path, 'data', 'scripts'),
249
- ... script_command='tdpython3 ./alice/mapper.py',
250
- ... data_order_column="Id",
251
- ... is_local_order=False,
252
- ... nulls_first=False,
253
- ... sort_ascending=False,
254
- ... charset='latin',
255
- ... returns=OrderedDict([("word", VARCHAR(15)),("count_input", VARCHAR(2))]))
256
-
257
- # Run user script locally within docker container and using data from csv.
258
- # This helps the user to fix script level issues outside Vantage.
259
- # Setup the environment by providing local path to docker image file.
260
- >>> sto.setup_sto_env(docker_image_location='/tmp/sto_sandbox_docker_image.tar'))
261
- Loading image from /tmp/sto_sandbox_docker_image.tar. It may take few minutes.
262
- Image loaded successfully.
263
- Starting a container for stosandbox:1.0 image.
264
- Container d7c73cb498c79a082180576bb5b10bb07b52efdd3026856146fc15e91147b19f
265
- started successfully.
266
-
267
- >>> sto.test_script(input_data_file='../barrier.csv', data_file_delimiter=',')
268
-
269
- ############ STDOUT Output ############
270
-
271
- word count_input
272
- 0 1 1
273
- 1 Old 1
274
- 2 Macdonald 1
275
- 3 Had 1
276
- 4 A 1
277
- 5 Farm 1
278
- >>>
279
-
280
- # Script results look good. Now install file on Vantage.
281
- >>> sto.install_file(file_identifier='mapper',
282
- ... file_name='mapper.py',
283
- ... is_binary=False)
284
- File mapper.py installed in Vantage
285
-
286
- # Execute the user script on Vantage.
287
- >>> sto.execute_script()
288
- ############ STDOUT Output ############
289
-
290
- word count_input
291
- 0 Macdonald 1
292
- 1 A 1
293
- 2 Farm 1
294
- 3 Had 1
295
- 4 Old 1
296
- 5 1 1
297
-
298
- # Remove the installed file from Vantage.
299
- >>> sto.remove_file(file_identifier='mapper', force_remove=True)
300
- File mapper removed from Vantage
301
- """
302
- self.result = None
303
- self.data = data
304
- self.script_name = script_name
305
- self.files_local_path = files_local_path
306
- self.script_command = script_command
307
- self.delimiter = delimiter
308
- self.returns = returns
309
- self.auth = auth
310
- self.charset = charset
311
- self.quotechar = quotechar
312
- self.data_partition_column = data_partition_column
313
- self.data_hash_column = data_hash_column
314
- self.data_order_column = data_order_column
315
- self.is_local_order = is_local_order
316
- self.sort_ascending = sort_ascending
317
- self.nulls_first = nulls_first
318
- self._check_reserved_keyword = True
319
-
320
- # Create AnalyticsWrapperUtils instance which contains validation functions.
321
- # This is required for is_default_or_not check.
322
- # Rest all validation is done using _Validators
323
- self.__awu = AnalyticsWrapperUtils()
324
-
325
- # Below matrix is a list of lists, where in each row contains following elements:
326
- # Let's take an example of following, just to get an idea:
327
- # [element1, element2, element3, element4, element5, element6]
328
- # e.g.
329
- # ["join", join, True, (str), True, concat_join_permitted_values]
330
-
331
- # 1. element1 --> Argument Name, a string. ["join" in above example.]
332
- # 2. element2 --> Argument itself. [join]
333
- # 3. element3 --> Specifies a flag that mentions argument is optional or not.
334
- # False, means required and True means optional.
335
- # 4. element4 --> Tuple of accepted types. (str) in above example.
336
- # 5. element5 --> True, means validate for empty value. Error will be raised,
337
- # if empty value is passed.
338
- # If not specified, means same as specifying False.
339
- # 6. element6 --> A list of permitted values, an argument can accept.
340
- # If not specified, it is as good as passing None.
341
- # If a list is passed, validation will be
342
- # performed for permitted values.
343
-
344
- self.awu_matrix = []
345
- self.awu_matrix.append(["data", self.data, True, (tdmldf.dataframe.DataFrame)])
346
- self.awu_matrix.append(["data_partition_column", self.data_partition_column, True,
347
- (str, list), True])
348
- self.awu_matrix.append(["data_hash_column", self.data_hash_column, True,
349
- (str, list), True])
350
- self.awu_matrix.append(["data_order_column", self.data_order_column, True,
351
- (str, list), True])
352
- self.awu_matrix.append(["is_local_order", self.is_local_order, True, (bool)])
353
- self.awu_matrix.append(["sort_ascending", self.sort_ascending, True, (bool)])
354
- self.awu_matrix.append(["nulls_first", self.nulls_first, True, (bool)])
355
- self.awu_matrix.append(["script_command", self.script_command, False, (str),
356
- True])
357
- self.awu_matrix.append(["script_name", self.script_name, True, (str), True])
358
- self.awu_matrix.append(["files_local_path", self.files_local_path, True, (str),
359
- True])
360
- self.awu_matrix.append(["delimiter", self.delimiter, True, (str), False])
361
- self.awu_matrix.append(["returns", self.returns, False, (dict), True])
362
- self.awu_matrix.append(["auth", self.auth, True, (str), True])
363
- self.awu_matrix.append(["charset", self.charset, True, (str), True,
364
- ["utf-16", "latin"]])
365
- self.awu_matrix.append(["quotechar", self.quotechar, True, (str), False])
366
-
367
- # Perform the function validations
368
- self.__validate()
369
-
370
- @property
371
- def check_reserved_keyword(self):
372
- """
373
- DESCRIPTION:
374
- Getter for self._check_reserved_keyword.
375
-
376
- RETURNS:
377
- bool
378
-
379
- RAISES:
380
- None
381
- """
382
- return self._check_reserved_keyword
383
-
384
- @check_reserved_keyword.setter
385
- def check_reserved_keyword(self, flag):
386
- """
387
- DESCRIPTION:
388
- Setter for self._check_reserved_keyword
389
-
390
- RETURNS:
391
- None
392
-
393
- RAISES:
394
- None
395
- """
396
- self._check_reserved_keyword = flag
397
-
398
- def __validate_for_reserved_keyword(self):
399
- """
400
- DESCRIPTION:
401
- Function to validate if the returns clause has teradata reserved keyword or not.
402
- If it contains reserved keyword, then raise an error.
403
-
404
- RETURNS:
405
- None
406
-
407
- RAISES:
408
- TeradataMlException
409
-
410
- """
411
- if self.check_reserved_keyword:
412
- from teradataml import list_td_reserved_keywords
413
- if get_connection():
414
- # Checking for reserved keywords and raising error if present.
415
- for column_name in self.returns:
416
- list_td_reserved_keywords(key=column_name, raise_error=True)
417
-
418
- def __validate(self):
419
- """
420
- Function to validate Table Operator Function arguments, which verifies missing
421
- arguments, input argument and table types. Also processes the argument values.
422
- """
423
- # Make sure that a non-NULL value has been supplied for all mandatory arguments
424
- _Validators._validate_missing_required_arguments(self.awu_matrix)
425
-
426
- # Validate argument types.
427
- _Validators._validate_function_arguments(self.awu_matrix,
428
- skip_empty_check={"quotechar" : ["\n", "\t"],
429
- "delimiter" : ["\n"]})
430
-
431
- # permissible_datatypes in returns
432
- allowed_datatypes = (BYTEINT, SMALLINT, INTEGER, BIGINT, DECIMAL, FLOAT, NUMBER,
433
- TIMESTAMP, DATE, TIME, CHAR, VARCHAR, CLOB, BYTE, VARBYTE,
434
- BLOB, PERIOD_DATE, PERIOD_TIME, PERIOD_TIMESTAMP,
435
- INTERVAL_YEAR, INTERVAL_YEAR_TO_MONTH, INTERVAL_MONTH,
436
- INTERVAL_DAY, INTERVAL_DAY_TO_HOUR, INTERVAL_DAY_TO_MINUTE,
437
- INTERVAL_DAY_TO_SECOND, INTERVAL_HOUR,
438
- INTERVAL_HOUR_TO_MINUTE, INTERVAL_HOUR_TO_SECOND,
439
- INTERVAL_MINUTE, INTERVAL_MINUTE_TO_SECOND, INTERVAL_SECOND
440
- )
441
-
442
- # Validate keys and datatypes in returns.
443
- self.awu_matrix_returns = []
444
- for key in self.returns.keys():
445
- self.awu_matrix_returns.append(["keys in returns", key, False, (str), True])
446
- self.awu_matrix_returns.append(["value in returns", self.returns[key], False,
447
- allowed_datatypes])
448
-
449
-
450
- _Validators._validate_function_arguments(self.awu_matrix_returns)
451
-
452
- if self.data is not None:
453
- # Hash and order by can be used together as long as is_local_order = True.
454
- if all([self.data_hash_column,
455
- self.data_order_column]) and not self.is_local_order:
456
- raise TeradataMlException(
457
- Messages.get_message(MessageCodes.CANNOT_USE_TOGETHER_WITH,
458
- "data_hash_column' and 'data_order_column",
459
- "is_local_order=False"),
460
- MessageCodes.CANNOT_USE_TOGETHER_WITH)
461
-
462
- # Either hash or partition can be used.
463
- _Validators._validate_mutually_exclusive_arguments(self.data_hash_column,
464
- "data_hash_column",
465
- self.data_partition_column,
466
- "data_partition_column",
467
- skip_all_none_check=True)
468
-
469
- # Either local order by or partition by can be used.
470
- _Validators._validate_mutually_exclusive_arguments(self.is_local_order,
471
- "is_local_order=True",
472
- self.data_partition_column,
473
- "data_partition_column",
474
- skip_all_none_check=True)
475
-
476
- # local order by requires column name.
477
- if self.is_local_order and self.data_order_column is None:
478
- message = Messages.get_message(MessageCodes.DEPENDENT_ARG_MISSING,
479
- "data_order_column", "is_local_order=True")
480
- raise TeradataMlException(message, MessageCodes.DEPENDENT_ARG_MISSING)
481
-
482
- if self.__awu._is_default_or_not(self.data_partition_column, "ANY"):
483
- self.__awu._validate_dataframe_has_argument_columns(
484
- self.data_partition_column, "data_partition_column", self.data, "data", True)
485
-
486
- if self.data_order_column is not None:
487
- self.__awu._validate_dataframe_has_argument_columns(
488
- self.data_order_column, "data_order_column",self.data, "data", False)
489
-
490
- if self.data_hash_column is not None:
491
- self.__awu._validate_dataframe_has_argument_columns(
492
- self.data_hash_column, "data_hash_column", self.data, "data", False)
493
-
494
- if self.data_partition_column is not None:
495
- self.__awu._validate_dataframe_has_argument_columns(
496
- self.data_partition_column, "data_partition_column", self.data, "data", False)
497
-
498
- # Check for length of the arguments "delimiter" and "quotechar".
499
- if self.delimiter is not None:
500
- _Validators._validate_str_arg_length('delimiter', self.delimiter, 'EQ', 1)
501
-
502
- if self.quotechar is not None:
503
- _Validators._validate_str_arg_length('quotechar', self.quotechar, 'EQ', 1)
504
-
505
- # The arguments 'quotechar' and 'delimiter' cannot take newline character.
506
- if self.delimiter == '\n':
507
- message = Messages.get_message(MessageCodes.NOT_ALLOWED_VALUES, "\n", "delimiter")
508
- raise TeradataMlException(message, MessageCodes.NOT_ALLOWED_VALUES)
509
- if self.quotechar == '\n':
510
- message = Messages.get_message(MessageCodes.NOT_ALLOWED_VALUES, "\n", "quotechar")
511
- raise TeradataMlException(message, MessageCodes.NOT_ALLOWED_VALUES)
512
-
513
- # The arguments 'quotechar' and 'delimiter' cannot have the same value.
514
- if self.delimiter == self.quotechar:
515
- message = Messages.get_message(MessageCodes.ARGUMENT_VALUE_SAME, "delimiter",
516
- "quotechar")
517
- raise TeradataMlException(message, MessageCodes.ARGUMENT_VALUE_SAME)
518
-
519
-
520
- def set_data(self,
521
- data,
522
- data_partition_column=None,
523
- data_hash_column=None,
524
- data_order_column=None,
525
- is_local_order=False,
526
- sort_ascending=True,
527
- nulls_first=True):
528
- """
529
- DESCRIPTION:
530
- Function enables user to set data and data related arguments without having to
531
- re-create Script object.
532
-
533
- PARAMETERS:
534
- data:
535
- Required Argument.
536
- Specifies a teradataml DataFrame containing the input data for the script.
537
-
538
- data_hash_column:
539
- Optional Argument.
540
- Specifies the column to be used for hashing.
541
- The rows in the data are redistributed to AMPs based on the
542
- hash value of the column specified.
543
- The user installed script then runs once on each AMP.
544
- If there is no data_partition_column, then the entire result set delivered
545
- by the function, constitutes a single group or partition.
546
- Types: str
547
- Note:
548
- "data_hash_column" can not be specified along with
549
- "data_partition_column", "is_local_order" and "data_order_column".
550
-
551
- data_partition_column:
552
- Optional Argument.
553
- Specifies Partition By columns for data.
554
- Values to this argument can be provided as a list, if multiple
555
- columns are used for partition.
556
- Default Value: ANY
557
- Types: str OR list of Strings (str)
558
- Note:
559
- 1) "data_partition_column" can not be specified along with
560
- "data_hash_column".
561
- 2) "data_partition_column" can not be specified along with
562
- "is_local_order = True".
563
-
564
- is_local_order:
565
- Optional Argument.
566
- Specifies a boolean value to determine whether the input data is to be
567
- ordered locally or not. Order by specifies the order in which the
568
- values in a group or partition are sorted. Local Order By specifies
569
- orders qualified rows on each AMP in preparation to be input to a table
570
- function. This argument is ignored, if "data_order_column" is None. When
571
- set to True, data is ordered locally.
572
- Default Value: False
573
- Types: bool
574
- Note:
575
- 1) "is_local_order" can not be specified along with
576
- "data_hash_column".
577
- 2) When "is_local_order" is set to True, "data_order_column" should be
578
- specified, and the columns specified in "data_order_column" are
579
- used for local ordering.
580
-
581
- data_order_column:
582
- Optional Argument.
583
- Specifies Order By columns for data.
584
- Values to this argument can be provided as a list, if multiple
585
- columns are used for ordering.
586
- This argument is used in both cases:
587
- "is_local_order = True" and "is_local_order = False".
588
- Types: str OR list of Strings (str)
589
- Note:
590
- "data_order_column" can not be specified along with
591
- "data_hash_column".
592
-
593
- sort_ascending:
594
- Optional Argument.
595
- Specifies a boolean value to determine if the result set is to be sorted
596
- on the column specified in "data_order_column", in ascending or descending
597
- order.
598
- The sorting is ascending when this argument is set to True, and descending
599
- when set to False.
600
- This argument is ignored, if "data_order_column" is None.
601
- Default Value: True
602
- Types: bool
603
-
604
- nulls_first:
605
- Optional Argument.
606
- Specifies a boolean value to determine whether NULLS are listed first or
607
- last during ordering.
608
- This argument is ignored, if "data_order_column" is None.
609
- NULLS are listed first when this argument is set to True, and
610
- last when set to False.
611
- Default Value: True
612
- Types: bool
613
-
614
- RETURNS:
615
- None.
616
-
617
- RAISES:
618
- TeradataMlException
619
-
620
- EXAMPLES:
621
- # Note - Refer to User Guide for setting search path and required permissions.
622
- # Load example data.
623
- load_example_data("Script", ["barrier"])
624
-
625
- # Example 1
626
- # Create teradataml DataFrame objects.
627
- >>> barrierdf = DataFrame.from_table("barrier")
628
- >>> barrierdf
629
- Name
630
- Id
631
- 1 Old Macdonald Had A Farm
632
- >>>
633
-
634
- # Set SEARCHUIFDBPATH
635
- >>> execute_sql("SET SESSION SEARCHUIFDBPATH = alice;")
636
- >>> import teradataml
637
- >>> from teradatasqlalchemy import VARCHAR
638
- >>> td_path = os.path.dirname(teradataml.__file__)
639
-
640
- # The script mapper.py reads in a line of text input
641
- # ("Old Macdonald Had A Farm") from csv and
642
- # splits the line into individual words, emitting a new row for each word.
643
- # Create a Script object without data and its arguments.
644
- >>> sto = Script(data = barrierdf,
645
- ... script_name='mapper.py',
646
- ... files_local_path= os.path.join(td_path,'data', 'scripts'),
647
- ... script_command='tdpython3 ./alice/mapper.py',
648
- ... charset='latin',
649
- ... returns=OrderedDict([("word", VARCHAR(15)),("count_input", VARCHAR(2))]))
650
-
651
- # Test script using data from file
652
- >>> sto.test_script(input_data_file='../barrier.csv', data_file_delimiter=',')
653
- ############ STDOUT Output ############
654
- word count_input
655
- 0 1 1
656
- 1 Old 1
657
- 2 Macdonald 1
658
- 3 Had 1
659
- 4 A 1
660
- 5 Farm 1
661
- >>>
662
-
663
- # Test script using data from DB.
664
- >>> sto.test_script(password='alice')
665
- ############ STDOUT Output ############
666
-
667
- word count_input
668
- 0 1 1
669
- 1 Old 1
670
- 2 Macdonald 1
671
- 3 Had 1
672
- 4 A 1
673
- 5 Farm 1
674
-
675
- # Test script using data from DB and with data_row_limit.
676
- >>> sto.test_script(password='alice', data_row_limit=5)
677
- ############ STDOUT Output ############
678
-
679
- word count_input
680
- 0 1 1
681
- 1 Old 1
682
- 2 Macdonald 1
683
- 3 Had 1
684
- 4 A 1
685
- 5 Farm 1
686
-
687
- # Now in order to test / run script on actual data on Vantage user must
688
- # set data and related arguments.
689
- # Note:
690
- # All data related arguments that are not specified in set_data() are
691
- # reset to default values.
692
- >>> sto.set_data(data=barrierdf,
693
- ... data_order_column="Id",
694
- ... is_local_order=False,
695
- ... nulls_first=False,
696
- ... sort_ascending=False)
697
-
698
- # Execute the user script on Vantage.
699
- >>> sto.execute_script()
700
- ############ STDOUT Output ############
701
- word count_input
702
- 0 Macdonald 1
703
- 1 A 1
704
- 2 Farm 1
705
- 3 Had 1
706
- 4 Old 1
707
- 5 1 1
708
-
709
- # Example 2 -
710
- # Script is tested using test_script and executed on Vantage.
711
- # use set_data() to reset arguments.
712
- # Create teradataml DataFrame objects.
713
- >>> load_example_data("Script", ["barrier_new"])
714
- >>> barrierdf_new = DataFrame.from_table("barrier_new")
715
- >>> barrierdf_new
716
- Name
717
- Id
718
- 2 On his farm he had a cow
719
- 1 Old Macdonald Had A Farm
720
- >>>
721
-
722
- # Create a Script object that allows us to execute script on Vantage.
723
- >>> sto = Script(data=barrierdf_new,
724
- ... script_name='mapper.py',
725
- ... files_local_path= os.path.join(td_path, 'data', 'scripts'),
726
- ... script_command='tdpython3 ./alice/mapper.py',
727
- ... data_order_column="Id",
728
- ... is_local_order=False,
729
- ... nulls_first=False,
730
- ... sort_ascending=False,
731
- ... charset='latin',
732
- ... returns=OrderedDict([("word", VARCHAR(15)),("count_input", VARCHAR(2))]))
733
- # Script is tested using test_script and executed on Vantage.
734
- >>> sto.execute_script()
735
- ############ STDOUT Output ############
736
- word count_input
737
- 0 his 1
738
- 1 he 1
739
- 2 had 1
740
- 3 a 1
741
- 4 1 1
742
- 5 Old 1
743
- 6 cow 1
744
- 7 farm 1
745
- 8 On 1
746
- 9 2 1
747
-
748
- # Now in order to run the script with a different dataset,
749
- # user can use set_data().
750
- # Re-set data and some data related parameters.
751
- # Note:
752
- # All data related arguments that are not specified in set_data() are
753
- # reset to default values.
754
- >>> sto.set_data(data=barrierdf,
755
- ... data_order_column='Id',
756
- ... is_local_order=True,
757
- ... nulls_first=True)
758
- >>> sto.execute_script()
759
- word count_input
760
- 0 Macdonald 1
761
- 1 A 1
762
- 2 Farm 1
763
- 3 Had 1
764
- 4 Old 1
765
- 5 1 1
766
-
767
- # Example 3
768
- # Script is tested using test_script and executed on Vantage.
769
- # In order to run the script with same dataset but different data related
770
- # arguments, use set_data() to reset arguments.
771
- # Note:
772
- # All data related arguments that are not specified in set_data() are
773
- # reset to default values.
774
- >>> sto.set_data(data=barrierdf_new,
775
- ... data_order_column='Id',
776
- ... is_local_order = True,
777
- ... nulls_first = True)
778
-
779
- >>> sto.execute_script()
780
- ############ STDOUT Output ############
781
-
782
- word count_input
783
- 0 Macdonald 1
784
- 1 A 1
785
- 2 Farm 1
786
- 3 2 1
787
- 4 his 1
788
- 5 farm 1
789
- 6 On 1
790
- 7 Had 1
791
- 8 Old 1
792
- 9 1 1
793
- """
794
- super(Script, self).set_data(data,
795
- data_partition_column,
796
- data_hash_column,
797
- data_order_column,
798
- is_local_order,
799
- sort_ascending,
800
- nulls_first)
801
- self.__validate()
802
-
803
- def setup_sto_env(self, docker_image_location):
804
- """
805
- DESCRIPTION:
806
- Function enables user to load already downloaded sandbox image.
807
-
808
- PARAMETERS:
809
- docker_image_location:
810
- Required Argument.
811
- Specifies the location of image on user's system.
812
- Types: str
813
-
814
- Note:
815
- For location to download docker image refer teradataml User Guide.
816
-
817
- RETURNS:
818
- None.
819
-
820
- RAISES:
821
- TeradataMlException
822
-
823
- EXAMPLES:
824
- # Note - Refer to User Guide for setting search path and required permissions.
825
- # Load example data.
826
- load_example_data("Script", ["barrier"])
827
-
828
- # Example - The script mapper.py reads in a line of text input
829
- # ("Old Macdonald Had A Farm") from csv and
830
- # splits the line into individual words, emitting a new row for each word.
831
-
832
- # Create teradataml DataFrame objects.
833
- >>> barrierdf = DataFrame.from_table("barrier")
834
-
835
- # Set SEARCHUIFDBPATH.
836
- >>> execute_sql("SET SESSION SEARCHUIFDBPATH = alice;")
837
-
838
- # Create a Script object that allows us to execute script on Vantage.
839
- >>> import os
840
- >>> td_path = os.path.dirname(teradataml.__file__)
841
- >>> from teradatasqlalchemy import VARCHAR
842
- >>> sto = Script(data=barrierdf,
843
- ... script_name='mapper.py',
844
- ... files_local_path= os.path.join(td_path, 'data', 'scripts'),
845
- ... script_command='tdpython3 ./alice/mapper.py',
846
- ... data_order_column="Id",
847
- ... is_local_order=False,
848
- ... nulls_first=False,
849
- ... sort_ascending=False,
850
- ... charset='latin',
851
- ... returns=OrderedDict([("word", VARCHAR(15)),("count_input", VARCHAR(2))]))
852
-
853
- # Run user script locally within docker container and using data from csv.
854
- # This helps the user to fix script level issues outside Vantage.
855
- # Setup the environment by providing local path to docker image file.
856
- >>> sto.setup_sto_env(docker_image_location='/tmp/sto_sandbox_docker_image.tar')
857
- Loading image from /tmp/sto_sandbox_docker_image.tar. It may take few minutes.
858
- Image loaded successfully.
859
- Starting a container for stosandbox:1.0 image.
860
- Container d7c73cb498c79a082180576bb5b10bb07b52efdd3026856146fc15e91147b19f
861
- started successfully.
862
-
863
- """
864
- self.awu_matrix_setup = []
865
- self.awu_matrix_setup.append((["docker_image_location", docker_image_location,
866
- False, (str), True]))
867
-
868
- # Validate missing arguments.
869
- _Validators._validate_missing_required_arguments(self.awu_matrix_setup)
870
-
871
- # Validate argument types.
872
- _Validators._validate_function_arguments(self.awu_matrix_setup)
873
-
874
- from teradataml.table_operators.sandbox_container_util import setup_sandbox_env
875
- setup_sandbox_env(sandbox_image_location=docker_image_location,
876
- sandbox_image_name='stosandbox:1.0')
877
-
878
- # Set _latest_sandbox_exists to True - which indicates sandbox image for STO
879
- # exists on the system.
880
- configure._latest_sandbox_exists = True
881
-
882
- def test_script(self, supporting_files=None, input_data_file=None, script_args="",
883
- exec_mode ='sandbox', **kwargs):
884
- """
885
- DESCRIPTION:
886
- Function enables user to run script in docker container environment outside
887
- Vantage.
888
- Input data for user script is either read from a file or from database.
889
- Note:
890
- 1. Purpose of test_script() function is to enable the user to test their scripts for any errors without
891
- installing it on Vantage, using the input data provided.
892
- 2. Data is not partitioned for testing the script if read from input data file.
893
- 3. Function can produce different output if input is read from a file than input from database.
894
-
895
- PARAMETERS:
896
- supporting_files:
897
- Optional Argument.
898
- Specifies a file or list of supporting files like model files to be
899
- copied to the container.
900
- Types: string or list of str
901
-
902
- input_data_file:
903
- Optional Argument.
904
- Specifies the name of the input data file.
905
- It should have a path relative to the location specified in
906
- "files_local_path" argument.
907
- If set to None, read data from AMP, else from file passed in the argument
908
- 'input_data_file'.
909
- File should have at least permissions of mode 644.
910
- Types: str
911
-
912
- script_args:
913
- Optional Argument.
914
- Specifies command line arguments required by the user script.
915
- Types: str
916
-
917
- exec_mode:
918
- Optional Argument.
919
- Specifies the mode in which user wants to test the script.
920
- If set to 'sandbox', the user script will run within the sandbox
921
- environment, else it will run locally on user's system.
922
- Permitted Values: 'sandbox', 'local'
923
- Default Value: 'sandbox'
924
- Types: str
925
-
926
- kwargs:
927
- Optional Argument.
928
- Specifies the keyword arguments required for testing.
929
- Keys can be:
930
- data_row_limit:
931
- Optional Argument. Ignored when data is read from file.
932
- Specifies the number of rows to be taken from all amps when
933
- reading from a table or view on Vantage.
934
- Default Value: 1000
935
- Types: int
936
-
937
- password:
938
- Optional Argument. Required when reading from database.
939
- Specifies the password to connect to vantage where the data
940
- resides.
941
- Types: str
942
-
943
- data_file_delimiter:
944
- Optional Argument.
945
- Specifies the delimiter used in the input data file. This
946
- argument can be specified when data is read from file.
947
- Default Value: '\t'
948
- Types: str
949
-
950
- data_file_header:
951
- Optional Argument.
952
- Specifies whether the input data file contains header. This
953
- argument can be specified when data is read from file.
954
- Default Value: True
955
- Types: bool
956
-
957
- timeout:
958
- Optional Argument.
959
- Specifies the timeout for docker API calls when running in
960
- sandbox mode.
961
- Default Value: 5000
962
- Types: int
963
-
964
- data_file_quote_char:
965
- Optional Argument.
966
- Specifies the quotechar used in the input data file.
967
- This argument can be specified when data is read from file.
968
- Default Value: '"'
969
-
970
- logmech:
971
- Optional Argument.
972
- Specifies the type of logon mechanism to establish a connection to
973
- Teradata Vantage.
974
- Permitted Values: 'TD2', 'TDNEGO', 'LDAP', 'KRB5' & 'JWT'.
975
- TD2:
976
- The Teradata 2 (TD2) mechanism provides authentication
977
- using a Vantage username and password. This is the default
978
- logon mechanism using which the connection is established
979
- to Vantage.
980
-
981
- TDNEGO:
982
- A security mechanism that automatically determines the
983
- actual mechanism required, based on policy, without user's
984
- involvement. The actual mechanism is determined by the
985
- TDGSS server configuration and by the security policy's
986
- mechanism restrictions.
987
-
988
- LDAP:
989
- A directory-based user logon to Vantage with a directory
990
- username and password and is authenticated by the directory.
991
-
992
- KRB5 (Kerberos):
993
- A directory-based user logon to Vantage with a domain
994
- username and password and is authenticated by
995
- Kerberos (KRB5 mechanism).
996
- Note:
997
- User must have a valid ticket-granting ticket in
998
- order to use this logon mechanism.
999
-
1000
- JWT:
1001
- The JSON Web Token (JWT) authentication mechanism enables
1002
- single sign-on (SSO) to the Vantage after the user
1003
- successfully authenticates to Teradata UDA User Service.
1004
- Note:
1005
- User must use logdata parameter when using 'JWT' as
1006
- the logon mechanism.
1007
- Default Value: TD2
1008
- Types: str
1009
-
1010
- Note:
1011
- teradataml expects the client environments are already setup with appropriate
1012
- security mechanisms and are in working conditions.
1013
- For more information please refer Teradata Vantage™ - Advanced SQL Engine
1014
- Security Administration at https://www.info.teradata.com/
1015
-
1016
- logdata:
1017
- Optional Argument.
1018
- Specifies parameters to the LOGMECH command beyond those needed by
1019
- the logon mechanism, such as user ID, password and tokens
1020
- (in case of JWT) to successfully authenticate the user.
1021
- Types: str
1022
-
1023
- Types: dict
1024
-
1025
- RETURNS:
1026
- Output from user script.
1027
-
1028
- RAISES:
1029
- TeradataMlException
1030
-
1031
- EXAMPLES:
1032
- # Assumption - sto is Script() object. Please refer to help(Script)
1033
- # for creating Script object.
1034
- # Run user script in sandbox mode with input from data file.
1035
-
1036
- >>> sto.test_script(input_data_file='../barrier.csv',
1037
- ... data_file_delimiter=',',
1038
- ... data_file_quote_char='"',
1039
- ... data_file_header=True,
1040
- ... exec_mode='sandbox')
1041
-
1042
- ############ STDOUT Output ############
1043
- word count_input
1044
- 0 1 1
1045
- 1 Old 1
1046
- 2 Macdonald 1
1047
- 3 Had 1
1048
- 4 A 1
1049
- 5 Farm 1
1050
- >>>
1051
-
1052
- # Run user script in local mode with input from table.
1053
- >>> sto.test_script(data_row_limit=300, password='alice', exec_mode='local')
1054
-
1055
- ############ STDOUT Output ############
1056
- word count_input
1057
- 0 1 1
1058
- 1 Old 1
1059
- 2 Macdonald 1
1060
- 3 Had 1
1061
- 4 A 1
1062
- 5 Farm 1
1063
-
1064
- # Run user script in sandbox mode with logmech as 'TD2'.
1065
- >>> sto.test_script(script_args="4 5 10 6 480", password="alice", logmech="TD2")
1066
-
1067
- # Run user script in sandbox mode with logmech as 'TDNEGO'.
1068
- >>> sto.test_script(script_args="4 5 10 6 480", password="alice", logmech="TDNEGO")
1069
-
1070
- # Run user script in sandbox mode with logmech as 'LDAP'.
1071
- >>> sto.test_script(script_args="4 5 10 6 480", password="alice", logmech="LDAP")
1072
-
1073
- # Run user script in sandbox mode with logmech as 'KRB5'.
1074
- >>> sto.test_script(script_args="4 5 10 6 480", password="alice", logmech="KRB5")
1075
-
1076
- # Run user script in sandbox mode with logmech as 'JWT'.
1077
- >>> sto.test_script(script_args="4 5 10 6 480", password="alice",
1078
- logmech='JWT', logdata='token=eyJpc...h8dA')
1079
-
1080
- """
1081
- logmech_valid_values = ['TD2', 'TDNEGO', 'LDAP', 'KRB5', 'JWT']
1082
-
1083
- awu_matrix_test=[]
1084
- awu_matrix_test.append((["supporting_files", supporting_files, True,
1085
- (str,list), True]))
1086
- awu_matrix_test.append((["input_data_file", input_data_file, True, (str), True]))
1087
- awu_matrix_test.append((["script_args", script_args, True, (str), False]))
1088
- awu_matrix_test.append((["exec_mode", exec_mode, True, (str), True,
1089
- [TableOperatorConstants.SANDBOX_EXEC.value,
1090
- TableOperatorConstants.LOCAL_EXEC.value]]))
1091
-
1092
- data_row_limit = kwargs.pop("data_row_limit", 1000)
1093
- awu_matrix_test.append((["data_row_limit", data_row_limit, True, (int), True]))
1094
-
1095
- data_file_delimiter = kwargs.pop("data_file_delimiter", '\t')
1096
- awu_matrix_test.append((["data_file_delimiter", data_file_delimiter, True,
1097
- (str), False]))
1098
-
1099
- data_file_quote_char = kwargs.pop("data_file_quote_char", '"')
1100
- awu_matrix_test.append((["data_file_quote_char", data_file_quote_char, True,
1101
- (str), False]))
1102
-
1103
- data_file_header = kwargs.pop("data_file_header", True)
1104
- awu_matrix_test.append((["data_file_header", data_file_header, True, (bool)]))
1105
-
1106
- timeout = kwargs.pop("timeout", 5000)
1107
- awu_matrix_test.append((["timeout", timeout, True, (int), True]))
1108
-
1109
- logmech = kwargs.pop("logmech", "TD2")
1110
- awu_matrix_test.append(
1111
- ["logmech", logmech, True, (str), True, logmech_valid_values])
1112
-
1113
- logdata = kwargs.pop("logdata", None)
1114
- awu_matrix_test.append(["logdata", logdata, True, (str), True])
1115
-
1116
- # Validate argument types.
1117
- _Validators._validate_function_arguments(awu_matrix_test)
1118
-
1119
- # Validate timeout value.
1120
- _Validators._validate_positive_int(timeout, "timeout")
1121
-
1122
- self.__validate()
1123
- self.__validate_for_reserved_keyword()
1124
-
1125
-
1126
- if logmech == "JWT" and not logdata:
1127
- raise TeradataMlException(
1128
- Messages.get_message(MessageCodes.DEPENDENT_ARG_MISSING, 'logdata',
1129
- 'logmech=JWT'),
1130
- MessageCodes.DEPENDENT_ARG_MISSING)
1131
-
1132
- if data_row_limit <= 0:
1133
- raise ValueError(Messages.get_message(MessageCodes.TDMLDF_POSITIVE_INT).
1134
- format("data_row_limit", "greater than"))
1135
-
1136
- # Either of 'input_data_file' or 'password' argument is required.
1137
- password = kwargs.pop("password", None)
1138
-
1139
- # The check of EITHER_THIS_OR_THAT_ARGUMENT is applicable only when the exec_mode is sandbox.
1140
- # Hence adding the check exec_mode != "local".
1141
- # When exec_mode is local, the connection object is used to get the values in the table.
1142
- if exec_mode != "local" and not (input_data_file or (self.data and password)):
1143
- message = Messages.get_message(MessageCodes.EITHER_THIS_OR_THAT_ARGUMENT,
1144
- "input_data_file", "Script data and password")
1145
- raise TeradataMlException(message, MessageCodes.EITHER_THIS_OR_THAT_ARGUMENT)
1146
- elif exec_mode == "local" and not (input_data_file or self.data):
1147
- message = Messages.get_message(MessageCodes.EITHER_THIS_OR_THAT_ARGUMENT,
1148
- "input_data_file", "Script data")
1149
- raise TeradataMlException(message, MessageCodes.EITHER_THIS_OR_THAT_ARGUMENT)
1150
-
1151
- if not self.script_name and self.files_local_path:
1152
- message = Messages.get_message(MessageCodes.MISSING_ARGS,
1153
- "script_name and files_local_path")
1154
- raise TeradataMlException(message, MessageCodes.MISSING_ARGS)
1155
-
1156
- if input_data_file:
1157
- if self.files_local_path is None:
1158
- message = Messages.get_message(MessageCodes.DEPENDENT_ARG_MISSING,
1159
- "files_local_path","input_data_file")
1160
- raise TeradataMlException(message, MessageCodes.DEPENDENT_ARG_MISSING)
1161
- else:
1162
- # Check if file exists.
1163
- fpath = os.path.join(self.files_local_path,
1164
- input_data_file)
1165
- _Validators._validate_file_exists(fpath)
1166
-
1167
- if self.script_name and self.files_local_path:
1168
- # Check if file exists.
1169
- fpath = os.path.join(self.files_local_path,
1170
- os.path.basename(self.script_name))
1171
- _Validators._validate_file_exists(fpath)
1172
-
1173
- if exec_mode.upper() == TableOperatorConstants.LOCAL_EXEC.value:
1174
- user_script_path = os.path.join(self.files_local_path, self.script_name)
1175
- import sys
1176
- cmd = [str(sys.executable), user_script_path]
1177
- cmd.extend(script_args)
1178
-
1179
- if input_data_file is not None:
1180
- input_file_path = os.path.join(self.files_local_path, input_data_file)
1181
-
1182
- # Run user script locally with input from a file.
1183
- exec_cmd_output = self.__local_run_user_script_input_file(
1184
- cmd, input_file_path, data_file_delimiter, data_file_quote_char, data_file_header)
1185
- try:
1186
- return self.__process_test_script_output(exec_cmd_output)
1187
- except Exception as exp:
1188
- raise
1189
-
1190
- else:
1191
- if self.data.shape[0] > data_row_limit:
1192
- raise ValueError(
1193
- Messages.get_message(MessageCodes.DATAFRAME_LIMIT_ERROR,
1194
- 'data_row_limit', 'data_row_limit',
1195
- data_row_limit))
1196
-
1197
- if not self.data._table_name:
1198
- self.data._table_name = df_utils._execute_node_return_db_object_name(
1199
- self.data._nodeid, self.data._metaexpr)
1200
-
1201
- table_name = UtilFuncs._extract_table_name(self.data._table_name)
1202
-
1203
- # Run user script locally with input from db.
1204
- exec_cmd_output = self.__local_run_user_script_input_db(cmd, table_name)
1205
- try:
1206
- return self.__process_test_script_output(exec_cmd_output)
1207
- except Exception as exp:
1208
- raise
1209
- else:
1210
- # Execution Mode - sandbox.
1211
-
1212
- # get the frame object of the function.
1213
- import inspect
1214
- frame = inspect.currentframe()
1215
-
1216
- # Validate argument types.
1217
- _Validators._validate_module_presence('docker', frame.f_code.co_name)
1218
-
1219
- import docker
1220
-
1221
- # Read container_id from configure.sandbox_container_id, if it is None then
1222
- # raise an exception
1223
-
1224
- container_id = configure.sandbox_container_id
1225
- if container_id is None:
1226
- message = Messages.get_message(MessageCodes.SANDBOX_CONTAINER_NOT_FOUND)
1227
- raise TeradataMlException(message,
1228
- MessageCodes.SANDBOX_CONTAINER_NOT_FOUND)
1229
-
1230
- # Set path inside docker container. This is where files will be copied to.
1231
- # os.path.join() will not work here because the path is not dependent on
1232
- # client platform. Sandbox environment is linux based here.
1233
- _path_in_docker_container = "/home/tdatuser"
1234
- user_script_path = "{}/{}".format(_path_in_docker_container, self.script_name)
1235
-
1236
- if input_data_file is not None:
1237
- input_file_name = os.path.basename(input_data_file)
1238
- input_file_path = "{}/{}".format(_path_in_docker_container,
1239
- input_file_name)
1240
- # Create script_executor.
1241
- self._create_executor_script(user_script_path=user_script_path,
1242
- user_script_args=script_args,
1243
- data_file_path=input_file_path,
1244
- data_file_delimiter=data_file_delimiter,
1245
- data_file_quote_char=data_file_quote_char,
1246
- data_file_header=data_file_header)
1247
- else:
1248
- # Read input from db.
1249
- if self.data.shape[0] > data_row_limit:
1250
- raise ValueError(
1251
- Messages.get_message(MessageCodes.DATAFRAME_LIMIT_ERROR,
1252
- 'data_row_limit', 'data_row_limit',
1253
- data_row_limit))
1254
- db_host = get_context().url.host
1255
-
1256
- user_name = get_context().url.username
1257
-
1258
- if not self.data._table_name:
1259
- self.data._table_name = df_utils._execute_node_return_db_object_name(
1260
- self.data._nodeid, self.data._metaexpr)
1261
- table_name = UtilFuncs._extract_table_name(self.data._table_name)
1262
-
1263
- db_name = _get_current_databasename()
1264
-
1265
- # Create script_executor.
1266
- self._create_executor_script(user_script_path=user_script_path,
1267
- user_script_args=script_args,
1268
- db_host=db_host,
1269
- user_name=user_name,
1270
- passwd=password,
1271
- table_name=table_name,
1272
- db_name=db_name,
1273
- logmech=logmech,
1274
- logdata=logdata)
1275
-
1276
- client = docker.APIClient(timeout=timeout)
1277
-
1278
- # Copy files to container indicated in configure.sandbox_container_id.
1279
- files_to_copy = [self.script_name]
1280
-
1281
- if supporting_files is not None:
1282
- if isinstance(supporting_files, str):
1283
- supporting_files = [supporting_files]
1284
-
1285
- if len(supporting_files) == 0 \
1286
- or any(file in [None, "None", ""] for file in supporting_files):
1287
- raise ValueError(
1288
- Messages.get_message(MessageCodes.LIST_SELECT_NONE_OR_EMPTY,
1289
- 'supporting_files'))
1290
- else:
1291
- files_to_copy.extend(supporting_files)
1292
-
1293
- if input_data_file is not None:
1294
- files_to_copy.append(input_data_file)
1295
-
1296
- for filename in files_to_copy:
1297
- file_path = os.path.join(self.files_local_path, filename)
1298
- # Check if file exists.
1299
- _Validators._validate_file_exists(file_path)
1300
-
1301
- # Copy file to docker container.
1302
- self._copy_to_docker_container(client, file_path,
1303
- _path_in_docker_container,
1304
- container_id)
1305
-
1306
- # Copy script_executor to docker container.
1307
- self._copy_to_docker_container(client, self.script_path,
1308
- _path_in_docker_container,
1309
- container_id)
1310
-
1311
- script_executor_file_name = os.path.basename(self.script_path)
1312
- exec_cmd = ("tdpython3 {0}/{1}".format(_path_in_docker_container,
1313
- script_executor_file_name))
1314
-
1315
- try:
1316
- # Setup an exec instance in the container.
1317
- exec_cmd_create = client.exec_create(container_id, exec_cmd)
1318
-
1319
- # Start exec instance and run user script.
1320
- exec_cmd_output = client.exec_start(exec_cmd_create, demux=True)
1321
-
1322
- # Inspect the output for success or failure.
1323
- inspect_out = client.exec_inspect(exec_cmd_create)
1324
-
1325
- # Extract the exit code.
1326
- exit_code = inspect_out['ExitCode']
1327
-
1328
- if exec_cmd_output[0] is not None:
1329
- executor_output = exec_cmd_output[0].decode()
1330
-
1331
- executor_error = ""
1332
- if exec_cmd_output[1] is not None:
1333
- executor_error = exec_cmd_output[1].decode()
1334
-
1335
- # Exit code 1 indicates any error thrown by subprocess.
1336
- # Exit code 126 indicates permission problem or command is not executable.
1337
- # Exit code 127 indicates possible typos in shell script with
1338
- # unrecognizable characters.
1339
- if exit_code == 1 or exit_code == 126 or exit_code == 127:
1340
- message = Messages.get_message(
1341
- MessageCodes.SANDBOX_SCRIPT_ERROR).format(executor_error)
1342
- raise TeradataMlException(message,
1343
- MessageCodes.SANDBOX_SCRIPT_ERROR)
1344
- # Exit code 2 indicates either username or password is invalid.
1345
- elif exit_code == 2:
1346
- message = Messages.get_message(
1347
- MessageCodes.SANDBOX_CONNECTION_ERROR).format(executor_error)
1348
- raise TeradataMlException(message,
1349
- MessageCodes.SANDBOX_CONNECTION_ERROR)
1350
- # Exit code 3 indicates problem with query.
1351
- elif exit_code == 3:
1352
- message = Messages.get_message(
1353
- MessageCodes.SANDBOX_QUERY_ERROR).format(executor_error)
1354
- raise TeradataMlException(message,
1355
- MessageCodes.SANDBOX_QUERY_ERROR)
1356
- # Exit code 4 indicates all other exceptions / errors.
1357
- elif exit_code == 4:
1358
- message = Messages.get_message(
1359
- MessageCodes.SANDBOX_CONTAINER_ERROR).format(executor_error)
1360
- raise TeradataMlException(message,
1361
- MessageCodes.SANDBOX_CONTAINER_ERROR)
1362
- elif exit_code != 0:
1363
- # Any error other than exit code 1, 2, 3, 4
1364
- message = Messages.get_message(
1365
- MessageCodes.SANDBOX_CONTAINER_ERROR).format(executor_error)
1366
- raise TeradataMlException(message,
1367
- MessageCodes.SANDBOX_CONTAINER_ERROR)
1368
- else:
1369
- return self.__process_test_script_output(executor_output)
1370
- except Exception as exp:
1371
- message = Messages.get_message(
1372
- MessageCodes.SANDBOX_CONTAINER_ERROR).format(str(exp))
1373
- raise TeradataMlException(message,
1374
- MessageCodes.SANDBOX_CONTAINER_ERROR)
1375
-
1376
- def __local_run_user_script_input_file(self, cmd, input_file_path,
1377
- data_file_delimiter='\t',
1378
- data_file_quote_char='"',
1379
- data_file_header=True):
1380
- """
1381
- DESCRIPTION:
1382
- Function to run the user script in local mode with input from file.
1383
-
1384
- PARAMETERS:
1385
- cmd:
1386
- Required Argument.
1387
- Specifies the command for running the user script.
1388
- Types: str
1389
-
1390
- input_file_path:
1391
- Required Argument.
1392
- Specifies the absolute local path of input data file.
1393
- Types: str
1394
-
1395
- data_file_delimiter:
1396
- Optional Argument.
1397
- Specifies the delimiter used in input data file.
1398
- Default Value: '\t'
1399
- Types: str
1400
-
1401
- data_file_quote_char:
1402
- Optional Argument.
1403
- Specifies the quote character used in input data file.
1404
- Default Value: '"'
1405
- Types: str
1406
-
1407
- data_file_header:
1408
- Optional Argument.
1409
- Specifies whether the input data file has header.
1410
- Default Value: True
1411
- Types: bool
1412
-
1413
- RETURNS:
1414
- The string output of the command that is run on input data file.
1415
-
1416
- RAISES:
1417
- Exception.
1418
-
1419
- EXAMPLES:
1420
- self.__local_run_user_script_input_file(cmd ="cmd",
1421
- input_file_path = "input_file_path",
1422
- data_file_delimiter = "data_file_delimiter",
1423
- data_file_quote_char = "data_file_quote_char",
1424
- data_file_header = True)
1425
-
1426
- """
1427
- with open(input_file_path) as data_file:
1428
- import csv
1429
- from pandas import isna as pd_isna
1430
-
1431
- data_handle = StringIO()
1432
-
1433
- # Read data from input file.
1434
- ip_data = csv.reader(data_file,
1435
- delimiter=data_file_delimiter,
1436
- quotechar=data_file_quote_char)
1437
- # Skip the first row of input file if data_file_header is True.
1438
- if data_file_header:
1439
- next(ip_data)
1440
- for row in ip_data:
1441
- if self.quotechar is not None:
1442
- # A NULL value should not be enclosed in quotes.
1443
- # The CSV module has no support for such output with writer,
1444
- # and hence the custom formatting.
1445
- line = ['' if pd_isna(s) else "{}{}{}".format(self.quotechar,
1446
- str(s),
1447
- self.quotechar)
1448
- for s in row]
1449
- else:
1450
- line = ['' if pd_isna(s) else str(s) for s in row]
1451
-
1452
- complete_line = (self.delimiter.join(line))
1453
-
1454
- data_handle.write(complete_line)
1455
- data_handle.write("\n")
1456
-
1457
- return self.__run_user_script_subprocess(cmd, data_handle)
1458
-
1459
- def __local_run_user_script_input_db(self, cmd, table_name):
1460
- """
1461
- DESCRIPTION:
1462
- Function to run the user script in local mode with input from db.
1463
-
1464
- PARAMETERS:
1465
- cmd:
1466
- Required Argument.
1467
- Specifies the command for running the user script.
1468
- Types: str
1469
-
1470
- table_name:
1471
- Required Argument.
1472
- Specifies the table name for input to user script.
1473
- Types: str
1474
-
1475
- RETURNS:
1476
- The string output of the command that is run on the Vantage table.
1477
-
1478
- RAISES:
1479
- Exception.
1480
-
1481
- EXAMPLES:
1482
- self.__local_run_user_script_input_db(cmd = "cmd", table_name = "table_name")
1483
-
1484
- """
1485
- db_data_handle = StringIO()
1486
- try:
1487
- con = get_connection()
1488
- # Query for reading data from DB.
1489
- query = ("SELECT * FROM {} ORDER BY 1;".format(table_name))
1490
- cur = execute_sql(query)
1491
- row = cur.fetchone()
1492
- from pandas import isna as pd_isna
1493
- while row:
1494
- if self.quotechar is not None:
1495
- # A NULL value should not be enclosed in quotes.
1496
- # The CSV module has no support for such output with writer,
1497
- # and hence the custom formatting.
1498
- line = ['' if pd_isna(s) else "{}{}{}".format(self.quotechar,
1499
- str(s),
1500
- self.quotechar)
1501
- for s in row]
1502
- else:
1503
- line = ['' if pd_isna(s) else str(s) for s in row]
1504
-
1505
- complete_line = (self.delimiter.join(line))
1506
- db_data_handle.write(complete_line)
1507
- db_data_handle.write("\n")
1508
- row = cur.fetchone()
1509
- except Exception as exp:
1510
- raise exp
1511
-
1512
- return self.__run_user_script_subprocess(cmd, db_data_handle)
1513
-
1514
- def __process_test_script_output(self, exec_cmd_output):
1515
- """
1516
- DESCRIPTION:
1517
- Function to format the output of the user script.
1518
-
1519
- PARAMETERS:
1520
- exec_cmd_output:
1521
- Required Argument.
1522
- Specifies the output returned by the user script.
1523
- Types: str
1524
-
1525
- RETURNS:
1526
- The test script output as Pandas DataFrame.
1527
-
1528
- RAISES:
1529
- Exception.
1530
-
1531
- EXAMPLES:
1532
- self.__process_test_script_output(exec_cmd_output = "exec_cmd_output")
1533
- """
1534
- try:
1535
- kwargs = dict()
1536
- if self.quotechar is not None:
1537
- kwargs['quotechar'] = self.quotechar
1538
- kwargs['quoting'] = 1 # QUOTE_ALL
1539
-
1540
- output = StringIO(exec_cmd_output)
1541
-
1542
- from pandas import read_csv as pd_read_csv
1543
-
1544
- # Form a pandas dataframe.
1545
- df = pd_read_csv(output, sep=self.delimiter, index_col=False, header=None,
1546
- names=list(self.returns.keys()), **kwargs)
1547
- return df
1548
-
1549
- except Exception as exp:
1550
- raise exp
1551
-
1552
- def __run_user_script_subprocess(self, cmd, data_handle):
1553
- """
1554
- DESCRIPTION:
1555
- Function to run the user script in a new process and return the output.
1556
-
1557
- PARAMETERS:
1558
- cmd:
1559
- Required Argument.
1560
- Specifies the command for running the script.
1561
- Types: str
1562
-
1563
- data_handle:
1564
- Required Argument.
1565
- Specifies the data handle for the input data required by the user script.
1566
-
1567
- RETURNS:
1568
- Output of user script on input data supplied in data_handle.
1569
-
1570
- RAISES:
1571
- None.
1572
-
1573
- EXAMPLES:
1574
- self.__run_user_script_subprocess(cmd = "exec_cmd_output",
1575
- data_handle = data_handle)
1576
-
1577
- """
1578
- # Launching new process to run the user script.
1579
- try:
1580
- proc = subprocess.Popen(cmd, stdin=subprocess.PIPE,
1581
- stdout=subprocess.PIPE,
1582
- stderr=subprocess.PIPE)
1583
- process_output, process_error = proc.communicate(data_handle.getvalue().encode())
1584
- data_handle.close()
1585
-
1586
- if proc.returncode == 0:
1587
- return process_output.decode("utf-8").rstrip("\r|\n")
1588
- else:
1589
- message = Messages.get_message(MessageCodes.SCRIPT_LOCAL_RUN_ERROR).\
1590
- format(process_error)
1591
- raise TeradataMlException(message, MessageCodes.SCRIPT_LOCAL_RUN_ERROR)
1592
- except Exception as e:
1593
- raise e
1594
-
1595
- def execute_script(self, output_style='VIEW'):
1596
- """
1597
- DESCRIPTION:
1598
- Function enables user to run script on Vantage.
1599
-
1600
- PARAMETERS:
1601
- output_style:
1602
- Specifies the type of output object to create - a table or a view.
1603
- Permitted Values: 'VIEW', 'TABLE'.
1604
- Default value: 'VIEW'
1605
- Types: str
1606
-
1607
- RETURNS:
1608
- Output teradataml DataFrames can be accessed using attribute
1609
- references, such as ScriptObj.<attribute_name>.
1610
- Output teradataml DataFrame attribute name is:
1611
- result
1612
-
1613
- RAISES:
1614
- TeradataMlException, ValueError
1615
-
1616
- EXAMPLES:
1617
- Refer to help(Script)
1618
- """
1619
- # Validate the output_style.
1620
- permitted_values = [OutputStyle.OUTPUT_TABLE.value,
1621
- OutputStyle.OUTPUT_VIEW.value]
1622
- _Validators._validate_permitted_values(output_style, permitted_values,
1623
- 'output_style',
1624
- case_insensitive=False, includeNone=False)
1625
-
1626
- # Validate arguments.
1627
- self.__validate()
1628
- # Validating for reserved keywords.
1629
- self.__validate_for_reserved_keyword()
1630
-
1631
- # Generate the Table Operator query.
1632
- self.__form_table_operator_query()
1633
-
1634
- # Execute Table Operator query and return results.
1635
- return self.__execute(output_style)
1636
-
1637
- def install_file(self, file_identifier, file_name, is_binary = False,
1638
- replace = False, force_replace = False):
1639
- """
1640
- DESCRIPTION:
1641
- Function to install script on Vantage.
1642
- On success, prints a message that file is installed.
1643
- This language script can be executed via execute_script() function.
1644
-
1645
- PARAMETERS:
1646
- file_identifier:
1647
- Required Argument.
1648
- Specifies the name associated with the user-installed file.
1649
- It cannot have a schema name associated with it,
1650
- as the file is always installed in the current schema.
1651
- The name should be unique within the schema. It can be any valid Teradata
1652
- identifier.
1653
- Types: str
1654
-
1655
- file_name:
1656
- Required Argument:
1657
- Specifies the name of the file user wnats to install.
1658
- Types: str
1659
-
1660
- is_binary:
1661
- Optional Argument.
1662
- Specifies if file to be installed is a binary file.
1663
- Default Value: False
1664
- Types: bool
1665
-
1666
- replace:
1667
- Optional Argument.
1668
- Specifies if the file is to be installed or replaced.
1669
- If set to True, then the file is replaced based on value the of
1670
- force_replace.
1671
- If set to False, then the file is installed.
1672
- Default Value: False
1673
- Types: bool
1674
-
1675
- force_replace:
1676
- Optional Argument.
1677
- Specifies if system should check for the file being used before
1678
- replacing it.
1679
- If set to True, then the file is replaced even if it is being executed.
1680
- If set to False, then an error is thrown if it is being executed.
1681
- Default Value: False
1682
- Types: bool
1683
-
1684
- RETURNS:
1685
- True, if success
1686
-
1687
- RAISES:
1688
- TeradataMLException.
1689
-
1690
- EXAMPLES:
1691
- # Note - Refer to User Guide for setting search path and required permissions.
1692
- # Example 1: Install the file mapper.py found at the relative path
1693
- # data/scripts/ using the default text mode.
1694
-
1695
- # Set SEARCHUIFDBPATH.
1696
- >>> execute_sql("SET SESSION SEARCHUIFDBPATH = alice;")
1697
-
1698
- # Create a Script object that allows us to execute script on Vantage.
1699
- >>> import os
1700
- >>> from teradatasqlalchemy import VARCHAR
1701
- >>> td_path = os.path.dirname(teradataml.__file__)
1702
- >>> sto = Script(data=barrierdf,
1703
- ... script_name='mapper.py',
1704
- ... files_local_path= os.path.join(td_path, 'data', "scripts"),
1705
- ... script_command='tdpython3 ./alice/mapper.py',
1706
- ... data_order_column="Id",
1707
- ... is_local_order=False,
1708
- ... nulls_first=False,
1709
- ... sort_ascending=False,
1710
- ... charset='latin',
1711
- ... returns=OrderedDict([("word", VARCHAR(15)),("count_input", VARCHAR(2))]))
1712
- >>>
1713
-
1714
- # Install file on Vantage.
1715
-
1716
- >>> sto.install_file(file_identifier='mapper',
1717
- ... file_name='mapper.py',
1718
- ... is_binary=False)
1719
- File mapper.py installed in Vantage
1720
-
1721
- # Replace file on Vantage.
1722
- >>> sto.install_file(file_identifier='mapper',
1723
- ... file_name='mapper.py',
1724
- ... is_binary=False,
1725
- ... replace=True,
1726
- ... force_replace=True)
1727
- File mapper.py replaced in Vantage
1728
- """
1729
- # Install/Replace file on Vantage.
1730
- try:
1731
- file_path = os.path.join(self.files_local_path, file_name)
1732
- # Install file on Vantage.
1733
- install_file(file_identifier=file_identifier, file_path=file_path,
1734
- is_binary=is_binary,
1735
- replace=replace, force_replace=force_replace)
1736
- except:
1737
- raise
1738
-
1739
- def remove_file(self, file_identifier, force_remove=False):
1740
- """
1741
- DESCRIPTION:
1742
- Function to remove user installed files/scripts from Vantage.
1743
-
1744
- PARAMETERS:
1745
- file_identifier:
1746
- Required Argument.
1747
- Specifies the name associated with the user-installed file.
1748
- It cannot have a database name associated with it,
1749
- as the file is always installed in the current database.
1750
- Types: str
1751
-
1752
- force_remove:
1753
- Required Argument.
1754
- Specifies if system should check for the file being used before
1755
- removing it.
1756
- If set to True, then the file is removed even if it is being executed.
1757
- If set to False, then an error is thrown if it is being executed.
1758
- Default value: False
1759
- Types: bool
1760
-
1761
- RETURNS:
1762
- True, if success.
1763
-
1764
- RAISES:
1765
- TeradataMLException.
1766
-
1767
- EXAMPLES:
1768
- # Note - Refer to User Guide for setting search path and required permissions.
1769
- # Run install_file example before removing file.
1770
-
1771
- # Set SEARCHUIFDBPATH.
1772
- >>> execute_sql("SET SESSION SEARCHUIFDBPATH = alice;")
1773
-
1774
- # Create a Script object that allows us to execute script on Vantage.
1775
- >>> sto = Script(data=barrierdf,
1776
- ... script_name='mapper.py',
1777
- ... files_local_path= os.path.join(td_path, 'data', "scripts"),
1778
- ... script_command='tdpython3 ./alice/mapper.py',
1779
- ... data_order_column="Id",
1780
- ... is_local_order=False,
1781
- ... nulls_first=False,
1782
- ... sort_ascending=False,
1783
- ... charset='latin',
1784
- ... returns=OrderedDict([("word", VARCHAR(15)),("count_input", VARCHAR(2))]))
1785
- >>>
1786
-
1787
- # Install file on Vantage.
1788
- >>> sto.install_file(file_identifier='mapper',
1789
- ... file_name='mapper.py',
1790
- ... is_binary=False,
1791
- ... replace=True,
1792
- ... force_replace=True)
1793
- File mapper.py replaced in Vantage
1794
-
1795
- # Remove the installed file.
1796
- >>> sto.remove_file(file_identifier='mapper', force_remove=True)
1797
- File mapper removed from Vantage
1798
-
1799
- """
1800
- # Remove file from Vantage
1801
- try:
1802
- remove_file(file_identifier, force_remove)
1803
- except:
1804
- raise
1805
-
1806
- def __form_table_operator_query(self):
1807
- """
1808
- Function to generate the Table Operator queries. The function defines
1809
- variables and list of arguments required to form the query.
1810
- """
1811
- # Output table arguments list.
1812
- self.__func_output_args_sql_names = []
1813
- self.__func_output_args = []
1814
-
1815
- # Generate lists for rest of the function arguments.
1816
- self.__func_other_arg_sql_names = []
1817
- self.__func_other_args = []
1818
- self.__func_other_arg_json_datatypes = []
1819
-
1820
- self.__func_other_arg_sql_names.append("SCRIPT_COMMAND")
1821
- self.__func_other_args.append(
1822
- UtilFuncs._teradata_collapse_arglist(self.script_command, "'"))
1823
- self.__func_other_arg_json_datatypes.append("STRING")
1824
-
1825
- if self.delimiter is not None:
1826
- self.__func_other_arg_sql_names.append("delimiter")
1827
- self.__func_other_args.append(
1828
- UtilFuncs._teradata_collapse_arglist(self.delimiter, "'"))
1829
- self.__func_other_arg_json_datatypes.append("STRING")
1830
-
1831
- # Generate returns clause.
1832
- ret_vals = []
1833
- returns_clause = ''
1834
- for key in self.returns.keys():
1835
- ret_vals.append('{} {}'.format(key, self.returns[key].compile(td_dialect())))
1836
- returns_clause = ', '.join(ret_vals)
1837
-
1838
- self.__func_other_arg_sql_names.append("returns")
1839
- self.__func_other_args.append(
1840
- UtilFuncs._teradata_collapse_arglist(returns_clause, "'"))
1841
- self.__func_other_arg_json_datatypes.append("STRING")
1842
-
1843
- if self.auth is not None:
1844
- self.__func_other_arg_sql_names.append("auth")
1845
- self.__func_other_args.append(
1846
- UtilFuncs._teradata_collapse_arglist(self.auth, "'"))
1847
- self.__func_other_arg_json_datatypes.append("STRING")
1848
-
1849
- if self.charset is not None:
1850
- self.__func_other_arg_sql_names.append("charset")
1851
- self.__func_other_args.append(
1852
- UtilFuncs._teradata_collapse_arglist(self.charset, "'"))
1853
- self.__func_other_arg_json_datatypes.append("STRING")
1854
-
1855
- if self.quotechar is not None:
1856
- self.__func_other_arg_sql_names.append("quotechar")
1857
- self.__func_other_args.append(
1858
- UtilFuncs._teradata_collapse_arglist(self.quotechar, "'"))
1859
- self.__func_other_arg_json_datatypes.append("STRING")
1860
-
1861
- # Declare empty lists to hold input table information.
1862
- self.__func_input_arg_sql_names = []
1863
- self.__func_input_table_view_query = []
1864
- self.__func_input_dataframe_type = []
1865
- self.__func_input_distribution = []
1866
- self.__func_input_partition_by_cols = []
1867
- self.__func_input_order_by_cols = []
1868
- self.__func_input_order_by_type = []
1869
- self.__func_input_sort_ascending = self.sort_ascending
1870
- self.__func_input_nulls_first = None
1871
-
1872
- # Process data.
1873
- if self.data is not None:
1874
- data_distribution = "FACT"
1875
- if self.data_hash_column is not None:
1876
- data_distribution = "HASH"
1877
- data_partition_column = UtilFuncs._teradata_collapse_arglist(
1878
- self.data_hash_column, "\"")
1879
- else:
1880
- if self.__awu._is_default_or_not(self.data_partition_column, "ANY"):
1881
- data_partition_column = UtilFuncs._teradata_collapse_arglist(
1882
- self.data_partition_column, "\"")
1883
- else:
1884
- data_partition_column = None
1885
- if self.data_order_column is not None:
1886
- if self.is_local_order:
1887
- self.__func_input_order_by_type.append("LOCAL")
1888
- if not self.data_hash_column:
1889
- data_distribution = None
1890
- else:
1891
- self.__func_input_order_by_type.append(None)
1892
- self.__func_input_order_by_cols.append(
1893
- UtilFuncs._teradata_collapse_arglist(self.data_order_column, "\""))
1894
- else:
1895
- self.__func_input_order_by_type.append(None)
1896
- self.__func_input_order_by_cols.append("NA_character_")
1897
-
1898
- self.__table_ref = self.__awu._teradata_on_clause_from_dataframe(self.data,
1899
- False)
1900
- self.__func_input_distribution.append(data_distribution)
1901
- self.__func_input_arg_sql_names.append("input")
1902
- self.__func_input_table_view_query.append(self.__table_ref["ref"])
1903
- self.__func_input_dataframe_type.append(self.__table_ref["ref_type"])
1904
- self.__func_input_partition_by_cols.append(data_partition_column)
1905
- self.__func_input_nulls_first = self.nulls_first
1906
-
1907
- function_name = "Script"
1908
- # Create instance to generate Table Operator Query.
1909
- aqg_obj = TableOperatorQueryGenerator(function_name,
1910
- self.__func_input_arg_sql_names,
1911
- self.__func_input_table_view_query,
1912
- self.__func_input_dataframe_type,
1913
- self.__func_input_distribution,
1914
- self.__func_input_partition_by_cols,
1915
- self.__func_input_order_by_cols,
1916
- self.__func_other_arg_sql_names,
1917
- self.__func_other_args,
1918
- self.__func_other_arg_json_datatypes,
1919
- self.__func_output_args_sql_names,
1920
- self.__func_output_args,
1921
- self.__func_input_order_by_type,
1922
- self.__func_input_sort_ascending,
1923
- self.__func_input_nulls_first,
1924
- engine="ENGINE_SQL"
1925
- )
1926
-
1927
- # Invoke call to Table operator query generation.
1928
- self._tblop_query = aqg_obj._gen_table_operator_select_stmt_sql()
1929
-
1930
- # Print Table Operator query if requested to do so.
1931
- if display.print_sqlmr_query:
1932
- print(self._tblop_query)
1933
-
1934
- def __execute(self, output_style='VIEW'):
1935
- """
1936
- DESCRIPTION:
1937
- Function to execute Table Operator queries.
1938
- Create DataFrames for the required Table Operator output.
1939
-
1940
- PARAMETERS:
1941
- output_style:
1942
- Specifies the type of output object to create - a table of a view.
1943
- Permitted Values: 'VIEW', 'TABLE'.
1944
- Default value: 'VIEW'
1945
- Types: str
1946
-
1947
- RAISES:
1948
- None.
1949
-
1950
- RETURNS:
1951
- None.
1952
-
1953
- EXAMPLES:
1954
- >>> return self.__execute(output_style)
1955
- """
1956
- # Generate STDOUT table name and add it to the output table list.
1957
- if output_style == OutputStyle.OUTPUT_TABLE.value:
1958
- table_type = TeradataConstants.TERADATA_TABLE
1959
- else:
1960
- table_type = TeradataConstants.TERADATA_VIEW
1961
-
1962
- tblop_stdout_temp_tablename = \
1963
- UtilFuncs._generate_temp_table_name(prefix="td_tblop_out_",
1964
- use_default_database=True,
1965
- gc_on_quit=True, quote=False,
1966
- table_type=table_type)
1967
- try:
1968
- if output_style == OutputStyle.OUTPUT_TABLE.value:
1969
- UtilFuncs._create_table(tblop_stdout_temp_tablename, self._tblop_query)
1970
- else:
1971
- UtilFuncs._create_view(tblop_stdout_temp_tablename, self._tblop_query)
1972
- except Exception as emsg:
1973
- raise TeradataMlException(
1974
- Messages.get_message(MessageCodes.TDMLDF_EXEC_SQL_FAILED, str(emsg)),
1975
- MessageCodes.TDMLDF_EXEC_SQL_FAILED)
1976
-
1977
- self.result = self.__awu._create_data_set_object(
1978
- df_input=UtilFuncs._extract_table_name(tblop_stdout_temp_tablename),
1979
- source_type="table",
1980
- database_name=UtilFuncs._extract_db_name(tblop_stdout_temp_tablename))
1981
-
1982
- return self.result
1
+ #!/usr/bin/python
2
+ # ##################################################################
3
+ #
4
+ # Copyright 2020 Teradata. All rights reserved. #
5
+ # TERADATA CONFIDENTIAL AND TRADE SECRET #
6
+ #
7
+ # Primary Owner: Gouri Patwardhan (gouri.patwardhan@teradata.com) #
8
+ # Secondary Owner: Trupti Purohit (trupti.purohit@teradata.com) #
9
+ #
10
+ # Function Version: 1.0 #
11
+ #
12
+ # Description: Script is a TeradataML wrapper around Teradata's #
13
+ # Script Table Operator #
14
+ # ##################################################################
15
+
16
+ import os
17
+ import teradataml.dataframe as tdmldf
18
+ import subprocess
19
+ import sys
20
+ from io import StringIO
21
+ from teradataml.common.constants import TableOperatorConstants
22
+ from teradataml.common.wrapper_utils import AnalyticsWrapperUtils
23
+ from teradataml.common.utils import UtilFuncs
24
+ from teradataml.common.constants import OutputStyle, TeradataConstants
25
+ from teradataml.context.context import _get_current_databasename
26
+ from teradataml.context.context import get_context, get_connection
27
+ from teradataml.dataframe.dataframe_utils import DataFrameUtils as df_utils
28
+ from teradataml.dbutils.filemgr import install_file
29
+ from teradataml.dbutils.filemgr import remove_file
30
+ from teradataml.table_operators.table_operator_query_generator import \
31
+ TableOperatorQueryGenerator
32
+ from teradataml.common.exceptions import TeradataMlException
33
+ from teradataml.common.messages import Messages
34
+ from teradataml.common.messagecodes import MessageCodes
35
+ from teradataml.utils.validators import _Validators
36
+ from teradataml.options.display import display
37
+ from teradataml.options.configure import configure
38
+ from teradataml.utils.utils import execute_sql
39
+ from teradatasqlalchemy.dialect import dialect as td_dialect
40
+ from teradatasqlalchemy import (BYTEINT, SMALLINT, INTEGER, BIGINT, DECIMAL, FLOAT,
41
+ NUMBER)
42
+ from teradatasqlalchemy import (TIMESTAMP, DATE, TIME)
43
+ from teradatasqlalchemy import (CHAR, VARCHAR, CLOB)
44
+ from teradatasqlalchemy import (BYTE, VARBYTE, BLOB)
45
+ from teradatasqlalchemy import (PERIOD_DATE, PERIOD_TIME, PERIOD_TIMESTAMP)
46
+ from teradatasqlalchemy import (INTERVAL_YEAR, INTERVAL_YEAR_TO_MONTH, INTERVAL_MONTH,
47
+ INTERVAL_DAY,INTERVAL_DAY_TO_HOUR, INTERVAL_DAY_TO_MINUTE,
48
+ INTERVAL_DAY_TO_SECOND, INTERVAL_HOUR,
49
+ INTERVAL_HOUR_TO_MINUTE, INTERVAL_HOUR_TO_SECOND,
50
+ INTERVAL_MINUTE, INTERVAL_MINUTE_TO_SECOND,
51
+ INTERVAL_SECOND)
52
+ from teradataml.table_operators.TableOperator import TableOperator
53
+
54
+ class Script(TableOperator):
55
+ def __init__(self,
56
+ data=None,
57
+ script_name=None,
58
+ files_local_path=None,
59
+ script_command=None,
60
+ delimiter="\t",
61
+ returns=None,
62
+ auth=None,
63
+ charset=None,
64
+ quotechar=None,
65
+ data_partition_column=None,
66
+ data_hash_column=None,
67
+ data_order_column=None,
68
+ is_local_order=False,
69
+ sort_ascending=True,
70
+ nulls_first=True):
71
+ """
72
+ DESCRIPTION:
73
+ The Script table operator function executes a user-installed script or
74
+ any LINUX command inside database on Teradata Vantage.
75
+
76
+ PARAMETERS:
77
+ script_command:
78
+ Required Argument.
79
+ Specifies the command/script to run.
80
+ Types: str
81
+
82
+ script_name:
83
+ Required Argument.
84
+ Specifies the name of user script.
85
+ User script should have at least permissions of mode 644.
86
+ Types: str
87
+
88
+ files_local_path:
89
+ Required Argument.
90
+ Specifies the absolute local path where user script and all supporting
91
+ files like model files, input data file reside.
92
+ Types: str
93
+
94
+ returns:
95
+ Required Argument.
96
+ Specifies output column definition.
97
+ Types: Dictionary specifying column name to teradatasqlalchemy type mapping.
98
+ Default: None
99
+ Note:
100
+ User can pass a dictionary (dict or OrderedDict) to the "returns" argument,
101
+ with the keys ordered to represent the order of the output columns.
102
+ Preferred type is OrderedDict.
103
+
104
+ data:
105
+ Optional Argument.
106
+ Specifies a teradataml DataFrame containing the input data for the
107
+ script.
108
+
109
+ data_hash_column:
110
+ Optional Argument.
111
+ Specifies the column to be used for hashing.
112
+ The rows in the data are redistributed to AMPs based on the hash value of
113
+ the column specified.
114
+ The user-installed script file then runs once on each AMP.
115
+ If there is no "data_partition_column", then the entire result set,
116
+ delivered by the function, constitutes a single group or partition.
117
+ Types: str
118
+ Note:
119
+ "data_hash_column" can not be specified along with
120
+ "data_partition_column", "is_local_order" and "data_order_column".
121
+
122
+ data_partition_column:
123
+ Optional Argument.
124
+ Specifies Partition By columns for "data".
125
+ Values to this argument can be provided as a list, if multiple
126
+ columns are used for partition.
127
+ Default Value: ANY
128
+ Types: str OR list of Strings (str)
129
+ Note:
130
+ 1) "data_partition_column" can not be specified along with
131
+ "data_hash_column".
132
+ 2) "data_partition_column" can not be specified along with
133
+ "is_local_order = True".
134
+
135
+ is_local_order:
136
+ Optional Argument.
137
+ Specifies a boolean value to determine whether the input data is to be
138
+ ordered locally or not. Order by specifies the order in which the
139
+ values in a group, or partition, are sorted. Local Order By specifies
140
+ orders qualified rows on each AMP in preparation to be input to a table
141
+ function. This argument is ignored, if "data_order_column" is None. When
142
+ set to True, data is ordered locally.
143
+ Default Value: False
144
+ Types: bool
145
+ Note:
146
+ 1) "is_local_order" can not be specified along with "data_hash_column".
147
+ 2) When "is_local_order" is set to True, "data_order_column" should be
148
+ specified, and the columns specified in "data_order_column" are
149
+ used for local ordering.
150
+
151
+ data_order_column:
152
+ Optional Argument.
153
+ Specifies Order By columns for "data".
154
+ Values to this argument can be provided as a list, if multiple
155
+ columns are used for ordering. This argument is used with in both cases:
156
+ "is_local_order = True" and "is_local_order = False".
157
+ Types: str OR list of Strings (str)
158
+ Note:
159
+ "data_order_column" can not be specified along with "data_hash_column".
160
+
161
+ sort_ascending:
162
+ Optional Argument.
163
+ Specifies a boolean value to determine if the result set is to be sorted
164
+ on the "data_order_column" column in ascending or descending order.
165
+ The sorting is ascending when this argument is set to True, and descending
166
+ when set to False. This argument is ignored, if "data_order_column" is
167
+ None.
168
+ Default Value: True
169
+ Types: bool
170
+
171
+ nulls_first:
172
+ Optional Argument.
173
+ Specifies a boolean value to determine whether NULLS are listed first or
174
+ last during ordering. This argument is ignored, if "data_order_column" is
175
+ None. NULLS are listed first when this argument is set to True, and last
176
+ when set to False.
177
+ Default Value: True
178
+ Types: bool
179
+
180
+ delimiter:
181
+ Optional Argument.
182
+ Specifies a delimiter to use when reading columns from a row and
183
+ writing result columns.
184
+ Default Value: "\t" (tab)
185
+ Types: str of length 1 character
186
+ Notes:
187
+ 1) This argument cannot be same as "quotechar" argument.
188
+ 2) This argument cannot be a newline character i.e., '\\n'.
189
+
190
+ auth:
191
+ Optional Argument.
192
+ Specifies an authorization to use when running the script.
193
+ Types: str
194
+
195
+ charset:
196
+ Optional Argument.
197
+ Specifies the character encoding for data.
198
+ Permitted Values: utf-16, latin
199
+ Types: str
200
+
201
+ quotechar:
202
+ Optional Argument.
203
+ Specifies a character that forces all input and output of the script
204
+ to be quoted using this specified character.
205
+ Using this argument enables the Advanced SQL Engine to distinguish
206
+ between NULL fields and empty strings. A string with length zero is
207
+ quoted, while NULL fields are not.
208
+ If this character is found in the data, it will be escaped by a second
209
+ quote character.
210
+ Types: character of length 1
211
+ Notes:
212
+ 1) This argument cannot be same as "delimiter" argument.
213
+ 2) This argument cannot be a newline character i.e., '\\n'.
214
+
215
+ RETURNS:
216
+ Script Object
217
+
218
+ RAISES:
219
+ TeradataMlException
220
+
221
+ EXAMPLES:
222
+ # Note - Refer to User Guide for setting search path and required permissions.
223
+ # Load example data.
224
+ load_example_data("Script", ["barrier"])
225
+
226
+ # Example - The script mapper.py reads in a line of text input
227
+ # ("Old Macdonald Had A Farm") from csv and splits the line into individual
228
+ # words, emitting a new row for each word.
229
+
230
+ # Create teradataml DataFrame objects.
231
+ >>> barrierdf = DataFrame.from_table("barrier")
232
+
233
+ # Set SEARCHUIFDBPATH.
234
+ >>> execute_sql("SET SESSION SEARCHUIFDBPATH = alice;")
235
+
236
+ # Create a Script object that allows us to execute script on Vantage.
237
+ >>> import teradataml, os
238
+ >>> from teradatasqlalchemy import VARCHAR
239
+ >>> td_path = os.path.dirname(teradataml.__file__)
240
+ >>> sto = Script(data=barrierdf,
241
+ ... script_name='mapper.py',
242
+ ... files_local_path= os.path.join(td_path, 'data', 'scripts'),
243
+ ... script_command='tdpython3 ./alice/mapper.py',
244
+ ... data_order_column="Id",
245
+ ... is_local_order=False,
246
+ ... nulls_first=False,
247
+ ... sort_ascending=False,
248
+ ... charset='latin',
249
+ ... returns=OrderedDict([("word", VARCHAR(15)),("count_input", VARCHAR(2))]))
250
+
251
+ # Run user script locally and using data from csv.
252
+
253
+ >>> sto.test_script(input_data_file='../barrier.csv', data_file_delimiter=',')
254
+
255
+ ############ STDOUT Output ############
256
+
257
+ word count_input
258
+ 0 1 1
259
+ 1 Old 1
260
+ 2 Macdonald 1
261
+ 3 Had 1
262
+ 4 A 1
263
+ 5 Farm 1
264
+ >>>
265
+
266
+ # Script results look good. Now install file on Vantage.
267
+ >>> sto.install_file(file_identifier='mapper',
268
+ ... file_name='mapper.py',
269
+ ... is_binary=False)
270
+ File mapper.py installed in Vantage
271
+
272
+ # Execute the user script on Vantage.
273
+ >>> sto.execute_script()
274
+ ############ STDOUT Output ############
275
+
276
+ word count_input
277
+ 0 Macdonald 1
278
+ 1 A 1
279
+ 2 Farm 1
280
+ 3 Had 1
281
+ 4 Old 1
282
+ 5 1 1
283
+
284
+ # Remove the installed file from Vantage.
285
+ >>> sto.remove_file(file_identifier='mapper', force_remove=True)
286
+ File mapper removed from Vantage
287
+ """
288
+ self.result = None
289
+ self.data = data
290
+ self.script_name = script_name
291
+ self.files_local_path = files_local_path
292
+ self.script_command = script_command
293
+ self.delimiter = delimiter
294
+ self.returns = returns
295
+ self.auth = auth
296
+ self.charset = charset
297
+ self.quotechar = quotechar
298
+ self.data_partition_column = data_partition_column
299
+ self.data_hash_column = data_hash_column
300
+ self.data_order_column = data_order_column
301
+ self.is_local_order = is_local_order
302
+ self.sort_ascending = sort_ascending
303
+ self.nulls_first = nulls_first
304
+ self._check_reserved_keyword = True
305
+ self._skip_argument_validation = False
306
+
307
+ # Create AnalyticsWrapperUtils instance which contains validation functions.
308
+ # This is required for is_default_or_not check.
309
+ # Rest all validation is done using _Validators
310
+ self.__awu = AnalyticsWrapperUtils()
311
+
312
+ # Below matrix is a list of lists, where in each row contains following elements:
313
+ # Let's take an example of following, just to get an idea:
314
+ # [element1, element2, element3, element4, element5, element6]
315
+ # e.g.
316
+ # ["join", join, True, (str), True, concat_join_permitted_values]
317
+
318
+ # 1. element1 --> Argument Name, a string. ["join" in above example.]
319
+ # 2. element2 --> Argument itself. [join]
320
+ # 3. element3 --> Specifies a flag that mentions argument is optional or not.
321
+ # False, means required and True means optional.
322
+ # 4. element4 --> Tuple of accepted types. (str) in above example.
323
+ # 5. element5 --> True, means validate for empty value. Error will be raised,
324
+ # if empty value is passed.
325
+ # If not specified, means same as specifying False.
326
+ # 6. element6 --> A list of permitted values, an argument can accept.
327
+ # If not specified, it is as good as passing None.
328
+ # If a list is passed, validation will be
329
+ # performed for permitted values.
330
+
331
+ self.awu_matrix = []
332
+ self.awu_matrix.append(["data", self.data, True, (tdmldf.dataframe.DataFrame)])
333
+ self.awu_matrix.append(["data_partition_column", self.data_partition_column, True,
334
+ (str, list), True])
335
+ self.awu_matrix.append(["data_hash_column", self.data_hash_column, True,
336
+ (str, list), True])
337
+ self.awu_matrix.append(["data_order_column", self.data_order_column, True,
338
+ (str, list), True])
339
+ self.awu_matrix.append(["is_local_order", self.is_local_order, True, (bool)])
340
+ self.awu_matrix.append(["sort_ascending", self.sort_ascending, True, (bool)])
341
+ self.awu_matrix.append(["nulls_first", self.nulls_first, True, (bool)])
342
+ self.awu_matrix.append(["script_command", self.script_command, False, (str),
343
+ True])
344
+ self.awu_matrix.append(["script_name", self.script_name, True, (str), True])
345
+ self.awu_matrix.append(["files_local_path", self.files_local_path, True, (str),
346
+ True])
347
+ self.awu_matrix.append(["delimiter", self.delimiter, True, (str), False])
348
+ self.awu_matrix.append(["returns", self.returns, False, (dict), True])
349
+ self.awu_matrix.append(["auth", self.auth, True, (str), True])
350
+ self.awu_matrix.append(["charset", self.charset, True, (str), True,
351
+ ["utf-16", "latin"]])
352
+ self.awu_matrix.append(["quotechar", self.quotechar, True, (str), False])
353
+
354
+ # Perform the function validations
355
+ self.__validate()
356
+
357
+ @property
358
+ def skip_argument_validation(self):
359
+ """
360
+ DESCRIPTION:
361
+ Getter for self._skip_argument_validation.
362
+
363
+ RETURNS:
364
+ bool
365
+
366
+ RAISES:
367
+ None
368
+ """
369
+ return self._skip_argument_validation
370
+
371
+ @skip_argument_validation.setter
372
+ def skip_argument_validation(self, flag):
373
+ """
374
+ DESCRIPTION:
375
+ Setter for self._skip_argument_validation
376
+
377
+ PARAMETERS:
378
+ flag Required Argument.
379
+ Specifies whether the arguments should be skipped or not.
380
+ Types: bool
381
+ RETURNS:
382
+ None
383
+
384
+ RAISES:
385
+ None
386
+ """
387
+ self._skip_argument_validation = flag
388
+
389
+ @property
390
+ def check_reserved_keyword(self):
391
+ """
392
+ DESCRIPTION:
393
+ Getter for self._check_reserved_keyword.
394
+
395
+ RETURNS:
396
+ bool
397
+
398
+ RAISES:
399
+ None
400
+ """
401
+ return self._check_reserved_keyword
402
+
403
+ @check_reserved_keyword.setter
404
+ def check_reserved_keyword(self, flag):
405
+ """
406
+ DESCRIPTION:
407
+ Setter for self._check_reserved_keyword
408
+
409
+ RETURNS:
410
+ None
411
+
412
+ RAISES:
413
+ None
414
+ """
415
+ self._check_reserved_keyword = flag
416
+
417
+ def __validate_for_reserved_keyword(self):
418
+ """
419
+ DESCRIPTION:
420
+ Function to validate if the returns clause has teradata reserved keyword or not.
421
+ If it contains reserved keyword, then raise an error.
422
+
423
+ RETURNS:
424
+ None
425
+
426
+ RAISES:
427
+ TeradataMlException
428
+
429
+ """
430
+ if self.check_reserved_keyword:
431
+ from teradataml import list_td_reserved_keywords
432
+ if get_connection():
433
+ # Checking for reserved keywords and raising error if present.
434
+ for column_name in self.returns:
435
+ list_td_reserved_keywords(key=column_name, raise_error=True)
436
+
437
+ def __validate(self):
438
+ """
439
+ Function to validate Table Operator Function arguments, which verifies missing
440
+ arguments, input argument and table types. Also processes the argument values.
441
+ """
442
+ if self.skip_argument_validation:
443
+ return
444
+ # Make sure that a non-NULL value has been supplied for all mandatory arguments
445
+ _Validators._validate_missing_required_arguments(self.awu_matrix)
446
+
447
+ # Validate argument types.
448
+ _Validators._validate_function_arguments(self.awu_matrix,
449
+ skip_empty_check={"quotechar" : ["\n", "\t"],
450
+ "delimiter" : ["\n"]})
451
+
452
+ # permissible_datatypes in returns
453
+ allowed_datatypes = (BYTEINT, SMALLINT, INTEGER, BIGINT, DECIMAL, FLOAT, NUMBER,
454
+ TIMESTAMP, DATE, TIME, CHAR, VARCHAR, CLOB, BYTE, VARBYTE,
455
+ BLOB, PERIOD_DATE, PERIOD_TIME, PERIOD_TIMESTAMP,
456
+ INTERVAL_YEAR, INTERVAL_YEAR_TO_MONTH, INTERVAL_MONTH,
457
+ INTERVAL_DAY, INTERVAL_DAY_TO_HOUR, INTERVAL_DAY_TO_MINUTE,
458
+ INTERVAL_DAY_TO_SECOND, INTERVAL_HOUR,
459
+ INTERVAL_HOUR_TO_MINUTE, INTERVAL_HOUR_TO_SECOND,
460
+ INTERVAL_MINUTE, INTERVAL_MINUTE_TO_SECOND, INTERVAL_SECOND
461
+ )
462
+
463
+ # Validate keys and datatypes in returns.
464
+ self.awu_matrix_returns = []
465
+ for key in self.returns.keys():
466
+ self.awu_matrix_returns.append(["keys in returns", key, False, (str), True])
467
+ self.awu_matrix_returns.append(["value in returns", self.returns[key], False,
468
+ allowed_datatypes])
469
+
470
+
471
+ _Validators._validate_function_arguments(self.awu_matrix_returns)
472
+
473
+ if self.data is not None:
474
+ # Hash and order by can be used together as long as is_local_order = True.
475
+ if all([self.data_hash_column,
476
+ self.data_order_column]) and not self.is_local_order:
477
+ raise TeradataMlException(
478
+ Messages.get_message(MessageCodes.CANNOT_USE_TOGETHER_WITH,
479
+ "data_hash_column' and 'data_order_column",
480
+ "is_local_order=False"),
481
+ MessageCodes.CANNOT_USE_TOGETHER_WITH)
482
+
483
+ # Either hash or partition can be used.
484
+ _Validators._validate_mutually_exclusive_arguments(self.data_hash_column,
485
+ "data_hash_column",
486
+ self.data_partition_column,
487
+ "data_partition_column",
488
+ skip_all_none_check=True)
489
+
490
+ # Either local order by or partition by can be used.
491
+ _Validators._validate_mutually_exclusive_arguments(self.is_local_order,
492
+ "is_local_order=True",
493
+ self.data_partition_column,
494
+ "data_partition_column",
495
+ skip_all_none_check=True)
496
+
497
+ # local order by requires column name.
498
+ if self.is_local_order and self.data_order_column is None:
499
+ message = Messages.get_message(MessageCodes.DEPENDENT_ARG_MISSING,
500
+ "data_order_column", "is_local_order=True")
501
+ raise TeradataMlException(message, MessageCodes.DEPENDENT_ARG_MISSING)
502
+
503
+ if self.__awu._is_default_or_not(self.data_partition_column, "ANY"):
504
+ self.__awu._validate_dataframe_has_argument_columns(
505
+ self.data_partition_column, "data_partition_column", self.data, "data", True)
506
+
507
+ if self.data_order_column is not None:
508
+ self.__awu._validate_dataframe_has_argument_columns(
509
+ self.data_order_column, "data_order_column",self.data, "data", False)
510
+
511
+ if self.data_hash_column is not None:
512
+ self.__awu._validate_dataframe_has_argument_columns(
513
+ self.data_hash_column, "data_hash_column", self.data, "data", False)
514
+
515
+ if self.data_partition_column is not None:
516
+ self.__awu._validate_dataframe_has_argument_columns(
517
+ self.data_partition_column, "data_partition_column", self.data, "data", False)
518
+
519
+ # Check for length of the arguments "delimiter" and "quotechar".
520
+ if self.delimiter is not None:
521
+ _Validators._validate_str_arg_length('delimiter', self.delimiter, 'EQ', 1)
522
+
523
+ if self.quotechar is not None:
524
+ _Validators._validate_str_arg_length('quotechar', self.quotechar, 'EQ', 1)
525
+
526
+ # The arguments 'quotechar' and 'delimiter' cannot take newline character.
527
+ if self.delimiter == '\n':
528
+ message = Messages.get_message(MessageCodes.NOT_ALLOWED_VALUES, "\n", "delimiter")
529
+ raise TeradataMlException(message, MessageCodes.NOT_ALLOWED_VALUES)
530
+ if self.quotechar == '\n':
531
+ message = Messages.get_message(MessageCodes.NOT_ALLOWED_VALUES, "\n", "quotechar")
532
+ raise TeradataMlException(message, MessageCodes.NOT_ALLOWED_VALUES)
533
+
534
+ # The arguments 'quotechar' and 'delimiter' cannot have the same value.
535
+ if self.delimiter == self.quotechar:
536
+ message = Messages.get_message(MessageCodes.ARGUMENT_VALUE_SAME, "delimiter",
537
+ "quotechar")
538
+ raise TeradataMlException(message, MessageCodes.ARGUMENT_VALUE_SAME)
539
+
540
+
541
+ def set_data(self,
542
+ data,
543
+ data_partition_column=None,
544
+ data_hash_column=None,
545
+ data_order_column=None,
546
+ is_local_order=False,
547
+ sort_ascending=True,
548
+ nulls_first=True):
549
+ """
550
+ DESCRIPTION:
551
+ Function enables user to set data and data related arguments without having to
552
+ re-create Script object.
553
+
554
+ PARAMETERS:
555
+ data:
556
+ Required Argument.
557
+ Specifies a teradataml DataFrame containing the input data for the script.
558
+
559
+ data_hash_column:
560
+ Optional Argument.
561
+ Specifies the column to be used for hashing.
562
+ The rows in the data are redistributed to AMPs based on the
563
+ hash value of the column specified.
564
+ The user installed script then runs once on each AMP.
565
+ If there is no data_partition_column, then the entire result set delivered
566
+ by the function, constitutes a single group or partition.
567
+ Types: str
568
+ Note:
569
+ "data_hash_column" can not be specified along with
570
+ "data_partition_column", "is_local_order" and "data_order_column".
571
+
572
+ data_partition_column:
573
+ Optional Argument.
574
+ Specifies Partition By columns for data.
575
+ Values to this argument can be provided as a list, if multiple
576
+ columns are used for partition.
577
+ Default Value: ANY
578
+ Types: str OR list of Strings (str)
579
+ Note:
580
+ 1) "data_partition_column" can not be specified along with
581
+ "data_hash_column".
582
+ 2) "data_partition_column" can not be specified along with
583
+ "is_local_order = True".
584
+
585
+ is_local_order:
586
+ Optional Argument.
587
+ Specifies a boolean value to determine whether the input data is to be
588
+ ordered locally or not. Order by specifies the order in which the
589
+ values in a group or partition are sorted. Local Order By specifies
590
+ orders qualified rows on each AMP in preparation to be input to a table
591
+ function. This argument is ignored, if "data_order_column" is None. When
592
+ set to True, data is ordered locally.
593
+ Default Value: False
594
+ Types: bool
595
+ Note:
596
+ 1) "is_local_order" can not be specified along with
597
+ "data_hash_column".
598
+ 2) When "is_local_order" is set to True, "data_order_column" should be
599
+ specified, and the columns specified in "data_order_column" are
600
+ used for local ordering.
601
+
602
+ data_order_column:
603
+ Optional Argument.
604
+ Specifies Order By columns for data.
605
+ Values to this argument can be provided as a list, if multiple
606
+ columns are used for ordering.
607
+ This argument is used in both cases:
608
+ "is_local_order = True" and "is_local_order = False".
609
+ Types: str OR list of Strings (str)
610
+ Note:
611
+ "data_order_column" can not be specified along with
612
+ "data_hash_column".
613
+
614
+ sort_ascending:
615
+ Optional Argument.
616
+ Specifies a boolean value to determine if the result set is to be sorted
617
+ on the column specified in "data_order_column", in ascending or descending
618
+ order.
619
+ The sorting is ascending when this argument is set to True, and descending
620
+ when set to False.
621
+ This argument is ignored, if "data_order_column" is None.
622
+ Default Value: True
623
+ Types: bool
624
+
625
+ nulls_first:
626
+ Optional Argument.
627
+ Specifies a boolean value to determine whether NULLS are listed first or
628
+ last during ordering.
629
+ This argument is ignored, if "data_order_column" is None.
630
+ NULLS are listed first when this argument is set to True, and
631
+ last when set to False.
632
+ Default Value: True
633
+ Types: bool
634
+
635
+ RETURNS:
636
+ None.
637
+
638
+ RAISES:
639
+ TeradataMlException
640
+
641
+ EXAMPLES:
642
+ # Note - Refer to User Guide for setting search path and required permissions.
643
+ # Load example data.
644
+ load_example_data("Script", ["barrier"])
645
+
646
+ # Example 1
647
+ # Create teradataml DataFrame objects.
648
+ >>> barrierdf = DataFrame.from_table("barrier")
649
+ >>> barrierdf
650
+ Name
651
+ Id
652
+ 1 Old Macdonald Had A Farm
653
+ >>>
654
+
655
+ # Set SEARCHUIFDBPATH
656
+ >>> execute_sql("SET SESSION SEARCHUIFDBPATH = alice;")
657
+ >>> import teradataml
658
+ >>> from teradatasqlalchemy import VARCHAR
659
+ >>> td_path = os.path.dirname(teradataml.__file__)
660
+
661
+ # The script mapper.py reads in a line of text input
662
+ # ("Old Macdonald Had A Farm") from csv and
663
+ # splits the line into individual words, emitting a new row for each word.
664
+ # Create a Script object without data and its arguments.
665
+ >>> sto = Script(data = barrierdf,
666
+ ... script_name='mapper.py',
667
+ ... files_local_path= os.path.join(td_path,'data', 'scripts'),
668
+ ... script_command='tdpython3 ./alice/mapper.py',
669
+ ... charset='latin',
670
+ ... returns=OrderedDict([("word", VARCHAR(15)),("count_input", VARCHAR(2))]))
671
+
672
+ # Test script using data from file
673
+ >>> sto.test_script(input_data_file='../barrier.csv', data_file_delimiter=',')
674
+ ############ STDOUT Output ############
675
+ word count_input
676
+ 0 1 1
677
+ 1 Old 1
678
+ 2 Macdonald 1
679
+ 3 Had 1
680
+ 4 A 1
681
+ 5 Farm 1
682
+ >>>
683
+
684
+ # Test script using data from DB.
685
+ >>> sto.test_script(password='alice')
686
+ ############ STDOUT Output ############
687
+
688
+ word count_input
689
+ 0 1 1
690
+ 1 Old 1
691
+ 2 Macdonald 1
692
+ 3 Had 1
693
+ 4 A 1
694
+ 5 Farm 1
695
+
696
+ # Test script using data from DB and with data_row_limit.
697
+ >>> sto.test_script(password='alice', data_row_limit=5)
698
+ ############ STDOUT Output ############
699
+
700
+ word count_input
701
+ 0 1 1
702
+ 1 Old 1
703
+ 2 Macdonald 1
704
+ 3 Had 1
705
+ 4 A 1
706
+ 5 Farm 1
707
+
708
+ # Now in order to test / run script on actual data on Vantage user must
709
+ # set data and related arguments.
710
+ # Note:
711
+ # All data related arguments that are not specified in set_data() are
712
+ # reset to default values.
713
+ >>> sto.set_data(data=barrierdf,
714
+ ... data_order_column="Id",
715
+ ... is_local_order=False,
716
+ ... nulls_first=False,
717
+ ... sort_ascending=False)
718
+
719
+ # Execute the user script on Vantage.
720
+ >>> sto.execute_script()
721
+ ############ STDOUT Output ############
722
+ word count_input
723
+ 0 Macdonald 1
724
+ 1 A 1
725
+ 2 Farm 1
726
+ 3 Had 1
727
+ 4 Old 1
728
+ 5 1 1
729
+
730
+ # Example 2 -
731
+ # Script is tested using test_script and executed on Vantage.
732
+ # use set_data() to reset arguments.
733
+ # Create teradataml DataFrame objects.
734
+ >>> load_example_data("Script", ["barrier_new"])
735
+ >>> barrierdf_new = DataFrame.from_table("barrier_new")
736
+ >>> barrierdf_new
737
+ Name
738
+ Id
739
+ 2 On his farm he had a cow
740
+ 1 Old Macdonald Had A Farm
741
+ >>>
742
+
743
+ # Create a Script object that allows us to execute script on Vantage.
744
+ >>> sto = Script(data=barrierdf_new,
745
+ ... script_name='mapper.py',
746
+ ... files_local_path= os.path.join(td_path, 'data', 'scripts'),
747
+ ... script_command='tdpython3 ./alice/mapper.py',
748
+ ... data_order_column="Id",
749
+ ... is_local_order=False,
750
+ ... nulls_first=False,
751
+ ... sort_ascending=False,
752
+ ... charset='latin',
753
+ ... returns=OrderedDict([("word", VARCHAR(15)),("count_input", VARCHAR(2))]))
754
+ # Script is tested using test_script and executed on Vantage.
755
+ >>> sto.execute_script()
756
+ ############ STDOUT Output ############
757
+ word count_input
758
+ 0 his 1
759
+ 1 he 1
760
+ 2 had 1
761
+ 3 a 1
762
+ 4 1 1
763
+ 5 Old 1
764
+ 6 cow 1
765
+ 7 farm 1
766
+ 8 On 1
767
+ 9 2 1
768
+
769
+ # Now in order to run the script with a different dataset,
770
+ # user can use set_data().
771
+ # Re-set data and some data related parameters.
772
+ # Note:
773
+ # All data related arguments that are not specified in set_data() are
774
+ # reset to default values.
775
+ >>> sto.set_data(data=barrierdf,
776
+ ... data_order_column='Id',
777
+ ... is_local_order=True,
778
+ ... nulls_first=True)
779
+ >>> sto.execute_script()
780
+ word count_input
781
+ 0 Macdonald 1
782
+ 1 A 1
783
+ 2 Farm 1
784
+ 3 Had 1
785
+ 4 Old 1
786
+ 5 1 1
787
+
788
+ # Example 3
789
+ # Script is tested using test_script and executed on Vantage.
790
+ # In order to run the script with same dataset but different data related
791
+ # arguments, use set_data() to reset arguments.
792
+ # Note:
793
+ # All data related arguments that are not specified in set_data() are
794
+ # reset to default values.
795
+ >>> sto.set_data(data=barrierdf_new,
796
+ ... data_order_column='Id',
797
+ ... is_local_order = True,
798
+ ... nulls_first = True)
799
+
800
+ >>> sto.execute_script()
801
+ ############ STDOUT Output ############
802
+
803
+ word count_input
804
+ 0 Macdonald 1
805
+ 1 A 1
806
+ 2 Farm 1
807
+ 3 2 1
808
+ 4 his 1
809
+ 5 farm 1
810
+ 6 On 1
811
+ 7 Had 1
812
+ 8 Old 1
813
+ 9 1 1
814
+ """
815
+ super(Script, self).set_data(data,
816
+ data_partition_column,
817
+ data_hash_column,
818
+ data_order_column,
819
+ is_local_order,
820
+ sort_ascending,
821
+ nulls_first)
822
+ self.__validate()
823
+
824
+ def test_script(self, supporting_files=None, input_data_file=None, script_args="",
825
+ exec_mode ='local', **kwargs):
826
+ """
827
+ DESCRIPTION:
828
+ Function enables user to run script locally outside Vantage.
829
+ Input data for user script is either read from a file or from database.
830
+ Note:
831
+ 1. Purpose of test_script() function is to enable the user to test their scripts for any errors without
832
+ installing it on Vantage, using the input data provided.
833
+ 2. Data is not partitioned for testing the script if read from input data file.
834
+ 3. Function can produce different output if input is read from a file than input from database.
835
+
836
+ PARAMETERS:
837
+ supporting_files:
838
+ Optional Argument.
839
+ Specifies a file or list of supporting files like model files to be
840
+ copied to the container.
841
+ Types: string or list of str
842
+
843
+ input_data_file:
844
+ Optional Argument.
845
+ Specifies the name of the input data file.
846
+ It should have a path relative to the location specified in
847
+ "files_local_path" argument.
848
+ If set to None, read data from AMP, else from file passed in the argument
849
+ 'input_data_file'.
850
+ File should have at least permissions of mode 644.
851
+ Types: str
852
+
853
+ script_args:
854
+ Optional Argument.
855
+ Specifies command line arguments required by the user script.
856
+ Types: str
857
+
858
+ exec_mode:
859
+ Optional Argument.
860
+ Specifies the mode in which user wants to test the script.
861
+ When set to 'local', the user script will run locally on user's system.
862
+ Permitted Values: 'local'
863
+ Default Value: 'local'
864
+ Types: str
865
+
866
+ kwargs:
867
+ Optional Argument.
868
+ Specifies the keyword arguments required for testing.
869
+ Keys can be:
870
+ data_row_limit:
871
+ Optional Argument. Ignored when data is read from file.
872
+ Specifies the number of rows to be taken from all amps when
873
+ reading from a table or view on Vantage.
874
+ Default Value: 1000
875
+ Types: int
876
+
877
+ password:
878
+ Optional Argument. Required when reading from database.
879
+ Specifies the password to connect to vantage where the data
880
+ resides.
881
+ Types: str
882
+
883
+ data_file_delimiter:
884
+ Optional Argument.
885
+ Specifies the delimiter used in the input data file. This
886
+ argument can be specified when data is read from file.
887
+ Default Value: '\t'
888
+ Types: str
889
+
890
+ data_file_header:
891
+ Optional Argument.
892
+ Specifies whether the input data file contains header. This
893
+ argument can be specified when data is read from file.
894
+ Default Value: True
895
+ Types: bool
896
+
897
+ data_file_quote_char:
898
+ Optional Argument.
899
+ Specifies the quotechar used in the input data file.
900
+ This argument can be specified when data is read from file.
901
+ Default Value: '"'
902
+
903
+ logmech:
904
+ Optional Argument.
905
+ Specifies the type of logon mechanism to establish a connection to
906
+ Teradata Vantage.
907
+ Permitted Values: 'TD2', 'TDNEGO', 'LDAP', 'KRB5' & 'JWT'.
908
+ TD2:
909
+ The Teradata 2 (TD2) mechanism provides authentication
910
+ using a Vantage username and password. This is the default
911
+ logon mechanism using which the connection is established
912
+ to Vantage.
913
+
914
+ TDNEGO:
915
+ A security mechanism that automatically determines the
916
+ actual mechanism required, based on policy, without user's
917
+ involvement. The actual mechanism is determined by the
918
+ TDGSS server configuration and by the security policy's
919
+ mechanism restrictions.
920
+
921
+ LDAP:
922
+ A directory-based user logon to Vantage with a directory
923
+ username and password and is authenticated by the directory.
924
+
925
+ KRB5 (Kerberos):
926
+ A directory-based user logon to Vantage with a domain
927
+ username and password and is authenticated by
928
+ Kerberos (KRB5 mechanism).
929
+ Note:
930
+ User must have a valid ticket-granting ticket in
931
+ order to use this logon mechanism.
932
+
933
+ JWT:
934
+ The JSON Web Token (JWT) authentication mechanism enables
935
+ single sign-on (SSO) to the Vantage after the user
936
+ successfully authenticates to Teradata UDA User Service.
937
+ Note:
938
+ User must use logdata parameter when using 'JWT' as
939
+ the logon mechanism.
940
+ Default Value: TD2
941
+ Types: str
942
+
943
+ Note:
944
+ teradataml expects the client environments are already setup with appropriate
945
+ security mechanisms and are in working conditions.
946
+ For more information please refer Teradata Vantage™ - Advanced SQL Engine
947
+ Security Administration at https://www.info.teradata.com/
948
+
949
+ logdata:
950
+ Optional Argument.
951
+ Specifies parameters to the LOGMECH command beyond those needed by
952
+ the logon mechanism, such as user ID, password and tokens
953
+ (in case of JWT) to successfully authenticate the user.
954
+ Types: str
955
+
956
+ Types: dict
957
+
958
+ RETURNS:
959
+ Output from user script.
960
+
961
+ RAISES:
962
+ TeradataMlException
963
+
964
+ EXAMPLES:
965
+ # Assumption - sto is Script() object. Please refer to help(Script)
966
+ # for creating Script object.
967
+ # Run user script in local mode with input from data file.
968
+
969
+ >>> sto.test_script(input_data_file='../barrier.csv',
970
+ ... data_file_delimiter=',',
971
+ ... data_file_quote_char='"',
972
+ ... data_file_header=True,
973
+ ... exec_mode='local')
974
+
975
+ ############ STDOUT Output ############
976
+ word count_input
977
+ 0 1 1
978
+ 1 Old 1
979
+ 2 Macdonald 1
980
+ 3 Had 1
981
+ 4 A 1
982
+ 5 Farm 1
983
+ >>>
984
+
985
+ # Run user script in local mode with input from table.
986
+ >>> sto.test_script(data_row_limit=300, password='alice', exec_mode='local')
987
+
988
+ ############ STDOUT Output ############
989
+ word count_input
990
+ 0 1 1
991
+ 1 Old 1
992
+ 2 Macdonald 1
993
+ 3 Had 1
994
+ 4 A 1
995
+ 5 Farm 1
996
+
997
+ """
998
+ logmech_valid_values = ['TD2', 'TDNEGO', 'LDAP', 'KRB5', 'JWT']
999
+
1000
+ awu_matrix_test=[]
1001
+ awu_matrix_test.append((["supporting_files", supporting_files, True,
1002
+ (str,list), True]))
1003
+ awu_matrix_test.append((["input_data_file", input_data_file, True, (str), True]))
1004
+ awu_matrix_test.append((["script_args", script_args, True, (str), False]))
1005
+ awu_matrix_test.append((["exec_mode", exec_mode, True, (str), True,
1006
+ [TableOperatorConstants.LOCAL_EXEC.value]]))
1007
+
1008
+ data_row_limit = kwargs.pop("data_row_limit", 1000)
1009
+ awu_matrix_test.append((["data_row_limit", data_row_limit, True, (int), True]))
1010
+
1011
+ data_file_delimiter = kwargs.pop("data_file_delimiter", '\t')
1012
+ awu_matrix_test.append((["data_file_delimiter", data_file_delimiter, True,
1013
+ (str), False]))
1014
+
1015
+ data_file_quote_char = kwargs.pop("data_file_quote_char", '"')
1016
+ awu_matrix_test.append((["data_file_quote_char", data_file_quote_char, True,
1017
+ (str), False]))
1018
+
1019
+ data_file_header = kwargs.pop("data_file_header", True)
1020
+ awu_matrix_test.append((["data_file_header", data_file_header, True, (bool)]))
1021
+
1022
+ logmech = kwargs.pop("logmech", "TD2")
1023
+ awu_matrix_test.append(
1024
+ ["logmech", logmech, True, (str), True, logmech_valid_values])
1025
+
1026
+ logdata = kwargs.pop("logdata", None)
1027
+ awu_matrix_test.append(["logdata", logdata, True, (str), True])
1028
+
1029
+ # Validate argument types.
1030
+ _Validators._validate_function_arguments(awu_matrix_test)
1031
+
1032
+ self.__validate()
1033
+ self.__validate_for_reserved_keyword()
1034
+
1035
+
1036
+ if logmech == "JWT" and not logdata:
1037
+ raise TeradataMlException(
1038
+ Messages.get_message(MessageCodes.DEPENDENT_ARG_MISSING, 'logdata',
1039
+ 'logmech=JWT'),
1040
+ MessageCodes.DEPENDENT_ARG_MISSING)
1041
+
1042
+ if data_row_limit <= 0:
1043
+ raise ValueError(Messages.get_message(MessageCodes.TDMLDF_POSITIVE_INT).
1044
+ format("data_row_limit", "greater than"))
1045
+
1046
+ # Either of 'input_data_file' or 'password' argument is required.
1047
+ password = kwargs.pop("password", None)
1048
+
1049
+ # When exec_mode is local, the connection object is used to get the values in the table.
1050
+ if exec_mode == "local" and not (input_data_file or self.data):
1051
+ message = Messages.get_message(MessageCodes.EITHER_THIS_OR_THAT_ARGUMENT,
1052
+ "input_data_file", "Script data")
1053
+ raise TeradataMlException(message, MessageCodes.EITHER_THIS_OR_THAT_ARGUMENT)
1054
+
1055
+ if not self.script_name and self.files_local_path:
1056
+ message = Messages.get_message(MessageCodes.MISSING_ARGS,
1057
+ "script_name and files_local_path")
1058
+ raise TeradataMlException(message, MessageCodes.MISSING_ARGS)
1059
+
1060
+ if input_data_file:
1061
+ if self.files_local_path is None:
1062
+ message = Messages.get_message(MessageCodes.DEPENDENT_ARG_MISSING,
1063
+ "files_local_path","input_data_file")
1064
+ raise TeradataMlException(message, MessageCodes.DEPENDENT_ARG_MISSING)
1065
+ else:
1066
+ # Check if file exists.
1067
+ fpath = os.path.join(self.files_local_path,
1068
+ input_data_file)
1069
+ _Validators._validate_file_exists(fpath)
1070
+
1071
+ if self.script_name and self.files_local_path:
1072
+ # Check if file exists.
1073
+ fpath = os.path.join(self.files_local_path,
1074
+ os.path.basename(self.script_name))
1075
+ _Validators._validate_file_exists(fpath)
1076
+
1077
+ if exec_mode.upper() == TableOperatorConstants.LOCAL_EXEC.value:
1078
+ user_script_path = os.path.join(self.files_local_path, self.script_name)
1079
+ cmd = [str(sys.executable), user_script_path]
1080
+ cmd.extend(script_args)
1081
+
1082
+ if input_data_file is not None:
1083
+ input_file_path = os.path.join(self.files_local_path, input_data_file)
1084
+
1085
+ # Run user script locally with input from a file.
1086
+ exec_cmd_output = self.__local_run_user_script_input_file(
1087
+ cmd, input_file_path, data_file_delimiter, data_file_quote_char, data_file_header)
1088
+ try:
1089
+ return self.__process_test_script_output(exec_cmd_output)
1090
+ except Exception as exp:
1091
+ raise
1092
+
1093
+ else:
1094
+ if self.data.shape[0] > data_row_limit:
1095
+ raise ValueError(
1096
+ Messages.get_message(MessageCodes.DATAFRAME_LIMIT_ERROR,
1097
+ 'data_row_limit', 'data_row_limit',
1098
+ data_row_limit))
1099
+
1100
+ if not self.data._table_name:
1101
+ self.data._table_name = df_utils._execute_node_return_db_object_name(
1102
+ self.data._nodeid, self.data._metaexpr)
1103
+
1104
+ table_name = UtilFuncs._extract_table_name(self.data._table_name)
1105
+
1106
+ # Run user script locally with input from db.
1107
+ exec_cmd_output = self.__local_run_user_script_input_db(cmd, table_name)
1108
+ try:
1109
+ return self.__process_test_script_output(exec_cmd_output)
1110
+ except Exception as exp:
1111
+ raise
1112
+
1113
+ def __local_run_user_script_input_file(self, cmd, input_file_path,
1114
+ data_file_delimiter='\t',
1115
+ data_file_quote_char='"',
1116
+ data_file_header=True):
1117
+ """
1118
+ DESCRIPTION:
1119
+ Function to run the user script in local mode with input from file.
1120
+
1121
+ PARAMETERS:
1122
+ cmd:
1123
+ Required Argument.
1124
+ Specifies the command for running the user script.
1125
+ Types: str
1126
+
1127
+ input_file_path:
1128
+ Required Argument.
1129
+ Specifies the absolute local path of input data file.
1130
+ Types: str
1131
+
1132
+ data_file_delimiter:
1133
+ Optional Argument.
1134
+ Specifies the delimiter used in input data file.
1135
+ Default Value: '\t'
1136
+ Types: str
1137
+
1138
+ data_file_quote_char:
1139
+ Optional Argument.
1140
+ Specifies the quote character used in input data file.
1141
+ Default Value: '"'
1142
+ Types: str
1143
+
1144
+ data_file_header:
1145
+ Optional Argument.
1146
+ Specifies whether the input data file has header.
1147
+ Default Value: True
1148
+ Types: bool
1149
+
1150
+ RETURNS:
1151
+ The string output of the command that is run on input data file.
1152
+
1153
+ RAISES:
1154
+ Exception.
1155
+
1156
+ EXAMPLES:
1157
+ self.__local_run_user_script_input_file(cmd ="cmd",
1158
+ input_file_path = "input_file_path",
1159
+ data_file_delimiter = "data_file_delimiter",
1160
+ data_file_quote_char = "data_file_quote_char",
1161
+ data_file_header = True)
1162
+
1163
+ """
1164
+ with open(input_file_path) as data_file:
1165
+ import csv
1166
+ from pandas import isna as pd_isna
1167
+
1168
+ data_handle = StringIO()
1169
+
1170
+ # Read data from input file.
1171
+ ip_data = csv.reader(data_file,
1172
+ delimiter=data_file_delimiter,
1173
+ quotechar=data_file_quote_char)
1174
+ # Skip the first row of input file if data_file_header is True.
1175
+ if data_file_header:
1176
+ next(ip_data)
1177
+ for row in ip_data:
1178
+ if self.quotechar is not None:
1179
+ # A NULL value should not be enclosed in quotes.
1180
+ # The CSV module has no support for such output with writer,
1181
+ # and hence the custom formatting.
1182
+ line = ['' if pd_isna(s) else "{}{}{}".format(self.quotechar,
1183
+ str(s),
1184
+ self.quotechar)
1185
+ for s in row]
1186
+ else:
1187
+ line = ['' if pd_isna(s) else str(s) for s in row]
1188
+
1189
+ complete_line = (self.delimiter.join(line))
1190
+
1191
+ data_handle.write(complete_line)
1192
+ data_handle.write("\n")
1193
+
1194
+ return self.__run_user_script_subprocess(cmd, data_handle)
1195
+
1196
+ def __local_run_user_script_input_db(self, cmd, table_name):
1197
+ """
1198
+ DESCRIPTION:
1199
+ Function to run the user script in local mode with input from db.
1200
+
1201
+ PARAMETERS:
1202
+ cmd:
1203
+ Required Argument.
1204
+ Specifies the command for running the user script.
1205
+ Types: str
1206
+
1207
+ table_name:
1208
+ Required Argument.
1209
+ Specifies the table name for input to user script.
1210
+ Types: str
1211
+
1212
+ RETURNS:
1213
+ The string output of the command that is run on the Vantage table.
1214
+
1215
+ RAISES:
1216
+ Exception.
1217
+
1218
+ EXAMPLES:
1219
+ self.__local_run_user_script_input_db(cmd = "cmd", table_name = "table_name")
1220
+
1221
+ """
1222
+ db_data_handle = StringIO()
1223
+ try:
1224
+ con = get_connection()
1225
+ # Query for reading data from DB.
1226
+ query = ("SELECT * FROM {} ORDER BY 1;".format(table_name))
1227
+ cur = execute_sql(query)
1228
+ row = cur.fetchone()
1229
+ from pandas import isna as pd_isna
1230
+ while row:
1231
+ if self.quotechar is not None:
1232
+ # A NULL value should not be enclosed in quotes.
1233
+ # The CSV module has no support for such output with writer,
1234
+ # and hence the custom formatting.
1235
+ line = ['' if pd_isna(s) else "{}{}{}".format(self.quotechar,
1236
+ str(s),
1237
+ self.quotechar)
1238
+ for s in row]
1239
+ else:
1240
+ line = ['' if pd_isna(s) else str(s) for s in row]
1241
+
1242
+ complete_line = (self.delimiter.join(line))
1243
+ db_data_handle.write(complete_line)
1244
+ db_data_handle.write("\n")
1245
+ row = cur.fetchone()
1246
+ except Exception as exp:
1247
+ raise exp
1248
+
1249
+ return self.__run_user_script_subprocess(cmd, db_data_handle)
1250
+
1251
+ def __process_test_script_output(self, exec_cmd_output):
1252
+ """
1253
+ DESCRIPTION:
1254
+ Function to format the output of the user script.
1255
+
1256
+ PARAMETERS:
1257
+ exec_cmd_output:
1258
+ Required Argument.
1259
+ Specifies the output returned by the user script.
1260
+ Types: str
1261
+
1262
+ RETURNS:
1263
+ The test script output as Pandas DataFrame.
1264
+
1265
+ RAISES:
1266
+ Exception.
1267
+
1268
+ EXAMPLES:
1269
+ self.__process_test_script_output(exec_cmd_output = "exec_cmd_output")
1270
+ """
1271
+ try:
1272
+ kwargs = dict()
1273
+ if self.quotechar is not None:
1274
+ kwargs['quotechar'] = self.quotechar
1275
+ kwargs['quoting'] = 1 # QUOTE_ALL
1276
+
1277
+ output = StringIO(exec_cmd_output)
1278
+
1279
+ from pandas import read_csv as pd_read_csv
1280
+
1281
+ # Form a pandas dataframe.
1282
+ df = pd_read_csv(output, sep=self.delimiter, index_col=False, header=None,
1283
+ names=list(self.returns.keys()), **kwargs)
1284
+ return df
1285
+
1286
+ except Exception as exp:
1287
+ raise exp
1288
+
1289
+ def __run_user_script_subprocess(self, cmd, data_handle):
1290
+ """
1291
+ DESCRIPTION:
1292
+ Function to run the user script in a new process and return the output.
1293
+
1294
+ PARAMETERS:
1295
+ cmd:
1296
+ Required Argument.
1297
+ Specifies the command for running the script.
1298
+ Types: str
1299
+
1300
+ data_handle:
1301
+ Required Argument.
1302
+ Specifies the data handle for the input data required by the user script.
1303
+
1304
+ RETURNS:
1305
+ Output of user script on input data supplied in data_handle.
1306
+
1307
+ RAISES:
1308
+ None.
1309
+
1310
+ EXAMPLES:
1311
+ self.__run_user_script_subprocess(cmd = "exec_cmd_output",
1312
+ data_handle = data_handle)
1313
+
1314
+ """
1315
+ # Launching new process to run the user script.
1316
+ try:
1317
+ proc = subprocess.Popen(cmd, stdin=subprocess.PIPE,
1318
+ stdout=subprocess.PIPE,
1319
+ stderr=subprocess.PIPE)
1320
+ process_output, process_error = proc.communicate(data_handle.getvalue().encode())
1321
+ data_handle.close()
1322
+
1323
+ if proc.returncode == 0:
1324
+ return process_output.decode("utf-8").rstrip("\r|\n")
1325
+ else:
1326
+ message = Messages.get_message(MessageCodes.SCRIPT_LOCAL_RUN_ERROR).\
1327
+ format(process_error)
1328
+ raise TeradataMlException(message, MessageCodes.SCRIPT_LOCAL_RUN_ERROR)
1329
+ except Exception as e:
1330
+ raise e
1331
+
1332
+ def execute_script(self, output_style='VIEW'):
1333
+ """
1334
+ DESCRIPTION:
1335
+ Function enables user to run script on Vantage.
1336
+
1337
+ PARAMETERS:
1338
+ output_style:
1339
+ Specifies the type of output object to create - a table or a view.
1340
+ Permitted Values: 'VIEW', 'TABLE'.
1341
+ Default value: 'VIEW'
1342
+ Types: str
1343
+
1344
+ RETURNS:
1345
+ Output teradataml DataFrames can be accessed using attribute
1346
+ references, such as ScriptObj.<attribute_name>.
1347
+ Output teradataml DataFrame attribute name is:
1348
+ result
1349
+
1350
+ RAISES:
1351
+ TeradataMlException, ValueError
1352
+
1353
+ EXAMPLES:
1354
+ Refer to help(Script)
1355
+ """
1356
+ # Validate the output_style.
1357
+ permitted_values = [OutputStyle.OUTPUT_TABLE.value,
1358
+ OutputStyle.OUTPUT_VIEW.value]
1359
+ _Validators._validate_permitted_values(output_style, permitted_values,
1360
+ 'output_style',
1361
+ case_insensitive=False, includeNone=False)
1362
+
1363
+ # Validate arguments.
1364
+ self.__validate()
1365
+ # Validating for reserved keywords.
1366
+ self.__validate_for_reserved_keyword()
1367
+
1368
+ # Generate the Table Operator query.
1369
+ self.__form_table_operator_query()
1370
+
1371
+ # Execute Table Operator query and return results.
1372
+ return self.__execute(output_style)
1373
+
1374
+ def install_file(self, file_identifier, file_name, is_binary = False,
1375
+ replace = False, force_replace = False):
1376
+ """
1377
+ DESCRIPTION:
1378
+ Function to install script on Vantage.
1379
+ On success, prints a message that file is installed.
1380
+ This language script can be executed via execute_script() function.
1381
+
1382
+ PARAMETERS:
1383
+ file_identifier:
1384
+ Required Argument.
1385
+ Specifies the name associated with the user-installed file.
1386
+ It cannot have a schema name associated with it,
1387
+ as the file is always installed in the current schema.
1388
+ The name should be unique within the schema. It can be any valid Teradata
1389
+ identifier.
1390
+ Types: str
1391
+
1392
+ file_name:
1393
+ Required Argument:
1394
+ Specifies the name of the file user wnats to install.
1395
+ Types: str
1396
+
1397
+ is_binary:
1398
+ Optional Argument.
1399
+ Specifies if file to be installed is a binary file.
1400
+ Default Value: False
1401
+ Types: bool
1402
+
1403
+ replace:
1404
+ Optional Argument.
1405
+ Specifies if the file is to be installed or replaced.
1406
+ If set to True, then the file is replaced based on value the of
1407
+ force_replace.
1408
+ If set to False, then the file is installed.
1409
+ Default Value: False
1410
+ Types: bool
1411
+
1412
+ force_replace:
1413
+ Optional Argument.
1414
+ Specifies if system should check for the file being used before
1415
+ replacing it.
1416
+ If set to True, then the file is replaced even if it is being executed.
1417
+ If set to False, then an error is thrown if it is being executed.
1418
+ Default Value: False
1419
+ Types: bool
1420
+
1421
+ RETURNS:
1422
+ True, if success
1423
+
1424
+ RAISES:
1425
+ TeradataMLException.
1426
+
1427
+ EXAMPLES:
1428
+ # Note - Refer to User Guide for setting search path and required permissions.
1429
+ # Example 1: Install the file mapper.py found at the relative path
1430
+ # data/scripts/ using the default text mode.
1431
+
1432
+ # Set SEARCHUIFDBPATH.
1433
+ >>> execute_sql("SET SESSION SEARCHUIFDBPATH = alice;")
1434
+
1435
+ # Create a Script object that allows us to execute script on Vantage.
1436
+ >>> import os
1437
+ >>> from teradatasqlalchemy import VARCHAR
1438
+ >>> td_path = os.path.dirname(teradataml.__file__)
1439
+ >>> sto = Script(data=barrierdf,
1440
+ ... script_name='mapper.py',
1441
+ ... files_local_path= os.path.join(td_path, 'data', "scripts"),
1442
+ ... script_command='tdpython3 ./alice/mapper.py',
1443
+ ... data_order_column="Id",
1444
+ ... is_local_order=False,
1445
+ ... nulls_first=False,
1446
+ ... sort_ascending=False,
1447
+ ... charset='latin',
1448
+ ... returns=OrderedDict([("word", VARCHAR(15)),("count_input", VARCHAR(2))]))
1449
+ >>>
1450
+
1451
+ # Install file on Vantage.
1452
+
1453
+ >>> sto.install_file(file_identifier='mapper',
1454
+ ... file_name='mapper.py',
1455
+ ... is_binary=False)
1456
+ File mapper.py installed in Vantage
1457
+
1458
+ # Replace file on Vantage.
1459
+ >>> sto.install_file(file_identifier='mapper',
1460
+ ... file_name='mapper.py',
1461
+ ... is_binary=False,
1462
+ ... replace=True,
1463
+ ... force_replace=True)
1464
+ File mapper.py replaced in Vantage
1465
+ """
1466
+ # Install/Replace file on Vantage.
1467
+ try:
1468
+ file_path = os.path.join(self.files_local_path, file_name)
1469
+ # Install file on Vantage.
1470
+ install_file(file_identifier=file_identifier, file_path=file_path,
1471
+ is_binary=is_binary,
1472
+ replace=replace, force_replace=force_replace)
1473
+ except:
1474
+ raise
1475
+
1476
+ def remove_file(self, file_identifier, force_remove=False):
1477
+ """
1478
+ DESCRIPTION:
1479
+ Function to remove user installed files/scripts from Vantage.
1480
+
1481
+ PARAMETERS:
1482
+ file_identifier:
1483
+ Required Argument.
1484
+ Specifies the name associated with the user-installed file.
1485
+ It cannot have a database name associated with it,
1486
+ as the file is always installed in the current database.
1487
+ Types: str
1488
+
1489
+ force_remove:
1490
+ Required Argument.
1491
+ Specifies if system should check for the file being used before
1492
+ removing it.
1493
+ If set to True, then the file is removed even if it is being executed.
1494
+ If set to False, then an error is thrown if it is being executed.
1495
+ Default value: False
1496
+ Types: bool
1497
+
1498
+ RETURNS:
1499
+ True, if success.
1500
+
1501
+ RAISES:
1502
+ TeradataMLException.
1503
+
1504
+ EXAMPLES:
1505
+ # Note - Refer to User Guide for setting search path and required permissions.
1506
+ # Run install_file example before removing file.
1507
+
1508
+ # Set SEARCHUIFDBPATH.
1509
+ >>> execute_sql("SET SESSION SEARCHUIFDBPATH = alice;")
1510
+
1511
+ # Create a Script object that allows us to execute script on Vantage.
1512
+ >>> sto = Script(data=barrierdf,
1513
+ ... script_name='mapper.py',
1514
+ ... files_local_path= os.path.join(td_path, 'data', "scripts"),
1515
+ ... script_command='tdpython3 ./alice/mapper.py',
1516
+ ... data_order_column="Id",
1517
+ ... is_local_order=False,
1518
+ ... nulls_first=False,
1519
+ ... sort_ascending=False,
1520
+ ... charset='latin',
1521
+ ... returns=OrderedDict([("word", VARCHAR(15)),("count_input", VARCHAR(2))]))
1522
+ >>>
1523
+
1524
+ # Install file on Vantage.
1525
+ >>> sto.install_file(file_identifier='mapper',
1526
+ ... file_name='mapper.py',
1527
+ ... is_binary=False,
1528
+ ... replace=True,
1529
+ ... force_replace=True)
1530
+ File mapper.py replaced in Vantage
1531
+
1532
+ # Remove the installed file.
1533
+ >>> sto.remove_file(file_identifier='mapper', force_remove=True)
1534
+ File mapper removed from Vantage
1535
+
1536
+ """
1537
+ # Remove file from Vantage
1538
+ try:
1539
+ remove_file(file_identifier, force_remove)
1540
+ except:
1541
+ raise
1542
+
1543
+ def __form_table_operator_query(self):
1544
+ """
1545
+ Function to generate the Table Operator queries. The function defines
1546
+ variables and list of arguments required to form the query.
1547
+ """
1548
+ # Output table arguments list.
1549
+ self.__func_output_args_sql_names = []
1550
+ self.__func_output_args = []
1551
+
1552
+ # Generate lists for rest of the function arguments.
1553
+ self.__func_other_arg_sql_names = []
1554
+ self.__func_other_args = []
1555
+ self.__func_other_arg_json_datatypes = []
1556
+
1557
+ self.__func_other_arg_sql_names.append("SCRIPT_COMMAND")
1558
+ self.__func_other_args.append(
1559
+ UtilFuncs._teradata_collapse_arglist(self.script_command, "'"))
1560
+ self.__func_other_arg_json_datatypes.append("STRING")
1561
+
1562
+ if self.delimiter is not None:
1563
+ self.__func_other_arg_sql_names.append("delimiter")
1564
+ self.__func_other_args.append(
1565
+ UtilFuncs._teradata_collapse_arglist(self.delimiter, "'"))
1566
+ self.__func_other_arg_json_datatypes.append("STRING")
1567
+
1568
+ # Generate returns clause.
1569
+ ret_vals = []
1570
+ returns_clause = ''
1571
+ for key in self.returns.keys():
1572
+ ret_vals.append('{} {}'.format(key, self.returns[key].compile(td_dialect())))
1573
+ returns_clause = ', '.join(ret_vals)
1574
+
1575
+ self.__func_other_arg_sql_names.append("returns")
1576
+ self.__func_other_args.append(
1577
+ UtilFuncs._teradata_collapse_arglist(returns_clause, "'"))
1578
+ self.__func_other_arg_json_datatypes.append("STRING")
1579
+
1580
+ if self.auth is not None:
1581
+ self.__func_other_arg_sql_names.append("auth")
1582
+ self.__func_other_args.append(
1583
+ UtilFuncs._teradata_collapse_arglist(self.auth, "'"))
1584
+ self.__func_other_arg_json_datatypes.append("STRING")
1585
+
1586
+ if self.charset is not None:
1587
+ self.__func_other_arg_sql_names.append("charset")
1588
+ self.__func_other_args.append(
1589
+ UtilFuncs._teradata_collapse_arglist(self.charset, "'"))
1590
+ self.__func_other_arg_json_datatypes.append("STRING")
1591
+
1592
+ if self.quotechar is not None:
1593
+ self.__func_other_arg_sql_names.append("quotechar")
1594
+ self.__func_other_args.append(
1595
+ UtilFuncs._teradata_collapse_arglist(self.quotechar, "'"))
1596
+ self.__func_other_arg_json_datatypes.append("STRING")
1597
+
1598
+ # Declare empty lists to hold input table information.
1599
+ self.__func_input_arg_sql_names = []
1600
+ self.__func_input_table_view_query = []
1601
+ self.__func_input_dataframe_type = []
1602
+ self.__func_input_distribution = []
1603
+ self.__func_input_partition_by_cols = []
1604
+ self.__func_input_order_by_cols = []
1605
+ self.__func_input_order_by_type = []
1606
+ self.__func_input_sort_ascending = self.sort_ascending
1607
+ self.__func_input_nulls_first = None
1608
+
1609
+ # Process data.
1610
+ if self.data is not None:
1611
+ data_distribution = "FACT"
1612
+ if self.data_hash_column is not None:
1613
+ data_distribution = "HASH"
1614
+ data_partition_column = UtilFuncs._teradata_collapse_arglist(
1615
+ self.data_hash_column, "\"")
1616
+ else:
1617
+ if self.__awu._is_default_or_not(self.data_partition_column, "ANY"):
1618
+ data_partition_column = UtilFuncs._teradata_collapse_arglist(
1619
+ self.data_partition_column, "\"")
1620
+ else:
1621
+ data_partition_column = None
1622
+ if self.data_order_column is not None:
1623
+ if self.is_local_order:
1624
+ self.__func_input_order_by_type.append("LOCAL")
1625
+ if not self.data_hash_column:
1626
+ data_distribution = None
1627
+ else:
1628
+ self.__func_input_order_by_type.append(None)
1629
+ self.__func_input_order_by_cols.append(
1630
+ UtilFuncs._teradata_collapse_arglist(self.data_order_column, "\""))
1631
+ else:
1632
+ self.__func_input_order_by_type.append(None)
1633
+ self.__func_input_order_by_cols.append("NA_character_")
1634
+
1635
+ self.__table_ref = self.__awu._teradata_on_clause_from_dataframe(self.data,
1636
+ False)
1637
+ self.__func_input_distribution.append(data_distribution)
1638
+ self.__func_input_arg_sql_names.append("input")
1639
+ self.__func_input_table_view_query.append(self.__table_ref["ref"])
1640
+ self.__func_input_dataframe_type.append(self.__table_ref["ref_type"])
1641
+ self.__func_input_partition_by_cols.append(data_partition_column)
1642
+ self.__func_input_nulls_first = self.nulls_first
1643
+
1644
+ function_name = "Script"
1645
+ # Create instance to generate Table Operator Query.
1646
+ aqg_obj = TableOperatorQueryGenerator(function_name,
1647
+ self.__func_input_arg_sql_names,
1648
+ self.__func_input_table_view_query,
1649
+ self.__func_input_dataframe_type,
1650
+ self.__func_input_distribution,
1651
+ self.__func_input_partition_by_cols,
1652
+ self.__func_input_order_by_cols,
1653
+ self.__func_other_arg_sql_names,
1654
+ self.__func_other_args,
1655
+ self.__func_other_arg_json_datatypes,
1656
+ self.__func_output_args_sql_names,
1657
+ self.__func_output_args,
1658
+ self.__func_input_order_by_type,
1659
+ self.__func_input_sort_ascending,
1660
+ self.__func_input_nulls_first,
1661
+ engine="ENGINE_SQL"
1662
+ )
1663
+
1664
+ # Invoke call to Table operator query generation.
1665
+ self._tblop_query = aqg_obj._gen_table_operator_select_stmt_sql()
1666
+
1667
+ # Print Table Operator query if requested to do so.
1668
+ if display.print_sqlmr_query:
1669
+ print(self._tblop_query)
1670
+
1671
+ def __execute(self, output_style='VIEW'):
1672
+ """
1673
+ DESCRIPTION:
1674
+ Function to execute Table Operator queries.
1675
+ Create DataFrames for the required Table Operator output.
1676
+
1677
+ PARAMETERS:
1678
+ output_style:
1679
+ Specifies the type of output object to create - a table of a view.
1680
+ Permitted Values: 'VIEW', 'TABLE'.
1681
+ Default value: 'VIEW'
1682
+ Types: str
1683
+
1684
+ RAISES:
1685
+ None.
1686
+
1687
+ RETURNS:
1688
+ None.
1689
+
1690
+ EXAMPLES:
1691
+ >>> return self.__execute(output_style)
1692
+ """
1693
+ # Generate STDOUT table name and add it to the output table list.
1694
+ if output_style == OutputStyle.OUTPUT_TABLE.value:
1695
+ table_type = TeradataConstants.TERADATA_TABLE
1696
+ else:
1697
+ table_type = TeradataConstants.TERADATA_VIEW
1698
+
1699
+ tblop_stdout_temp_tablename = \
1700
+ UtilFuncs._generate_temp_table_name(prefix="td_tblop_out_",
1701
+ use_default_database=True,
1702
+ gc_on_quit=True, quote=False,
1703
+ table_type=table_type)
1704
+ try:
1705
+ if output_style == OutputStyle.OUTPUT_TABLE.value:
1706
+ UtilFuncs._create_table(tblop_stdout_temp_tablename, self._tblop_query)
1707
+ else:
1708
+ UtilFuncs._create_view(tblop_stdout_temp_tablename, self._tblop_query)
1709
+ except Exception as emsg:
1710
+ raise TeradataMlException(
1711
+ Messages.get_message(MessageCodes.TDMLDF_EXEC_SQL_FAILED, str(emsg)),
1712
+ MessageCodes.TDMLDF_EXEC_SQL_FAILED)
1713
+
1714
+ self.result = self.__awu._create_data_set_object(
1715
+ df_input=UtilFuncs._extract_table_name(tblop_stdout_temp_tablename),
1716
+ source_type="table",
1717
+ database_name=UtilFuncs._extract_db_name(tblop_stdout_temp_tablename))
1718
+
1719
+ return self.result