teradataml 17.20.0.7__py3-none-any.whl → 20.0.0.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of teradataml might be problematic. Click here for more details.

Files changed (1303) hide show
  1. teradataml/LICENSE-3RD-PARTY.pdf +0 -0
  2. teradataml/LICENSE.pdf +0 -0
  3. teradataml/README.md +1935 -1640
  4. teradataml/__init__.py +70 -60
  5. teradataml/_version.py +11 -11
  6. teradataml/analytics/Transformations.py +2995 -2995
  7. teradataml/analytics/__init__.py +81 -83
  8. teradataml/analytics/analytic_function_executor.py +2040 -2010
  9. teradataml/analytics/analytic_query_generator.py +958 -958
  10. teradataml/analytics/byom/H2OPredict.py +514 -514
  11. teradataml/analytics/byom/PMMLPredict.py +437 -437
  12. teradataml/analytics/byom/__init__.py +14 -14
  13. teradataml/analytics/json_parser/__init__.py +130 -130
  14. teradataml/analytics/json_parser/analytic_functions_argument.py +1707 -1707
  15. teradataml/analytics/json_parser/json_store.py +191 -191
  16. teradataml/analytics/json_parser/metadata.py +1637 -1637
  17. teradataml/analytics/json_parser/utils.py +798 -803
  18. teradataml/analytics/meta_class.py +196 -196
  19. teradataml/analytics/sqle/DecisionTreePredict.py +455 -470
  20. teradataml/analytics/sqle/NaiveBayesPredict.py +419 -428
  21. teradataml/analytics/sqle/__init__.py +97 -110
  22. teradataml/analytics/sqle/json/decisiontreepredict_sqle.json +78 -78
  23. teradataml/analytics/sqle/json/naivebayespredict_sqle.json +62 -62
  24. teradataml/analytics/table_operator/__init__.py +10 -10
  25. teradataml/analytics/uaf/__init__.py +63 -63
  26. teradataml/analytics/utils.py +693 -692
  27. teradataml/analytics/valib.py +1603 -1600
  28. teradataml/automl/__init__.py +1683 -0
  29. teradataml/automl/custom_json_utils.py +1270 -0
  30. teradataml/automl/data_preparation.py +1011 -0
  31. teradataml/automl/data_transformation.py +789 -0
  32. teradataml/automl/feature_engineering.py +1580 -0
  33. teradataml/automl/feature_exploration.py +554 -0
  34. teradataml/automl/model_evaluation.py +151 -0
  35. teradataml/automl/model_training.py +1026 -0
  36. teradataml/catalog/__init__.py +1 -3
  37. teradataml/catalog/byom.py +1759 -1716
  38. teradataml/catalog/function_argument_mapper.py +859 -861
  39. teradataml/catalog/model_cataloging_utils.py +491 -1510
  40. teradataml/clients/auth_client.py +133 -0
  41. teradataml/clients/pkce_client.py +481 -481
  42. teradataml/common/aed_utils.py +7 -2
  43. teradataml/common/bulk_exposed_utils.py +111 -111
  44. teradataml/common/constants.py +1438 -1441
  45. teradataml/common/deprecations.py +160 -0
  46. teradataml/common/exceptions.py +73 -73
  47. teradataml/common/formula.py +742 -742
  48. teradataml/common/garbagecollector.py +597 -635
  49. teradataml/common/messagecodes.py +424 -431
  50. teradataml/common/messages.py +228 -231
  51. teradataml/common/sqlbundle.py +693 -693
  52. teradataml/common/td_coltype_code_to_tdtype.py +48 -48
  53. teradataml/common/utils.py +2424 -2500
  54. teradataml/common/warnings.py +25 -25
  55. teradataml/common/wrapper_utils.py +1 -110
  56. teradataml/config/dummy_file1.cfg +4 -4
  57. teradataml/config/dummy_file2.cfg +2 -2
  58. teradataml/config/sqlengine_alias_definitions_v1.0 +13 -13
  59. teradataml/config/sqlengine_alias_definitions_v1.1 +19 -19
  60. teradataml/config/sqlengine_alias_definitions_v1.3 +18 -18
  61. teradataml/context/aed_context.py +217 -217
  62. teradataml/context/context.py +1091 -999
  63. teradataml/data/A_loan.csv +19 -19
  64. teradataml/data/BINARY_REALS_LEFT.csv +11 -11
  65. teradataml/data/BINARY_REALS_RIGHT.csv +11 -11
  66. teradataml/data/B_loan.csv +49 -49
  67. teradataml/data/BuoyData2.csv +17 -17
  68. teradataml/data/CONVOLVE2_COMPLEX_LEFT.csv +5 -5
  69. teradataml/data/CONVOLVE2_COMPLEX_RIGHT.csv +5 -5
  70. teradataml/data/Convolve2RealsLeft.csv +5 -5
  71. teradataml/data/Convolve2RealsRight.csv +5 -5
  72. teradataml/data/Convolve2ValidLeft.csv +11 -11
  73. teradataml/data/Convolve2ValidRight.csv +11 -11
  74. teradataml/data/DFFTConv_Real_8_8.csv +65 -65
  75. teradataml/data/Orders1_12mf.csv +24 -24
  76. teradataml/data/Pi_loan.csv +7 -7
  77. teradataml/data/SMOOTHED_DATA.csv +7 -7
  78. teradataml/data/TestDFFT8.csv +9 -9
  79. teradataml/data/TestRiver.csv +109 -109
  80. teradataml/data/Traindata.csv +28 -28
  81. teradataml/data/acf.csv +17 -17
  82. teradataml/data/adaboost_example.json +34 -34
  83. teradataml/data/adaboostpredict_example.json +24 -24
  84. teradataml/data/additional_table.csv +10 -10
  85. teradataml/data/admissions_test.csv +21 -21
  86. teradataml/data/admissions_train.csv +41 -41
  87. teradataml/data/admissions_train_nulls.csv +41 -41
  88. teradataml/data/advertising.csv +201 -0
  89. teradataml/data/ageandheight.csv +13 -13
  90. teradataml/data/ageandpressure.csv +31 -31
  91. teradataml/data/antiselect_example.json +36 -36
  92. teradataml/data/antiselect_input.csv +8 -8
  93. teradataml/data/antiselect_input_mixed_case.csv +8 -8
  94. teradataml/data/applicant_external.csv +6 -6
  95. teradataml/data/applicant_reference.csv +6 -6
  96. teradataml/data/arima_example.json +9 -9
  97. teradataml/data/assortedtext_input.csv +8 -8
  98. teradataml/data/attribution_example.json +33 -33
  99. teradataml/data/attribution_sample_table.csv +27 -27
  100. teradataml/data/attribution_sample_table1.csv +6 -6
  101. teradataml/data/attribution_sample_table2.csv +11 -11
  102. teradataml/data/bank_churn.csv +10001 -0
  103. teradataml/data/bank_marketing.csv +11163 -0
  104. teradataml/data/bank_web_clicks1.csv +42 -42
  105. teradataml/data/bank_web_clicks2.csv +91 -91
  106. teradataml/data/bank_web_url.csv +85 -85
  107. teradataml/data/barrier.csv +2 -2
  108. teradataml/data/barrier_new.csv +3 -3
  109. teradataml/data/betweenness_example.json +13 -13
  110. teradataml/data/bike_sharing.csv +732 -0
  111. teradataml/data/bin_breaks.csv +8 -8
  112. teradataml/data/bin_fit_ip.csv +3 -3
  113. teradataml/data/binary_complex_left.csv +11 -11
  114. teradataml/data/binary_complex_right.csv +11 -11
  115. teradataml/data/binary_matrix_complex_left.csv +21 -21
  116. teradataml/data/binary_matrix_complex_right.csv +21 -21
  117. teradataml/data/binary_matrix_real_left.csv +21 -21
  118. teradataml/data/binary_matrix_real_right.csv +21 -21
  119. teradataml/data/blood2ageandweight.csv +26 -26
  120. teradataml/data/bmi.csv +501 -0
  121. teradataml/data/boston.csv +507 -507
  122. teradataml/data/boston2cols.csv +721 -0
  123. teradataml/data/breast_cancer.csv +570 -0
  124. teradataml/data/buoydata_mix.csv +11 -11
  125. teradataml/data/burst_data.csv +5 -5
  126. teradataml/data/burst_example.json +20 -20
  127. teradataml/data/byom_example.json +17 -17
  128. teradataml/data/bytes_table.csv +3 -3
  129. teradataml/data/cal_housing_ex_raw.csv +70 -70
  130. teradataml/data/callers.csv +7 -7
  131. teradataml/data/calls.csv +10 -10
  132. teradataml/data/cars_hist.csv +33 -33
  133. teradataml/data/cat_table.csv +24 -24
  134. teradataml/data/ccm_example.json +31 -31
  135. teradataml/data/ccm_input.csv +91 -91
  136. teradataml/data/ccm_input2.csv +13 -13
  137. teradataml/data/ccmexample.csv +101 -101
  138. teradataml/data/ccmprepare_example.json +8 -8
  139. teradataml/data/ccmprepare_input.csv +91 -91
  140. teradataml/data/cfilter_example.json +12 -12
  141. teradataml/data/changepointdetection_example.json +18 -18
  142. teradataml/data/changepointdetectionrt_example.json +8 -8
  143. teradataml/data/chi_sq.csv +2 -2
  144. teradataml/data/churn_data.csv +14 -14
  145. teradataml/data/churn_emission.csv +35 -35
  146. teradataml/data/churn_initial.csv +3 -3
  147. teradataml/data/churn_state_transition.csv +5 -5
  148. teradataml/data/citedges_2.csv +745 -745
  149. teradataml/data/citvertices_2.csv +1210 -1210
  150. teradataml/data/clicks2.csv +16 -16
  151. teradataml/data/clickstream.csv +12 -12
  152. teradataml/data/clickstream1.csv +11 -11
  153. teradataml/data/closeness_example.json +15 -15
  154. teradataml/data/complaints.csv +21 -21
  155. teradataml/data/complaints_mini.csv +3 -3
  156. teradataml/data/complaints_testtoken.csv +224 -224
  157. teradataml/data/complaints_tokens_test.csv +353 -353
  158. teradataml/data/complaints_traintoken.csv +472 -472
  159. teradataml/data/computers_category.csv +1001 -1001
  160. teradataml/data/computers_test1.csv +1252 -1252
  161. teradataml/data/computers_train1.csv +5009 -5009
  162. teradataml/data/computers_train1_clustered.csv +5009 -5009
  163. teradataml/data/confusionmatrix_example.json +9 -9
  164. teradataml/data/conversion_event_table.csv +3 -3
  165. teradataml/data/corr_input.csv +17 -17
  166. teradataml/data/correlation_example.json +11 -11
  167. teradataml/data/coxhazardratio_example.json +39 -39
  168. teradataml/data/coxph_example.json +15 -15
  169. teradataml/data/coxsurvival_example.json +28 -28
  170. teradataml/data/cpt.csv +41 -41
  171. teradataml/data/credit_ex_merged.csv +45 -45
  172. teradataml/data/customer_loyalty.csv +301 -301
  173. teradataml/data/customer_loyalty_newseq.csv +31 -31
  174. teradataml/data/customer_segmentation_test.csv +2628 -0
  175. teradataml/data/customer_segmentation_train.csv +8069 -0
  176. teradataml/data/dataframe_example.json +146 -146
  177. teradataml/data/decisionforest_example.json +37 -37
  178. teradataml/data/decisionforestpredict_example.json +38 -38
  179. teradataml/data/decisiontree_example.json +21 -21
  180. teradataml/data/decisiontreepredict_example.json +45 -45
  181. teradataml/data/dfft2_size4_real.csv +17 -17
  182. teradataml/data/dfft2_test_matrix16.csv +17 -17
  183. teradataml/data/dfft2conv_real_4_4.csv +65 -65
  184. teradataml/data/diabetes.csv +443 -443
  185. teradataml/data/diabetes_test.csv +89 -89
  186. teradataml/data/dict_table.csv +5 -5
  187. teradataml/data/docperterm_table.csv +4 -4
  188. teradataml/data/docs/__init__.py +1 -1
  189. teradataml/data/docs/byom/docs/DataRobotPredict.py +180 -180
  190. teradataml/data/docs/byom/docs/DataikuPredict.py +177 -177
  191. teradataml/data/docs/byom/docs/H2OPredict.py +324 -324
  192. teradataml/data/docs/byom/docs/ONNXPredict.py +283 -283
  193. teradataml/data/docs/byom/docs/PMMLPredict.py +277 -277
  194. teradataml/data/docs/sqle/docs_17_10/Antiselect.py +82 -82
  195. teradataml/data/docs/sqle/docs_17_10/Attribution.py +199 -199
  196. teradataml/data/docs/sqle/docs_17_10/BincodeFit.py +171 -171
  197. teradataml/data/docs/sqle/docs_17_10/BincodeTransform.py +131 -130
  198. teradataml/data/docs/sqle/docs_17_10/CategoricalSummary.py +86 -86
  199. teradataml/data/docs/sqle/docs_17_10/ChiSq.py +90 -90
  200. teradataml/data/docs/sqle/docs_17_10/ColumnSummary.py +85 -85
  201. teradataml/data/docs/sqle/docs_17_10/ConvertTo.py +95 -95
  202. teradataml/data/docs/sqle/docs_17_10/DecisionForestPredict.py +139 -139
  203. teradataml/data/docs/sqle/docs_17_10/DecisionTreePredict.py +151 -151
  204. teradataml/data/docs/sqle/docs_17_10/FTest.py +160 -160
  205. teradataml/data/docs/sqle/docs_17_10/FillRowId.py +82 -82
  206. teradataml/data/docs/sqle/docs_17_10/Fit.py +87 -87
  207. teradataml/data/docs/sqle/docs_17_10/GLMPredict.py +144 -144
  208. teradataml/data/docs/sqle/docs_17_10/GetRowsWithMissingValues.py +84 -84
  209. teradataml/data/docs/sqle/docs_17_10/GetRowsWithoutMissingValues.py +81 -81
  210. teradataml/data/docs/sqle/docs_17_10/Histogram.py +164 -164
  211. teradataml/data/docs/sqle/docs_17_10/MovingAverage.py +134 -134
  212. teradataml/data/docs/sqle/docs_17_10/NGramSplitter.py +208 -208
  213. teradataml/data/docs/sqle/docs_17_10/NPath.py +265 -265
  214. teradataml/data/docs/sqle/docs_17_10/NaiveBayesPredict.py +116 -116
  215. teradataml/data/docs/sqle/docs_17_10/NaiveBayesTextClassifierPredict.py +176 -176
  216. teradataml/data/docs/sqle/docs_17_10/NumApply.py +147 -147
  217. teradataml/data/docs/sqle/docs_17_10/OneHotEncodingFit.py +134 -132
  218. teradataml/data/docs/sqle/docs_17_10/OneHotEncodingTransform.py +109 -103
  219. teradataml/data/docs/sqle/docs_17_10/OutlierFilterFit.py +165 -165
  220. teradataml/data/docs/sqle/docs_17_10/OutlierFilterTransform.py +105 -101
  221. teradataml/data/docs/sqle/docs_17_10/Pack.py +128 -128
  222. teradataml/data/docs/sqle/docs_17_10/PolynomialFeaturesFit.py +111 -111
  223. teradataml/data/docs/sqle/docs_17_10/PolynomialFeaturesTransform.py +102 -102
  224. teradataml/data/docs/sqle/docs_17_10/QQNorm.py +104 -104
  225. teradataml/data/docs/sqle/docs_17_10/RoundColumns.py +109 -109
  226. teradataml/data/docs/sqle/docs_17_10/RowNormalizeFit.py +117 -117
  227. teradataml/data/docs/sqle/docs_17_10/RowNormalizeTransform.py +99 -98
  228. teradataml/data/docs/sqle/docs_17_10/SVMSparsePredict.py +152 -152
  229. teradataml/data/docs/sqle/docs_17_10/ScaleFit.py +197 -197
  230. teradataml/data/docs/sqle/docs_17_10/ScaleTransform.py +99 -98
  231. teradataml/data/docs/sqle/docs_17_10/Sessionize.py +113 -113
  232. teradataml/data/docs/sqle/docs_17_10/SimpleImputeFit.py +116 -116
  233. teradataml/data/docs/sqle/docs_17_10/SimpleImputeTransform.py +98 -98
  234. teradataml/data/docs/sqle/docs_17_10/StrApply.py +187 -187
  235. teradataml/data/docs/sqle/docs_17_10/StringSimilarity.py +145 -145
  236. teradataml/data/docs/sqle/docs_17_10/Transform.py +105 -104
  237. teradataml/data/docs/sqle/docs_17_10/UnivariateStatistics.py +141 -141
  238. teradataml/data/docs/sqle/docs_17_10/Unpack.py +214 -214
  239. teradataml/data/docs/sqle/docs_17_10/WhichMax.py +83 -83
  240. teradataml/data/docs/sqle/docs_17_10/WhichMin.py +83 -83
  241. teradataml/data/docs/sqle/docs_17_10/ZTest.py +155 -155
  242. teradataml/data/docs/sqle/docs_17_20/ANOVA.py +186 -126
  243. teradataml/data/docs/sqle/docs_17_20/Antiselect.py +82 -82
  244. teradataml/data/docs/sqle/docs_17_20/Attribution.py +200 -200
  245. teradataml/data/docs/sqle/docs_17_20/BincodeFit.py +171 -171
  246. teradataml/data/docs/sqle/docs_17_20/BincodeTransform.py +139 -138
  247. teradataml/data/docs/sqle/docs_17_20/CategoricalSummary.py +86 -86
  248. teradataml/data/docs/sqle/docs_17_20/ChiSq.py +90 -90
  249. teradataml/data/docs/sqle/docs_17_20/ClassificationEvaluator.py +166 -166
  250. teradataml/data/docs/sqle/docs_17_20/ColumnSummary.py +85 -85
  251. teradataml/data/docs/sqle/docs_17_20/ColumnTransformer.py +245 -243
  252. teradataml/data/docs/sqle/docs_17_20/ConvertTo.py +113 -113
  253. teradataml/data/docs/sqle/docs_17_20/DecisionForest.py +279 -279
  254. teradataml/data/docs/sqle/docs_17_20/DecisionForestPredict.py +144 -144
  255. teradataml/data/docs/sqle/docs_17_20/DecisionTreePredict.py +135 -135
  256. teradataml/data/docs/sqle/docs_17_20/FTest.py +239 -160
  257. teradataml/data/docs/sqle/docs_17_20/FillRowId.py +82 -82
  258. teradataml/data/docs/sqle/docs_17_20/Fit.py +87 -87
  259. teradataml/data/docs/sqle/docs_17_20/GLM.py +541 -380
  260. teradataml/data/docs/sqle/docs_17_20/GLMPerSegment.py +414 -414
  261. teradataml/data/docs/sqle/docs_17_20/GLMPredict.py +144 -144
  262. teradataml/data/docs/sqle/docs_17_20/GLMPredictPerSegment.py +233 -234
  263. teradataml/data/docs/sqle/docs_17_20/GetFutileColumns.py +125 -123
  264. teradataml/data/docs/sqle/docs_17_20/GetRowsWithMissingValues.py +108 -108
  265. teradataml/data/docs/sqle/docs_17_20/GetRowsWithoutMissingValues.py +105 -105
  266. teradataml/data/docs/sqle/docs_17_20/Histogram.py +223 -223
  267. teradataml/data/docs/sqle/docs_17_20/KMeans.py +251 -204
  268. teradataml/data/docs/sqle/docs_17_20/KMeansPredict.py +144 -143
  269. teradataml/data/docs/sqle/docs_17_20/KNN.py +214 -214
  270. teradataml/data/docs/sqle/docs_17_20/MovingAverage.py +134 -134
  271. teradataml/data/docs/sqle/docs_17_20/NGramSplitter.py +208 -208
  272. teradataml/data/docs/sqle/docs_17_20/NPath.py +265 -265
  273. teradataml/data/docs/sqle/docs_17_20/NaiveBayesPredict.py +116 -116
  274. teradataml/data/docs/sqle/docs_17_20/NaiveBayesTextClassifierPredict.py +177 -176
  275. teradataml/data/docs/sqle/docs_17_20/NaiveBayesTextClassifierTrainer.py +126 -126
  276. teradataml/data/docs/sqle/docs_17_20/NonLinearCombineFit.py +118 -117
  277. teradataml/data/docs/sqle/docs_17_20/NonLinearCombineTransform.py +112 -112
  278. teradataml/data/docs/sqle/docs_17_20/NumApply.py +147 -147
  279. teradataml/data/docs/sqle/docs_17_20/OneClassSVM.py +307 -307
  280. teradataml/data/docs/sqle/docs_17_20/OneClassSVMPredict.py +185 -184
  281. teradataml/data/docs/sqle/docs_17_20/OneHotEncodingFit.py +230 -225
  282. teradataml/data/docs/sqle/docs_17_20/OneHotEncodingTransform.py +121 -115
  283. teradataml/data/docs/sqle/docs_17_20/OrdinalEncodingFit.py +219 -219
  284. teradataml/data/docs/sqle/docs_17_20/OrdinalEncodingTransform.py +127 -127
  285. teradataml/data/docs/sqle/docs_17_20/OutlierFilterFit.py +189 -189
  286. teradataml/data/docs/sqle/docs_17_20/OutlierFilterTransform.py +117 -112
  287. teradataml/data/docs/sqle/docs_17_20/Pack.py +128 -128
  288. teradataml/data/docs/sqle/docs_17_20/PolynomialFeaturesFit.py +111 -111
  289. teradataml/data/docs/sqle/docs_17_20/PolynomialFeaturesTransform.py +112 -111
  290. teradataml/data/docs/sqle/docs_17_20/QQNorm.py +104 -104
  291. teradataml/data/docs/sqle/docs_17_20/ROC.py +164 -163
  292. teradataml/data/docs/sqle/docs_17_20/RandomProjectionFit.py +154 -154
  293. teradataml/data/docs/sqle/docs_17_20/RandomProjectionMinComponents.py +106 -106
  294. teradataml/data/docs/sqle/docs_17_20/RandomProjectionTransform.py +120 -120
  295. teradataml/data/docs/sqle/docs_17_20/RegressionEvaluator.py +211 -211
  296. teradataml/data/docs/sqle/docs_17_20/RoundColumns.py +108 -108
  297. teradataml/data/docs/sqle/docs_17_20/RowNormalizeFit.py +117 -117
  298. teradataml/data/docs/sqle/docs_17_20/RowNormalizeTransform.py +111 -110
  299. teradataml/data/docs/sqle/docs_17_20/SVM.py +413 -413
  300. teradataml/data/docs/sqle/docs_17_20/SVMPredict.py +213 -202
  301. teradataml/data/docs/sqle/docs_17_20/SVMSparsePredict.py +152 -152
  302. teradataml/data/docs/sqle/docs_17_20/ScaleFit.py +315 -197
  303. teradataml/data/docs/sqle/docs_17_20/ScaleTransform.py +202 -109
  304. teradataml/data/docs/sqle/docs_17_20/SentimentExtractor.py +206 -206
  305. teradataml/data/docs/sqle/docs_17_20/Sessionize.py +113 -113
  306. teradataml/data/docs/sqle/docs_17_20/Silhouette.py +152 -152
  307. teradataml/data/docs/sqle/docs_17_20/SimpleImputeFit.py +116 -116
  308. teradataml/data/docs/sqle/docs_17_20/SimpleImputeTransform.py +109 -108
  309. teradataml/data/docs/sqle/docs_17_20/StrApply.py +187 -187
  310. teradataml/data/docs/sqle/docs_17_20/StringSimilarity.py +145 -145
  311. teradataml/data/docs/sqle/docs_17_20/TDDecisionForestPredict.py +207 -207
  312. teradataml/data/docs/sqle/docs_17_20/TDGLMPredict.py +333 -171
  313. teradataml/data/docs/sqle/docs_17_20/TargetEncodingFit.py +266 -266
  314. teradataml/data/docs/sqle/docs_17_20/TargetEncodingTransform.py +141 -140
  315. teradataml/data/docs/sqle/docs_17_20/TextParser.py +172 -172
  316. teradataml/data/docs/sqle/docs_17_20/TrainTestSplit.py +159 -159
  317. teradataml/data/docs/sqle/docs_17_20/Transform.py +123 -123
  318. teradataml/data/docs/sqle/docs_17_20/UnivariateStatistics.py +141 -141
  319. teradataml/data/docs/sqle/docs_17_20/Unpack.py +214 -214
  320. teradataml/data/docs/sqle/docs_17_20/VectorDistance.py +168 -168
  321. teradataml/data/docs/sqle/docs_17_20/WhichMax.py +83 -83
  322. teradataml/data/docs/sqle/docs_17_20/WhichMin.py +83 -83
  323. teradataml/data/docs/sqle/docs_17_20/WordEmbeddings.py +236 -236
  324. teradataml/data/docs/sqle/docs_17_20/XGBoost.py +361 -353
  325. teradataml/data/docs/sqle/docs_17_20/XGBoostPredict.py +281 -275
  326. teradataml/data/docs/sqle/docs_17_20/ZTest.py +220 -155
  327. teradataml/data/docs/tableoperator/docs_17_00/ReadNOS.py +429 -429
  328. teradataml/data/docs/tableoperator/docs_17_05/ReadNOS.py +429 -429
  329. teradataml/data/docs/tableoperator/docs_17_05/WriteNOS.py +347 -347
  330. teradataml/data/docs/tableoperator/docs_17_10/ReadNOS.py +428 -428
  331. teradataml/data/docs/tableoperator/docs_17_10/WriteNOS.py +347 -347
  332. teradataml/data/docs/tableoperator/docs_17_20/ReadNOS.py +439 -439
  333. teradataml/data/docs/tableoperator/docs_17_20/WriteNOS.py +386 -386
  334. teradataml/data/docs/uaf/docs_17_20/ACF.py +195 -195
  335. teradataml/data/docs/uaf/docs_17_20/ArimaEstimate.py +369 -369
  336. teradataml/data/docs/uaf/docs_17_20/ArimaForecast.py +142 -142
  337. teradataml/data/docs/uaf/docs_17_20/ArimaValidate.py +159 -159
  338. teradataml/data/docs/uaf/docs_17_20/BinaryMatrixOp.py +247 -247
  339. teradataml/data/docs/uaf/docs_17_20/BinarySeriesOp.py +252 -252
  340. teradataml/data/docs/uaf/docs_17_20/BreuschGodfrey.py +177 -177
  341. teradataml/data/docs/uaf/docs_17_20/BreuschPaganGodfrey.py +174 -174
  342. teradataml/data/docs/uaf/docs_17_20/Convolve.py +226 -226
  343. teradataml/data/docs/uaf/docs_17_20/Convolve2.py +214 -214
  344. teradataml/data/docs/uaf/docs_17_20/CumulPeriodogram.py +183 -183
  345. teradataml/data/docs/uaf/docs_17_20/DFFT.py +203 -203
  346. teradataml/data/docs/uaf/docs_17_20/DFFT2.py +216 -216
  347. teradataml/data/docs/uaf/docs_17_20/DFFT2Conv.py +215 -215
  348. teradataml/data/docs/uaf/docs_17_20/DFFTConv.py +191 -191
  349. teradataml/data/docs/uaf/docs_17_20/DTW.py +179 -179
  350. teradataml/data/docs/uaf/docs_17_20/DickeyFuller.py +144 -144
  351. teradataml/data/docs/uaf/docs_17_20/DurbinWatson.py +183 -183
  352. teradataml/data/docs/uaf/docs_17_20/ExtractResults.py +184 -184
  353. teradataml/data/docs/uaf/docs_17_20/FitMetrics.py +172 -172
  354. teradataml/data/docs/uaf/docs_17_20/GenseriesFormula.py +205 -205
  355. teradataml/data/docs/uaf/docs_17_20/GenseriesSinusoids.py +142 -142
  356. teradataml/data/docs/uaf/docs_17_20/HoltWintersForecaster.py +258 -258
  357. teradataml/data/docs/uaf/docs_17_20/IDFFT.py +164 -164
  358. teradataml/data/docs/uaf/docs_17_20/IDFFT2.py +198 -198
  359. teradataml/data/docs/uaf/docs_17_20/InputValidator.py +120 -120
  360. teradataml/data/docs/uaf/docs_17_20/LineSpec.py +155 -155
  361. teradataml/data/docs/uaf/docs_17_20/LinearRegr.py +214 -214
  362. teradataml/data/docs/uaf/docs_17_20/MAMean.py +173 -173
  363. teradataml/data/docs/uaf/docs_17_20/MInfo.py +133 -133
  364. teradataml/data/docs/uaf/docs_17_20/MatrixMultiply.py +135 -135
  365. teradataml/data/docs/uaf/docs_17_20/MultivarRegr.py +190 -190
  366. teradataml/data/docs/uaf/docs_17_20/PACF.py +158 -158
  367. teradataml/data/docs/uaf/docs_17_20/Portman.py +216 -216
  368. teradataml/data/docs/uaf/docs_17_20/PowerTransform.py +154 -154
  369. teradataml/data/docs/uaf/docs_17_20/Resample.py +228 -228
  370. teradataml/data/docs/uaf/docs_17_20/SInfo.py +122 -122
  371. teradataml/data/docs/uaf/docs_17_20/SeasonalNormalize.py +165 -165
  372. teradataml/data/docs/uaf/docs_17_20/SelectionCriteria.py +173 -173
  373. teradataml/data/docs/uaf/docs_17_20/SignifPeriodicities.py +170 -170
  374. teradataml/data/docs/uaf/docs_17_20/SignifResidmean.py +163 -163
  375. teradataml/data/docs/uaf/docs_17_20/SimpleExp.py +179 -179
  376. teradataml/data/docs/uaf/docs_17_20/Smoothma.py +207 -207
  377. teradataml/data/docs/uaf/docs_17_20/TrackingOp.py +150 -150
  378. teradataml/data/docs/uaf/docs_17_20/UNDIFF.py +171 -171
  379. teradataml/data/docs/uaf/docs_17_20/Unnormalize.py +201 -201
  380. teradataml/data/docs/uaf/docs_17_20/WhitesGeneral.py +169 -169
  381. teradataml/data/dtw_example.json +17 -17
  382. teradataml/data/dtw_t1.csv +11 -11
  383. teradataml/data/dtw_t2.csv +4 -4
  384. teradataml/data/dwt2d_example.json +15 -15
  385. teradataml/data/dwt_example.json +14 -14
  386. teradataml/data/dwt_filter_dim.csv +5 -5
  387. teradataml/data/emission.csv +9 -9
  388. teradataml/data/emp_table_by_dept.csv +19 -19
  389. teradataml/data/employee_info.csv +4 -4
  390. teradataml/data/employee_table.csv +6 -6
  391. teradataml/data/excluding_event_table.csv +2 -2
  392. teradataml/data/finance_data.csv +6 -6
  393. teradataml/data/finance_data2.csv +61 -61
  394. teradataml/data/finance_data3.csv +93 -93
  395. teradataml/data/fish.csv +160 -0
  396. teradataml/data/fm_blood2ageandweight.csv +26 -26
  397. teradataml/data/fmeasure_example.json +11 -11
  398. teradataml/data/followers_leaders.csv +10 -10
  399. teradataml/data/fpgrowth_example.json +12 -12
  400. teradataml/data/frequentpaths_example.json +29 -29
  401. teradataml/data/friends.csv +9 -9
  402. teradataml/data/fs_input.csv +33 -33
  403. teradataml/data/fs_input1.csv +33 -33
  404. teradataml/data/genData.csv +513 -513
  405. teradataml/data/geodataframe_example.json +39 -39
  406. teradataml/data/glass_types.csv +215 -0
  407. teradataml/data/glm_admissions_model.csv +12 -12
  408. teradataml/data/glm_example.json +56 -29
  409. teradataml/data/glml1l2_example.json +28 -28
  410. teradataml/data/glml1l2predict_example.json +54 -54
  411. teradataml/data/glmpredict_example.json +54 -54
  412. teradataml/data/gq_t1.csv +21 -21
  413. teradataml/data/hconvolve_complex_right.csv +5 -5
  414. teradataml/data/hconvolve_complex_rightmulti.csv +5 -5
  415. teradataml/data/histogram_example.json +11 -11
  416. teradataml/data/hmmdecoder_example.json +78 -78
  417. teradataml/data/hmmevaluator_example.json +24 -24
  418. teradataml/data/hmmsupervised_example.json +10 -10
  419. teradataml/data/hmmunsupervised_example.json +7 -7
  420. teradataml/data/house_values.csv +12 -12
  421. teradataml/data/house_values2.csv +13 -13
  422. teradataml/data/housing_cat.csv +7 -7
  423. teradataml/data/housing_data.csv +9 -9
  424. teradataml/data/housing_test.csv +47 -47
  425. teradataml/data/housing_test_binary.csv +47 -47
  426. teradataml/data/housing_train.csv +493 -493
  427. teradataml/data/housing_train_attribute.csv +4 -4
  428. teradataml/data/housing_train_binary.csv +437 -437
  429. teradataml/data/housing_train_parameter.csv +2 -2
  430. teradataml/data/housing_train_response.csv +493 -493
  431. teradataml/data/housing_train_segment.csv +201 -0
  432. teradataml/data/ibm_stock.csv +370 -370
  433. teradataml/data/ibm_stock1.csv +370 -370
  434. teradataml/data/identitymatch_example.json +21 -21
  435. teradataml/data/idf_table.csv +4 -4
  436. teradataml/data/impressions.csv +101 -101
  437. teradataml/data/inflation.csv +21 -21
  438. teradataml/data/initial.csv +3 -3
  439. teradataml/data/insect2Cols.csv +61 -0
  440. teradataml/data/insect_sprays.csv +12 -12
  441. teradataml/data/insurance.csv +1339 -1339
  442. teradataml/data/interpolator_example.json +12 -12
  443. teradataml/data/iris_altinput.csv +481 -481
  444. teradataml/data/iris_attribute_output.csv +8 -8
  445. teradataml/data/iris_attribute_test.csv +121 -121
  446. teradataml/data/iris_attribute_train.csv +481 -481
  447. teradataml/data/iris_category_expect_predict.csv +31 -31
  448. teradataml/data/iris_data.csv +151 -0
  449. teradataml/data/iris_input.csv +151 -151
  450. teradataml/data/iris_response_train.csv +121 -121
  451. teradataml/data/iris_test.csv +31 -31
  452. teradataml/data/iris_train.csv +121 -121
  453. teradataml/data/join_table1.csv +4 -4
  454. teradataml/data/join_table2.csv +4 -4
  455. teradataml/data/jsons/anly_function_name.json +6 -6
  456. teradataml/data/jsons/byom/dataikupredict.json +147 -147
  457. teradataml/data/jsons/byom/datarobotpredict.json +146 -146
  458. teradataml/data/jsons/byom/h2opredict.json +194 -194
  459. teradataml/data/jsons/byom/onnxpredict.json +186 -186
  460. teradataml/data/jsons/byom/pmmlpredict.json +146 -146
  461. teradataml/data/jsons/paired_functions.json +435 -435
  462. teradataml/data/jsons/sqle/16.20/Antiselect.json +56 -56
  463. teradataml/data/jsons/sqle/16.20/Attribution.json +249 -249
  464. teradataml/data/jsons/sqle/16.20/DecisionForestPredict.json +156 -156
  465. teradataml/data/jsons/sqle/16.20/DecisionTreePredict.json +170 -170
  466. teradataml/data/jsons/sqle/16.20/GLMPredict.json +122 -122
  467. teradataml/data/jsons/sqle/16.20/MovingAverage.json +367 -367
  468. teradataml/data/jsons/sqle/16.20/NGramSplitter.json +239 -239
  469. teradataml/data/jsons/sqle/16.20/NaiveBayesPredict.json +136 -136
  470. teradataml/data/jsons/sqle/16.20/NaiveBayesTextClassifierPredict.json +235 -235
  471. teradataml/data/jsons/sqle/16.20/Pack.json +98 -98
  472. teradataml/data/jsons/sqle/16.20/SVMSparsePredict.json +162 -162
  473. teradataml/data/jsons/sqle/16.20/Sessionize.json +105 -105
  474. teradataml/data/jsons/sqle/16.20/StringSimilarity.json +86 -86
  475. teradataml/data/jsons/sqle/16.20/Unpack.json +166 -166
  476. teradataml/data/jsons/sqle/16.20/nPath.json +269 -269
  477. teradataml/data/jsons/sqle/17.00/Antiselect.json +56 -56
  478. teradataml/data/jsons/sqle/17.00/Attribution.json +249 -249
  479. teradataml/data/jsons/sqle/17.00/DecisionForestPredict.json +156 -156
  480. teradataml/data/jsons/sqle/17.00/DecisionTreePredict.json +170 -170
  481. teradataml/data/jsons/sqle/17.00/GLMPredict.json +122 -122
  482. teradataml/data/jsons/sqle/17.00/MovingAverage.json +367 -367
  483. teradataml/data/jsons/sqle/17.00/NGramSplitter.json +239 -239
  484. teradataml/data/jsons/sqle/17.00/NaiveBayesPredict.json +136 -136
  485. teradataml/data/jsons/sqle/17.00/NaiveBayesTextClassifierPredict.json +235 -235
  486. teradataml/data/jsons/sqle/17.00/Pack.json +98 -98
  487. teradataml/data/jsons/sqle/17.00/SVMSparsePredict.json +162 -162
  488. teradataml/data/jsons/sqle/17.00/Sessionize.json +105 -105
  489. teradataml/data/jsons/sqle/17.00/StringSimilarity.json +86 -86
  490. teradataml/data/jsons/sqle/17.00/Unpack.json +166 -166
  491. teradataml/data/jsons/sqle/17.00/nPath.json +269 -269
  492. teradataml/data/jsons/sqle/17.05/Antiselect.json +56 -56
  493. teradataml/data/jsons/sqle/17.05/Attribution.json +249 -249
  494. teradataml/data/jsons/sqle/17.05/DecisionForestPredict.json +156 -156
  495. teradataml/data/jsons/sqle/17.05/DecisionTreePredict.json +170 -170
  496. teradataml/data/jsons/sqle/17.05/GLMPredict.json +122 -122
  497. teradataml/data/jsons/sqle/17.05/MovingAverage.json +367 -367
  498. teradataml/data/jsons/sqle/17.05/NGramSplitter.json +239 -239
  499. teradataml/data/jsons/sqle/17.05/NaiveBayesPredict.json +136 -136
  500. teradataml/data/jsons/sqle/17.05/NaiveBayesTextClassifierPredict.json +235 -235
  501. teradataml/data/jsons/sqle/17.05/Pack.json +98 -98
  502. teradataml/data/jsons/sqle/17.05/SVMSparsePredict.json +162 -162
  503. teradataml/data/jsons/sqle/17.05/Sessionize.json +105 -105
  504. teradataml/data/jsons/sqle/17.05/StringSimilarity.json +86 -86
  505. teradataml/data/jsons/sqle/17.05/Unpack.json +166 -166
  506. teradataml/data/jsons/sqle/17.05/nPath.json +269 -269
  507. teradataml/data/jsons/sqle/17.10/Antiselect.json +56 -56
  508. teradataml/data/jsons/sqle/17.10/Attribution.json +249 -249
  509. teradataml/data/jsons/sqle/17.10/DecisionForestPredict.json +185 -185
  510. teradataml/data/jsons/sqle/17.10/DecisionTreePredict.json +171 -171
  511. teradataml/data/jsons/sqle/17.10/GLMPredict.json +151 -151
  512. teradataml/data/jsons/sqle/17.10/MovingAverage.json +368 -368
  513. teradataml/data/jsons/sqle/17.10/NGramSplitter.json +239 -239
  514. teradataml/data/jsons/sqle/17.10/NaiveBayesPredict.json +149 -149
  515. teradataml/data/jsons/sqle/17.10/NaiveBayesTextClassifierPredict.json +288 -288
  516. teradataml/data/jsons/sqle/17.10/Pack.json +133 -133
  517. teradataml/data/jsons/sqle/17.10/SVMSparsePredict.json +193 -193
  518. teradataml/data/jsons/sqle/17.10/Sessionize.json +105 -105
  519. teradataml/data/jsons/sqle/17.10/StringSimilarity.json +86 -86
  520. teradataml/data/jsons/sqle/17.10/TD_BinCodeFit.json +239 -239
  521. teradataml/data/jsons/sqle/17.10/TD_BinCodeTransform.json +70 -70
  522. teradataml/data/jsons/sqle/17.10/TD_CategoricalSummary.json +53 -53
  523. teradataml/data/jsons/sqle/17.10/TD_Chisq.json +67 -67
  524. teradataml/data/jsons/sqle/17.10/TD_ColumnSummary.json +53 -53
  525. teradataml/data/jsons/sqle/17.10/TD_ConvertTo.json +68 -68
  526. teradataml/data/jsons/sqle/17.10/TD_FTest.json +187 -187
  527. teradataml/data/jsons/sqle/17.10/TD_FillRowID.json +51 -51
  528. teradataml/data/jsons/sqle/17.10/TD_FunctionFit.json +46 -46
  529. teradataml/data/jsons/sqle/17.10/TD_FunctionTransform.json +72 -71
  530. teradataml/data/jsons/sqle/17.10/TD_GetRowsWithMissingValues.json +52 -52
  531. teradataml/data/jsons/sqle/17.10/TD_GetRowsWithoutMissingValues.json +52 -52
  532. teradataml/data/jsons/sqle/17.10/TD_Histogram.json +132 -132
  533. teradataml/data/jsons/sqle/17.10/TD_NumApply.json +147 -147
  534. teradataml/data/jsons/sqle/17.10/TD_OneHotEncodingFit.json +182 -182
  535. teradataml/data/jsons/sqle/17.10/TD_OneHotEncodingTransform.json +65 -64
  536. teradataml/data/jsons/sqle/17.10/TD_OutlierFilterFit.json +196 -196
  537. teradataml/data/jsons/sqle/17.10/TD_OutlierFilterTransform.json +48 -47
  538. teradataml/data/jsons/sqle/17.10/TD_PolynomialFeaturesFit.json +114 -114
  539. teradataml/data/jsons/sqle/17.10/TD_PolynomialFeaturesTransform.json +72 -71
  540. teradataml/data/jsons/sqle/17.10/TD_QQNorm.json +111 -111
  541. teradataml/data/jsons/sqle/17.10/TD_RoundColumns.json +93 -93
  542. teradataml/data/jsons/sqle/17.10/TD_RowNormalizeFit.json +127 -127
  543. teradataml/data/jsons/sqle/17.10/TD_RowNormalizeTransform.json +70 -69
  544. teradataml/data/jsons/sqle/17.10/TD_ScaleFit.json +156 -156
  545. teradataml/data/jsons/sqle/17.10/TD_ScaleTransform.json +70 -69
  546. teradataml/data/jsons/sqle/17.10/TD_SimpleImputeFit.json +147 -147
  547. teradataml/data/jsons/sqle/17.10/TD_SimpleImputeTransform.json +48 -47
  548. teradataml/data/jsons/sqle/17.10/TD_StrApply.json +240 -240
  549. teradataml/data/jsons/sqle/17.10/TD_UnivariateStatistics.json +118 -118
  550. teradataml/data/jsons/sqle/17.10/TD_WhichMax.json +52 -52
  551. teradataml/data/jsons/sqle/17.10/TD_WhichMin.json +52 -52
  552. teradataml/data/jsons/sqle/17.10/TD_ZTest.json +171 -171
  553. teradataml/data/jsons/sqle/17.10/Unpack.json +188 -188
  554. teradataml/data/jsons/sqle/17.10/nPath.json +269 -269
  555. teradataml/data/jsons/sqle/17.20/Antiselect.json +56 -56
  556. teradataml/data/jsons/sqle/17.20/Attribution.json +249 -249
  557. teradataml/data/jsons/sqle/17.20/DecisionForestPredict.json +185 -185
  558. teradataml/data/jsons/sqle/17.20/DecisionTreePredict.json +172 -172
  559. teradataml/data/jsons/sqle/17.20/GLMPredict.json +151 -151
  560. teradataml/data/jsons/sqle/17.20/MovingAverage.json +367 -367
  561. teradataml/data/jsons/sqle/17.20/NGramSplitter.json +239 -239
  562. teradataml/data/jsons/sqle/17.20/NaiveBayesPredict.json +149 -149
  563. teradataml/data/jsons/sqle/17.20/NaiveBayesTextClassifierPredict.json +287 -287
  564. teradataml/data/jsons/sqle/17.20/Pack.json +133 -133
  565. teradataml/data/jsons/sqle/17.20/SVMSparsePredict.json +192 -192
  566. teradataml/data/jsons/sqle/17.20/Sessionize.json +105 -105
  567. teradataml/data/jsons/sqle/17.20/StringSimilarity.json +86 -86
  568. teradataml/data/jsons/sqle/17.20/TD_ANOVA.json +148 -76
  569. teradataml/data/jsons/sqle/17.20/TD_BinCodeFit.json +239 -239
  570. teradataml/data/jsons/sqle/17.20/TD_BinCodeTransform.json +71 -71
  571. teradataml/data/jsons/sqle/17.20/TD_CategoricalSummary.json +53 -53
  572. teradataml/data/jsons/sqle/17.20/TD_Chisq.json +67 -67
  573. teradataml/data/jsons/sqle/17.20/TD_ClassificationEvaluator.json +145 -145
  574. teradataml/data/jsons/sqle/17.20/TD_ColumnSummary.json +53 -53
  575. teradataml/data/jsons/sqle/17.20/TD_ColumnTransformer.json +218 -218
  576. teradataml/data/jsons/sqle/17.20/TD_ConvertTo.json +92 -92
  577. teradataml/data/jsons/sqle/17.20/TD_DecisionForest.json +259 -259
  578. teradataml/data/jsons/sqle/17.20/TD_DecisionForestPredict.json +139 -139
  579. teradataml/data/jsons/sqle/17.20/TD_FTest.json +269 -186
  580. teradataml/data/jsons/sqle/17.20/TD_FillRowID.json +52 -52
  581. teradataml/data/jsons/sqle/17.20/TD_FunctionFit.json +46 -46
  582. teradataml/data/jsons/sqle/17.20/TD_FunctionTransform.json +72 -72
  583. teradataml/data/jsons/sqle/17.20/TD_GLM.json +507 -431
  584. teradataml/data/jsons/sqle/17.20/TD_GLMPREDICT.json +168 -125
  585. teradataml/data/jsons/sqle/17.20/TD_GLMPerSegment.json +411 -411
  586. teradataml/data/jsons/sqle/17.20/TD_GLMPredictPerSegment.json +146 -146
  587. teradataml/data/jsons/sqle/17.20/TD_GetFutileColumns.json +93 -91
  588. teradataml/data/jsons/sqle/17.20/TD_GetRowsWithMissingValues.json +76 -76
  589. teradataml/data/jsons/sqle/17.20/TD_GetRowsWithoutMissingValues.json +76 -76
  590. teradataml/data/jsons/sqle/17.20/TD_Histogram.json +152 -152
  591. teradataml/data/jsons/sqle/17.20/TD_KMeans.json +231 -211
  592. teradataml/data/jsons/sqle/17.20/TD_KMeansPredict.json +86 -86
  593. teradataml/data/jsons/sqle/17.20/TD_KNN.json +262 -262
  594. teradataml/data/jsons/sqle/17.20/TD_NaiveBayesTextClassifierTrainer.json +137 -137
  595. teradataml/data/jsons/sqle/17.20/TD_NonLinearCombineFit.json +102 -101
  596. teradataml/data/jsons/sqle/17.20/TD_NonLinearCombineTransform.json +71 -71
  597. teradataml/data/jsons/sqle/17.20/TD_NumApply.json +147 -147
  598. teradataml/data/jsons/sqle/17.20/TD_OneClassSVM.json +315 -315
  599. teradataml/data/jsons/sqle/17.20/TD_OneClassSVMPredict.json +123 -123
  600. teradataml/data/jsons/sqle/17.20/TD_OneHotEncodingFit.json +271 -271
  601. teradataml/data/jsons/sqle/17.20/TD_OneHotEncodingTransform.json +65 -65
  602. teradataml/data/jsons/sqle/17.20/TD_OrdinalEncodingFit.json +229 -229
  603. teradataml/data/jsons/sqle/17.20/TD_OrdinalEncodingTransform.json +75 -75
  604. teradataml/data/jsons/sqle/17.20/TD_OutlierFilterFit.json +217 -217
  605. teradataml/data/jsons/sqle/17.20/TD_OutlierFilterTransform.json +48 -48
  606. teradataml/data/jsons/sqle/17.20/TD_PolynomialFeaturesFit.json +114 -114
  607. teradataml/data/jsons/sqle/17.20/TD_PolynomialFeaturesTransform.json +72 -72
  608. teradataml/data/jsons/sqle/17.20/TD_QQNorm.json +111 -111
  609. teradataml/data/jsons/sqle/17.20/TD_ROC.json +178 -177
  610. teradataml/data/jsons/sqle/17.20/TD_RandomProjectionFit.json +178 -178
  611. teradataml/data/jsons/sqle/17.20/TD_RandomProjectionMinComponents.json +73 -73
  612. teradataml/data/jsons/sqle/17.20/TD_RandomProjectionTransform.json +74 -74
  613. teradataml/data/jsons/sqle/17.20/TD_RegressionEvaluator.json +137 -137
  614. teradataml/data/jsons/sqle/17.20/TD_RoundColumns.json +93 -93
  615. teradataml/data/jsons/sqle/17.20/TD_RowNormalizeFit.json +127 -127
  616. teradataml/data/jsons/sqle/17.20/TD_RowNormalizeTransform.json +70 -70
  617. teradataml/data/jsons/sqle/17.20/TD_SVM.json +389 -389
  618. teradataml/data/jsons/sqle/17.20/TD_SVMPredict.json +142 -124
  619. teradataml/data/jsons/sqle/17.20/TD_ScaleFit.json +309 -156
  620. teradataml/data/jsons/sqle/17.20/TD_ScaleTransform.json +119 -70
  621. teradataml/data/jsons/sqle/17.20/TD_SentimentExtractor.json +193 -193
  622. teradataml/data/jsons/sqle/17.20/TD_Silhouette.json +142 -142
  623. teradataml/data/jsons/sqle/17.20/TD_SimpleImputeFit.json +147 -147
  624. teradataml/data/jsons/sqle/17.20/TD_SimpleImputeTransform.json +48 -48
  625. teradataml/data/jsons/sqle/17.20/TD_StrApply.json +240 -240
  626. teradataml/data/jsons/sqle/17.20/TD_TargetEncodingFit.json +248 -248
  627. teradataml/data/jsons/sqle/17.20/TD_TargetEncodingTransform.json +75 -75
  628. teradataml/data/jsons/sqle/17.20/TD_TextParser.json +192 -192
  629. teradataml/data/jsons/sqle/17.20/TD_TrainTestSplit.json +142 -142
  630. teradataml/data/jsons/sqle/17.20/TD_UnivariateStatistics.json +117 -117
  631. teradataml/data/jsons/sqle/17.20/TD_VectorDistance.json +182 -182
  632. teradataml/data/jsons/sqle/17.20/TD_WhichMax.json +52 -52
  633. teradataml/data/jsons/sqle/17.20/TD_WhichMin.json +52 -52
  634. teradataml/data/jsons/sqle/17.20/TD_WordEmbeddings.json +241 -241
  635. teradataml/data/jsons/sqle/17.20/TD_XGBoost.json +330 -312
  636. teradataml/data/jsons/sqle/17.20/TD_XGBoostPredict.json +195 -182
  637. teradataml/data/jsons/sqle/17.20/TD_ZTest.json +247 -170
  638. teradataml/data/jsons/sqle/17.20/Unpack.json +188 -188
  639. teradataml/data/jsons/sqle/17.20/nPath.json +269 -269
  640. teradataml/data/jsons/tableoperator/17.00/read_nos.json +197 -197
  641. teradataml/data/jsons/tableoperator/17.05/read_nos.json +197 -197
  642. teradataml/data/jsons/tableoperator/17.05/write_nos.json +194 -194
  643. teradataml/data/jsons/tableoperator/17.10/read_nos.json +183 -183
  644. teradataml/data/jsons/tableoperator/17.10/write_nos.json +194 -194
  645. teradataml/data/jsons/tableoperator/17.20/read_nos.json +182 -182
  646. teradataml/data/jsons/tableoperator/17.20/write_nos.json +223 -223
  647. teradataml/data/jsons/uaf/17.20/TD_ACF.json +149 -149
  648. teradataml/data/jsons/uaf/17.20/TD_ARIMAESTIMATE.json +409 -409
  649. teradataml/data/jsons/uaf/17.20/TD_ARIMAFORECAST.json +79 -79
  650. teradataml/data/jsons/uaf/17.20/TD_ARIMAVALIDATE.json +151 -151
  651. teradataml/data/jsons/uaf/17.20/TD_BINARYMATRIXOP.json +109 -109
  652. teradataml/data/jsons/uaf/17.20/TD_BINARYSERIESOP.json +107 -107
  653. teradataml/data/jsons/uaf/17.20/TD_BREUSCH_GODFREY.json +87 -87
  654. teradataml/data/jsons/uaf/17.20/TD_BREUSCH_PAGAN_GODFREY.json +106 -106
  655. teradataml/data/jsons/uaf/17.20/TD_CONVOLVE.json +80 -80
  656. teradataml/data/jsons/uaf/17.20/TD_CONVOLVE2.json +67 -67
  657. teradataml/data/jsons/uaf/17.20/TD_CUMUL_PERIODOGRAM.json +91 -91
  658. teradataml/data/jsons/uaf/17.20/TD_DFFT.json +136 -136
  659. teradataml/data/jsons/uaf/17.20/TD_DFFT2.json +148 -148
  660. teradataml/data/jsons/uaf/17.20/TD_DFFT2CONV.json +108 -108
  661. teradataml/data/jsons/uaf/17.20/TD_DFFTCONV.json +109 -109
  662. teradataml/data/jsons/uaf/17.20/TD_DICKEY_FULLER.json +86 -86
  663. teradataml/data/jsons/uaf/17.20/TD_DIFF.json +91 -91
  664. teradataml/data/jsons/uaf/17.20/TD_DTW.json +116 -116
  665. teradataml/data/jsons/uaf/17.20/TD_DURBIN_WATSON.json +100 -100
  666. teradataml/data/jsons/uaf/17.20/TD_EXTRACT_RESULTS.json +38 -38
  667. teradataml/data/jsons/uaf/17.20/TD_FITMETRICS.json +100 -100
  668. teradataml/data/jsons/uaf/17.20/TD_GENSERIES4FORMULA.json +84 -84
  669. teradataml/data/jsons/uaf/17.20/TD_GENSERIES4SINUSOIDS.json +70 -70
  670. teradataml/data/jsons/uaf/17.20/TD_GOLDFELD_QUANDT.json +152 -152
  671. teradataml/data/jsons/uaf/17.20/TD_HOLT_WINTERS_FORECAST.json +313 -313
  672. teradataml/data/jsons/uaf/17.20/TD_IDFFT.json +57 -57
  673. teradataml/data/jsons/uaf/17.20/TD_IDFFT2.json +94 -94
  674. teradataml/data/jsons/uaf/17.20/TD_INPUTVALIDATOR.json +63 -63
  675. teradataml/data/jsons/uaf/17.20/TD_LINEAR_REGR.json +181 -181
  676. teradataml/data/jsons/uaf/17.20/TD_LINESPEC.json +102 -102
  677. teradataml/data/jsons/uaf/17.20/TD_MAMEAN.json +182 -182
  678. teradataml/data/jsons/uaf/17.20/TD_MATRIXMULTIPLY.json +67 -67
  679. teradataml/data/jsons/uaf/17.20/TD_MINFO.json +66 -66
  680. teradataml/data/jsons/uaf/17.20/TD_MULTIVAR_REGR.json +178 -178
  681. teradataml/data/jsons/uaf/17.20/TD_PACF.json +114 -114
  682. teradataml/data/jsons/uaf/17.20/TD_PORTMAN.json +118 -118
  683. teradataml/data/jsons/uaf/17.20/TD_POWERSPEC.json +175 -175
  684. teradataml/data/jsons/uaf/17.20/TD_POWERTRANSFORM.json +97 -97
  685. teradataml/data/jsons/uaf/17.20/TD_RESAMPLE.json +173 -173
  686. teradataml/data/jsons/uaf/17.20/TD_SEASONALNORMALIZE.json +136 -136
  687. teradataml/data/jsons/uaf/17.20/TD_SELECTION_CRITERIA.json +89 -89
  688. teradataml/data/jsons/uaf/17.20/TD_SIGNIF_PERIODICITIES.json +79 -79
  689. teradataml/data/jsons/uaf/17.20/TD_SIGNIF_RESIDMEAN.json +67 -67
  690. teradataml/data/jsons/uaf/17.20/TD_SIMPLEEXP.json +184 -184
  691. teradataml/data/jsons/uaf/17.20/TD_SINFO.json +57 -57
  692. teradataml/data/jsons/uaf/17.20/TD_SMOOTHMA.json +162 -162
  693. teradataml/data/jsons/uaf/17.20/TD_TRACKINGOP.json +100 -100
  694. teradataml/data/jsons/uaf/17.20/TD_UNDIFF.json +111 -111
  695. teradataml/data/jsons/uaf/17.20/TD_UNNORMALIZE.json +95 -95
  696. teradataml/data/jsons/uaf/17.20/TD_WHITES_GENERAL.json +77 -77
  697. teradataml/data/kmeans_example.json +22 -17
  698. teradataml/data/kmeans_table.csv +10 -0
  699. teradataml/data/kmeans_us_arrests_data.csv +0 -0
  700. teradataml/data/knn_example.json +18 -18
  701. teradataml/data/knnrecommender_example.json +6 -6
  702. teradataml/data/knnrecommenderpredict_example.json +12 -12
  703. teradataml/data/lar_example.json +17 -17
  704. teradataml/data/larpredict_example.json +30 -30
  705. teradataml/data/lc_new_predictors.csv +5 -5
  706. teradataml/data/lc_new_reference.csv +9 -9
  707. teradataml/data/lda_example.json +8 -8
  708. teradataml/data/ldainference_example.json +14 -14
  709. teradataml/data/ldatopicsummary_example.json +8 -8
  710. teradataml/data/levendist_input.csv +13 -13
  711. teradataml/data/levenshteindistance_example.json +10 -10
  712. teradataml/data/linreg_example.json +9 -9
  713. teradataml/data/load_example_data.py +326 -323
  714. teradataml/data/loan_prediction.csv +295 -295
  715. teradataml/data/lungcancer.csv +138 -138
  716. teradataml/data/mappingdata.csv +12 -12
  717. teradataml/data/milk_timeseries.csv +157 -157
  718. teradataml/data/min_max_titanic.csv +4 -4
  719. teradataml/data/minhash_example.json +6 -6
  720. teradataml/data/ml_ratings.csv +7547 -7547
  721. teradataml/data/ml_ratings_10.csv +2445 -2445
  722. teradataml/data/model1_table.csv +5 -5
  723. teradataml/data/model2_table.csv +5 -5
  724. teradataml/data/models/iris_db_glm_model.pmml +56 -56
  725. teradataml/data/models/iris_db_xgb_model.pmml +4471 -4471
  726. teradataml/data/modularity_example.json +12 -12
  727. teradataml/data/movavg_example.json +7 -7
  728. teradataml/data/mtx1.csv +7 -7
  729. teradataml/data/mtx2.csv +13 -13
  730. teradataml/data/multi_model_classification.csv +401 -0
  731. teradataml/data/multi_model_regression.csv +401 -0
  732. teradataml/data/mvdfft8.csv +9 -9
  733. teradataml/data/naivebayes_example.json +9 -9
  734. teradataml/data/naivebayespredict_example.json +19 -19
  735. teradataml/data/naivebayestextclassifier2_example.json +6 -6
  736. teradataml/data/naivebayestextclassifier_example.json +8 -8
  737. teradataml/data/naivebayestextclassifierpredict_example.json +20 -20
  738. teradataml/data/name_Find_configure.csv +10 -10
  739. teradataml/data/namedentityfinder_example.json +14 -14
  740. teradataml/data/namedentityfinderevaluator_example.json +10 -10
  741. teradataml/data/namedentityfindertrainer_example.json +6 -6
  742. teradataml/data/nb_iris_input_test.csv +31 -31
  743. teradataml/data/nb_iris_input_train.csv +121 -121
  744. teradataml/data/nbp_iris_model.csv +13 -13
  745. teradataml/data/ner_extractor_text.csv +2 -2
  746. teradataml/data/ner_sports_test2.csv +29 -29
  747. teradataml/data/ner_sports_train.csv +501 -501
  748. teradataml/data/nerevaluator_example.json +5 -5
  749. teradataml/data/nerextractor_example.json +18 -18
  750. teradataml/data/nermem_sports_test.csv +17 -17
  751. teradataml/data/nermem_sports_train.csv +50 -50
  752. teradataml/data/nertrainer_example.json +6 -6
  753. teradataml/data/ngrams_example.json +6 -6
  754. teradataml/data/notebooks/sqlalchemy/Teradata Vantage Aggregate Functions using SQLAlchemy.ipynb +1455 -1455
  755. teradataml/data/notebooks/sqlalchemy/Teradata Vantage Arithmetic Functions Using SQLAlchemy.ipynb +1993 -1993
  756. teradataml/data/notebooks/sqlalchemy/Teradata Vantage Bit-Byte Manipulation Functions using SQLAlchemy.ipynb +1492 -1492
  757. teradataml/data/notebooks/sqlalchemy/Teradata Vantage Built-in functions using SQLAlchemy.ipynb +536 -536
  758. teradataml/data/notebooks/sqlalchemy/Teradata Vantage Regular Expressions Using SQLAlchemy.ipynb +570 -570
  759. teradataml/data/notebooks/sqlalchemy/Teradata Vantage String Functions Using SQLAlchemy.ipynb +2559 -2559
  760. teradataml/data/notebooks/sqlalchemy/Teradata Vantage Window Aggregate Functions using SQLAlchemy.ipynb +2911 -2911
  761. teradataml/data/notebooks/sqlalchemy/Using Generic SQLAlchemy ClauseElements teradataml DataFrame assign method.ipynb +698 -698
  762. teradataml/data/notebooks/sqlalchemy/teradataml filtering using SQLAlchemy ClauseElements.ipynb +784 -784
  763. teradataml/data/npath_example.json +23 -23
  764. teradataml/data/ntree_example.json +14 -14
  765. teradataml/data/numeric_strings.csv +4 -4
  766. teradataml/data/numerics.csv +4 -4
  767. teradataml/data/ocean_buoy.csv +17 -17
  768. teradataml/data/ocean_buoy2.csv +17 -17
  769. teradataml/data/ocean_buoys.csv +27 -27
  770. teradataml/data/ocean_buoys2.csv +10 -10
  771. teradataml/data/ocean_buoys_nonpti.csv +28 -28
  772. teradataml/data/ocean_buoys_seq.csv +29 -29
  773. teradataml/data/onehot_encoder_train.csv +4 -0
  774. teradataml/data/openml_example.json +92 -0
  775. teradataml/data/optional_event_table.csv +4 -4
  776. teradataml/data/orders1.csv +11 -11
  777. teradataml/data/orders1_12.csv +12 -12
  778. teradataml/data/orders_ex.csv +4 -4
  779. teradataml/data/pack_example.json +8 -8
  780. teradataml/data/package_tracking.csv +19 -19
  781. teradataml/data/package_tracking_pti.csv +18 -18
  782. teradataml/data/pagerank_example.json +13 -13
  783. teradataml/data/paragraphs_input.csv +6 -6
  784. teradataml/data/pathanalyzer_example.json +7 -7
  785. teradataml/data/pathgenerator_example.json +7 -7
  786. teradataml/data/phrases.csv +7 -7
  787. teradataml/data/pivot_example.json +8 -8
  788. teradataml/data/pivot_input.csv +22 -22
  789. teradataml/data/playerRating.csv +31 -31
  790. teradataml/data/postagger_example.json +6 -6
  791. teradataml/data/posttagger_output.csv +44 -44
  792. teradataml/data/production_data.csv +16 -16
  793. teradataml/data/production_data2.csv +7 -7
  794. teradataml/data/randomsample_example.json +31 -31
  795. teradataml/data/randomwalksample_example.json +8 -8
  796. teradataml/data/rank_table.csv +6 -6
  797. teradataml/data/ref_mobile_data.csv +4 -4
  798. teradataml/data/ref_mobile_data_dense.csv +2 -2
  799. teradataml/data/ref_url.csv +17 -17
  800. teradataml/data/restaurant_reviews.csv +7 -7
  801. teradataml/data/river_data.csv +145 -145
  802. teradataml/data/roc_example.json +7 -7
  803. teradataml/data/roc_input.csv +101 -101
  804. teradataml/data/rule_inputs.csv +6 -6
  805. teradataml/data/rule_table.csv +2 -2
  806. teradataml/data/sales.csv +7 -7
  807. teradataml/data/sales_transaction.csv +501 -501
  808. teradataml/data/salesdata.csv +342 -342
  809. teradataml/data/sample_cities.csv +2 -2
  810. teradataml/data/sample_shapes.csv +10 -10
  811. teradataml/data/sample_streets.csv +2 -2
  812. teradataml/data/sampling_example.json +15 -15
  813. teradataml/data/sax_example.json +8 -8
  814. teradataml/data/scale_attributes.csv +3 -0
  815. teradataml/data/scale_example.json +74 -23
  816. teradataml/data/scale_housing.csv +11 -11
  817. teradataml/data/scale_housing_test.csv +6 -6
  818. teradataml/data/scale_input_part_sparse.csv +31 -0
  819. teradataml/data/scale_input_partitioned.csv +16 -0
  820. teradataml/data/scale_input_sparse.csv +11 -0
  821. teradataml/data/scale_parameters.csv +3 -0
  822. teradataml/data/scale_stat.csv +11 -11
  823. teradataml/data/scalebypartition_example.json +13 -13
  824. teradataml/data/scalemap_example.json +13 -13
  825. teradataml/data/scalesummary_example.json +12 -12
  826. teradataml/data/score_category.csv +101 -101
  827. teradataml/data/score_summary.csv +4 -4
  828. teradataml/data/script_example.json +9 -9
  829. teradataml/data/scripts/deploy_script.py +84 -0
  830. teradataml/data/scripts/mapper.R +20 -0
  831. teradataml/data/scripts/mapper.py +15 -15
  832. teradataml/data/scripts/mapper_replace.py +15 -15
  833. teradataml/data/scripts/sklearn/__init__.py +0 -0
  834. teradataml/data/scripts/sklearn/sklearn_fit.py +171 -0
  835. teradataml/data/scripts/sklearn/sklearn_fit_predict.py +127 -0
  836. teradataml/data/scripts/sklearn/sklearn_function.template +108 -0
  837. teradataml/data/scripts/sklearn/sklearn_model_selection_split.py +148 -0
  838. teradataml/data/scripts/sklearn/sklearn_neighbors.py +143 -0
  839. teradataml/data/scripts/sklearn/sklearn_score.py +119 -0
  840. teradataml/data/scripts/sklearn/sklearn_transform.py +171 -0
  841. teradataml/data/seeds.csv +10 -10
  842. teradataml/data/sentenceextractor_example.json +6 -6
  843. teradataml/data/sentiment_extract_input.csv +11 -11
  844. teradataml/data/sentiment_train.csv +16 -16
  845. teradataml/data/sentiment_word.csv +20 -20
  846. teradataml/data/sentiment_word_input.csv +19 -19
  847. teradataml/data/sentimentextractor_example.json +24 -24
  848. teradataml/data/sentimenttrainer_example.json +8 -8
  849. teradataml/data/sequence_table.csv +10 -10
  850. teradataml/data/seriessplitter_example.json +7 -7
  851. teradataml/data/sessionize_example.json +17 -17
  852. teradataml/data/sessionize_table.csv +116 -116
  853. teradataml/data/setop_test1.csv +24 -24
  854. teradataml/data/setop_test2.csv +22 -22
  855. teradataml/data/soc_nw_edges.csv +10 -10
  856. teradataml/data/soc_nw_vertices.csv +7 -7
  857. teradataml/data/souvenir_timeseries.csv +167 -167
  858. teradataml/data/sparse_iris_attribute.csv +5 -5
  859. teradataml/data/sparse_iris_test.csv +121 -121
  860. teradataml/data/sparse_iris_train.csv +601 -601
  861. teradataml/data/star1.csv +6 -6
  862. teradataml/data/state_transition.csv +5 -5
  863. teradataml/data/stock_data.csv +53 -53
  864. teradataml/data/stock_movement.csv +11 -11
  865. teradataml/data/stock_vol.csv +76 -76
  866. teradataml/data/stop_words.csv +8 -8
  867. teradataml/data/store_sales.csv +37 -37
  868. teradataml/data/stringsimilarity_example.json +7 -7
  869. teradataml/data/strsimilarity_input.csv +13 -13
  870. teradataml/data/students.csv +101 -101
  871. teradataml/data/svm_iris_input_test.csv +121 -121
  872. teradataml/data/svm_iris_input_train.csv +481 -481
  873. teradataml/data/svm_iris_model.csv +7 -7
  874. teradataml/data/svmdense_example.json +9 -9
  875. teradataml/data/svmdensepredict_example.json +18 -18
  876. teradataml/data/svmsparse_example.json +7 -7
  877. teradataml/data/svmsparsepredict_example.json +13 -13
  878. teradataml/data/svmsparsesummary_example.json +7 -7
  879. teradataml/data/target_mobile_data.csv +13 -13
  880. teradataml/data/target_mobile_data_dense.csv +5 -5
  881. teradataml/data/templatedata.csv +1201 -1201
  882. teradataml/data/templates/open_source_ml.json +9 -0
  883. teradataml/data/teradataml_example.json +150 -1
  884. teradataml/data/test_classification.csv +101 -0
  885. teradataml/data/test_loan_prediction.csv +53 -53
  886. teradataml/data/test_pacf_12.csv +37 -37
  887. teradataml/data/test_prediction.csv +101 -0
  888. teradataml/data/test_regression.csv +101 -0
  889. teradataml/data/test_river2.csv +109 -109
  890. teradataml/data/text_inputs.csv +6 -6
  891. teradataml/data/textchunker_example.json +7 -7
  892. teradataml/data/textclassifier_example.json +6 -6
  893. teradataml/data/textclassifier_input.csv +7 -7
  894. teradataml/data/textclassifiertrainer_example.json +6 -6
  895. teradataml/data/textmorph_example.json +5 -5
  896. teradataml/data/textparser_example.json +15 -15
  897. teradataml/data/texttagger_example.json +11 -11
  898. teradataml/data/texttokenizer_example.json +6 -6
  899. teradataml/data/texttrainer_input.csv +11 -11
  900. teradataml/data/tf_example.json +6 -6
  901. teradataml/data/tfidf_example.json +13 -13
  902. teradataml/data/tfidf_input1.csv +201 -201
  903. teradataml/data/tfidf_train.csv +6 -6
  904. teradataml/data/time_table1.csv +535 -535
  905. teradataml/data/time_table2.csv +14 -14
  906. teradataml/data/timeseriesdata.csv +1601 -1601
  907. teradataml/data/timeseriesdatasetsd4.csv +105 -105
  908. teradataml/data/titanic.csv +892 -892
  909. teradataml/data/token_table.csv +696 -696
  910. teradataml/data/train_multiclass.csv +101 -0
  911. teradataml/data/train_regression.csv +101 -0
  912. teradataml/data/train_regression_multiple_labels.csv +101 -0
  913. teradataml/data/train_tracking.csv +27 -27
  914. teradataml/data/transformation_table.csv +5 -5
  915. teradataml/data/transformation_table_new.csv +1 -1
  916. teradataml/data/tv_spots.csv +16 -16
  917. teradataml/data/twod_climate_data.csv +117 -117
  918. teradataml/data/uaf_example.json +475 -475
  919. teradataml/data/univariatestatistics_example.json +8 -8
  920. teradataml/data/unpack_example.json +9 -9
  921. teradataml/data/unpivot_example.json +9 -9
  922. teradataml/data/unpivot_input.csv +8 -8
  923. teradataml/data/us_air_pass.csv +36 -36
  924. teradataml/data/us_population.csv +624 -624
  925. teradataml/data/us_states_shapes.csv +52 -52
  926. teradataml/data/varmax_example.json +17 -17
  927. teradataml/data/vectordistance_example.json +25 -25
  928. teradataml/data/ville_climatedata.csv +121 -121
  929. teradataml/data/ville_tempdata.csv +12 -12
  930. teradataml/data/ville_tempdata1.csv +12 -12
  931. teradataml/data/ville_temperature.csv +11 -11
  932. teradataml/data/waveletTable.csv +1605 -1605
  933. teradataml/data/waveletTable2.csv +1605 -1605
  934. teradataml/data/weightedmovavg_example.json +8 -8
  935. teradataml/data/wft_testing.csv +5 -5
  936. teradataml/data/wine_data.csv +1600 -0
  937. teradataml/data/word_embed_input_table1.csv +5 -5
  938. teradataml/data/word_embed_input_table2.csv +4 -4
  939. teradataml/data/word_embed_model.csv +22 -22
  940. teradataml/data/words_input.csv +13 -13
  941. teradataml/data/xconvolve_complex_left.csv +6 -6
  942. teradataml/data/xconvolve_complex_leftmulti.csv +6 -6
  943. teradataml/data/xgboost_example.json +35 -35
  944. teradataml/data/xgboostpredict_example.json +31 -31
  945. teradataml/data/ztest_example.json +16 -0
  946. teradataml/dataframe/copy_to.py +1769 -1698
  947. teradataml/dataframe/data_transfer.py +2812 -2745
  948. teradataml/dataframe/dataframe.py +17630 -16946
  949. teradataml/dataframe/dataframe_utils.py +1875 -1740
  950. teradataml/dataframe/fastload.py +794 -603
  951. teradataml/dataframe/indexer.py +424 -424
  952. teradataml/dataframe/setop.py +1179 -1166
  953. teradataml/dataframe/sql.py +10174 -6432
  954. teradataml/dataframe/sql_function_parameters.py +439 -388
  955. teradataml/dataframe/sql_functions.py +652 -652
  956. teradataml/dataframe/sql_interfaces.py +220 -220
  957. teradataml/dataframe/vantage_function_types.py +674 -630
  958. teradataml/dataframe/window.py +693 -692
  959. teradataml/dbutils/__init__.py +3 -3
  960. teradataml/dbutils/dbutils.py +1167 -1150
  961. teradataml/dbutils/filemgr.py +267 -267
  962. teradataml/gen_ai/__init__.py +2 -2
  963. teradataml/gen_ai/convAI.py +472 -472
  964. teradataml/geospatial/__init__.py +3 -3
  965. teradataml/geospatial/geodataframe.py +1105 -1094
  966. teradataml/geospatial/geodataframecolumn.py +392 -387
  967. teradataml/geospatial/geometry_types.py +925 -925
  968. teradataml/hyperparameter_tuner/__init__.py +1 -1
  969. teradataml/hyperparameter_tuner/optimizer.py +3783 -2993
  970. teradataml/hyperparameter_tuner/utils.py +281 -187
  971. teradataml/lib/aed_0_1.dll +0 -0
  972. teradataml/lib/libaed_0_1.dylib +0 -0
  973. teradataml/lib/libaed_0_1.so +0 -0
  974. teradataml/libaed_0_1.dylib +0 -0
  975. teradataml/libaed_0_1.so +0 -0
  976. teradataml/opensource/__init__.py +1 -0
  977. teradataml/opensource/sklearn/__init__.py +1 -0
  978. teradataml/opensource/sklearn/_class.py +255 -0
  979. teradataml/opensource/sklearn/_sklearn_wrapper.py +1715 -0
  980. teradataml/opensource/sklearn/_wrapper_utils.py +268 -0
  981. teradataml/opensource/sklearn/constants.py +54 -0
  982. teradataml/options/__init__.py +130 -124
  983. teradataml/options/configure.py +358 -336
  984. teradataml/options/display.py +176 -176
  985. teradataml/plot/__init__.py +2 -2
  986. teradataml/plot/axis.py +1388 -1388
  987. teradataml/plot/constants.py +15 -15
  988. teradataml/plot/figure.py +398 -398
  989. teradataml/plot/plot.py +760 -760
  990. teradataml/plot/query_generator.py +83 -83
  991. teradataml/plot/subplot.py +216 -216
  992. teradataml/scriptmgmt/UserEnv.py +3791 -3761
  993. teradataml/scriptmgmt/__init__.py +3 -3
  994. teradataml/scriptmgmt/lls_utils.py +1719 -1604
  995. teradataml/series/series.py +532 -532
  996. teradataml/series/series_utils.py +71 -71
  997. teradataml/table_operators/Apply.py +949 -917
  998. teradataml/table_operators/Script.py +1718 -1982
  999. teradataml/table_operators/TableOperator.py +1255 -1616
  1000. teradataml/table_operators/__init__.py +2 -3
  1001. teradataml/table_operators/apply_query_generator.py +262 -262
  1002. teradataml/table_operators/query_generator.py +507 -507
  1003. teradataml/table_operators/table_operator_query_generator.py +460 -460
  1004. teradataml/table_operators/table_operator_util.py +631 -639
  1005. teradataml/table_operators/templates/dataframe_apply.template +184 -184
  1006. teradataml/table_operators/templates/dataframe_map.template +176 -176
  1007. teradataml/table_operators/templates/script_executor.template +170 -170
  1008. teradataml/utils/dtypes.py +684 -684
  1009. teradataml/utils/internal_buffer.py +84 -84
  1010. teradataml/utils/print_versions.py +205 -205
  1011. teradataml/utils/utils.py +410 -410
  1012. teradataml/utils/validators.py +2277 -2115
  1013. {teradataml-17.20.0.7.dist-info → teradataml-20.0.0.1.dist-info}/METADATA +346 -45
  1014. teradataml-20.0.0.1.dist-info/RECORD +1056 -0
  1015. {teradataml-17.20.0.7.dist-info → teradataml-20.0.0.1.dist-info}/WHEEL +1 -1
  1016. {teradataml-17.20.0.7.dist-info → teradataml-20.0.0.1.dist-info}/zip-safe +1 -1
  1017. teradataml/analytics/mle/AdaBoost.py +0 -651
  1018. teradataml/analytics/mle/AdaBoostPredict.py +0 -564
  1019. teradataml/analytics/mle/Antiselect.py +0 -342
  1020. teradataml/analytics/mle/Arima.py +0 -641
  1021. teradataml/analytics/mle/ArimaPredict.py +0 -477
  1022. teradataml/analytics/mle/Attribution.py +0 -1070
  1023. teradataml/analytics/mle/Betweenness.py +0 -658
  1024. teradataml/analytics/mle/Burst.py +0 -711
  1025. teradataml/analytics/mle/CCM.py +0 -600
  1026. teradataml/analytics/mle/CCMPrepare.py +0 -324
  1027. teradataml/analytics/mle/CFilter.py +0 -460
  1028. teradataml/analytics/mle/ChangePointDetection.py +0 -572
  1029. teradataml/analytics/mle/ChangePointDetectionRT.py +0 -477
  1030. teradataml/analytics/mle/Closeness.py +0 -737
  1031. teradataml/analytics/mle/ConfusionMatrix.py +0 -420
  1032. teradataml/analytics/mle/Correlation.py +0 -477
  1033. teradataml/analytics/mle/Correlation2.py +0 -573
  1034. teradataml/analytics/mle/CoxHazardRatio.py +0 -679
  1035. teradataml/analytics/mle/CoxPH.py +0 -556
  1036. teradataml/analytics/mle/CoxSurvival.py +0 -478
  1037. teradataml/analytics/mle/CumulativeMovAvg.py +0 -363
  1038. teradataml/analytics/mle/DTW.py +0 -623
  1039. teradataml/analytics/mle/DWT.py +0 -564
  1040. teradataml/analytics/mle/DWT2D.py +0 -599
  1041. teradataml/analytics/mle/DecisionForest.py +0 -716
  1042. teradataml/analytics/mle/DecisionForestEvaluator.py +0 -363
  1043. teradataml/analytics/mle/DecisionForestPredict.py +0 -561
  1044. teradataml/analytics/mle/DecisionTree.py +0 -830
  1045. teradataml/analytics/mle/DecisionTreePredict.py +0 -528
  1046. teradataml/analytics/mle/ExponentialMovAvg.py +0 -418
  1047. teradataml/analytics/mle/FMeasure.py +0 -402
  1048. teradataml/analytics/mle/FPGrowth.py +0 -734
  1049. teradataml/analytics/mle/FrequentPaths.py +0 -695
  1050. teradataml/analytics/mle/GLM.py +0 -558
  1051. teradataml/analytics/mle/GLML1L2.py +0 -547
  1052. teradataml/analytics/mle/GLML1L2Predict.py +0 -519
  1053. teradataml/analytics/mle/GLMPredict.py +0 -529
  1054. teradataml/analytics/mle/HMMDecoder.py +0 -945
  1055. teradataml/analytics/mle/HMMEvaluator.py +0 -901
  1056. teradataml/analytics/mle/HMMSupervised.py +0 -521
  1057. teradataml/analytics/mle/HMMUnsupervised.py +0 -572
  1058. teradataml/analytics/mle/Histogram.py +0 -561
  1059. teradataml/analytics/mle/IDWT.py +0 -476
  1060. teradataml/analytics/mle/IDWT2D.py +0 -493
  1061. teradataml/analytics/mle/IdentityMatch.py +0 -763
  1062. teradataml/analytics/mle/Interpolator.py +0 -918
  1063. teradataml/analytics/mle/KMeans.py +0 -485
  1064. teradataml/analytics/mle/KNN.py +0 -627
  1065. teradataml/analytics/mle/KNNRecommender.py +0 -488
  1066. teradataml/analytics/mle/KNNRecommenderPredict.py +0 -581
  1067. teradataml/analytics/mle/LAR.py +0 -439
  1068. teradataml/analytics/mle/LARPredict.py +0 -478
  1069. teradataml/analytics/mle/LDA.py +0 -548
  1070. teradataml/analytics/mle/LDAInference.py +0 -492
  1071. teradataml/analytics/mle/LDATopicSummary.py +0 -464
  1072. teradataml/analytics/mle/LevenshteinDistance.py +0 -450
  1073. teradataml/analytics/mle/LinReg.py +0 -433
  1074. teradataml/analytics/mle/LinRegPredict.py +0 -438
  1075. teradataml/analytics/mle/MinHash.py +0 -544
  1076. teradataml/analytics/mle/Modularity.py +0 -587
  1077. teradataml/analytics/mle/NEREvaluator.py +0 -410
  1078. teradataml/analytics/mle/NERExtractor.py +0 -595
  1079. teradataml/analytics/mle/NERTrainer.py +0 -458
  1080. teradataml/analytics/mle/NGrams.py +0 -570
  1081. teradataml/analytics/mle/NPath.py +0 -634
  1082. teradataml/analytics/mle/NTree.py +0 -549
  1083. teradataml/analytics/mle/NaiveBayes.py +0 -462
  1084. teradataml/analytics/mle/NaiveBayesPredict.py +0 -513
  1085. teradataml/analytics/mle/NaiveBayesTextClassifier.py +0 -607
  1086. teradataml/analytics/mle/NaiveBayesTextClassifier2.py +0 -531
  1087. teradataml/analytics/mle/NaiveBayesTextClassifierPredict.py +0 -799
  1088. teradataml/analytics/mle/NamedEntityFinder.py +0 -529
  1089. teradataml/analytics/mle/NamedEntityFinderEvaluator.py +0 -414
  1090. teradataml/analytics/mle/NamedEntityFinderTrainer.py +0 -396
  1091. teradataml/analytics/mle/POSTagger.py +0 -417
  1092. teradataml/analytics/mle/Pack.py +0 -411
  1093. teradataml/analytics/mle/PageRank.py +0 -535
  1094. teradataml/analytics/mle/PathAnalyzer.py +0 -426
  1095. teradataml/analytics/mle/PathGenerator.py +0 -367
  1096. teradataml/analytics/mle/PathStart.py +0 -464
  1097. teradataml/analytics/mle/PathSummarizer.py +0 -470
  1098. teradataml/analytics/mle/Pivot.py +0 -471
  1099. teradataml/analytics/mle/ROC.py +0 -425
  1100. teradataml/analytics/mle/RandomSample.py +0 -637
  1101. teradataml/analytics/mle/RandomWalkSample.py +0 -490
  1102. teradataml/analytics/mle/SAX.py +0 -779
  1103. teradataml/analytics/mle/SVMDense.py +0 -677
  1104. teradataml/analytics/mle/SVMDensePredict.py +0 -536
  1105. teradataml/analytics/mle/SVMDenseSummary.py +0 -437
  1106. teradataml/analytics/mle/SVMSparse.py +0 -557
  1107. teradataml/analytics/mle/SVMSparsePredict.py +0 -553
  1108. teradataml/analytics/mle/SVMSparseSummary.py +0 -435
  1109. teradataml/analytics/mle/Sampling.py +0 -549
  1110. teradataml/analytics/mle/Scale.py +0 -565
  1111. teradataml/analytics/mle/ScaleByPartition.py +0 -496
  1112. teradataml/analytics/mle/ScaleMap.py +0 -378
  1113. teradataml/analytics/mle/ScaleSummary.py +0 -320
  1114. teradataml/analytics/mle/SentenceExtractor.py +0 -363
  1115. teradataml/analytics/mle/SentimentEvaluator.py +0 -432
  1116. teradataml/analytics/mle/SentimentExtractor.py +0 -578
  1117. teradataml/analytics/mle/SentimentTrainer.py +0 -405
  1118. teradataml/analytics/mle/SeriesSplitter.py +0 -641
  1119. teradataml/analytics/mle/Sessionize.py +0 -475
  1120. teradataml/analytics/mle/SimpleMovAvg.py +0 -397
  1121. teradataml/analytics/mle/StringSimilarity.py +0 -425
  1122. teradataml/analytics/mle/TF.py +0 -389
  1123. teradataml/analytics/mle/TFIDF.py +0 -504
  1124. teradataml/analytics/mle/TextChunker.py +0 -414
  1125. teradataml/analytics/mle/TextClassifier.py +0 -399
  1126. teradataml/analytics/mle/TextClassifierEvaluator.py +0 -413
  1127. teradataml/analytics/mle/TextClassifierTrainer.py +0 -565
  1128. teradataml/analytics/mle/TextMorph.py +0 -494
  1129. teradataml/analytics/mle/TextParser.py +0 -623
  1130. teradataml/analytics/mle/TextTagger.py +0 -530
  1131. teradataml/analytics/mle/TextTokenizer.py +0 -502
  1132. teradataml/analytics/mle/UnivariateStatistics.py +0 -488
  1133. teradataml/analytics/mle/Unpack.py +0 -526
  1134. teradataml/analytics/mle/Unpivot.py +0 -438
  1135. teradataml/analytics/mle/VarMax.py +0 -776
  1136. teradataml/analytics/mle/VectorDistance.py +0 -762
  1137. teradataml/analytics/mle/WeightedMovAvg.py +0 -400
  1138. teradataml/analytics/mle/XGBoost.py +0 -842
  1139. teradataml/analytics/mle/XGBoostPredict.py +0 -627
  1140. teradataml/analytics/mle/__init__.py +0 -123
  1141. teradataml/analytics/mle/json/adaboost_mle.json +0 -135
  1142. teradataml/analytics/mle/json/adaboostpredict_mle.json +0 -85
  1143. teradataml/analytics/mle/json/antiselect_mle.json +0 -34
  1144. teradataml/analytics/mle/json/antiselect_mle_mle.json +0 -34
  1145. teradataml/analytics/mle/json/arima_mle.json +0 -172
  1146. teradataml/analytics/mle/json/arimapredict_mle.json +0 -52
  1147. teradataml/analytics/mle/json/attribution_mle_mle.json +0 -143
  1148. teradataml/analytics/mle/json/betweenness_mle.json +0 -97
  1149. teradataml/analytics/mle/json/burst_mle.json +0 -140
  1150. teradataml/analytics/mle/json/ccm_mle.json +0 -124
  1151. teradataml/analytics/mle/json/ccmprepare_mle.json +0 -14
  1152. teradataml/analytics/mle/json/cfilter_mle.json +0 -93
  1153. teradataml/analytics/mle/json/changepointdetection_mle.json +0 -92
  1154. teradataml/analytics/mle/json/changepointdetectionrt_mle.json +0 -78
  1155. teradataml/analytics/mle/json/closeness_mle.json +0 -104
  1156. teradataml/analytics/mle/json/confusionmatrix_mle.json +0 -79
  1157. teradataml/analytics/mle/json/correlation_mle.json +0 -86
  1158. teradataml/analytics/mle/json/correlationreduce_mle.json +0 -49
  1159. teradataml/analytics/mle/json/coxhazardratio_mle.json +0 -89
  1160. teradataml/analytics/mle/json/coxph_mle.json +0 -98
  1161. teradataml/analytics/mle/json/coxsurvival_mle.json +0 -79
  1162. teradataml/analytics/mle/json/cumulativemovavg_mle.json +0 -34
  1163. teradataml/analytics/mle/json/decisionforest_mle.json +0 -167
  1164. teradataml/analytics/mle/json/decisionforestevaluator_mle.json +0 -33
  1165. teradataml/analytics/mle/json/decisionforestpredict_mle_mle.json +0 -74
  1166. teradataml/analytics/mle/json/decisiontree_mle.json +0 -194
  1167. teradataml/analytics/mle/json/decisiontreepredict_mle_mle.json +0 -86
  1168. teradataml/analytics/mle/json/dtw_mle.json +0 -97
  1169. teradataml/analytics/mle/json/dwt2d_mle.json +0 -116
  1170. teradataml/analytics/mle/json/dwt_mle.json +0 -101
  1171. teradataml/analytics/mle/json/exponentialmovavg_mle.json +0 -55
  1172. teradataml/analytics/mle/json/fmeasure_mle.json +0 -58
  1173. teradataml/analytics/mle/json/fpgrowth_mle.json +0 -159
  1174. teradataml/analytics/mle/json/frequentpaths_mle.json +0 -129
  1175. teradataml/analytics/mle/json/glm_mle.json +0 -111
  1176. teradataml/analytics/mle/json/glml1l2_mle.json +0 -106
  1177. teradataml/analytics/mle/json/glml1l2predict_mle.json +0 -57
  1178. teradataml/analytics/mle/json/glmpredict_mle_mle.json +0 -74
  1179. teradataml/analytics/mle/json/histogram_mle.json +0 -100
  1180. teradataml/analytics/mle/json/hmmdecoder_mle.json +0 -192
  1181. teradataml/analytics/mle/json/hmmevaluator_mle.json +0 -206
  1182. teradataml/analytics/mle/json/hmmsupervised_mle.json +0 -91
  1183. teradataml/analytics/mle/json/hmmunsupervised_mle.json +0 -114
  1184. teradataml/analytics/mle/json/identitymatch_mle.json +0 -88
  1185. teradataml/analytics/mle/json/idwt2d_mle.json +0 -73
  1186. teradataml/analytics/mle/json/idwt_mle.json +0 -66
  1187. teradataml/analytics/mle/json/interpolator_mle.json +0 -151
  1188. teradataml/analytics/mle/json/kmeans_mle.json +0 -97
  1189. teradataml/analytics/mle/json/knn_mle.json +0 -141
  1190. teradataml/analytics/mle/json/knnrecommender_mle.json +0 -111
  1191. teradataml/analytics/mle/json/knnrecommenderpredict_mle.json +0 -75
  1192. teradataml/analytics/mle/json/lar_mle.json +0 -78
  1193. teradataml/analytics/mle/json/larpredict_mle.json +0 -69
  1194. teradataml/analytics/mle/json/lda_mle.json +0 -130
  1195. teradataml/analytics/mle/json/ldainference_mle.json +0 -78
  1196. teradataml/analytics/mle/json/ldatopicsummary_mle.json +0 -64
  1197. teradataml/analytics/mle/json/levenshteindistance_mle.json +0 -92
  1198. teradataml/analytics/mle/json/linreg_mle.json +0 -42
  1199. teradataml/analytics/mle/json/linregpredict_mle.json +0 -56
  1200. teradataml/analytics/mle/json/minhash_mle.json +0 -113
  1201. teradataml/analytics/mle/json/modularity_mle.json +0 -91
  1202. teradataml/analytics/mle/json/naivebayespredict_mle_mle.json +0 -85
  1203. teradataml/analytics/mle/json/naivebayesreduce_mle.json +0 -52
  1204. teradataml/analytics/mle/json/naivebayestextclassifierpredict_mle_mle.json +0 -147
  1205. teradataml/analytics/mle/json/naivebayestextclassifiertrainer2_mle.json +0 -108
  1206. teradataml/analytics/mle/json/naivebayestextclassifiertrainer_mle.json +0 -102
  1207. teradataml/analytics/mle/json/namedentityfinder_mle.json +0 -84
  1208. teradataml/analytics/mle/json/namedentityfinderevaluatorreduce_mle.json +0 -43
  1209. teradataml/analytics/mle/json/namedentityfindertrainer_mle.json +0 -64
  1210. teradataml/analytics/mle/json/nerevaluator_mle.json +0 -54
  1211. teradataml/analytics/mle/json/nerextractor_mle.json +0 -87
  1212. teradataml/analytics/mle/json/nertrainer_mle.json +0 -89
  1213. teradataml/analytics/mle/json/ngrams_mle.json +0 -137
  1214. teradataml/analytics/mle/json/ngramsplitter_mle_mle.json +0 -137
  1215. teradataml/analytics/mle/json/npath@coprocessor_mle.json +0 -73
  1216. teradataml/analytics/mle/json/ntree@coprocessor_mle.json +0 -123
  1217. teradataml/analytics/mle/json/pack_mle.json +0 -58
  1218. teradataml/analytics/mle/json/pack_mle_mle.json +0 -58
  1219. teradataml/analytics/mle/json/pagerank_mle.json +0 -81
  1220. teradataml/analytics/mle/json/pathanalyzer_mle.json +0 -63
  1221. teradataml/analytics/mle/json/pathgenerator_mle.json +0 -40
  1222. teradataml/analytics/mle/json/pathstart_mle.json +0 -62
  1223. teradataml/analytics/mle/json/pathsummarizer_mle.json +0 -72
  1224. teradataml/analytics/mle/json/pivoting_mle.json +0 -71
  1225. teradataml/analytics/mle/json/postagger_mle.json +0 -51
  1226. teradataml/analytics/mle/json/randomsample_mle.json +0 -131
  1227. teradataml/analytics/mle/json/randomwalksample_mle.json +0 -85
  1228. teradataml/analytics/mle/json/roc_mle.json +0 -73
  1229. teradataml/analytics/mle/json/sampling_mle.json +0 -75
  1230. teradataml/analytics/mle/json/sax_mle.json +0 -154
  1231. teradataml/analytics/mle/json/scale_mle.json +0 -93
  1232. teradataml/analytics/mle/json/scalebypartition_mle.json +0 -89
  1233. teradataml/analytics/mle/json/scalemap_mle.json +0 -44
  1234. teradataml/analytics/mle/json/scalesummary_mle.json +0 -14
  1235. teradataml/analytics/mle/json/sentenceextractor_mle.json +0 -41
  1236. teradataml/analytics/mle/json/sentimentevaluator_mle.json +0 -43
  1237. teradataml/analytics/mle/json/sentimentextractor_mle.json +0 -100
  1238. teradataml/analytics/mle/json/sentimenttrainer_mle.json +0 -68
  1239. teradataml/analytics/mle/json/seriessplitter_mle.json +0 -133
  1240. teradataml/analytics/mle/json/sessionize_mle_mle.json +0 -62
  1241. teradataml/analytics/mle/json/simplemovavg_mle.json +0 -48
  1242. teradataml/analytics/mle/json/stringsimilarity_mle.json +0 -50
  1243. teradataml/analytics/mle/json/stringsimilarity_mle_mle.json +0 -50
  1244. teradataml/analytics/mle/json/svmdense_mle.json +0 -165
  1245. teradataml/analytics/mle/json/svmdensepredict_mle.json +0 -95
  1246. teradataml/analytics/mle/json/svmdensesummary_mle.json +0 -58
  1247. teradataml/analytics/mle/json/svmsparse_mle.json +0 -148
  1248. teradataml/analytics/mle/json/svmsparsepredict_mle_mle.json +0 -103
  1249. teradataml/analytics/mle/json/svmsparsesummary_mle.json +0 -57
  1250. teradataml/analytics/mle/json/textchunker_mle.json +0 -40
  1251. teradataml/analytics/mle/json/textclassifier_mle.json +0 -51
  1252. teradataml/analytics/mle/json/textclassifierevaluator_mle.json +0 -43
  1253. teradataml/analytics/mle/json/textclassifiertrainer_mle.json +0 -103
  1254. teradataml/analytics/mle/json/textmorph_mle.json +0 -63
  1255. teradataml/analytics/mle/json/textparser_mle.json +0 -166
  1256. teradataml/analytics/mle/json/texttagger_mle.json +0 -81
  1257. teradataml/analytics/mle/json/texttokenizer_mle.json +0 -91
  1258. teradataml/analytics/mle/json/tf_mle.json +0 -33
  1259. teradataml/analytics/mle/json/tfidf_mle.json +0 -34
  1260. teradataml/analytics/mle/json/univariatestatistics_mle.json +0 -81
  1261. teradataml/analytics/mle/json/unpack_mle.json +0 -91
  1262. teradataml/analytics/mle/json/unpack_mle_mle.json +0 -91
  1263. teradataml/analytics/mle/json/unpivoting_mle.json +0 -63
  1264. teradataml/analytics/mle/json/varmax_mle.json +0 -176
  1265. teradataml/analytics/mle/json/vectordistance_mle.json +0 -179
  1266. teradataml/analytics/mle/json/weightedmovavg_mle.json +0 -48
  1267. teradataml/analytics/mle/json/xgboost_mle.json +0 -178
  1268. teradataml/analytics/mle/json/xgboostpredict_mle.json +0 -104
  1269. teradataml/analytics/sqle/Antiselect.py +0 -321
  1270. teradataml/analytics/sqle/Attribution.py +0 -603
  1271. teradataml/analytics/sqle/DecisionForestPredict.py +0 -408
  1272. teradataml/analytics/sqle/GLMPredict.py +0 -430
  1273. teradataml/analytics/sqle/MovingAverage.py +0 -543
  1274. teradataml/analytics/sqle/NGramSplitter.py +0 -548
  1275. teradataml/analytics/sqle/NPath.py +0 -632
  1276. teradataml/analytics/sqle/NaiveBayesTextClassifierPredict.py +0 -515
  1277. teradataml/analytics/sqle/Pack.py +0 -388
  1278. teradataml/analytics/sqle/SVMSparsePredict.py +0 -464
  1279. teradataml/analytics/sqle/Sessionize.py +0 -390
  1280. teradataml/analytics/sqle/StringSimilarity.py +0 -400
  1281. teradataml/analytics/sqle/Unpack.py +0 -503
  1282. teradataml/analytics/sqle/json/antiselect_sqle.json +0 -21
  1283. teradataml/analytics/sqle/json/attribution_sqle.json +0 -92
  1284. teradataml/analytics/sqle/json/decisionforestpredict_sqle.json +0 -48
  1285. teradataml/analytics/sqle/json/glmpredict_sqle.json +0 -48
  1286. teradataml/analytics/sqle/json/h2opredict_sqle.json +0 -63
  1287. teradataml/analytics/sqle/json/movingaverage_sqle.json +0 -58
  1288. teradataml/analytics/sqle/json/naivebayestextclassifierpredict_sqle.json +0 -76
  1289. teradataml/analytics/sqle/json/ngramsplitter_sqle.json +0 -126
  1290. teradataml/analytics/sqle/json/npath_sqle.json +0 -67
  1291. teradataml/analytics/sqle/json/pack_sqle.json +0 -47
  1292. teradataml/analytics/sqle/json/pmmlpredict_sqle.json +0 -55
  1293. teradataml/analytics/sqle/json/sessionize_sqle.json +0 -43
  1294. teradataml/analytics/sqle/json/stringsimilarity_sqle.json +0 -39
  1295. teradataml/analytics/sqle/json/svmsparsepredict_sqle.json +0 -74
  1296. teradataml/analytics/sqle/json/unpack_sqle.json +0 -80
  1297. teradataml/catalog/model_cataloging.py +0 -980
  1298. teradataml/config/mlengine_alias_definitions_v1.0 +0 -118
  1299. teradataml/config/mlengine_alias_definitions_v1.1 +0 -127
  1300. teradataml/config/mlengine_alias_definitions_v1.3 +0 -129
  1301. teradataml/table_operators/sandbox_container_util.py +0 -643
  1302. teradataml-17.20.0.7.dist-info/RECORD +0 -1280
  1303. {teradataml-17.20.0.7.dist-info → teradataml-20.0.0.1.dist-info}/top_level.txt +0 -0
@@ -1,1982 +1,1718 @@
1
- #!/usr/bin/python
2
- # ##################################################################
3
- #
4
- # Copyright 2020 Teradata. All rights reserved. #
5
- # TERADATA CONFIDENTIAL AND TRADE SECRET #
6
- #
7
- # Primary Owner: Gouri Patwardhan (gouri.patwardhan@teradata.com) #
8
- # Secondary Owner: Trupti Purohit (trupti.purohit@teradata.com) #
9
- #
10
- # Function Version: 1.0 #
11
- #
12
- # Description: Script is a TeradataML wrapper around Teradata's #
13
- # Script Table Operator #
14
- # ##################################################################
15
-
16
- import os
17
- import tarfile
18
- import teradataml.dataframe as tdmldf
19
- import subprocess
20
- import sys
21
- from pathlib import Path
22
- from datetime import datetime
23
- from io import StringIO
24
- from teradataml.common.constants import TableOperatorConstants
25
- from teradataml.common.wrapper_utils import AnalyticsWrapperUtils
26
- from teradataml.common.utils import UtilFuncs
27
- from teradataml.common.constants import OutputStyle, TeradataConstants
28
- from teradataml.context.context import _get_current_databasename
29
- from teradataml.context.context import get_context, get_connection
30
- from teradataml.dataframe.dataframe_utils import DataFrameUtils as df_utils
31
- from teradataml.common.garbagecollector import GarbageCollector
32
- from teradataml.dbutils.filemgr import install_file
33
- from teradataml.dbutils.filemgr import remove_file
34
- from teradataml.table_operators.table_operator_query_generator import \
35
- TableOperatorQueryGenerator
36
- from teradataml.common.exceptions import TeradataMlException
37
- from teradataml.common.messages import Messages
38
- from teradataml.common.messagecodes import MessageCodes
39
- from teradataml.utils.validators import _Validators
40
- from teradataml.options.display import display
41
- from teradataml.options.configure import configure
42
- from teradataml.table_operators.sandbox_container_util import setup_sandbox_env
43
- from teradataml.utils.utils import execute_sql
44
- from teradatasqlalchemy.dialect import dialect as td_dialect
45
- from teradatasqlalchemy import (BYTEINT, SMALLINT, INTEGER, BIGINT, DECIMAL, FLOAT,
46
- NUMBER)
47
- from teradatasqlalchemy import (TIMESTAMP, DATE, TIME)
48
- from teradatasqlalchemy import (CHAR, VARCHAR, CLOB)
49
- from teradatasqlalchemy import (BYTE, VARBYTE, BLOB)
50
- from teradatasqlalchemy import (PERIOD_DATE, PERIOD_TIME, PERIOD_TIMESTAMP)
51
- from teradatasqlalchemy import (INTERVAL_YEAR, INTERVAL_YEAR_TO_MONTH, INTERVAL_MONTH,
52
- INTERVAL_DAY,INTERVAL_DAY_TO_HOUR, INTERVAL_DAY_TO_MINUTE,
53
- INTERVAL_DAY_TO_SECOND, INTERVAL_HOUR,
54
- INTERVAL_HOUR_TO_MINUTE, INTERVAL_HOUR_TO_SECOND,
55
- INTERVAL_MINUTE, INTERVAL_MINUTE_TO_SECOND,
56
- INTERVAL_SECOND)
57
- from teradataml.table_operators.TableOperator import TableOperator
58
-
59
- class Script(TableOperator):
60
-
61
- def __init__(self,
62
- data=None,
63
- script_name=None,
64
- files_local_path=None,
65
- script_command=None,
66
- delimiter="\t",
67
- returns=None,
68
- auth=None,
69
- charset=None,
70
- quotechar=None,
71
- data_partition_column=None,
72
- data_hash_column=None,
73
- data_order_column=None,
74
- is_local_order=False,
75
- sort_ascending=True,
76
- nulls_first=True):
77
- """
78
- DESCRIPTION:
79
- The Script table operator function executes a user-installed script or
80
- any LINUX command inside database on Teradata Vantage.
81
-
82
- PARAMETERS:
83
- script_command:
84
- Required Argument.
85
- Specifies the command/script to run.
86
- Types: str
87
-
88
- script_name:
89
- Required Argument.
90
- Specifies the name of user script.
91
- User script should have at least permissions of mode 644.
92
- Types: str
93
-
94
- files_local_path:
95
- Required Argument.
96
- Specifies the absolute local path where user script and all supporting
97
- files like model files, input data file reside.
98
- Types: str
99
-
100
- returns:
101
- Required Argument.
102
- Specifies output column definition.
103
- Types: Dictionary specifying column name to teradatasqlalchemy type mapping.
104
- Default: None
105
- Note:
106
- User can pass a dictionary (dict or OrderedDict) to the "returns" argument,
107
- with the keys ordered to represent the order of the output columns.
108
- Preferred type is OrderedDict.
109
-
110
- data:
111
- Optional Argument.
112
- Specifies a teradataml DataFrame containing the input data for the
113
- script.
114
-
115
- data_hash_column:
116
- Optional Argument.
117
- Specifies the column to be used for hashing.
118
- The rows in the data are redistributed to AMPs based on the hash value of
119
- the column specified.
120
- The user-installed script file then runs once on each AMP.
121
- If there is no "data_partition_column", then the entire result set,
122
- delivered by the function, constitutes a single group or partition.
123
- Types: str
124
- Note:
125
- "data_hash_column" can not be specified along with
126
- "data_partition_column", "is_local_order" and "data_order_column".
127
-
128
- data_partition_column:
129
- Optional Argument.
130
- Specifies Partition By columns for "data".
131
- Values to this argument can be provided as a list, if multiple
132
- columns are used for partition.
133
- Default Value: ANY
134
- Types: str OR list of Strings (str)
135
- Note:
136
- 1) "data_partition_column" can not be specified along with
137
- "data_hash_column".
138
- 2) "data_partition_column" can not be specified along with
139
- "is_local_order = True".
140
-
141
- is_local_order:
142
- Optional Argument.
143
- Specifies a boolean value to determine whether the input data is to be
144
- ordered locally or not. Order by specifies the order in which the
145
- values in a group, or partition, are sorted. Local Order By specifies
146
- orders qualified rows on each AMP in preparation to be input to a table
147
- function. This argument is ignored, if "data_order_column" is None. When
148
- set to True, data is ordered locally.
149
- Default Value: False
150
- Types: bool
151
- Note:
152
- 1) "is_local_order" can not be specified along with "data_hash_column".
153
- 2) When "is_local_order" is set to True, "data_order_column" should be
154
- specified, and the columns specified in "data_order_column" are
155
- used for local ordering.
156
-
157
- data_order_column:
158
- Optional Argument.
159
- Specifies Order By columns for "data".
160
- Values to this argument can be provided as a list, if multiple
161
- columns are used for ordering. This argument is used with in both cases:
162
- "is_local_order = True" and "is_local_order = False".
163
- Types: str OR list of Strings (str)
164
- Note:
165
- "data_order_column" can not be specified along with "data_hash_column".
166
-
167
- sort_ascending:
168
- Optional Argument.
169
- Specifies a boolean value to determine if the result set is to be sorted
170
- on the "data_order_column" column in ascending or descending order.
171
- The sorting is ascending when this argument is set to True, and descending
172
- when set to False. This argument is ignored, if "data_order_column" is
173
- None.
174
- Default Value: True
175
- Types: bool
176
-
177
- nulls_first:
178
- Optional Argument.
179
- Specifies a boolean value to determine whether NULLS are listed first or
180
- last during ordering. This argument is ignored, if "data_order_column" is
181
- None. NULLS are listed first when this argument is set to True, and last
182
- when set to False.
183
- Default Value: True
184
- Types: bool
185
-
186
- delimiter:
187
- Optional Argument.
188
- Specifies a delimiter to use when reading columns from a row and
189
- writing result columns.
190
- Default Value: "\t" (tab)
191
- Types: str of length 1 character
192
- Notes:
193
- 1) This argument cannot be same as "quotechar" argument.
194
- 2) This argument cannot be a newline character i.e., '\\n'.
195
-
196
- auth:
197
- Optional Argument.
198
- Specifies an authorization to use when running the script.
199
- Types: str
200
-
201
- charset:
202
- Optional Argument.
203
- Specifies the character encoding for data.
204
- Permitted Values: utf-16, latin
205
- Types: str
206
-
207
- quotechar:
208
- Optional Argument.
209
- Specifies a character that forces all input and output of the script
210
- to be quoted using this specified character.
211
- Using this argument enables the Advanced SQL Engine to distinguish
212
- between NULL fields and empty strings. A string with length zero is
213
- quoted, while NULL fields are not.
214
- If this character is found in the data, it will be escaped by a second
215
- quote character.
216
- Types: character of length 1
217
- Notes:
218
- 1) This argument cannot be same as "delimiter" argument.
219
- 2) This argument cannot be a newline character i.e., '\\n'.
220
-
221
- RETURNS:
222
- Script Object
223
-
224
- RAISES:
225
- TeradataMlException
226
-
227
- EXAMPLES:
228
- # Note - Refer to User Guide for setting search path and required permissions.
229
- # Load example data.
230
- load_example_data("Script", ["barrier"])
231
-
232
- # Example - The script mapper.py reads in a line of text input
233
- # ("Old Macdonald Had A Farm") from csv and splits the line into individual
234
- # words, emitting a new row for each word.
235
-
236
- # Create teradataml DataFrame objects.
237
- >>> barrierdf = DataFrame.from_table("barrier")
238
-
239
- # Set SEARCHUIFDBPATH.
240
- >>> execute_sql("SET SESSION SEARCHUIFDBPATH = alice;")
241
-
242
- # Create a Script object that allows us to execute script on Vantage.
243
- >>> import teradataml, os
244
- >>> from teradatasqlalchemy import VARCHAR
245
- >>> td_path = os.path.dirname(teradataml.__file__)
246
- >>> sto = Script(data=barrierdf,
247
- ... script_name='mapper.py',
248
- ... files_local_path= os.path.join(td_path, 'data', 'scripts'),
249
- ... script_command='tdpython3 ./alice/mapper.py',
250
- ... data_order_column="Id",
251
- ... is_local_order=False,
252
- ... nulls_first=False,
253
- ... sort_ascending=False,
254
- ... charset='latin',
255
- ... returns=OrderedDict([("word", VARCHAR(15)),("count_input", VARCHAR(2))]))
256
-
257
- # Run user script locally within docker container and using data from csv.
258
- # This helps the user to fix script level issues outside Vantage.
259
- # Setup the environment by providing local path to docker image file.
260
- >>> sto.setup_sto_env(docker_image_location='/tmp/sto_sandbox_docker_image.tar'))
261
- Loading image from /tmp/sto_sandbox_docker_image.tar. It may take few minutes.
262
- Image loaded successfully.
263
- Starting a container for stosandbox:1.0 image.
264
- Container d7c73cb498c79a082180576bb5b10bb07b52efdd3026856146fc15e91147b19f
265
- started successfully.
266
-
267
- >>> sto.test_script(input_data_file='../barrier.csv', data_file_delimiter=',')
268
-
269
- ############ STDOUT Output ############
270
-
271
- word count_input
272
- 0 1 1
273
- 1 Old 1
274
- 2 Macdonald 1
275
- 3 Had 1
276
- 4 A 1
277
- 5 Farm 1
278
- >>>
279
-
280
- # Script results look good. Now install file on Vantage.
281
- >>> sto.install_file(file_identifier='mapper',
282
- ... file_name='mapper.py',
283
- ... is_binary=False)
284
- File mapper.py installed in Vantage
285
-
286
- # Execute the user script on Vantage.
287
- >>> sto.execute_script()
288
- ############ STDOUT Output ############
289
-
290
- word count_input
291
- 0 Macdonald 1
292
- 1 A 1
293
- 2 Farm 1
294
- 3 Had 1
295
- 4 Old 1
296
- 5 1 1
297
-
298
- # Remove the installed file from Vantage.
299
- >>> sto.remove_file(file_identifier='mapper', force_remove=True)
300
- File mapper removed from Vantage
301
- """
302
- self.result = None
303
- self.data = data
304
- self.script_name = script_name
305
- self.files_local_path = files_local_path
306
- self.script_command = script_command
307
- self.delimiter = delimiter
308
- self.returns = returns
309
- self.auth = auth
310
- self.charset = charset
311
- self.quotechar = quotechar
312
- self.data_partition_column = data_partition_column
313
- self.data_hash_column = data_hash_column
314
- self.data_order_column = data_order_column
315
- self.is_local_order = is_local_order
316
- self.sort_ascending = sort_ascending
317
- self.nulls_first = nulls_first
318
- self._check_reserved_keyword = True
319
-
320
- # Create AnalyticsWrapperUtils instance which contains validation functions.
321
- # This is required for is_default_or_not check.
322
- # Rest all validation is done using _Validators
323
- self.__awu = AnalyticsWrapperUtils()
324
-
325
- # Below matrix is a list of lists, where in each row contains following elements:
326
- # Let's take an example of following, just to get an idea:
327
- # [element1, element2, element3, element4, element5, element6]
328
- # e.g.
329
- # ["join", join, True, (str), True, concat_join_permitted_values]
330
-
331
- # 1. element1 --> Argument Name, a string. ["join" in above example.]
332
- # 2. element2 --> Argument itself. [join]
333
- # 3. element3 --> Specifies a flag that mentions argument is optional or not.
334
- # False, means required and True means optional.
335
- # 4. element4 --> Tuple of accepted types. (str) in above example.
336
- # 5. element5 --> True, means validate for empty value. Error will be raised,
337
- # if empty value is passed.
338
- # If not specified, means same as specifying False.
339
- # 6. element6 --> A list of permitted values, an argument can accept.
340
- # If not specified, it is as good as passing None.
341
- # If a list is passed, validation will be
342
- # performed for permitted values.
343
-
344
- self.awu_matrix = []
345
- self.awu_matrix.append(["data", self.data, True, (tdmldf.dataframe.DataFrame)])
346
- self.awu_matrix.append(["data_partition_column", self.data_partition_column, True,
347
- (str, list), True])
348
- self.awu_matrix.append(["data_hash_column", self.data_hash_column, True,
349
- (str, list), True])
350
- self.awu_matrix.append(["data_order_column", self.data_order_column, True,
351
- (str, list), True])
352
- self.awu_matrix.append(["is_local_order", self.is_local_order, True, (bool)])
353
- self.awu_matrix.append(["sort_ascending", self.sort_ascending, True, (bool)])
354
- self.awu_matrix.append(["nulls_first", self.nulls_first, True, (bool)])
355
- self.awu_matrix.append(["script_command", self.script_command, False, (str),
356
- True])
357
- self.awu_matrix.append(["script_name", self.script_name, True, (str), True])
358
- self.awu_matrix.append(["files_local_path", self.files_local_path, True, (str),
359
- True])
360
- self.awu_matrix.append(["delimiter", self.delimiter, True, (str), False])
361
- self.awu_matrix.append(["returns", self.returns, False, (dict), True])
362
- self.awu_matrix.append(["auth", self.auth, True, (str), True])
363
- self.awu_matrix.append(["charset", self.charset, True, (str), True,
364
- ["utf-16", "latin"]])
365
- self.awu_matrix.append(["quotechar", self.quotechar, True, (str), False])
366
-
367
- # Perform the function validations
368
- self.__validate()
369
-
370
- @property
371
- def check_reserved_keyword(self):
372
- """
373
- DESCRIPTION:
374
- Getter for self._check_reserved_keyword.
375
-
376
- RETURNS:
377
- bool
378
-
379
- RAISES:
380
- None
381
- """
382
- return self._check_reserved_keyword
383
-
384
- @check_reserved_keyword.setter
385
- def check_reserved_keyword(self, flag):
386
- """
387
- DESCRIPTION:
388
- Setter for self._check_reserved_keyword
389
-
390
- RETURNS:
391
- None
392
-
393
- RAISES:
394
- None
395
- """
396
- self._check_reserved_keyword = flag
397
-
398
- def __validate_for_reserved_keyword(self):
399
- """
400
- DESCRIPTION:
401
- Function to validate if the returns clause has teradata reserved keyword or not.
402
- If it contains reserved keyword, then raise an error.
403
-
404
- RETURNS:
405
- None
406
-
407
- RAISES:
408
- TeradataMlException
409
-
410
- """
411
- if self.check_reserved_keyword:
412
- from teradataml import list_td_reserved_keywords
413
- if get_connection():
414
- # Checking for reserved keywords and raising error if present.
415
- for column_name in self.returns:
416
- list_td_reserved_keywords(key=column_name, raise_error=True)
417
-
418
- def __validate(self):
419
- """
420
- Function to validate Table Operator Function arguments, which verifies missing
421
- arguments, input argument and table types. Also processes the argument values.
422
- """
423
- # Make sure that a non-NULL value has been supplied for all mandatory arguments
424
- _Validators._validate_missing_required_arguments(self.awu_matrix)
425
-
426
- # Validate argument types.
427
- _Validators._validate_function_arguments(self.awu_matrix,
428
- skip_empty_check={"quotechar" : ["\n", "\t"],
429
- "delimiter" : ["\n"]})
430
-
431
- # permissible_datatypes in returns
432
- allowed_datatypes = (BYTEINT, SMALLINT, INTEGER, BIGINT, DECIMAL, FLOAT, NUMBER,
433
- TIMESTAMP, DATE, TIME, CHAR, VARCHAR, CLOB, BYTE, VARBYTE,
434
- BLOB, PERIOD_DATE, PERIOD_TIME, PERIOD_TIMESTAMP,
435
- INTERVAL_YEAR, INTERVAL_YEAR_TO_MONTH, INTERVAL_MONTH,
436
- INTERVAL_DAY, INTERVAL_DAY_TO_HOUR, INTERVAL_DAY_TO_MINUTE,
437
- INTERVAL_DAY_TO_SECOND, INTERVAL_HOUR,
438
- INTERVAL_HOUR_TO_MINUTE, INTERVAL_HOUR_TO_SECOND,
439
- INTERVAL_MINUTE, INTERVAL_MINUTE_TO_SECOND, INTERVAL_SECOND
440
- )
441
-
442
- # Validate keys and datatypes in returns.
443
- self.awu_matrix_returns = []
444
- for key in self.returns.keys():
445
- self.awu_matrix_returns.append(["keys in returns", key, False, (str), True])
446
- self.awu_matrix_returns.append(["value in returns", self.returns[key], False,
447
- allowed_datatypes])
448
-
449
-
450
- _Validators._validate_function_arguments(self.awu_matrix_returns)
451
-
452
- if self.data is not None:
453
- # Hash and order by can be used together as long as is_local_order = True.
454
- if all([self.data_hash_column,
455
- self.data_order_column]) and not self.is_local_order:
456
- raise TeradataMlException(
457
- Messages.get_message(MessageCodes.CANNOT_USE_TOGETHER_WITH,
458
- "data_hash_column' and 'data_order_column",
459
- "is_local_order=False"),
460
- MessageCodes.CANNOT_USE_TOGETHER_WITH)
461
-
462
- # Either hash or partition can be used.
463
- _Validators._validate_mutually_exclusive_arguments(self.data_hash_column,
464
- "data_hash_column",
465
- self.data_partition_column,
466
- "data_partition_column",
467
- skip_all_none_check=True)
468
-
469
- # Either local order by or partition by can be used.
470
- _Validators._validate_mutually_exclusive_arguments(self.is_local_order,
471
- "is_local_order=True",
472
- self.data_partition_column,
473
- "data_partition_column",
474
- skip_all_none_check=True)
475
-
476
- # local order by requires column name.
477
- if self.is_local_order and self.data_order_column is None:
478
- message = Messages.get_message(MessageCodes.DEPENDENT_ARG_MISSING,
479
- "data_order_column", "is_local_order=True")
480
- raise TeradataMlException(message, MessageCodes.DEPENDENT_ARG_MISSING)
481
-
482
- if self.__awu._is_default_or_not(self.data_partition_column, "ANY"):
483
- self.__awu._validate_dataframe_has_argument_columns(
484
- self.data_partition_column, "data_partition_column", self.data, "data", True)
485
-
486
- if self.data_order_column is not None:
487
- self.__awu._validate_dataframe_has_argument_columns(
488
- self.data_order_column, "data_order_column",self.data, "data", False)
489
-
490
- if self.data_hash_column is not None:
491
- self.__awu._validate_dataframe_has_argument_columns(
492
- self.data_hash_column, "data_hash_column", self.data, "data", False)
493
-
494
- if self.data_partition_column is not None:
495
- self.__awu._validate_dataframe_has_argument_columns(
496
- self.data_partition_column, "data_partition_column", self.data, "data", False)
497
-
498
- # Check for length of the arguments "delimiter" and "quotechar".
499
- if self.delimiter is not None:
500
- _Validators._validate_str_arg_length('delimiter', self.delimiter, 'EQ', 1)
501
-
502
- if self.quotechar is not None:
503
- _Validators._validate_str_arg_length('quotechar', self.quotechar, 'EQ', 1)
504
-
505
- # The arguments 'quotechar' and 'delimiter' cannot take newline character.
506
- if self.delimiter == '\n':
507
- message = Messages.get_message(MessageCodes.NOT_ALLOWED_VALUES, "\n", "delimiter")
508
- raise TeradataMlException(message, MessageCodes.NOT_ALLOWED_VALUES)
509
- if self.quotechar == '\n':
510
- message = Messages.get_message(MessageCodes.NOT_ALLOWED_VALUES, "\n", "quotechar")
511
- raise TeradataMlException(message, MessageCodes.NOT_ALLOWED_VALUES)
512
-
513
- # The arguments 'quotechar' and 'delimiter' cannot have the same value.
514
- if self.delimiter == self.quotechar:
515
- message = Messages.get_message(MessageCodes.ARGUMENT_VALUE_SAME, "delimiter",
516
- "quotechar")
517
- raise TeradataMlException(message, MessageCodes.ARGUMENT_VALUE_SAME)
518
-
519
-
520
- def set_data(self,
521
- data,
522
- data_partition_column=None,
523
- data_hash_column=None,
524
- data_order_column=None,
525
- is_local_order=False,
526
- sort_ascending=True,
527
- nulls_first=True):
528
- """
529
- DESCRIPTION:
530
- Function enables user to set data and data related arguments without having to
531
- re-create Script object.
532
-
533
- PARAMETERS:
534
- data:
535
- Required Argument.
536
- Specifies a teradataml DataFrame containing the input data for the script.
537
-
538
- data_hash_column:
539
- Optional Argument.
540
- Specifies the column to be used for hashing.
541
- The rows in the data are redistributed to AMPs based on the
542
- hash value of the column specified.
543
- The user installed script then runs once on each AMP.
544
- If there is no data_partition_column, then the entire result set delivered
545
- by the function, constitutes a single group or partition.
546
- Types: str
547
- Note:
548
- "data_hash_column" can not be specified along with
549
- "data_partition_column", "is_local_order" and "data_order_column".
550
-
551
- data_partition_column:
552
- Optional Argument.
553
- Specifies Partition By columns for data.
554
- Values to this argument can be provided as a list, if multiple
555
- columns are used for partition.
556
- Default Value: ANY
557
- Types: str OR list of Strings (str)
558
- Note:
559
- 1) "data_partition_column" can not be specified along with
560
- "data_hash_column".
561
- 2) "data_partition_column" can not be specified along with
562
- "is_local_order = True".
563
-
564
- is_local_order:
565
- Optional Argument.
566
- Specifies a boolean value to determine whether the input data is to be
567
- ordered locally or not. Order by specifies the order in which the
568
- values in a group or partition are sorted. Local Order By specifies
569
- orders qualified rows on each AMP in preparation to be input to a table
570
- function. This argument is ignored, if "data_order_column" is None. When
571
- set to True, data is ordered locally.
572
- Default Value: False
573
- Types: bool
574
- Note:
575
- 1) "is_local_order" can not be specified along with
576
- "data_hash_column".
577
- 2) When "is_local_order" is set to True, "data_order_column" should be
578
- specified, and the columns specified in "data_order_column" are
579
- used for local ordering.
580
-
581
- data_order_column:
582
- Optional Argument.
583
- Specifies Order By columns for data.
584
- Values to this argument can be provided as a list, if multiple
585
- columns are used for ordering.
586
- This argument is used in both cases:
587
- "is_local_order = True" and "is_local_order = False".
588
- Types: str OR list of Strings (str)
589
- Note:
590
- "data_order_column" can not be specified along with
591
- "data_hash_column".
592
-
593
- sort_ascending:
594
- Optional Argument.
595
- Specifies a boolean value to determine if the result set is to be sorted
596
- on the column specified in "data_order_column", in ascending or descending
597
- order.
598
- The sorting is ascending when this argument is set to True, and descending
599
- when set to False.
600
- This argument is ignored, if "data_order_column" is None.
601
- Default Value: True
602
- Types: bool
603
-
604
- nulls_first:
605
- Optional Argument.
606
- Specifies a boolean value to determine whether NULLS are listed first or
607
- last during ordering.
608
- This argument is ignored, if "data_order_column" is None.
609
- NULLS are listed first when this argument is set to True, and
610
- last when set to False.
611
- Default Value: True
612
- Types: bool
613
-
614
- RETURNS:
615
- None.
616
-
617
- RAISES:
618
- TeradataMlException
619
-
620
- EXAMPLES:
621
- # Note - Refer to User Guide for setting search path and required permissions.
622
- # Load example data.
623
- load_example_data("Script", ["barrier"])
624
-
625
- # Example 1
626
- # Create teradataml DataFrame objects.
627
- >>> barrierdf = DataFrame.from_table("barrier")
628
- >>> barrierdf
629
- Name
630
- Id
631
- 1 Old Macdonald Had A Farm
632
- >>>
633
-
634
- # Set SEARCHUIFDBPATH
635
- >>> execute_sql("SET SESSION SEARCHUIFDBPATH = alice;")
636
- >>> import teradataml
637
- >>> from teradatasqlalchemy import VARCHAR
638
- >>> td_path = os.path.dirname(teradataml.__file__)
639
-
640
- # The script mapper.py reads in a line of text input
641
- # ("Old Macdonald Had A Farm") from csv and
642
- # splits the line into individual words, emitting a new row for each word.
643
- # Create a Script object without data and its arguments.
644
- >>> sto = Script(data = barrierdf,
645
- ... script_name='mapper.py',
646
- ... files_local_path= os.path.join(td_path,'data', 'scripts'),
647
- ... script_command='tdpython3 ./alice/mapper.py',
648
- ... charset='latin',
649
- ... returns=OrderedDict([("word", VARCHAR(15)),("count_input", VARCHAR(2))]))
650
-
651
- # Test script using data from file
652
- >>> sto.test_script(input_data_file='../barrier.csv', data_file_delimiter=',')
653
- ############ STDOUT Output ############
654
- word count_input
655
- 0 1 1
656
- 1 Old 1
657
- 2 Macdonald 1
658
- 3 Had 1
659
- 4 A 1
660
- 5 Farm 1
661
- >>>
662
-
663
- # Test script using data from DB.
664
- >>> sto.test_script(password='alice')
665
- ############ STDOUT Output ############
666
-
667
- word count_input
668
- 0 1 1
669
- 1 Old 1
670
- 2 Macdonald 1
671
- 3 Had 1
672
- 4 A 1
673
- 5 Farm 1
674
-
675
- # Test script using data from DB and with data_row_limit.
676
- >>> sto.test_script(password='alice', data_row_limit=5)
677
- ############ STDOUT Output ############
678
-
679
- word count_input
680
- 0 1 1
681
- 1 Old 1
682
- 2 Macdonald 1
683
- 3 Had 1
684
- 4 A 1
685
- 5 Farm 1
686
-
687
- # Now in order to test / run script on actual data on Vantage user must
688
- # set data and related arguments.
689
- # Note:
690
- # All data related arguments that are not specified in set_data() are
691
- # reset to default values.
692
- >>> sto.set_data(data=barrierdf,
693
- ... data_order_column="Id",
694
- ... is_local_order=False,
695
- ... nulls_first=False,
696
- ... sort_ascending=False)
697
-
698
- # Execute the user script on Vantage.
699
- >>> sto.execute_script()
700
- ############ STDOUT Output ############
701
- word count_input
702
- 0 Macdonald 1
703
- 1 A 1
704
- 2 Farm 1
705
- 3 Had 1
706
- 4 Old 1
707
- 5 1 1
708
-
709
- # Example 2 -
710
- # Script is tested using test_script and executed on Vantage.
711
- # use set_data() to reset arguments.
712
- # Create teradataml DataFrame objects.
713
- >>> load_example_data("Script", ["barrier_new"])
714
- >>> barrierdf_new = DataFrame.from_table("barrier_new")
715
- >>> barrierdf_new
716
- Name
717
- Id
718
- 2 On his farm he had a cow
719
- 1 Old Macdonald Had A Farm
720
- >>>
721
-
722
- # Create a Script object that allows us to execute script on Vantage.
723
- >>> sto = Script(data=barrierdf_new,
724
- ... script_name='mapper.py',
725
- ... files_local_path= os.path.join(td_path, 'data', 'scripts'),
726
- ... script_command='tdpython3 ./alice/mapper.py',
727
- ... data_order_column="Id",
728
- ... is_local_order=False,
729
- ... nulls_first=False,
730
- ... sort_ascending=False,
731
- ... charset='latin',
732
- ... returns=OrderedDict([("word", VARCHAR(15)),("count_input", VARCHAR(2))]))
733
- # Script is tested using test_script and executed on Vantage.
734
- >>> sto.execute_script()
735
- ############ STDOUT Output ############
736
- word count_input
737
- 0 his 1
738
- 1 he 1
739
- 2 had 1
740
- 3 a 1
741
- 4 1 1
742
- 5 Old 1
743
- 6 cow 1
744
- 7 farm 1
745
- 8 On 1
746
- 9 2 1
747
-
748
- # Now in order to run the script with a different dataset,
749
- # user can use set_data().
750
- # Re-set data and some data related parameters.
751
- # Note:
752
- # All data related arguments that are not specified in set_data() are
753
- # reset to default values.
754
- >>> sto.set_data(data=barrierdf,
755
- ... data_order_column='Id',
756
- ... is_local_order=True,
757
- ... nulls_first=True)
758
- >>> sto.execute_script()
759
- word count_input
760
- 0 Macdonald 1
761
- 1 A 1
762
- 2 Farm 1
763
- 3 Had 1
764
- 4 Old 1
765
- 5 1 1
766
-
767
- # Example 3
768
- # Script is tested using test_script and executed on Vantage.
769
- # In order to run the script with same dataset but different data related
770
- # arguments, use set_data() to reset arguments.
771
- # Note:
772
- # All data related arguments that are not specified in set_data() are
773
- # reset to default values.
774
- >>> sto.set_data(data=barrierdf_new,
775
- ... data_order_column='Id',
776
- ... is_local_order = True,
777
- ... nulls_first = True)
778
-
779
- >>> sto.execute_script()
780
- ############ STDOUT Output ############
781
-
782
- word count_input
783
- 0 Macdonald 1
784
- 1 A 1
785
- 2 Farm 1
786
- 3 2 1
787
- 4 his 1
788
- 5 farm 1
789
- 6 On 1
790
- 7 Had 1
791
- 8 Old 1
792
- 9 1 1
793
- """
794
- super(Script, self).set_data(data,
795
- data_partition_column,
796
- data_hash_column,
797
- data_order_column,
798
- is_local_order,
799
- sort_ascending,
800
- nulls_first)
801
- self.__validate()
802
-
803
- def setup_sto_env(self, docker_image_location):
804
- """
805
- DESCRIPTION:
806
- Function enables user to load already downloaded sandbox image.
807
-
808
- PARAMETERS:
809
- docker_image_location:
810
- Required Argument.
811
- Specifies the location of image on user's system.
812
- Types: str
813
-
814
- Note:
815
- For location to download docker image refer teradataml User Guide.
816
-
817
- RETURNS:
818
- None.
819
-
820
- RAISES:
821
- TeradataMlException
822
-
823
- EXAMPLES:
824
- # Note - Refer to User Guide for setting search path and required permissions.
825
- # Load example data.
826
- load_example_data("Script", ["barrier"])
827
-
828
- # Example - The script mapper.py reads in a line of text input
829
- # ("Old Macdonald Had A Farm") from csv and
830
- # splits the line into individual words, emitting a new row for each word.
831
-
832
- # Create teradataml DataFrame objects.
833
- >>> barrierdf = DataFrame.from_table("barrier")
834
-
835
- # Set SEARCHUIFDBPATH.
836
- >>> execute_sql("SET SESSION SEARCHUIFDBPATH = alice;")
837
-
838
- # Create a Script object that allows us to execute script on Vantage.
839
- >>> import os
840
- >>> td_path = os.path.dirname(teradataml.__file__)
841
- >>> from teradatasqlalchemy import VARCHAR
842
- >>> sto = Script(data=barrierdf,
843
- ... script_name='mapper.py',
844
- ... files_local_path= os.path.join(td_path, 'data', 'scripts'),
845
- ... script_command='tdpython3 ./alice/mapper.py',
846
- ... data_order_column="Id",
847
- ... is_local_order=False,
848
- ... nulls_first=False,
849
- ... sort_ascending=False,
850
- ... charset='latin',
851
- ... returns=OrderedDict([("word", VARCHAR(15)),("count_input", VARCHAR(2))]))
852
-
853
- # Run user script locally within docker container and using data from csv.
854
- # This helps the user to fix script level issues outside Vantage.
855
- # Setup the environment by providing local path to docker image file.
856
- >>> sto.setup_sto_env(docker_image_location='/tmp/sto_sandbox_docker_image.tar')
857
- Loading image from /tmp/sto_sandbox_docker_image.tar. It may take few minutes.
858
- Image loaded successfully.
859
- Starting a container for stosandbox:1.0 image.
860
- Container d7c73cb498c79a082180576bb5b10bb07b52efdd3026856146fc15e91147b19f
861
- started successfully.
862
-
863
- """
864
- self.awu_matrix_setup = []
865
- self.awu_matrix_setup.append((["docker_image_location", docker_image_location,
866
- False, (str), True]))
867
-
868
- # Validate missing arguments.
869
- _Validators._validate_missing_required_arguments(self.awu_matrix_setup)
870
-
871
- # Validate argument types.
872
- _Validators._validate_function_arguments(self.awu_matrix_setup)
873
-
874
- from teradataml.table_operators.sandbox_container_util import setup_sandbox_env
875
- setup_sandbox_env(sandbox_image_location=docker_image_location,
876
- sandbox_image_name='stosandbox:1.0')
877
-
878
- # Set _latest_sandbox_exists to True - which indicates sandbox image for STO
879
- # exists on the system.
880
- configure._latest_sandbox_exists = True
881
-
882
- def test_script(self, supporting_files=None, input_data_file=None, script_args="",
883
- exec_mode ='sandbox', **kwargs):
884
- """
885
- DESCRIPTION:
886
- Function enables user to run script in docker container environment outside
887
- Vantage.
888
- Input data for user script is either read from a file or from database.
889
- Note:
890
- 1. Purpose of test_script() function is to enable the user to test their scripts for any errors without
891
- installing it on Vantage, using the input data provided.
892
- 2. Data is not partitioned for testing the script if read from input data file.
893
- 3. Function can produce different output if input is read from a file than input from database.
894
-
895
- PARAMETERS:
896
- supporting_files:
897
- Optional Argument.
898
- Specifies a file or list of supporting files like model files to be
899
- copied to the container.
900
- Types: string or list of str
901
-
902
- input_data_file:
903
- Optional Argument.
904
- Specifies the name of the input data file.
905
- It should have a path relative to the location specified in
906
- "files_local_path" argument.
907
- If set to None, read data from AMP, else from file passed in the argument
908
- 'input_data_file'.
909
- File should have at least permissions of mode 644.
910
- Types: str
911
-
912
- script_args:
913
- Optional Argument.
914
- Specifies command line arguments required by the user script.
915
- Types: str
916
-
917
- exec_mode:
918
- Optional Argument.
919
- Specifies the mode in which user wants to test the script.
920
- If set to 'sandbox', the user script will run within the sandbox
921
- environment, else it will run locally on user's system.
922
- Permitted Values: 'sandbox', 'local'
923
- Default Value: 'sandbox'
924
- Types: str
925
-
926
- kwargs:
927
- Optional Argument.
928
- Specifies the keyword arguments required for testing.
929
- Keys can be:
930
- data_row_limit:
931
- Optional Argument. Ignored when data is read from file.
932
- Specifies the number of rows to be taken from all amps when
933
- reading from a table or view on Vantage.
934
- Default Value: 1000
935
- Types: int
936
-
937
- password:
938
- Optional Argument. Required when reading from database.
939
- Specifies the password to connect to vantage where the data
940
- resides.
941
- Types: str
942
-
943
- data_file_delimiter:
944
- Optional Argument.
945
- Specifies the delimiter used in the input data file. This
946
- argument can be specified when data is read from file.
947
- Default Value: '\t'
948
- Types: str
949
-
950
- data_file_header:
951
- Optional Argument.
952
- Specifies whether the input data file contains header. This
953
- argument can be specified when data is read from file.
954
- Default Value: True
955
- Types: bool
956
-
957
- timeout:
958
- Optional Argument.
959
- Specifies the timeout for docker API calls when running in
960
- sandbox mode.
961
- Default Value: 5000
962
- Types: int
963
-
964
- data_file_quote_char:
965
- Optional Argument.
966
- Specifies the quotechar used in the input data file.
967
- This argument can be specified when data is read from file.
968
- Default Value: '"'
969
-
970
- logmech:
971
- Optional Argument.
972
- Specifies the type of logon mechanism to establish a connection to
973
- Teradata Vantage.
974
- Permitted Values: 'TD2', 'TDNEGO', 'LDAP', 'KRB5' & 'JWT'.
975
- TD2:
976
- The Teradata 2 (TD2) mechanism provides authentication
977
- using a Vantage username and password. This is the default
978
- logon mechanism using which the connection is established
979
- to Vantage.
980
-
981
- TDNEGO:
982
- A security mechanism that automatically determines the
983
- actual mechanism required, based on policy, without user's
984
- involvement. The actual mechanism is determined by the
985
- TDGSS server configuration and by the security policy's
986
- mechanism restrictions.
987
-
988
- LDAP:
989
- A directory-based user logon to Vantage with a directory
990
- username and password and is authenticated by the directory.
991
-
992
- KRB5 (Kerberos):
993
- A directory-based user logon to Vantage with a domain
994
- username and password and is authenticated by
995
- Kerberos (KRB5 mechanism).
996
- Note:
997
- User must have a valid ticket-granting ticket in
998
- order to use this logon mechanism.
999
-
1000
- JWT:
1001
- The JSON Web Token (JWT) authentication mechanism enables
1002
- single sign-on (SSO) to the Vantage after the user
1003
- successfully authenticates to Teradata UDA User Service.
1004
- Note:
1005
- User must use logdata parameter when using 'JWT' as
1006
- the logon mechanism.
1007
- Default Value: TD2
1008
- Types: str
1009
-
1010
- Note:
1011
- teradataml expects the client environments are already setup with appropriate
1012
- security mechanisms and are in working conditions.
1013
- For more information please refer Teradata Vantage™ - Advanced SQL Engine
1014
- Security Administration at https://www.info.teradata.com/
1015
-
1016
- logdata:
1017
- Optional Argument.
1018
- Specifies parameters to the LOGMECH command beyond those needed by
1019
- the logon mechanism, such as user ID, password and tokens
1020
- (in case of JWT) to successfully authenticate the user.
1021
- Types: str
1022
-
1023
- Types: dict
1024
-
1025
- RETURNS:
1026
- Output from user script.
1027
-
1028
- RAISES:
1029
- TeradataMlException
1030
-
1031
- EXAMPLES:
1032
- # Assumption - sto is Script() object. Please refer to help(Script)
1033
- # for creating Script object.
1034
- # Run user script in sandbox mode with input from data file.
1035
-
1036
- >>> sto.test_script(input_data_file='../barrier.csv',
1037
- ... data_file_delimiter=',',
1038
- ... data_file_quote_char='"',
1039
- ... data_file_header=True,
1040
- ... exec_mode='sandbox')
1041
-
1042
- ############ STDOUT Output ############
1043
- word count_input
1044
- 0 1 1
1045
- 1 Old 1
1046
- 2 Macdonald 1
1047
- 3 Had 1
1048
- 4 A 1
1049
- 5 Farm 1
1050
- >>>
1051
-
1052
- # Run user script in local mode with input from table.
1053
- >>> sto.test_script(data_row_limit=300, password='alice', exec_mode='local')
1054
-
1055
- ############ STDOUT Output ############
1056
- word count_input
1057
- 0 1 1
1058
- 1 Old 1
1059
- 2 Macdonald 1
1060
- 3 Had 1
1061
- 4 A 1
1062
- 5 Farm 1
1063
-
1064
- # Run user script in sandbox mode with logmech as 'TD2'.
1065
- >>> sto.test_script(script_args="4 5 10 6 480", password="alice", logmech="TD2")
1066
-
1067
- # Run user script in sandbox mode with logmech as 'TDNEGO'.
1068
- >>> sto.test_script(script_args="4 5 10 6 480", password="alice", logmech="TDNEGO")
1069
-
1070
- # Run user script in sandbox mode with logmech as 'LDAP'.
1071
- >>> sto.test_script(script_args="4 5 10 6 480", password="alice", logmech="LDAP")
1072
-
1073
- # Run user script in sandbox mode with logmech as 'KRB5'.
1074
- >>> sto.test_script(script_args="4 5 10 6 480", password="alice", logmech="KRB5")
1075
-
1076
- # Run user script in sandbox mode with logmech as 'JWT'.
1077
- >>> sto.test_script(script_args="4 5 10 6 480", password="alice",
1078
- logmech='JWT', logdata='token=eyJpc...h8dA')
1079
-
1080
- """
1081
- logmech_valid_values = ['TD2', 'TDNEGO', 'LDAP', 'KRB5', 'JWT']
1082
-
1083
- awu_matrix_test=[]
1084
- awu_matrix_test.append((["supporting_files", supporting_files, True,
1085
- (str,list), True]))
1086
- awu_matrix_test.append((["input_data_file", input_data_file, True, (str), True]))
1087
- awu_matrix_test.append((["script_args", script_args, True, (str), False]))
1088
- awu_matrix_test.append((["exec_mode", exec_mode, True, (str), True,
1089
- [TableOperatorConstants.SANDBOX_EXEC.value,
1090
- TableOperatorConstants.LOCAL_EXEC.value]]))
1091
-
1092
- data_row_limit = kwargs.pop("data_row_limit", 1000)
1093
- awu_matrix_test.append((["data_row_limit", data_row_limit, True, (int), True]))
1094
-
1095
- data_file_delimiter = kwargs.pop("data_file_delimiter", '\t')
1096
- awu_matrix_test.append((["data_file_delimiter", data_file_delimiter, True,
1097
- (str), False]))
1098
-
1099
- data_file_quote_char = kwargs.pop("data_file_quote_char", '"')
1100
- awu_matrix_test.append((["data_file_quote_char", data_file_quote_char, True,
1101
- (str), False]))
1102
-
1103
- data_file_header = kwargs.pop("data_file_header", True)
1104
- awu_matrix_test.append((["data_file_header", data_file_header, True, (bool)]))
1105
-
1106
- timeout = kwargs.pop("timeout", 5000)
1107
- awu_matrix_test.append((["timeout", timeout, True, (int), True]))
1108
-
1109
- logmech = kwargs.pop("logmech", "TD2")
1110
- awu_matrix_test.append(
1111
- ["logmech", logmech, True, (str), True, logmech_valid_values])
1112
-
1113
- logdata = kwargs.pop("logdata", None)
1114
- awu_matrix_test.append(["logdata", logdata, True, (str), True])
1115
-
1116
- # Validate argument types.
1117
- _Validators._validate_function_arguments(awu_matrix_test)
1118
-
1119
- # Validate timeout value.
1120
- _Validators._validate_positive_int(timeout, "timeout")
1121
-
1122
- self.__validate()
1123
- self.__validate_for_reserved_keyword()
1124
-
1125
-
1126
- if logmech == "JWT" and not logdata:
1127
- raise TeradataMlException(
1128
- Messages.get_message(MessageCodes.DEPENDENT_ARG_MISSING, 'logdata',
1129
- 'logmech=JWT'),
1130
- MessageCodes.DEPENDENT_ARG_MISSING)
1131
-
1132
- if data_row_limit <= 0:
1133
- raise ValueError(Messages.get_message(MessageCodes.TDMLDF_POSITIVE_INT).
1134
- format("data_row_limit", "greater than"))
1135
-
1136
- # Either of 'input_data_file' or 'password' argument is required.
1137
- password = kwargs.pop("password", None)
1138
-
1139
- # The check of EITHER_THIS_OR_THAT_ARGUMENT is applicable only when the exec_mode is sandbox.
1140
- # Hence adding the check exec_mode != "local".
1141
- # When exec_mode is local, the connection object is used to get the values in the table.
1142
- if exec_mode != "local" and not (input_data_file or (self.data and password)):
1143
- message = Messages.get_message(MessageCodes.EITHER_THIS_OR_THAT_ARGUMENT,
1144
- "input_data_file", "Script data and password")
1145
- raise TeradataMlException(message, MessageCodes.EITHER_THIS_OR_THAT_ARGUMENT)
1146
- elif exec_mode == "local" and not (input_data_file or self.data):
1147
- message = Messages.get_message(MessageCodes.EITHER_THIS_OR_THAT_ARGUMENT,
1148
- "input_data_file", "Script data")
1149
- raise TeradataMlException(message, MessageCodes.EITHER_THIS_OR_THAT_ARGUMENT)
1150
-
1151
- if not self.script_name and self.files_local_path:
1152
- message = Messages.get_message(MessageCodes.MISSING_ARGS,
1153
- "script_name and files_local_path")
1154
- raise TeradataMlException(message, MessageCodes.MISSING_ARGS)
1155
-
1156
- if input_data_file:
1157
- if self.files_local_path is None:
1158
- message = Messages.get_message(MessageCodes.DEPENDENT_ARG_MISSING,
1159
- "files_local_path","input_data_file")
1160
- raise TeradataMlException(message, MessageCodes.DEPENDENT_ARG_MISSING)
1161
- else:
1162
- # Check if file exists.
1163
- fpath = os.path.join(self.files_local_path,
1164
- input_data_file)
1165
- _Validators._validate_file_exists(fpath)
1166
-
1167
- if self.script_name and self.files_local_path:
1168
- # Check if file exists.
1169
- fpath = os.path.join(self.files_local_path,
1170
- os.path.basename(self.script_name))
1171
- _Validators._validate_file_exists(fpath)
1172
-
1173
- if exec_mode.upper() == TableOperatorConstants.LOCAL_EXEC.value:
1174
- user_script_path = os.path.join(self.files_local_path, self.script_name)
1175
- import sys
1176
- cmd = [str(sys.executable), user_script_path]
1177
- cmd.extend(script_args)
1178
-
1179
- if input_data_file is not None:
1180
- input_file_path = os.path.join(self.files_local_path, input_data_file)
1181
-
1182
- # Run user script locally with input from a file.
1183
- exec_cmd_output = self.__local_run_user_script_input_file(
1184
- cmd, input_file_path, data_file_delimiter, data_file_quote_char, data_file_header)
1185
- try:
1186
- return self.__process_test_script_output(exec_cmd_output)
1187
- except Exception as exp:
1188
- raise
1189
-
1190
- else:
1191
- if self.data.shape[0] > data_row_limit:
1192
- raise ValueError(
1193
- Messages.get_message(MessageCodes.DATAFRAME_LIMIT_ERROR,
1194
- 'data_row_limit', 'data_row_limit',
1195
- data_row_limit))
1196
-
1197
- if not self.data._table_name:
1198
- self.data._table_name = df_utils._execute_node_return_db_object_name(
1199
- self.data._nodeid, self.data._metaexpr)
1200
-
1201
- table_name = UtilFuncs._extract_table_name(self.data._table_name)
1202
-
1203
- # Run user script locally with input from db.
1204
- exec_cmd_output = self.__local_run_user_script_input_db(cmd, table_name)
1205
- try:
1206
- return self.__process_test_script_output(exec_cmd_output)
1207
- except Exception as exp:
1208
- raise
1209
- else:
1210
- # Execution Mode - sandbox.
1211
-
1212
- # get the frame object of the function.
1213
- import inspect
1214
- frame = inspect.currentframe()
1215
-
1216
- # Validate argument types.
1217
- _Validators._validate_module_presence('docker', frame.f_code.co_name)
1218
-
1219
- import docker
1220
-
1221
- # Read container_id from configure.sandbox_container_id, if it is None then
1222
- # raise an exception
1223
-
1224
- container_id = configure.sandbox_container_id
1225
- if container_id is None:
1226
- message = Messages.get_message(MessageCodes.SANDBOX_CONTAINER_NOT_FOUND)
1227
- raise TeradataMlException(message,
1228
- MessageCodes.SANDBOX_CONTAINER_NOT_FOUND)
1229
-
1230
- # Set path inside docker container. This is where files will be copied to.
1231
- # os.path.join() will not work here because the path is not dependent on
1232
- # client platform. Sandbox environment is linux based here.
1233
- _path_in_docker_container = "/home/tdatuser"
1234
- user_script_path = "{}/{}".format(_path_in_docker_container, self.script_name)
1235
-
1236
- if input_data_file is not None:
1237
- input_file_name = os.path.basename(input_data_file)
1238
- input_file_path = "{}/{}".format(_path_in_docker_container,
1239
- input_file_name)
1240
- # Create script_executor.
1241
- self._create_executor_script(user_script_path=user_script_path,
1242
- user_script_args=script_args,
1243
- data_file_path=input_file_path,
1244
- data_file_delimiter=data_file_delimiter,
1245
- data_file_quote_char=data_file_quote_char,
1246
- data_file_header=data_file_header)
1247
- else:
1248
- # Read input from db.
1249
- if self.data.shape[0] > data_row_limit:
1250
- raise ValueError(
1251
- Messages.get_message(MessageCodes.DATAFRAME_LIMIT_ERROR,
1252
- 'data_row_limit', 'data_row_limit',
1253
- data_row_limit))
1254
- db_host = get_context().url.host
1255
-
1256
- user_name = get_context().url.username
1257
-
1258
- if not self.data._table_name:
1259
- self.data._table_name = df_utils._execute_node_return_db_object_name(
1260
- self.data._nodeid, self.data._metaexpr)
1261
- table_name = UtilFuncs._extract_table_name(self.data._table_name)
1262
-
1263
- db_name = _get_current_databasename()
1264
-
1265
- # Create script_executor.
1266
- self._create_executor_script(user_script_path=user_script_path,
1267
- user_script_args=script_args,
1268
- db_host=db_host,
1269
- user_name=user_name,
1270
- passwd=password,
1271
- table_name=table_name,
1272
- db_name=db_name,
1273
- logmech=logmech,
1274
- logdata=logdata)
1275
-
1276
- client = docker.APIClient(timeout=timeout)
1277
-
1278
- # Copy files to container indicated in configure.sandbox_container_id.
1279
- files_to_copy = [self.script_name]
1280
-
1281
- if supporting_files is not None:
1282
- if isinstance(supporting_files, str):
1283
- supporting_files = [supporting_files]
1284
-
1285
- if len(supporting_files) == 0 \
1286
- or any(file in [None, "None", ""] for file in supporting_files):
1287
- raise ValueError(
1288
- Messages.get_message(MessageCodes.LIST_SELECT_NONE_OR_EMPTY,
1289
- 'supporting_files'))
1290
- else:
1291
- files_to_copy.extend(supporting_files)
1292
-
1293
- if input_data_file is not None:
1294
- files_to_copy.append(input_data_file)
1295
-
1296
- for filename in files_to_copy:
1297
- file_path = os.path.join(self.files_local_path, filename)
1298
- # Check if file exists.
1299
- _Validators._validate_file_exists(file_path)
1300
-
1301
- # Copy file to docker container.
1302
- self._copy_to_docker_container(client, file_path,
1303
- _path_in_docker_container,
1304
- container_id)
1305
-
1306
- # Copy script_executor to docker container.
1307
- self._copy_to_docker_container(client, self.script_path,
1308
- _path_in_docker_container,
1309
- container_id)
1310
-
1311
- script_executor_file_name = os.path.basename(self.script_path)
1312
- exec_cmd = ("tdpython3 {0}/{1}".format(_path_in_docker_container,
1313
- script_executor_file_name))
1314
-
1315
- try:
1316
- # Setup an exec instance in the container.
1317
- exec_cmd_create = client.exec_create(container_id, exec_cmd)
1318
-
1319
- # Start exec instance and run user script.
1320
- exec_cmd_output = client.exec_start(exec_cmd_create, demux=True)
1321
-
1322
- # Inspect the output for success or failure.
1323
- inspect_out = client.exec_inspect(exec_cmd_create)
1324
-
1325
- # Extract the exit code.
1326
- exit_code = inspect_out['ExitCode']
1327
-
1328
- if exec_cmd_output[0] is not None:
1329
- executor_output = exec_cmd_output[0].decode()
1330
-
1331
- executor_error = ""
1332
- if exec_cmd_output[1] is not None:
1333
- executor_error = exec_cmd_output[1].decode()
1334
-
1335
- # Exit code 1 indicates any error thrown by subprocess.
1336
- # Exit code 126 indicates permission problem or command is not executable.
1337
- # Exit code 127 indicates possible typos in shell script with
1338
- # unrecognizable characters.
1339
- if exit_code == 1 or exit_code == 126 or exit_code == 127:
1340
- message = Messages.get_message(
1341
- MessageCodes.SANDBOX_SCRIPT_ERROR).format(executor_error)
1342
- raise TeradataMlException(message,
1343
- MessageCodes.SANDBOX_SCRIPT_ERROR)
1344
- # Exit code 2 indicates either username or password is invalid.
1345
- elif exit_code == 2:
1346
- message = Messages.get_message(
1347
- MessageCodes.SANDBOX_CONNECTION_ERROR).format(executor_error)
1348
- raise TeradataMlException(message,
1349
- MessageCodes.SANDBOX_CONNECTION_ERROR)
1350
- # Exit code 3 indicates problem with query.
1351
- elif exit_code == 3:
1352
- message = Messages.get_message(
1353
- MessageCodes.SANDBOX_QUERY_ERROR).format(executor_error)
1354
- raise TeradataMlException(message,
1355
- MessageCodes.SANDBOX_QUERY_ERROR)
1356
- # Exit code 4 indicates all other exceptions / errors.
1357
- elif exit_code == 4:
1358
- message = Messages.get_message(
1359
- MessageCodes.SANDBOX_CONTAINER_ERROR).format(executor_error)
1360
- raise TeradataMlException(message,
1361
- MessageCodes.SANDBOX_CONTAINER_ERROR)
1362
- elif exit_code != 0:
1363
- # Any error other than exit code 1, 2, 3, 4
1364
- message = Messages.get_message(
1365
- MessageCodes.SANDBOX_CONTAINER_ERROR).format(executor_error)
1366
- raise TeradataMlException(message,
1367
- MessageCodes.SANDBOX_CONTAINER_ERROR)
1368
- else:
1369
- return self.__process_test_script_output(executor_output)
1370
- except Exception as exp:
1371
- message = Messages.get_message(
1372
- MessageCodes.SANDBOX_CONTAINER_ERROR).format(str(exp))
1373
- raise TeradataMlException(message,
1374
- MessageCodes.SANDBOX_CONTAINER_ERROR)
1375
-
1376
- def __local_run_user_script_input_file(self, cmd, input_file_path,
1377
- data_file_delimiter='\t',
1378
- data_file_quote_char='"',
1379
- data_file_header=True):
1380
- """
1381
- DESCRIPTION:
1382
- Function to run the user script in local mode with input from file.
1383
-
1384
- PARAMETERS:
1385
- cmd:
1386
- Required Argument.
1387
- Specifies the command for running the user script.
1388
- Types: str
1389
-
1390
- input_file_path:
1391
- Required Argument.
1392
- Specifies the absolute local path of input data file.
1393
- Types: str
1394
-
1395
- data_file_delimiter:
1396
- Optional Argument.
1397
- Specifies the delimiter used in input data file.
1398
- Default Value: '\t'
1399
- Types: str
1400
-
1401
- data_file_quote_char:
1402
- Optional Argument.
1403
- Specifies the quote character used in input data file.
1404
- Default Value: '"'
1405
- Types: str
1406
-
1407
- data_file_header:
1408
- Optional Argument.
1409
- Specifies whether the input data file has header.
1410
- Default Value: True
1411
- Types: bool
1412
-
1413
- RETURNS:
1414
- The string output of the command that is run on input data file.
1415
-
1416
- RAISES:
1417
- Exception.
1418
-
1419
- EXAMPLES:
1420
- self.__local_run_user_script_input_file(cmd ="cmd",
1421
- input_file_path = "input_file_path",
1422
- data_file_delimiter = "data_file_delimiter",
1423
- data_file_quote_char = "data_file_quote_char",
1424
- data_file_header = True)
1425
-
1426
- """
1427
- with open(input_file_path) as data_file:
1428
- import csv
1429
- from pandas import isna as pd_isna
1430
-
1431
- data_handle = StringIO()
1432
-
1433
- # Read data from input file.
1434
- ip_data = csv.reader(data_file,
1435
- delimiter=data_file_delimiter,
1436
- quotechar=data_file_quote_char)
1437
- # Skip the first row of input file if data_file_header is True.
1438
- if data_file_header:
1439
- next(ip_data)
1440
- for row in ip_data:
1441
- if self.quotechar is not None:
1442
- # A NULL value should not be enclosed in quotes.
1443
- # The CSV module has no support for such output with writer,
1444
- # and hence the custom formatting.
1445
- line = ['' if pd_isna(s) else "{}{}{}".format(self.quotechar,
1446
- str(s),
1447
- self.quotechar)
1448
- for s in row]
1449
- else:
1450
- line = ['' if pd_isna(s) else str(s) for s in row]
1451
-
1452
- complete_line = (self.delimiter.join(line))
1453
-
1454
- data_handle.write(complete_line)
1455
- data_handle.write("\n")
1456
-
1457
- return self.__run_user_script_subprocess(cmd, data_handle)
1458
-
1459
- def __local_run_user_script_input_db(self, cmd, table_name):
1460
- """
1461
- DESCRIPTION:
1462
- Function to run the user script in local mode with input from db.
1463
-
1464
- PARAMETERS:
1465
- cmd:
1466
- Required Argument.
1467
- Specifies the command for running the user script.
1468
- Types: str
1469
-
1470
- table_name:
1471
- Required Argument.
1472
- Specifies the table name for input to user script.
1473
- Types: str
1474
-
1475
- RETURNS:
1476
- The string output of the command that is run on the Vantage table.
1477
-
1478
- RAISES:
1479
- Exception.
1480
-
1481
- EXAMPLES:
1482
- self.__local_run_user_script_input_db(cmd = "cmd", table_name = "table_name")
1483
-
1484
- """
1485
- db_data_handle = StringIO()
1486
- try:
1487
- con = get_connection()
1488
- # Query for reading data from DB.
1489
- query = ("SELECT * FROM {} ORDER BY 1;".format(table_name))
1490
- cur = execute_sql(query)
1491
- row = cur.fetchone()
1492
- from pandas import isna as pd_isna
1493
- while row:
1494
- if self.quotechar is not None:
1495
- # A NULL value should not be enclosed in quotes.
1496
- # The CSV module has no support for such output with writer,
1497
- # and hence the custom formatting.
1498
- line = ['' if pd_isna(s) else "{}{}{}".format(self.quotechar,
1499
- str(s),
1500
- self.quotechar)
1501
- for s in row]
1502
- else:
1503
- line = ['' if pd_isna(s) else str(s) for s in row]
1504
-
1505
- complete_line = (self.delimiter.join(line))
1506
- db_data_handle.write(complete_line)
1507
- db_data_handle.write("\n")
1508
- row = cur.fetchone()
1509
- except Exception as exp:
1510
- raise exp
1511
-
1512
- return self.__run_user_script_subprocess(cmd, db_data_handle)
1513
-
1514
- def __process_test_script_output(self, exec_cmd_output):
1515
- """
1516
- DESCRIPTION:
1517
- Function to format the output of the user script.
1518
-
1519
- PARAMETERS:
1520
- exec_cmd_output:
1521
- Required Argument.
1522
- Specifies the output returned by the user script.
1523
- Types: str
1524
-
1525
- RETURNS:
1526
- The test script output as Pandas DataFrame.
1527
-
1528
- RAISES:
1529
- Exception.
1530
-
1531
- EXAMPLES:
1532
- self.__process_test_script_output(exec_cmd_output = "exec_cmd_output")
1533
- """
1534
- try:
1535
- kwargs = dict()
1536
- if self.quotechar is not None:
1537
- kwargs['quotechar'] = self.quotechar
1538
- kwargs['quoting'] = 1 # QUOTE_ALL
1539
-
1540
- output = StringIO(exec_cmd_output)
1541
-
1542
- from pandas import read_csv as pd_read_csv
1543
-
1544
- # Form a pandas dataframe.
1545
- df = pd_read_csv(output, sep=self.delimiter, index_col=False, header=None,
1546
- names=list(self.returns.keys()), **kwargs)
1547
- return df
1548
-
1549
- except Exception as exp:
1550
- raise exp
1551
-
1552
- def __run_user_script_subprocess(self, cmd, data_handle):
1553
- """
1554
- DESCRIPTION:
1555
- Function to run the user script in a new process and return the output.
1556
-
1557
- PARAMETERS:
1558
- cmd:
1559
- Required Argument.
1560
- Specifies the command for running the script.
1561
- Types: str
1562
-
1563
- data_handle:
1564
- Required Argument.
1565
- Specifies the data handle for the input data required by the user script.
1566
-
1567
- RETURNS:
1568
- Output of user script on input data supplied in data_handle.
1569
-
1570
- RAISES:
1571
- None.
1572
-
1573
- EXAMPLES:
1574
- self.__run_user_script_subprocess(cmd = "exec_cmd_output",
1575
- data_handle = data_handle)
1576
-
1577
- """
1578
- # Launching new process to run the user script.
1579
- try:
1580
- proc = subprocess.Popen(cmd, stdin=subprocess.PIPE,
1581
- stdout=subprocess.PIPE,
1582
- stderr=subprocess.PIPE)
1583
- process_output, process_error = proc.communicate(data_handle.getvalue().encode())
1584
- data_handle.close()
1585
-
1586
- if proc.returncode == 0:
1587
- return process_output.decode("utf-8").rstrip("\r|\n")
1588
- else:
1589
- message = Messages.get_message(MessageCodes.SCRIPT_LOCAL_RUN_ERROR).\
1590
- format(process_error)
1591
- raise TeradataMlException(message, MessageCodes.SCRIPT_LOCAL_RUN_ERROR)
1592
- except Exception as e:
1593
- raise e
1594
-
1595
- def execute_script(self, output_style='VIEW'):
1596
- """
1597
- DESCRIPTION:
1598
- Function enables user to run script on Vantage.
1599
-
1600
- PARAMETERS:
1601
- output_style:
1602
- Specifies the type of output object to create - a table or a view.
1603
- Permitted Values: 'VIEW', 'TABLE'.
1604
- Default value: 'VIEW'
1605
- Types: str
1606
-
1607
- RETURNS:
1608
- Output teradataml DataFrames can be accessed using attribute
1609
- references, such as ScriptObj.<attribute_name>.
1610
- Output teradataml DataFrame attribute name is:
1611
- result
1612
-
1613
- RAISES:
1614
- TeradataMlException, ValueError
1615
-
1616
- EXAMPLES:
1617
- Refer to help(Script)
1618
- """
1619
- # Validate the output_style.
1620
- permitted_values = [OutputStyle.OUTPUT_TABLE.value,
1621
- OutputStyle.OUTPUT_VIEW.value]
1622
- _Validators._validate_permitted_values(output_style, permitted_values,
1623
- 'output_style',
1624
- case_insensitive=False, includeNone=False)
1625
-
1626
- # Validate arguments.
1627
- self.__validate()
1628
- # Validating for reserved keywords.
1629
- self.__validate_for_reserved_keyword()
1630
-
1631
- # Generate the Table Operator query.
1632
- self.__form_table_operator_query()
1633
-
1634
- # Execute Table Operator query and return results.
1635
- return self.__execute(output_style)
1636
-
1637
- def install_file(self, file_identifier, file_name, is_binary = False,
1638
- replace = False, force_replace = False):
1639
- """
1640
- DESCRIPTION:
1641
- Function to install script on Vantage.
1642
- On success, prints a message that file is installed.
1643
- This language script can be executed via execute_script() function.
1644
-
1645
- PARAMETERS:
1646
- file_identifier:
1647
- Required Argument.
1648
- Specifies the name associated with the user-installed file.
1649
- It cannot have a schema name associated with it,
1650
- as the file is always installed in the current schema.
1651
- The name should be unique within the schema. It can be any valid Teradata
1652
- identifier.
1653
- Types: str
1654
-
1655
- file_name:
1656
- Required Argument:
1657
- Specifies the name of the file user wnats to install.
1658
- Types: str
1659
-
1660
- is_binary:
1661
- Optional Argument.
1662
- Specifies if file to be installed is a binary file.
1663
- Default Value: False
1664
- Types: bool
1665
-
1666
- replace:
1667
- Optional Argument.
1668
- Specifies if the file is to be installed or replaced.
1669
- If set to True, then the file is replaced based on value the of
1670
- force_replace.
1671
- If set to False, then the file is installed.
1672
- Default Value: False
1673
- Types: bool
1674
-
1675
- force_replace:
1676
- Optional Argument.
1677
- Specifies if system should check for the file being used before
1678
- replacing it.
1679
- If set to True, then the file is replaced even if it is being executed.
1680
- If set to False, then an error is thrown if it is being executed.
1681
- Default Value: False
1682
- Types: bool
1683
-
1684
- RETURNS:
1685
- True, if success
1686
-
1687
- RAISES:
1688
- TeradataMLException.
1689
-
1690
- EXAMPLES:
1691
- # Note - Refer to User Guide for setting search path and required permissions.
1692
- # Example 1: Install the file mapper.py found at the relative path
1693
- # data/scripts/ using the default text mode.
1694
-
1695
- # Set SEARCHUIFDBPATH.
1696
- >>> execute_sql("SET SESSION SEARCHUIFDBPATH = alice;")
1697
-
1698
- # Create a Script object that allows us to execute script on Vantage.
1699
- >>> import os
1700
- >>> from teradatasqlalchemy import VARCHAR
1701
- >>> td_path = os.path.dirname(teradataml.__file__)
1702
- >>> sto = Script(data=barrierdf,
1703
- ... script_name='mapper.py',
1704
- ... files_local_path= os.path.join(td_path, 'data', "scripts"),
1705
- ... script_command='tdpython3 ./alice/mapper.py',
1706
- ... data_order_column="Id",
1707
- ... is_local_order=False,
1708
- ... nulls_first=False,
1709
- ... sort_ascending=False,
1710
- ... charset='latin',
1711
- ... returns=OrderedDict([("word", VARCHAR(15)),("count_input", VARCHAR(2))]))
1712
- >>>
1713
-
1714
- # Install file on Vantage.
1715
-
1716
- >>> sto.install_file(file_identifier='mapper',
1717
- ... file_name='mapper.py',
1718
- ... is_binary=False)
1719
- File mapper.py installed in Vantage
1720
-
1721
- # Replace file on Vantage.
1722
- >>> sto.install_file(file_identifier='mapper',
1723
- ... file_name='mapper.py',
1724
- ... is_binary=False,
1725
- ... replace=True,
1726
- ... force_replace=True)
1727
- File mapper.py replaced in Vantage
1728
- """
1729
- # Install/Replace file on Vantage.
1730
- try:
1731
- file_path = os.path.join(self.files_local_path, file_name)
1732
- # Install file on Vantage.
1733
- install_file(file_identifier=file_identifier, file_path=file_path,
1734
- is_binary=is_binary,
1735
- replace=replace, force_replace=force_replace)
1736
- except:
1737
- raise
1738
-
1739
- def remove_file(self, file_identifier, force_remove=False):
1740
- """
1741
- DESCRIPTION:
1742
- Function to remove user installed files/scripts from Vantage.
1743
-
1744
- PARAMETERS:
1745
- file_identifier:
1746
- Required Argument.
1747
- Specifies the name associated with the user-installed file.
1748
- It cannot have a database name associated with it,
1749
- as the file is always installed in the current database.
1750
- Types: str
1751
-
1752
- force_remove:
1753
- Required Argument.
1754
- Specifies if system should check for the file being used before
1755
- removing it.
1756
- If set to True, then the file is removed even if it is being executed.
1757
- If set to False, then an error is thrown if it is being executed.
1758
- Default value: False
1759
- Types: bool
1760
-
1761
- RETURNS:
1762
- True, if success.
1763
-
1764
- RAISES:
1765
- TeradataMLException.
1766
-
1767
- EXAMPLES:
1768
- # Note - Refer to User Guide for setting search path and required permissions.
1769
- # Run install_file example before removing file.
1770
-
1771
- # Set SEARCHUIFDBPATH.
1772
- >>> execute_sql("SET SESSION SEARCHUIFDBPATH = alice;")
1773
-
1774
- # Create a Script object that allows us to execute script on Vantage.
1775
- >>> sto = Script(data=barrierdf,
1776
- ... script_name='mapper.py',
1777
- ... files_local_path= os.path.join(td_path, 'data', "scripts"),
1778
- ... script_command='tdpython3 ./alice/mapper.py',
1779
- ... data_order_column="Id",
1780
- ... is_local_order=False,
1781
- ... nulls_first=False,
1782
- ... sort_ascending=False,
1783
- ... charset='latin',
1784
- ... returns=OrderedDict([("word", VARCHAR(15)),("count_input", VARCHAR(2))]))
1785
- >>>
1786
-
1787
- # Install file on Vantage.
1788
- >>> sto.install_file(file_identifier='mapper',
1789
- ... file_name='mapper.py',
1790
- ... is_binary=False,
1791
- ... replace=True,
1792
- ... force_replace=True)
1793
- File mapper.py replaced in Vantage
1794
-
1795
- # Remove the installed file.
1796
- >>> sto.remove_file(file_identifier='mapper', force_remove=True)
1797
- File mapper removed from Vantage
1798
-
1799
- """
1800
- # Remove file from Vantage
1801
- try:
1802
- remove_file(file_identifier, force_remove)
1803
- except:
1804
- raise
1805
-
1806
- def __form_table_operator_query(self):
1807
- """
1808
- Function to generate the Table Operator queries. The function defines
1809
- variables and list of arguments required to form the query.
1810
- """
1811
- # Output table arguments list.
1812
- self.__func_output_args_sql_names = []
1813
- self.__func_output_args = []
1814
-
1815
- # Generate lists for rest of the function arguments.
1816
- self.__func_other_arg_sql_names = []
1817
- self.__func_other_args = []
1818
- self.__func_other_arg_json_datatypes = []
1819
-
1820
- self.__func_other_arg_sql_names.append("SCRIPT_COMMAND")
1821
- self.__func_other_args.append(
1822
- UtilFuncs._teradata_collapse_arglist(self.script_command, "'"))
1823
- self.__func_other_arg_json_datatypes.append("STRING")
1824
-
1825
- if self.delimiter is not None:
1826
- self.__func_other_arg_sql_names.append("delimiter")
1827
- self.__func_other_args.append(
1828
- UtilFuncs._teradata_collapse_arglist(self.delimiter, "'"))
1829
- self.__func_other_arg_json_datatypes.append("STRING")
1830
-
1831
- # Generate returns clause.
1832
- ret_vals = []
1833
- returns_clause = ''
1834
- for key in self.returns.keys():
1835
- ret_vals.append('{} {}'.format(key, self.returns[key].compile(td_dialect())))
1836
- returns_clause = ', '.join(ret_vals)
1837
-
1838
- self.__func_other_arg_sql_names.append("returns")
1839
- self.__func_other_args.append(
1840
- UtilFuncs._teradata_collapse_arglist(returns_clause, "'"))
1841
- self.__func_other_arg_json_datatypes.append("STRING")
1842
-
1843
- if self.auth is not None:
1844
- self.__func_other_arg_sql_names.append("auth")
1845
- self.__func_other_args.append(
1846
- UtilFuncs._teradata_collapse_arglist(self.auth, "'"))
1847
- self.__func_other_arg_json_datatypes.append("STRING")
1848
-
1849
- if self.charset is not None:
1850
- self.__func_other_arg_sql_names.append("charset")
1851
- self.__func_other_args.append(
1852
- UtilFuncs._teradata_collapse_arglist(self.charset, "'"))
1853
- self.__func_other_arg_json_datatypes.append("STRING")
1854
-
1855
- if self.quotechar is not None:
1856
- self.__func_other_arg_sql_names.append("quotechar")
1857
- self.__func_other_args.append(
1858
- UtilFuncs._teradata_collapse_arglist(self.quotechar, "'"))
1859
- self.__func_other_arg_json_datatypes.append("STRING")
1860
-
1861
- # Declare empty lists to hold input table information.
1862
- self.__func_input_arg_sql_names = []
1863
- self.__func_input_table_view_query = []
1864
- self.__func_input_dataframe_type = []
1865
- self.__func_input_distribution = []
1866
- self.__func_input_partition_by_cols = []
1867
- self.__func_input_order_by_cols = []
1868
- self.__func_input_order_by_type = []
1869
- self.__func_input_sort_ascending = self.sort_ascending
1870
- self.__func_input_nulls_first = None
1871
-
1872
- # Process data.
1873
- if self.data is not None:
1874
- data_distribution = "FACT"
1875
- if self.data_hash_column is not None:
1876
- data_distribution = "HASH"
1877
- data_partition_column = UtilFuncs._teradata_collapse_arglist(
1878
- self.data_hash_column, "\"")
1879
- else:
1880
- if self.__awu._is_default_or_not(self.data_partition_column, "ANY"):
1881
- data_partition_column = UtilFuncs._teradata_collapse_arglist(
1882
- self.data_partition_column, "\"")
1883
- else:
1884
- data_partition_column = None
1885
- if self.data_order_column is not None:
1886
- if self.is_local_order:
1887
- self.__func_input_order_by_type.append("LOCAL")
1888
- if not self.data_hash_column:
1889
- data_distribution = None
1890
- else:
1891
- self.__func_input_order_by_type.append(None)
1892
- self.__func_input_order_by_cols.append(
1893
- UtilFuncs._teradata_collapse_arglist(self.data_order_column, "\""))
1894
- else:
1895
- self.__func_input_order_by_type.append(None)
1896
- self.__func_input_order_by_cols.append("NA_character_")
1897
-
1898
- self.__table_ref = self.__awu._teradata_on_clause_from_dataframe(self.data,
1899
- False)
1900
- self.__func_input_distribution.append(data_distribution)
1901
- self.__func_input_arg_sql_names.append("input")
1902
- self.__func_input_table_view_query.append(self.__table_ref["ref"])
1903
- self.__func_input_dataframe_type.append(self.__table_ref["ref_type"])
1904
- self.__func_input_partition_by_cols.append(data_partition_column)
1905
- self.__func_input_nulls_first = self.nulls_first
1906
-
1907
- function_name = "Script"
1908
- # Create instance to generate Table Operator Query.
1909
- aqg_obj = TableOperatorQueryGenerator(function_name,
1910
- self.__func_input_arg_sql_names,
1911
- self.__func_input_table_view_query,
1912
- self.__func_input_dataframe_type,
1913
- self.__func_input_distribution,
1914
- self.__func_input_partition_by_cols,
1915
- self.__func_input_order_by_cols,
1916
- self.__func_other_arg_sql_names,
1917
- self.__func_other_args,
1918
- self.__func_other_arg_json_datatypes,
1919
- self.__func_output_args_sql_names,
1920
- self.__func_output_args,
1921
- self.__func_input_order_by_type,
1922
- self.__func_input_sort_ascending,
1923
- self.__func_input_nulls_first,
1924
- engine="ENGINE_SQL"
1925
- )
1926
-
1927
- # Invoke call to Table operator query generation.
1928
- self._tblop_query = aqg_obj._gen_table_operator_select_stmt_sql()
1929
-
1930
- # Print Table Operator query if requested to do so.
1931
- if display.print_sqlmr_query:
1932
- print(self._tblop_query)
1933
-
1934
- def __execute(self, output_style='VIEW'):
1935
- """
1936
- DESCRIPTION:
1937
- Function to execute Table Operator queries.
1938
- Create DataFrames for the required Table Operator output.
1939
-
1940
- PARAMETERS:
1941
- output_style:
1942
- Specifies the type of output object to create - a table of a view.
1943
- Permitted Values: 'VIEW', 'TABLE'.
1944
- Default value: 'VIEW'
1945
- Types: str
1946
-
1947
- RAISES:
1948
- None.
1949
-
1950
- RETURNS:
1951
- None.
1952
-
1953
- EXAMPLES:
1954
- >>> return self.__execute(output_style)
1955
- """
1956
- # Generate STDOUT table name and add it to the output table list.
1957
- if output_style == OutputStyle.OUTPUT_TABLE.value:
1958
- table_type = TeradataConstants.TERADATA_TABLE
1959
- else:
1960
- table_type = TeradataConstants.TERADATA_VIEW
1961
-
1962
- tblop_stdout_temp_tablename = \
1963
- UtilFuncs._generate_temp_table_name(prefix="td_tblop_out_",
1964
- use_default_database=True,
1965
- gc_on_quit=True, quote=False,
1966
- table_type=table_type)
1967
- try:
1968
- if output_style == OutputStyle.OUTPUT_TABLE.value:
1969
- UtilFuncs._create_table(tblop_stdout_temp_tablename, self._tblop_query)
1970
- else:
1971
- UtilFuncs._create_view(tblop_stdout_temp_tablename, self._tblop_query)
1972
- except Exception as emsg:
1973
- raise TeradataMlException(
1974
- Messages.get_message(MessageCodes.TDMLDF_EXEC_SQL_FAILED, str(emsg)),
1975
- MessageCodes.TDMLDF_EXEC_SQL_FAILED)
1976
-
1977
- self.result = self.__awu._create_data_set_object(
1978
- df_input=UtilFuncs._extract_table_name(tblop_stdout_temp_tablename),
1979
- source_type="table",
1980
- database_name=UtilFuncs._extract_db_name(tblop_stdout_temp_tablename))
1981
-
1982
- return self.result
1
+ #!/usr/bin/python
2
+ # ##################################################################
3
+ #
4
+ # Copyright 2020 Teradata. All rights reserved. #
5
+ # TERADATA CONFIDENTIAL AND TRADE SECRET #
6
+ #
7
+ # Primary Owner: Gouri Patwardhan (gouri.patwardhan@teradata.com) #
8
+ # Secondary Owner: Trupti Purohit (trupti.purohit@teradata.com) #
9
+ #
10
+ # Function Version: 1.0 #
11
+ #
12
+ # Description: Script is a TeradataML wrapper around Teradata's #
13
+ # Script Table Operator #
14
+ # ##################################################################
15
+
16
+ import os
17
+ import teradataml.dataframe as tdmldf
18
+ import subprocess
19
+ import sys
20
+ from io import StringIO
21
+ from teradataml.common.constants import TableOperatorConstants
22
+ from teradataml.common.wrapper_utils import AnalyticsWrapperUtils
23
+ from teradataml.common.utils import UtilFuncs
24
+ from teradataml.common.constants import OutputStyle, TeradataConstants
25
+ from teradataml.context.context import _get_current_databasename
26
+ from teradataml.context.context import get_context, get_connection
27
+ from teradataml.dataframe.dataframe_utils import DataFrameUtils as df_utils
28
+ from teradataml.dbutils.filemgr import install_file
29
+ from teradataml.dbutils.filemgr import remove_file
30
+ from teradataml.table_operators.table_operator_query_generator import \
31
+ TableOperatorQueryGenerator
32
+ from teradataml.common.exceptions import TeradataMlException
33
+ from teradataml.common.messages import Messages
34
+ from teradataml.common.messagecodes import MessageCodes
35
+ from teradataml.utils.validators import _Validators
36
+ from teradataml.options.display import display
37
+ from teradataml.options.configure import configure
38
+ from teradataml.utils.utils import execute_sql
39
+ from teradatasqlalchemy.dialect import dialect as td_dialect
40
+ from teradatasqlalchemy import (BYTEINT, SMALLINT, INTEGER, BIGINT, DECIMAL, FLOAT,
41
+ NUMBER)
42
+ from teradatasqlalchemy import (TIMESTAMP, DATE, TIME)
43
+ from teradatasqlalchemy import (CHAR, VARCHAR, CLOB)
44
+ from teradatasqlalchemy import (BYTE, VARBYTE, BLOB)
45
+ from teradatasqlalchemy import (PERIOD_DATE, PERIOD_TIME, PERIOD_TIMESTAMP)
46
+ from teradatasqlalchemy import (INTERVAL_YEAR, INTERVAL_YEAR_TO_MONTH, INTERVAL_MONTH,
47
+ INTERVAL_DAY,INTERVAL_DAY_TO_HOUR, INTERVAL_DAY_TO_MINUTE,
48
+ INTERVAL_DAY_TO_SECOND, INTERVAL_HOUR,
49
+ INTERVAL_HOUR_TO_MINUTE, INTERVAL_HOUR_TO_SECOND,
50
+ INTERVAL_MINUTE, INTERVAL_MINUTE_TO_SECOND,
51
+ INTERVAL_SECOND)
52
+ from teradataml.table_operators.TableOperator import TableOperator
53
+
54
+ class Script(TableOperator):
55
+ def __init__(self,
56
+ data=None,
57
+ script_name=None,
58
+ files_local_path=None,
59
+ script_command=None,
60
+ delimiter="\t",
61
+ returns=None,
62
+ auth=None,
63
+ charset=None,
64
+ quotechar=None,
65
+ data_partition_column=None,
66
+ data_hash_column=None,
67
+ data_order_column=None,
68
+ is_local_order=False,
69
+ sort_ascending=True,
70
+ nulls_first=True):
71
+ """
72
+ DESCRIPTION:
73
+ The Script table operator function executes a user-installed script or
74
+ any LINUX command inside database on Teradata Vantage.
75
+
76
+ PARAMETERS:
77
+ script_command:
78
+ Required Argument.
79
+ Specifies the command/script to run.
80
+ Types: str
81
+
82
+ script_name:
83
+ Required Argument.
84
+ Specifies the name of user script.
85
+ User script should have at least permissions of mode 644.
86
+ Types: str
87
+
88
+ files_local_path:
89
+ Required Argument.
90
+ Specifies the absolute local path where user script and all supporting
91
+ files like model files, input data file reside.
92
+ Types: str
93
+
94
+ returns:
95
+ Required Argument.
96
+ Specifies output column definition.
97
+ Types: Dictionary specifying column name to teradatasqlalchemy type mapping.
98
+ Default: None
99
+ Note:
100
+ User can pass a dictionary (dict or OrderedDict) to the "returns" argument,
101
+ with the keys ordered to represent the order of the output columns.
102
+ Preferred type is OrderedDict.
103
+
104
+ data:
105
+ Optional Argument.
106
+ Specifies a teradataml DataFrame containing the input data for the
107
+ script.
108
+
109
+ data_hash_column:
110
+ Optional Argument.
111
+ Specifies the column to be used for hashing.
112
+ The rows in the data are redistributed to AMPs based on the hash value of
113
+ the column specified.
114
+ The user-installed script file then runs once on each AMP.
115
+ If there is no "data_partition_column", then the entire result set,
116
+ delivered by the function, constitutes a single group or partition.
117
+ Types: str
118
+ Note:
119
+ "data_hash_column" can not be specified along with
120
+ "data_partition_column", "is_local_order" and "data_order_column".
121
+
122
+ data_partition_column:
123
+ Optional Argument.
124
+ Specifies Partition By columns for "data".
125
+ Values to this argument can be provided as a list, if multiple
126
+ columns are used for partition.
127
+ Default Value: ANY
128
+ Types: str OR list of Strings (str)
129
+ Note:
130
+ 1) "data_partition_column" can not be specified along with
131
+ "data_hash_column".
132
+ 2) "data_partition_column" can not be specified along with
133
+ "is_local_order = True".
134
+
135
+ is_local_order:
136
+ Optional Argument.
137
+ Specifies a boolean value to determine whether the input data is to be
138
+ ordered locally or not. Order by specifies the order in which the
139
+ values in a group, or partition, are sorted. Local Order By specifies
140
+ orders qualified rows on each AMP in preparation to be input to a table
141
+ function. This argument is ignored, if "data_order_column" is None. When
142
+ set to True, data is ordered locally.
143
+ Default Value: False
144
+ Types: bool
145
+ Note:
146
+ 1) "is_local_order" can not be specified along with "data_hash_column".
147
+ 2) When "is_local_order" is set to True, "data_order_column" should be
148
+ specified, and the columns specified in "data_order_column" are
149
+ used for local ordering.
150
+
151
+ data_order_column:
152
+ Optional Argument.
153
+ Specifies Order By columns for "data".
154
+ Values to this argument can be provided as a list, if multiple
155
+ columns are used for ordering. This argument is used with in both cases:
156
+ "is_local_order = True" and "is_local_order = False".
157
+ Types: str OR list of Strings (str)
158
+ Note:
159
+ "data_order_column" can not be specified along with "data_hash_column".
160
+
161
+ sort_ascending:
162
+ Optional Argument.
163
+ Specifies a boolean value to determine if the result set is to be sorted
164
+ on the "data_order_column" column in ascending or descending order.
165
+ The sorting is ascending when this argument is set to True, and descending
166
+ when set to False. This argument is ignored, if "data_order_column" is
167
+ None.
168
+ Default Value: True
169
+ Types: bool
170
+
171
+ nulls_first:
172
+ Optional Argument.
173
+ Specifies a boolean value to determine whether NULLS are listed first or
174
+ last during ordering. This argument is ignored, if "data_order_column" is
175
+ None. NULLS are listed first when this argument is set to True, and last
176
+ when set to False.
177
+ Default Value: True
178
+ Types: bool
179
+
180
+ delimiter:
181
+ Optional Argument.
182
+ Specifies a delimiter to use when reading columns from a row and
183
+ writing result columns.
184
+ Default Value: "\t" (tab)
185
+ Types: str of length 1 character
186
+ Notes:
187
+ 1) This argument cannot be same as "quotechar" argument.
188
+ 2) This argument cannot be a newline character i.e., '\\n'.
189
+
190
+ auth:
191
+ Optional Argument.
192
+ Specifies an authorization to use when running the script.
193
+ Types: str
194
+
195
+ charset:
196
+ Optional Argument.
197
+ Specifies the character encoding for data.
198
+ Permitted Values: utf-16, latin
199
+ Types: str
200
+
201
+ quotechar:
202
+ Optional Argument.
203
+ Specifies a character that forces all input and output of the script
204
+ to be quoted using this specified character.
205
+ Using this argument enables the Advanced SQL Engine to distinguish
206
+ between NULL fields and empty strings. A string with length zero is
207
+ quoted, while NULL fields are not.
208
+ If this character is found in the data, it will be escaped by a second
209
+ quote character.
210
+ Types: character of length 1
211
+ Notes:
212
+ 1) This argument cannot be same as "delimiter" argument.
213
+ 2) This argument cannot be a newline character i.e., '\\n'.
214
+
215
+ RETURNS:
216
+ Script Object
217
+
218
+ RAISES:
219
+ TeradataMlException
220
+
221
+ EXAMPLES:
222
+ # Note - Refer to User Guide for setting search path and required permissions.
223
+ # Load example data.
224
+ load_example_data("Script", ["barrier"])
225
+
226
+ # Example - The script mapper.py reads in a line of text input
227
+ # ("Old Macdonald Had A Farm") from csv and splits the line into individual
228
+ # words, emitting a new row for each word.
229
+
230
+ # Create teradataml DataFrame objects.
231
+ >>> barrierdf = DataFrame.from_table("barrier")
232
+
233
+ # Set SEARCHUIFDBPATH.
234
+ >>> execute_sql("SET SESSION SEARCHUIFDBPATH = alice;")
235
+
236
+ # Create a Script object that allows us to execute script on Vantage.
237
+ >>> import teradataml, os
238
+ >>> from teradatasqlalchemy import VARCHAR
239
+ >>> td_path = os.path.dirname(teradataml.__file__)
240
+ >>> sto = Script(data=barrierdf,
241
+ ... script_name='mapper.py',
242
+ ... files_local_path= os.path.join(td_path, 'data', 'scripts'),
243
+ ... script_command='tdpython3 ./alice/mapper.py',
244
+ ... data_order_column="Id",
245
+ ... is_local_order=False,
246
+ ... nulls_first=False,
247
+ ... sort_ascending=False,
248
+ ... charset='latin',
249
+ ... returns=OrderedDict([("word", VARCHAR(15)),("count_input", VARCHAR(2))]))
250
+
251
+ # Run user script locally and using data from csv.
252
+
253
+ >>> sto.test_script(input_data_file='../barrier.csv', data_file_delimiter=',')
254
+
255
+ ############ STDOUT Output ############
256
+
257
+ word count_input
258
+ 0 1 1
259
+ 1 Old 1
260
+ 2 Macdonald 1
261
+ 3 Had 1
262
+ 4 A 1
263
+ 5 Farm 1
264
+ >>>
265
+
266
+ # Script results look good. Now install file on Vantage.
267
+ >>> sto.install_file(file_identifier='mapper',
268
+ ... file_name='mapper.py',
269
+ ... is_binary=False)
270
+ File mapper.py installed in Vantage
271
+
272
+ # Execute the user script on Vantage.
273
+ >>> sto.execute_script()
274
+ ############ STDOUT Output ############
275
+
276
+ word count_input
277
+ 0 Macdonald 1
278
+ 1 A 1
279
+ 2 Farm 1
280
+ 3 Had 1
281
+ 4 Old 1
282
+ 5 1 1
283
+
284
+ # Remove the installed file from Vantage.
285
+ >>> sto.remove_file(file_identifier='mapper', force_remove=True)
286
+ File mapper removed from Vantage
287
+ """
288
+ self.result = None
289
+ self.data = data
290
+ self.script_name = script_name
291
+ self.files_local_path = files_local_path
292
+ self.script_command = script_command
293
+ self.delimiter = delimiter
294
+ self.returns = returns
295
+ self.auth = auth
296
+ self.charset = charset
297
+ self.quotechar = quotechar
298
+ self.data_partition_column = data_partition_column
299
+ self.data_hash_column = data_hash_column
300
+ self.data_order_column = data_order_column
301
+ self.is_local_order = is_local_order
302
+ self.sort_ascending = sort_ascending
303
+ self.nulls_first = nulls_first
304
+ self._check_reserved_keyword = True
305
+ self._skip_argument_validation = False
306
+
307
+ # Create AnalyticsWrapperUtils instance which contains validation functions.
308
+ # This is required for is_default_or_not check.
309
+ # Rest all validation is done using _Validators
310
+ self.__awu = AnalyticsWrapperUtils()
311
+
312
+ # Below matrix is a list of lists, where in each row contains following elements:
313
+ # Let's take an example of following, just to get an idea:
314
+ # [element1, element2, element3, element4, element5, element6]
315
+ # e.g.
316
+ # ["join", join, True, (str), True, concat_join_permitted_values]
317
+
318
+ # 1. element1 --> Argument Name, a string. ["join" in above example.]
319
+ # 2. element2 --> Argument itself. [join]
320
+ # 3. element3 --> Specifies a flag that mentions argument is optional or not.
321
+ # False, means required and True means optional.
322
+ # 4. element4 --> Tuple of accepted types. (str) in above example.
323
+ # 5. element5 --> True, means validate for empty value. Error will be raised,
324
+ # if empty value is passed.
325
+ # If not specified, means same as specifying False.
326
+ # 6. element6 --> A list of permitted values, an argument can accept.
327
+ # If not specified, it is as good as passing None.
328
+ # If a list is passed, validation will be
329
+ # performed for permitted values.
330
+
331
+ self.awu_matrix = []
332
+ self.awu_matrix.append(["data", self.data, True, (tdmldf.dataframe.DataFrame)])
333
+ self.awu_matrix.append(["data_partition_column", self.data_partition_column, True,
334
+ (str, list), True])
335
+ self.awu_matrix.append(["data_hash_column", self.data_hash_column, True,
336
+ (str, list), True])
337
+ self.awu_matrix.append(["data_order_column", self.data_order_column, True,
338
+ (str, list), True])
339
+ self.awu_matrix.append(["is_local_order", self.is_local_order, True, (bool)])
340
+ self.awu_matrix.append(["sort_ascending", self.sort_ascending, True, (bool)])
341
+ self.awu_matrix.append(["nulls_first", self.nulls_first, True, (bool)])
342
+ self.awu_matrix.append(["script_command", self.script_command, False, (str),
343
+ True])
344
+ self.awu_matrix.append(["script_name", self.script_name, True, (str), True])
345
+ self.awu_matrix.append(["files_local_path", self.files_local_path, True, (str),
346
+ True])
347
+ self.awu_matrix.append(["delimiter", self.delimiter, True, (str), False])
348
+ self.awu_matrix.append(["returns", self.returns, False, (dict), True])
349
+ self.awu_matrix.append(["auth", self.auth, True, (str), True])
350
+ self.awu_matrix.append(["charset", self.charset, True, (str), True,
351
+ ["utf-16", "latin"]])
352
+ self.awu_matrix.append(["quotechar", self.quotechar, True, (str), False])
353
+
354
+ # Perform the function validations
355
+ self.__validate()
356
+
357
+ @property
358
+ def skip_argument_validation(self):
359
+ """
360
+ DESCRIPTION:
361
+ Getter for self._skip_argument_validation.
362
+
363
+ RETURNS:
364
+ bool
365
+
366
+ RAISES:
367
+ None
368
+ """
369
+ return self._skip_argument_validation
370
+
371
+ @skip_argument_validation.setter
372
+ def skip_argument_validation(self, flag):
373
+ """
374
+ DESCRIPTION:
375
+ Setter for self._skip_argument_validation
376
+
377
+ PARAMETERS:
378
+ flag Required Argument.
379
+ Specifies whether the arguments should be skipped or not.
380
+ Types: bool
381
+ RETURNS:
382
+ None
383
+
384
+ RAISES:
385
+ None
386
+ """
387
+ self._skip_argument_validation = flag
388
+
389
+ @property
390
+ def check_reserved_keyword(self):
391
+ """
392
+ DESCRIPTION:
393
+ Getter for self._check_reserved_keyword.
394
+
395
+ RETURNS:
396
+ bool
397
+
398
+ RAISES:
399
+ None
400
+ """
401
+ return self._check_reserved_keyword
402
+
403
+ @check_reserved_keyword.setter
404
+ def check_reserved_keyword(self, flag):
405
+ """
406
+ DESCRIPTION:
407
+ Setter for self._check_reserved_keyword
408
+
409
+ RETURNS:
410
+ None
411
+
412
+ RAISES:
413
+ None
414
+ """
415
+ self._check_reserved_keyword = flag
416
+
417
+ def __validate_for_reserved_keyword(self):
418
+ """
419
+ DESCRIPTION:
420
+ Function to validate if the returns clause has teradata reserved keyword or not.
421
+ If it contains reserved keyword, then raise an error.
422
+
423
+ RETURNS:
424
+ None
425
+
426
+ RAISES:
427
+ TeradataMlException
428
+
429
+ """
430
+ if self.check_reserved_keyword:
431
+ from teradataml import list_td_reserved_keywords
432
+ if get_connection():
433
+ # Checking for reserved keywords and raising error if present.
434
+ for column_name in self.returns:
435
+ list_td_reserved_keywords(key=column_name, raise_error=True)
436
+
437
+ def __validate(self):
438
+ """
439
+ Function to validate Table Operator Function arguments, which verifies missing
440
+ arguments, input argument and table types. Also processes the argument values.
441
+ """
442
+ if self.skip_argument_validation:
443
+ return
444
+ # Make sure that a non-NULL value has been supplied for all mandatory arguments
445
+ _Validators._validate_missing_required_arguments(self.awu_matrix)
446
+
447
+ # Validate argument types.
448
+ _Validators._validate_function_arguments(self.awu_matrix,
449
+ skip_empty_check={"quotechar" : ["\n", "\t"],
450
+ "delimiter" : ["\n"]})
451
+
452
+ # permissible_datatypes in returns
453
+ allowed_datatypes = (BYTEINT, SMALLINT, INTEGER, BIGINT, DECIMAL, FLOAT, NUMBER,
454
+ TIMESTAMP, DATE, TIME, CHAR, VARCHAR, CLOB, BYTE, VARBYTE,
455
+ BLOB, PERIOD_DATE, PERIOD_TIME, PERIOD_TIMESTAMP,
456
+ INTERVAL_YEAR, INTERVAL_YEAR_TO_MONTH, INTERVAL_MONTH,
457
+ INTERVAL_DAY, INTERVAL_DAY_TO_HOUR, INTERVAL_DAY_TO_MINUTE,
458
+ INTERVAL_DAY_TO_SECOND, INTERVAL_HOUR,
459
+ INTERVAL_HOUR_TO_MINUTE, INTERVAL_HOUR_TO_SECOND,
460
+ INTERVAL_MINUTE, INTERVAL_MINUTE_TO_SECOND, INTERVAL_SECOND
461
+ )
462
+
463
+ # Validate keys and datatypes in returns.
464
+ self.awu_matrix_returns = []
465
+ for key in self.returns.keys():
466
+ self.awu_matrix_returns.append(["keys in returns", key, False, (str), True])
467
+ self.awu_matrix_returns.append(["value in returns", self.returns[key], False,
468
+ allowed_datatypes])
469
+
470
+
471
+ _Validators._validate_function_arguments(self.awu_matrix_returns)
472
+
473
+ if self.data is not None:
474
+ # Hash and order by can be used together as long as is_local_order = True.
475
+ if all([self.data_hash_column,
476
+ self.data_order_column]) and not self.is_local_order:
477
+ raise TeradataMlException(
478
+ Messages.get_message(MessageCodes.CANNOT_USE_TOGETHER_WITH,
479
+ "data_hash_column' and 'data_order_column",
480
+ "is_local_order=False"),
481
+ MessageCodes.CANNOT_USE_TOGETHER_WITH)
482
+
483
+ # Either hash or partition can be used.
484
+ _Validators._validate_mutually_exclusive_arguments(self.data_hash_column,
485
+ "data_hash_column",
486
+ self.data_partition_column,
487
+ "data_partition_column",
488
+ skip_all_none_check=True)
489
+
490
+ # Either local order by or partition by can be used.
491
+ _Validators._validate_mutually_exclusive_arguments(self.is_local_order,
492
+ "is_local_order=True",
493
+ self.data_partition_column,
494
+ "data_partition_column",
495
+ skip_all_none_check=True)
496
+
497
+ # local order by requires column name.
498
+ if self.is_local_order and self.data_order_column is None:
499
+ message = Messages.get_message(MessageCodes.DEPENDENT_ARG_MISSING,
500
+ "data_order_column", "is_local_order=True")
501
+ raise TeradataMlException(message, MessageCodes.DEPENDENT_ARG_MISSING)
502
+
503
+ if self.__awu._is_default_or_not(self.data_partition_column, "ANY"):
504
+ self.__awu._validate_dataframe_has_argument_columns(
505
+ self.data_partition_column, "data_partition_column", self.data, "data", True)
506
+
507
+ if self.data_order_column is not None:
508
+ self.__awu._validate_dataframe_has_argument_columns(
509
+ self.data_order_column, "data_order_column",self.data, "data", False)
510
+
511
+ if self.data_hash_column is not None:
512
+ self.__awu._validate_dataframe_has_argument_columns(
513
+ self.data_hash_column, "data_hash_column", self.data, "data", False)
514
+
515
+ if self.data_partition_column is not None:
516
+ self.__awu._validate_dataframe_has_argument_columns(
517
+ self.data_partition_column, "data_partition_column", self.data, "data", False)
518
+
519
+ # Check for length of the arguments "delimiter" and "quotechar".
520
+ if self.delimiter is not None:
521
+ _Validators._validate_str_arg_length('delimiter', self.delimiter, 'EQ', 1)
522
+
523
+ if self.quotechar is not None:
524
+ _Validators._validate_str_arg_length('quotechar', self.quotechar, 'EQ', 1)
525
+
526
+ # The arguments 'quotechar' and 'delimiter' cannot take newline character.
527
+ if self.delimiter == '\n':
528
+ message = Messages.get_message(MessageCodes.NOT_ALLOWED_VALUES, "\n", "delimiter")
529
+ raise TeradataMlException(message, MessageCodes.NOT_ALLOWED_VALUES)
530
+ if self.quotechar == '\n':
531
+ message = Messages.get_message(MessageCodes.NOT_ALLOWED_VALUES, "\n", "quotechar")
532
+ raise TeradataMlException(message, MessageCodes.NOT_ALLOWED_VALUES)
533
+
534
+ # The arguments 'quotechar' and 'delimiter' cannot have the same value.
535
+ if self.delimiter == self.quotechar:
536
+ message = Messages.get_message(MessageCodes.ARGUMENT_VALUE_SAME, "delimiter",
537
+ "quotechar")
538
+ raise TeradataMlException(message, MessageCodes.ARGUMENT_VALUE_SAME)
539
+
540
+
541
+ def set_data(self,
542
+ data,
543
+ data_partition_column=None,
544
+ data_hash_column=None,
545
+ data_order_column=None,
546
+ is_local_order=False,
547
+ sort_ascending=True,
548
+ nulls_first=True):
549
+ """
550
+ DESCRIPTION:
551
+ Function enables user to set data and data related arguments without having to
552
+ re-create Script object.
553
+
554
+ PARAMETERS:
555
+ data:
556
+ Required Argument.
557
+ Specifies a teradataml DataFrame containing the input data for the script.
558
+
559
+ data_hash_column:
560
+ Optional Argument.
561
+ Specifies the column to be used for hashing.
562
+ The rows in the data are redistributed to AMPs based on the
563
+ hash value of the column specified.
564
+ The user installed script then runs once on each AMP.
565
+ If there is no data_partition_column, then the entire result set delivered
566
+ by the function, constitutes a single group or partition.
567
+ Types: str
568
+ Note:
569
+ "data_hash_column" can not be specified along with
570
+ "data_partition_column", "is_local_order" and "data_order_column".
571
+
572
+ data_partition_column:
573
+ Optional Argument.
574
+ Specifies Partition By columns for data.
575
+ Values to this argument can be provided as a list, if multiple
576
+ columns are used for partition.
577
+ Default Value: ANY
578
+ Types: str OR list of Strings (str)
579
+ Note:
580
+ 1) "data_partition_column" can not be specified along with
581
+ "data_hash_column".
582
+ 2) "data_partition_column" can not be specified along with
583
+ "is_local_order = True".
584
+
585
+ is_local_order:
586
+ Optional Argument.
587
+ Specifies a boolean value to determine whether the input data is to be
588
+ ordered locally or not. Order by specifies the order in which the
589
+ values in a group or partition are sorted. Local Order By specifies
590
+ orders qualified rows on each AMP in preparation to be input to a table
591
+ function. This argument is ignored, if "data_order_column" is None. When
592
+ set to True, data is ordered locally.
593
+ Default Value: False
594
+ Types: bool
595
+ Note:
596
+ 1) "is_local_order" can not be specified along with
597
+ "data_hash_column".
598
+ 2) When "is_local_order" is set to True, "data_order_column" should be
599
+ specified, and the columns specified in "data_order_column" are
600
+ used for local ordering.
601
+
602
+ data_order_column:
603
+ Optional Argument.
604
+ Specifies Order By columns for data.
605
+ Values to this argument can be provided as a list, if multiple
606
+ columns are used for ordering.
607
+ This argument is used in both cases:
608
+ "is_local_order = True" and "is_local_order = False".
609
+ Types: str OR list of Strings (str)
610
+ Note:
611
+ "data_order_column" can not be specified along with
612
+ "data_hash_column".
613
+
614
+ sort_ascending:
615
+ Optional Argument.
616
+ Specifies a boolean value to determine if the result set is to be sorted
617
+ on the column specified in "data_order_column", in ascending or descending
618
+ order.
619
+ The sorting is ascending when this argument is set to True, and descending
620
+ when set to False.
621
+ This argument is ignored, if "data_order_column" is None.
622
+ Default Value: True
623
+ Types: bool
624
+
625
+ nulls_first:
626
+ Optional Argument.
627
+ Specifies a boolean value to determine whether NULLS are listed first or
628
+ last during ordering.
629
+ This argument is ignored, if "data_order_column" is None.
630
+ NULLS are listed first when this argument is set to True, and
631
+ last when set to False.
632
+ Default Value: True
633
+ Types: bool
634
+
635
+ RETURNS:
636
+ None.
637
+
638
+ RAISES:
639
+ TeradataMlException
640
+
641
+ EXAMPLES:
642
+ # Note - Refer to User Guide for setting search path and required permissions.
643
+ # Load example data.
644
+ load_example_data("Script", ["barrier"])
645
+
646
+ # Example 1
647
+ # Create teradataml DataFrame objects.
648
+ >>> barrierdf = DataFrame.from_table("barrier")
649
+ >>> barrierdf
650
+ Name
651
+ Id
652
+ 1 Old Macdonald Had A Farm
653
+ >>>
654
+
655
+ # Set SEARCHUIFDBPATH
656
+ >>> execute_sql("SET SESSION SEARCHUIFDBPATH = alice;")
657
+ >>> import teradataml
658
+ >>> from teradatasqlalchemy import VARCHAR
659
+ >>> td_path = os.path.dirname(teradataml.__file__)
660
+
661
+ # The script mapper.py reads in a line of text input
662
+ # ("Old Macdonald Had A Farm") from csv and
663
+ # splits the line into individual words, emitting a new row for each word.
664
+ # Create a Script object without data and its arguments.
665
+ >>> sto = Script(data = barrierdf,
666
+ ... script_name='mapper.py',
667
+ ... files_local_path= os.path.join(td_path,'data', 'scripts'),
668
+ ... script_command='tdpython3 ./alice/mapper.py',
669
+ ... charset='latin',
670
+ ... returns=OrderedDict([("word", VARCHAR(15)),("count_input", VARCHAR(2))]))
671
+
672
+ # Test script using data from file
673
+ >>> sto.test_script(input_data_file='../barrier.csv', data_file_delimiter=',')
674
+ ############ STDOUT Output ############
675
+ word count_input
676
+ 0 1 1
677
+ 1 Old 1
678
+ 2 Macdonald 1
679
+ 3 Had 1
680
+ 4 A 1
681
+ 5 Farm 1
682
+ >>>
683
+
684
+ # Test script using data from DB.
685
+ >>> sto.test_script(password='alice')
686
+ ############ STDOUT Output ############
687
+
688
+ word count_input
689
+ 0 1 1
690
+ 1 Old 1
691
+ 2 Macdonald 1
692
+ 3 Had 1
693
+ 4 A 1
694
+ 5 Farm 1
695
+
696
+ # Test script using data from DB and with data_row_limit.
697
+ >>> sto.test_script(password='alice', data_row_limit=5)
698
+ ############ STDOUT Output ############
699
+
700
+ word count_input
701
+ 0 1 1
702
+ 1 Old 1
703
+ 2 Macdonald 1
704
+ 3 Had 1
705
+ 4 A 1
706
+ 5 Farm 1
707
+
708
+ # Now in order to test / run script on actual data on Vantage user must
709
+ # set data and related arguments.
710
+ # Note:
711
+ # All data related arguments that are not specified in set_data() are
712
+ # reset to default values.
713
+ >>> sto.set_data(data=barrierdf,
714
+ ... data_order_column="Id",
715
+ ... is_local_order=False,
716
+ ... nulls_first=False,
717
+ ... sort_ascending=False)
718
+
719
+ # Execute the user script on Vantage.
720
+ >>> sto.execute_script()
721
+ ############ STDOUT Output ############
722
+ word count_input
723
+ 0 Macdonald 1
724
+ 1 A 1
725
+ 2 Farm 1
726
+ 3 Had 1
727
+ 4 Old 1
728
+ 5 1 1
729
+
730
+ # Example 2 -
731
+ # Input data is barrier_new and script is executed on Vantage.
732
+ # use set_data() to reset arguments.
733
+ # Create teradataml DataFrame objects.
734
+ >>> load_example_data("Script", ["barrier_new"])
735
+ >>> barrierdf_new = DataFrame.from_table("barrier_new")
736
+ >>> barrierdf_new
737
+ Name
738
+ Id
739
+ 2 On his farm he had a cow
740
+ 1 Old Macdonald Had A Farm
741
+ >>>
742
+
743
+ # Create a Script object that allows us to execute script on Vantage.
744
+ >>> sto = Script(data=barrierdf_new,
745
+ ... script_name='mapper.py',
746
+ ... files_local_path= os.path.join(td_path, 'data', 'scripts'),
747
+ ... script_command='tdpython3 ./alice/mapper.py',
748
+ ... data_order_column="Id",
749
+ ... is_local_order=False,
750
+ ... nulls_first=False,
751
+ ... sort_ascending=False,
752
+ ... charset='latin',
753
+ ... returns=OrderedDict([("word", VARCHAR(15)),("count_input", VARCHAR(2))]))
754
+ # Script is executed on Vantage.
755
+ >>> sto.execute_script()
756
+ ############ STDOUT Output ############
757
+ word count_input
758
+ 0 his 1
759
+ 1 he 1
760
+ 2 had 1
761
+ 3 a 1
762
+ 4 1 1
763
+ 5 Old 1
764
+ 6 cow 1
765
+ 7 farm 1
766
+ 8 On 1
767
+ 9 2 1
768
+
769
+ # Now in order to run the script with a different dataset,
770
+ # user can use set_data().
771
+ # Re-set data and some data related parameters.
772
+ # Note:
773
+ # All data related arguments that are not specified in set_data() are
774
+ # reset to default values.
775
+ >>> sto.set_data(data=barrierdf,
776
+ ... data_order_column='Id',
777
+ ... is_local_order=True,
778
+ ... nulls_first=True)
779
+ >>> sto.execute_script()
780
+ word count_input
781
+ 0 Macdonald 1
782
+ 1 A 1
783
+ 2 Farm 1
784
+ 3 Had 1
785
+ 4 Old 1
786
+ 5 1 1
787
+
788
+ # Example 3
789
+ # In order to run the script with same dataset but different data related
790
+ # arguments, use set_data() to reset arguments.
791
+ # Note:
792
+ # All data related arguments that are not specified in set_data() are
793
+ # reset to default values.
794
+ >>> sto.set_data(data=barrierdf_new,
795
+ ... data_order_column='Id',
796
+ ... is_local_order = True,
797
+ ... nulls_first = True)
798
+
799
+ >>> sto.execute_script()
800
+ ############ STDOUT Output ############
801
+
802
+ word count_input
803
+ 0 Macdonald 1
804
+ 1 A 1
805
+ 2 Farm 1
806
+ 3 2 1
807
+ 4 his 1
808
+ 5 farm 1
809
+ 6 On 1
810
+ 7 Had 1
811
+ 8 Old 1
812
+ 9 1 1
813
+ """
814
+ super(Script, self).set_data(data,
815
+ data_partition_column,
816
+ data_hash_column,
817
+ data_order_column,
818
+ is_local_order,
819
+ sort_ascending,
820
+ nulls_first)
821
+ self.__validate()
822
+
823
+ def test_script(self, supporting_files=None, input_data_file=None, script_args="",
824
+ exec_mode ='local', **kwargs):
825
+ """
826
+ DESCRIPTION:
827
+ Function enables user to run script locally outside Vantage.
828
+ Input data for user script is either read from a file or from database.
829
+ Note:
830
+ 1. Purpose of test_script() function is to enable the user to test their scripts for any errors without
831
+ installing it on Vantage, using the input data provided.
832
+ 2. Data is not partitioned for testing the script if read from input data file.
833
+ 3. Function can produce different output if input is read from a file than input from database.
834
+
835
+ PARAMETERS:
836
+ supporting_files:
837
+ Optional Argument.
838
+ Specifies a file or list of supporting files like model files to be
839
+ copied to the container.
840
+ Types: string or list of str
841
+
842
+ input_data_file:
843
+ Optional Argument.
844
+ Specifies the name of the input data file.
845
+ It should have a path relative to the location specified in
846
+ "files_local_path" argument.
847
+ If set to None, read data from AMP, else from file passed in the argument
848
+ 'input_data_file'.
849
+ File should have at least permissions of mode 644.
850
+ Types: str
851
+
852
+ script_args:
853
+ Optional Argument.
854
+ Specifies command line arguments required by the user script.
855
+ Types: str
856
+
857
+ exec_mode:
858
+ Optional Argument.
859
+ Specifies the mode in which user wants to test the script.
860
+ When set to 'local', the user script will run locally on user's system.
861
+ Permitted Values: 'local'
862
+ Default Value: 'local'
863
+ Types: str
864
+
865
+ kwargs:
866
+ Optional Argument.
867
+ Specifies the keyword arguments required for testing.
868
+ Keys can be:
869
+ data_row_limit:
870
+ Optional Argument. Ignored when data is read from file.
871
+ Specifies the number of rows to be taken from all amps when
872
+ reading from a table or view on Vantage.
873
+ Default Value: 1000
874
+ Types: int
875
+
876
+ password:
877
+ Optional Argument. Required when reading from database.
878
+ Specifies the password to connect to vantage where the data
879
+ resides.
880
+ Types: str
881
+
882
+ data_file_delimiter:
883
+ Optional Argument.
884
+ Specifies the delimiter used in the input data file. This
885
+ argument can be specified when data is read from file.
886
+ Default Value: '\t'
887
+ Types: str
888
+
889
+ data_file_header:
890
+ Optional Argument.
891
+ Specifies whether the input data file contains header. This
892
+ argument can be specified when data is read from file.
893
+ Default Value: True
894
+ Types: bool
895
+
896
+ data_file_quote_char:
897
+ Optional Argument.
898
+ Specifies the quotechar used in the input data file.
899
+ This argument can be specified when data is read from file.
900
+ Default Value: '"'
901
+
902
+ logmech:
903
+ Optional Argument.
904
+ Specifies the type of logon mechanism to establish a connection to
905
+ Teradata Vantage.
906
+ Permitted Values: 'TD2', 'TDNEGO', 'LDAP', 'KRB5' & 'JWT'.
907
+ TD2:
908
+ The Teradata 2 (TD2) mechanism provides authentication
909
+ using a Vantage username and password. This is the default
910
+ logon mechanism using which the connection is established
911
+ to Vantage.
912
+
913
+ TDNEGO:
914
+ A security mechanism that automatically determines the
915
+ actual mechanism required, based on policy, without user's
916
+ involvement. The actual mechanism is determined by the
917
+ TDGSS server configuration and by the security policy's
918
+ mechanism restrictions.
919
+
920
+ LDAP:
921
+ A directory-based user logon to Vantage with a directory
922
+ username and password and is authenticated by the directory.
923
+
924
+ KRB5 (Kerberos):
925
+ A directory-based user logon to Vantage with a domain
926
+ username and password and is authenticated by
927
+ Kerberos (KRB5 mechanism).
928
+ Note:
929
+ User must have a valid ticket-granting ticket in
930
+ order to use this logon mechanism.
931
+
932
+ JWT:
933
+ The JSON Web Token (JWT) authentication mechanism enables
934
+ single sign-on (SSO) to the Vantage after the user
935
+ successfully authenticates to Teradata UDA User Service.
936
+ Note:
937
+ User must use logdata parameter when using 'JWT' as
938
+ the logon mechanism.
939
+ Default Value: TD2
940
+ Types: str
941
+
942
+ Note:
943
+ teradataml expects the client environments are already setup with appropriate
944
+ security mechanisms and are in working conditions.
945
+ For more information please refer Teradata Vantage™ - Advanced SQL Engine
946
+ Security Administration at https://www.info.teradata.com/
947
+
948
+ logdata:
949
+ Optional Argument.
950
+ Specifies parameters to the LOGMECH command beyond those needed by
951
+ the logon mechanism, such as user ID, password and tokens
952
+ (in case of JWT) to successfully authenticate the user.
953
+ Types: str
954
+
955
+ Types: dict
956
+
957
+ RETURNS:
958
+ Output from user script.
959
+
960
+ RAISES:
961
+ TeradataMlException
962
+
963
+ EXAMPLES:
964
+ # Assumption - sto is Script() object. Please refer to help(Script)
965
+ # for creating Script object.
966
+ # Run user script in local mode with input from data file.
967
+
968
+ >>> sto.test_script(input_data_file='../barrier.csv',
969
+ ... data_file_delimiter=',',
970
+ ... data_file_quote_char='"',
971
+ ... data_file_header=True,
972
+ ... exec_mode='local')
973
+
974
+ ############ STDOUT Output ############
975
+ word count_input
976
+ 0 1 1
977
+ 1 Old 1
978
+ 2 Macdonald 1
979
+ 3 Had 1
980
+ 4 A 1
981
+ 5 Farm 1
982
+ >>>
983
+
984
+ # Run user script in local mode with input from table.
985
+ >>> sto.test_script(data_row_limit=300, password='alice', exec_mode='local')
986
+
987
+ ############ STDOUT Output ############
988
+ word count_input
989
+ 0 1 1
990
+ 1 Old 1
991
+ 2 Macdonald 1
992
+ 3 Had 1
993
+ 4 A 1
994
+ 5 Farm 1
995
+
996
+ """
997
+ logmech_valid_values = ['TD2', 'TDNEGO', 'LDAP', 'KRB5', 'JWT']
998
+
999
+ awu_matrix_test=[]
1000
+ awu_matrix_test.append((["supporting_files", supporting_files, True,
1001
+ (str,list), True]))
1002
+ awu_matrix_test.append((["input_data_file", input_data_file, True, (str), True]))
1003
+ awu_matrix_test.append((["script_args", script_args, True, (str), False]))
1004
+ awu_matrix_test.append((["exec_mode", exec_mode, True, (str), True,
1005
+ [TableOperatorConstants.LOCAL_EXEC.value]]))
1006
+
1007
+ data_row_limit = kwargs.pop("data_row_limit", 1000)
1008
+ awu_matrix_test.append((["data_row_limit", data_row_limit, True, (int), True]))
1009
+
1010
+ data_file_delimiter = kwargs.pop("data_file_delimiter", '\t')
1011
+ awu_matrix_test.append((["data_file_delimiter", data_file_delimiter, True,
1012
+ (str), False]))
1013
+
1014
+ data_file_quote_char = kwargs.pop("data_file_quote_char", '"')
1015
+ awu_matrix_test.append((["data_file_quote_char", data_file_quote_char, True,
1016
+ (str), False]))
1017
+
1018
+ data_file_header = kwargs.pop("data_file_header", True)
1019
+ awu_matrix_test.append((["data_file_header", data_file_header, True, (bool)]))
1020
+
1021
+ logmech = kwargs.pop("logmech", "TD2")
1022
+ awu_matrix_test.append(
1023
+ ["logmech", logmech, True, (str), True, logmech_valid_values])
1024
+
1025
+ logdata = kwargs.pop("logdata", None)
1026
+ awu_matrix_test.append(["logdata", logdata, True, (str), True])
1027
+
1028
+ # Validate argument types.
1029
+ _Validators._validate_function_arguments(awu_matrix_test)
1030
+
1031
+ self.__validate()
1032
+ self.__validate_for_reserved_keyword()
1033
+
1034
+
1035
+ if logmech == "JWT" and not logdata:
1036
+ raise TeradataMlException(
1037
+ Messages.get_message(MessageCodes.DEPENDENT_ARG_MISSING, 'logdata',
1038
+ 'logmech=JWT'),
1039
+ MessageCodes.DEPENDENT_ARG_MISSING)
1040
+
1041
+ if data_row_limit <= 0:
1042
+ raise ValueError(Messages.get_message(MessageCodes.TDMLDF_POSITIVE_INT).
1043
+ format("data_row_limit", "greater than"))
1044
+
1045
+ # Either of 'input_data_file' or 'password' argument is required.
1046
+ password = kwargs.pop("password", None)
1047
+
1048
+ # When exec_mode is local, the connection object is used to get the values in the table.
1049
+ if exec_mode == "local" and not (input_data_file or self.data):
1050
+ message = Messages.get_message(MessageCodes.EITHER_THIS_OR_THAT_ARGUMENT,
1051
+ "input_data_file", "Script data")
1052
+ raise TeradataMlException(message, MessageCodes.EITHER_THIS_OR_THAT_ARGUMENT)
1053
+
1054
+ if not self.script_name and self.files_local_path:
1055
+ message = Messages.get_message(MessageCodes.MISSING_ARGS,
1056
+ "script_name and files_local_path")
1057
+ raise TeradataMlException(message, MessageCodes.MISSING_ARGS)
1058
+
1059
+ if input_data_file:
1060
+ if self.files_local_path is None:
1061
+ message = Messages.get_message(MessageCodes.DEPENDENT_ARG_MISSING,
1062
+ "files_local_path","input_data_file")
1063
+ raise TeradataMlException(message, MessageCodes.DEPENDENT_ARG_MISSING)
1064
+ else:
1065
+ # Check if file exists.
1066
+ fpath = os.path.join(self.files_local_path,
1067
+ input_data_file)
1068
+ _Validators._validate_file_exists(fpath)
1069
+
1070
+ if self.script_name and self.files_local_path:
1071
+ # Check if file exists.
1072
+ fpath = os.path.join(self.files_local_path,
1073
+ os.path.basename(self.script_name))
1074
+ _Validators._validate_file_exists(fpath)
1075
+
1076
+ if exec_mode.upper() == TableOperatorConstants.LOCAL_EXEC.value:
1077
+ user_script_path = os.path.join(self.files_local_path, self.script_name)
1078
+ cmd = [str(sys.executable), user_script_path]
1079
+ cmd.extend(script_args)
1080
+
1081
+ if input_data_file is not None:
1082
+ input_file_path = os.path.join(self.files_local_path, input_data_file)
1083
+
1084
+ # Run user script locally with input from a file.
1085
+ exec_cmd_output = self.__local_run_user_script_input_file(
1086
+ cmd, input_file_path, data_file_delimiter, data_file_quote_char, data_file_header)
1087
+ try:
1088
+ return self.__process_test_script_output(exec_cmd_output)
1089
+ except Exception as exp:
1090
+ raise
1091
+
1092
+ else:
1093
+ if self.data.shape[0] > data_row_limit:
1094
+ raise ValueError(
1095
+ Messages.get_message(MessageCodes.DATAFRAME_LIMIT_ERROR,
1096
+ 'data_row_limit', 'data_row_limit',
1097
+ data_row_limit))
1098
+
1099
+ if not self.data._table_name:
1100
+ self.data._table_name = df_utils._execute_node_return_db_object_name(
1101
+ self.data._nodeid, self.data._metaexpr)
1102
+
1103
+ table_name = UtilFuncs._extract_table_name(self.data._table_name)
1104
+
1105
+ # Run user script locally with input from db.
1106
+ exec_cmd_output = self.__local_run_user_script_input_db(cmd, table_name)
1107
+ try:
1108
+ return self.__process_test_script_output(exec_cmd_output)
1109
+ except Exception as exp:
1110
+ raise
1111
+
1112
+ def __local_run_user_script_input_file(self, cmd, input_file_path,
1113
+ data_file_delimiter='\t',
1114
+ data_file_quote_char='"',
1115
+ data_file_header=True):
1116
+ """
1117
+ DESCRIPTION:
1118
+ Function to run the user script in local mode with input from file.
1119
+
1120
+ PARAMETERS:
1121
+ cmd:
1122
+ Required Argument.
1123
+ Specifies the command for running the user script.
1124
+ Types: str
1125
+
1126
+ input_file_path:
1127
+ Required Argument.
1128
+ Specifies the absolute local path of input data file.
1129
+ Types: str
1130
+
1131
+ data_file_delimiter:
1132
+ Optional Argument.
1133
+ Specifies the delimiter used in input data file.
1134
+ Default Value: '\t'
1135
+ Types: str
1136
+
1137
+ data_file_quote_char:
1138
+ Optional Argument.
1139
+ Specifies the quote character used in input data file.
1140
+ Default Value: '"'
1141
+ Types: str
1142
+
1143
+ data_file_header:
1144
+ Optional Argument.
1145
+ Specifies whether the input data file has header.
1146
+ Default Value: True
1147
+ Types: bool
1148
+
1149
+ RETURNS:
1150
+ The string output of the command that is run on input data file.
1151
+
1152
+ RAISES:
1153
+ Exception.
1154
+
1155
+ EXAMPLES:
1156
+ self.__local_run_user_script_input_file(cmd ="cmd",
1157
+ input_file_path = "input_file_path",
1158
+ data_file_delimiter = "data_file_delimiter",
1159
+ data_file_quote_char = "data_file_quote_char",
1160
+ data_file_header = True)
1161
+
1162
+ """
1163
+ with open(input_file_path) as data_file:
1164
+ import csv
1165
+ from pandas import isna as pd_isna
1166
+
1167
+ data_handle = StringIO()
1168
+
1169
+ # Read data from input file.
1170
+ ip_data = csv.reader(data_file,
1171
+ delimiter=data_file_delimiter,
1172
+ quotechar=data_file_quote_char)
1173
+ # Skip the first row of input file if data_file_header is True.
1174
+ if data_file_header:
1175
+ next(ip_data)
1176
+ for row in ip_data:
1177
+ if self.quotechar is not None:
1178
+ # A NULL value should not be enclosed in quotes.
1179
+ # The CSV module has no support for such output with writer,
1180
+ # and hence the custom formatting.
1181
+ line = ['' if pd_isna(s) else "{}{}{}".format(self.quotechar,
1182
+ str(s),
1183
+ self.quotechar)
1184
+ for s in row]
1185
+ else:
1186
+ line = ['' if pd_isna(s) else str(s) for s in row]
1187
+
1188
+ complete_line = (self.delimiter.join(line))
1189
+
1190
+ data_handle.write(complete_line)
1191
+ data_handle.write("\n")
1192
+
1193
+ return self.__run_user_script_subprocess(cmd, data_handle)
1194
+
1195
+ def __local_run_user_script_input_db(self, cmd, table_name):
1196
+ """
1197
+ DESCRIPTION:
1198
+ Function to run the user script in local mode with input from db.
1199
+
1200
+ PARAMETERS:
1201
+ cmd:
1202
+ Required Argument.
1203
+ Specifies the command for running the user script.
1204
+ Types: str
1205
+
1206
+ table_name:
1207
+ Required Argument.
1208
+ Specifies the table name for input to user script.
1209
+ Types: str
1210
+
1211
+ RETURNS:
1212
+ The string output of the command that is run on the Vantage table.
1213
+
1214
+ RAISES:
1215
+ Exception.
1216
+
1217
+ EXAMPLES:
1218
+ self.__local_run_user_script_input_db(cmd = "cmd", table_name = "table_name")
1219
+
1220
+ """
1221
+ db_data_handle = StringIO()
1222
+ try:
1223
+ con = get_connection()
1224
+ # Query for reading data from DB.
1225
+ query = ("SELECT * FROM {} ORDER BY 1;".format(table_name))
1226
+ cur = execute_sql(query)
1227
+ row = cur.fetchone()
1228
+ from pandas import isna as pd_isna
1229
+ while row:
1230
+ if self.quotechar is not None:
1231
+ # A NULL value should not be enclosed in quotes.
1232
+ # The CSV module has no support for such output with writer,
1233
+ # and hence the custom formatting.
1234
+ line = ['' if pd_isna(s) else "{}{}{}".format(self.quotechar,
1235
+ str(s),
1236
+ self.quotechar)
1237
+ for s in row]
1238
+ else:
1239
+ line = ['' if pd_isna(s) else str(s) for s in row]
1240
+
1241
+ complete_line = (self.delimiter.join(line))
1242
+ db_data_handle.write(complete_line)
1243
+ db_data_handle.write("\n")
1244
+ row = cur.fetchone()
1245
+ except Exception as exp:
1246
+ raise exp
1247
+
1248
+ return self.__run_user_script_subprocess(cmd, db_data_handle)
1249
+
1250
+ def __process_test_script_output(self, exec_cmd_output):
1251
+ """
1252
+ DESCRIPTION:
1253
+ Function to format the output of the user script.
1254
+
1255
+ PARAMETERS:
1256
+ exec_cmd_output:
1257
+ Required Argument.
1258
+ Specifies the output returned by the user script.
1259
+ Types: str
1260
+
1261
+ RETURNS:
1262
+ The test script output as Pandas DataFrame.
1263
+
1264
+ RAISES:
1265
+ Exception.
1266
+
1267
+ EXAMPLES:
1268
+ self.__process_test_script_output(exec_cmd_output = "exec_cmd_output")
1269
+ """
1270
+ try:
1271
+ kwargs = dict()
1272
+ if self.quotechar is not None:
1273
+ kwargs['quotechar'] = self.quotechar
1274
+ kwargs['quoting'] = 1 # QUOTE_ALL
1275
+
1276
+ output = StringIO(exec_cmd_output)
1277
+
1278
+ from pandas import read_csv as pd_read_csv
1279
+
1280
+ # Form a pandas dataframe.
1281
+ df = pd_read_csv(output, sep=self.delimiter, index_col=False, header=None,
1282
+ names=list(self.returns.keys()), **kwargs)
1283
+ return df
1284
+
1285
+ except Exception as exp:
1286
+ raise exp
1287
+
1288
+ def __run_user_script_subprocess(self, cmd, data_handle):
1289
+ """
1290
+ DESCRIPTION:
1291
+ Function to run the user script in a new process and return the output.
1292
+
1293
+ PARAMETERS:
1294
+ cmd:
1295
+ Required Argument.
1296
+ Specifies the command for running the script.
1297
+ Types: str
1298
+
1299
+ data_handle:
1300
+ Required Argument.
1301
+ Specifies the data handle for the input data required by the user script.
1302
+
1303
+ RETURNS:
1304
+ Output of user script on input data supplied in data_handle.
1305
+
1306
+ RAISES:
1307
+ None.
1308
+
1309
+ EXAMPLES:
1310
+ self.__run_user_script_subprocess(cmd = "exec_cmd_output",
1311
+ data_handle = data_handle)
1312
+
1313
+ """
1314
+ # Launching new process to run the user script.
1315
+ try:
1316
+ proc = subprocess.Popen(cmd, stdin=subprocess.PIPE,
1317
+ stdout=subprocess.PIPE,
1318
+ stderr=subprocess.PIPE)
1319
+ process_output, process_error = proc.communicate(data_handle.getvalue().encode())
1320
+ data_handle.close()
1321
+
1322
+ if proc.returncode == 0:
1323
+ return process_output.decode("utf-8").rstrip("\r|\n")
1324
+ else:
1325
+ message = Messages.get_message(MessageCodes.SCRIPT_LOCAL_RUN_ERROR).\
1326
+ format(process_error)
1327
+ raise TeradataMlException(message, MessageCodes.SCRIPT_LOCAL_RUN_ERROR)
1328
+ except Exception as e:
1329
+ raise e
1330
+
1331
+ def execute_script(self, output_style='VIEW'):
1332
+ """
1333
+ DESCRIPTION:
1334
+ Function enables user to run script on Vantage.
1335
+
1336
+ PARAMETERS:
1337
+ output_style:
1338
+ Specifies the type of output object to create - a table or a view.
1339
+ Permitted Values: 'VIEW', 'TABLE'.
1340
+ Default value: 'VIEW'
1341
+ Types: str
1342
+
1343
+ RETURNS:
1344
+ Output teradataml DataFrames can be accessed using attribute
1345
+ references, such as ScriptObj.<attribute_name>.
1346
+ Output teradataml DataFrame attribute name is:
1347
+ result
1348
+
1349
+ RAISES:
1350
+ TeradataMlException, ValueError
1351
+
1352
+ EXAMPLES:
1353
+ Refer to help(Script)
1354
+ """
1355
+ # Validate the output_style.
1356
+ permitted_values = [OutputStyle.OUTPUT_TABLE.value,
1357
+ OutputStyle.OUTPUT_VIEW.value]
1358
+ _Validators._validate_permitted_values(output_style, permitted_values,
1359
+ 'output_style',
1360
+ case_insensitive=False, includeNone=False)
1361
+
1362
+ # Validate arguments.
1363
+ self.__validate()
1364
+ # Validating for reserved keywords.
1365
+ self.__validate_for_reserved_keyword()
1366
+
1367
+ # Generate the Table Operator query.
1368
+ self.__form_table_operator_query()
1369
+
1370
+ # Execute Table Operator query and return results.
1371
+ return self.__execute(output_style)
1372
+
1373
+ def install_file(self, file_identifier, file_name, is_binary = False,
1374
+ replace = False, force_replace = False):
1375
+ """
1376
+ DESCRIPTION:
1377
+ Function to install script on Vantage.
1378
+ On success, prints a message that file is installed.
1379
+ This language script can be executed via execute_script() function.
1380
+
1381
+ PARAMETERS:
1382
+ file_identifier:
1383
+ Required Argument.
1384
+ Specifies the name associated with the user-installed file.
1385
+ It cannot have a schema name associated with it,
1386
+ as the file is always installed in the current schema.
1387
+ The name should be unique within the schema. It can be any valid Teradata
1388
+ identifier.
1389
+ Types: str
1390
+
1391
+ file_name:
1392
+ Required Argument:
1393
+ Specifies the name of the file user wnats to install.
1394
+ Types: str
1395
+
1396
+ is_binary:
1397
+ Optional Argument.
1398
+ Specifies if file to be installed is a binary file.
1399
+ Default Value: False
1400
+ Types: bool
1401
+
1402
+ replace:
1403
+ Optional Argument.
1404
+ Specifies if the file is to be installed or replaced.
1405
+ If set to True, then the file is replaced based on value the of
1406
+ force_replace.
1407
+ If set to False, then the file is installed.
1408
+ Default Value: False
1409
+ Types: bool
1410
+
1411
+ force_replace:
1412
+ Optional Argument.
1413
+ Specifies if system should check for the file being used before
1414
+ replacing it.
1415
+ If set to True, then the file is replaced even if it is being executed.
1416
+ If set to False, then an error is thrown if it is being executed.
1417
+ Default Value: False
1418
+ Types: bool
1419
+
1420
+ RETURNS:
1421
+ True, if success
1422
+
1423
+ RAISES:
1424
+ TeradataMLException.
1425
+
1426
+ EXAMPLES:
1427
+ # Note - Refer to User Guide for setting search path and required permissions.
1428
+ # Example 1: Install the file mapper.py found at the relative path
1429
+ # data/scripts/ using the default text mode.
1430
+
1431
+ # Set SEARCHUIFDBPATH.
1432
+ >>> execute_sql("SET SESSION SEARCHUIFDBPATH = alice;")
1433
+
1434
+ # Create a Script object that allows us to execute script on Vantage.
1435
+ >>> import os
1436
+ >>> from teradatasqlalchemy import VARCHAR
1437
+ >>> td_path = os.path.dirname(teradataml.__file__)
1438
+ >>> sto = Script(data=barrierdf,
1439
+ ... script_name='mapper.py',
1440
+ ... files_local_path= os.path.join(td_path, 'data', "scripts"),
1441
+ ... script_command='tdpython3 ./alice/mapper.py',
1442
+ ... data_order_column="Id",
1443
+ ... is_local_order=False,
1444
+ ... nulls_first=False,
1445
+ ... sort_ascending=False,
1446
+ ... charset='latin',
1447
+ ... returns=OrderedDict([("word", VARCHAR(15)),("count_input", VARCHAR(2))]))
1448
+ >>>
1449
+
1450
+ # Install file on Vantage.
1451
+
1452
+ >>> sto.install_file(file_identifier='mapper',
1453
+ ... file_name='mapper.py',
1454
+ ... is_binary=False)
1455
+ File mapper.py installed in Vantage
1456
+
1457
+ # Replace file on Vantage.
1458
+ >>> sto.install_file(file_identifier='mapper',
1459
+ ... file_name='mapper.py',
1460
+ ... is_binary=False,
1461
+ ... replace=True,
1462
+ ... force_replace=True)
1463
+ File mapper.py replaced in Vantage
1464
+ """
1465
+ # Install/Replace file on Vantage.
1466
+ try:
1467
+ file_path = os.path.join(self.files_local_path, file_name)
1468
+ # Install file on Vantage.
1469
+ install_file(file_identifier=file_identifier, file_path=file_path,
1470
+ is_binary=is_binary,
1471
+ replace=replace, force_replace=force_replace)
1472
+ except:
1473
+ raise
1474
+
1475
+ def remove_file(self, file_identifier, force_remove=False):
1476
+ """
1477
+ DESCRIPTION:
1478
+ Function to remove user installed files/scripts from Vantage.
1479
+
1480
+ PARAMETERS:
1481
+ file_identifier:
1482
+ Required Argument.
1483
+ Specifies the name associated with the user-installed file.
1484
+ It cannot have a database name associated with it,
1485
+ as the file is always installed in the current database.
1486
+ Types: str
1487
+
1488
+ force_remove:
1489
+ Required Argument.
1490
+ Specifies if system should check for the file being used before
1491
+ removing it.
1492
+ If set to True, then the file is removed even if it is being executed.
1493
+ If set to False, then an error is thrown if it is being executed.
1494
+ Default value: False
1495
+ Types: bool
1496
+
1497
+ RETURNS:
1498
+ True, if success.
1499
+
1500
+ RAISES:
1501
+ TeradataMLException.
1502
+
1503
+ EXAMPLES:
1504
+ # Note - Refer to User Guide for setting search path and required permissions.
1505
+ # Run install_file example before removing file.
1506
+
1507
+ # Set SEARCHUIFDBPATH.
1508
+ >>> execute_sql("SET SESSION SEARCHUIFDBPATH = alice;")
1509
+
1510
+ # Create a Script object that allows us to execute script on Vantage.
1511
+ >>> sto = Script(data=barrierdf,
1512
+ ... script_name='mapper.py',
1513
+ ... files_local_path= os.path.join(td_path, 'data', "scripts"),
1514
+ ... script_command='tdpython3 ./alice/mapper.py',
1515
+ ... data_order_column="Id",
1516
+ ... is_local_order=False,
1517
+ ... nulls_first=False,
1518
+ ... sort_ascending=False,
1519
+ ... charset='latin',
1520
+ ... returns=OrderedDict([("word", VARCHAR(15)),("count_input", VARCHAR(2))]))
1521
+ >>>
1522
+
1523
+ # Install file on Vantage.
1524
+ >>> sto.install_file(file_identifier='mapper',
1525
+ ... file_name='mapper.py',
1526
+ ... is_binary=False,
1527
+ ... replace=True,
1528
+ ... force_replace=True)
1529
+ File mapper.py replaced in Vantage
1530
+
1531
+ # Remove the installed file.
1532
+ >>> sto.remove_file(file_identifier='mapper', force_remove=True)
1533
+ File mapper removed from Vantage
1534
+
1535
+ """
1536
+ # Remove file from Vantage
1537
+ try:
1538
+ remove_file(file_identifier, force_remove)
1539
+ except:
1540
+ raise
1541
+
1542
+ def __form_table_operator_query(self):
1543
+ """
1544
+ Function to generate the Table Operator queries. The function defines
1545
+ variables and list of arguments required to form the query.
1546
+ """
1547
+ # Output table arguments list.
1548
+ self.__func_output_args_sql_names = []
1549
+ self.__func_output_args = []
1550
+
1551
+ # Generate lists for rest of the function arguments.
1552
+ self.__func_other_arg_sql_names = []
1553
+ self.__func_other_args = []
1554
+ self.__func_other_arg_json_datatypes = []
1555
+
1556
+ self.__func_other_arg_sql_names.append("SCRIPT_COMMAND")
1557
+ self.__func_other_args.append(
1558
+ UtilFuncs._teradata_collapse_arglist(self.script_command, "'"))
1559
+ self.__func_other_arg_json_datatypes.append("STRING")
1560
+
1561
+ if self.delimiter is not None:
1562
+ self.__func_other_arg_sql_names.append("delimiter")
1563
+ self.__func_other_args.append(
1564
+ UtilFuncs._teradata_collapse_arglist(self.delimiter, "'"))
1565
+ self.__func_other_arg_json_datatypes.append("STRING")
1566
+
1567
+ # Generate returns clause.
1568
+ ret_vals = []
1569
+ returns_clause = ''
1570
+ for key in self.returns.keys():
1571
+ ret_vals.append('{} {}'.format(key, self.returns[key].compile(td_dialect())))
1572
+ returns_clause = ', '.join(ret_vals)
1573
+
1574
+ self.__func_other_arg_sql_names.append("returns")
1575
+ self.__func_other_args.append(
1576
+ UtilFuncs._teradata_collapse_arglist(returns_clause, "'"))
1577
+ self.__func_other_arg_json_datatypes.append("STRING")
1578
+
1579
+ if self.auth is not None:
1580
+ self.__func_other_arg_sql_names.append("auth")
1581
+ self.__func_other_args.append(
1582
+ UtilFuncs._teradata_collapse_arglist(self.auth, "'"))
1583
+ self.__func_other_arg_json_datatypes.append("STRING")
1584
+
1585
+ if self.charset is not None:
1586
+ self.__func_other_arg_sql_names.append("charset")
1587
+ self.__func_other_args.append(
1588
+ UtilFuncs._teradata_collapse_arglist(self.charset, "'"))
1589
+ self.__func_other_arg_json_datatypes.append("STRING")
1590
+
1591
+ if self.quotechar is not None:
1592
+ self.__func_other_arg_sql_names.append("quotechar")
1593
+ self.__func_other_args.append(
1594
+ UtilFuncs._teradata_collapse_arglist(self.quotechar, "'"))
1595
+ self.__func_other_arg_json_datatypes.append("STRING")
1596
+
1597
+ # Declare empty lists to hold input table information.
1598
+ self.__func_input_arg_sql_names = []
1599
+ self.__func_input_table_view_query = []
1600
+ self.__func_input_dataframe_type = []
1601
+ self.__func_input_distribution = []
1602
+ self.__func_input_partition_by_cols = []
1603
+ self.__func_input_order_by_cols = []
1604
+ self.__func_input_order_by_type = []
1605
+ self.__func_input_sort_ascending = self.sort_ascending
1606
+ self.__func_input_nulls_first = None
1607
+
1608
+ # Process data.
1609
+ if self.data is not None:
1610
+ data_distribution = "FACT"
1611
+ if self.data_hash_column is not None:
1612
+ data_distribution = "HASH"
1613
+ data_partition_column = UtilFuncs._teradata_collapse_arglist(
1614
+ self.data_hash_column, "\"")
1615
+ else:
1616
+ if self.__awu._is_default_or_not(self.data_partition_column, "ANY"):
1617
+ data_partition_column = UtilFuncs._teradata_collapse_arglist(
1618
+ self.data_partition_column, "\"")
1619
+ else:
1620
+ data_partition_column = None
1621
+ if self.data_order_column is not None:
1622
+ if self.is_local_order:
1623
+ self.__func_input_order_by_type.append("LOCAL")
1624
+ if not self.data_hash_column:
1625
+ data_distribution = None
1626
+ else:
1627
+ self.__func_input_order_by_type.append(None)
1628
+ self.__func_input_order_by_cols.append(
1629
+ UtilFuncs._teradata_collapse_arglist(self.data_order_column, "\""))
1630
+ else:
1631
+ self.__func_input_order_by_type.append(None)
1632
+ self.__func_input_order_by_cols.append("NA_character_")
1633
+
1634
+ self.__table_ref = self.__awu._teradata_on_clause_from_dataframe(self.data,
1635
+ False)
1636
+ self.__func_input_distribution.append(data_distribution)
1637
+ self.__func_input_arg_sql_names.append("input")
1638
+ self.__func_input_table_view_query.append(self.__table_ref["ref"])
1639
+ self.__func_input_dataframe_type.append(self.__table_ref["ref_type"])
1640
+ self.__func_input_partition_by_cols.append(data_partition_column)
1641
+ self.__func_input_nulls_first = self.nulls_first
1642
+
1643
+ function_name = "Script"
1644
+ # Create instance to generate Table Operator Query.
1645
+ aqg_obj = TableOperatorQueryGenerator(function_name,
1646
+ self.__func_input_arg_sql_names,
1647
+ self.__func_input_table_view_query,
1648
+ self.__func_input_dataframe_type,
1649
+ self.__func_input_distribution,
1650
+ self.__func_input_partition_by_cols,
1651
+ self.__func_input_order_by_cols,
1652
+ self.__func_other_arg_sql_names,
1653
+ self.__func_other_args,
1654
+ self.__func_other_arg_json_datatypes,
1655
+ self.__func_output_args_sql_names,
1656
+ self.__func_output_args,
1657
+ self.__func_input_order_by_type,
1658
+ self.__func_input_sort_ascending,
1659
+ self.__func_input_nulls_first,
1660
+ engine="ENGINE_SQL"
1661
+ )
1662
+
1663
+ # Invoke call to Table operator query generation.
1664
+ self._tblop_query = aqg_obj._gen_table_operator_select_stmt_sql()
1665
+
1666
+ # Print Table Operator query if requested to do so.
1667
+ if display.print_sqlmr_query:
1668
+ print(self._tblop_query)
1669
+
1670
+ def __execute(self, output_style='VIEW'):
1671
+ """
1672
+ DESCRIPTION:
1673
+ Function to execute Table Operator queries.
1674
+ Create DataFrames for the required Table Operator output.
1675
+
1676
+ PARAMETERS:
1677
+ output_style:
1678
+ Specifies the type of output object to create - a table of a view.
1679
+ Permitted Values: 'VIEW', 'TABLE'.
1680
+ Default value: 'VIEW'
1681
+ Types: str
1682
+
1683
+ RAISES:
1684
+ None.
1685
+
1686
+ RETURNS:
1687
+ None.
1688
+
1689
+ EXAMPLES:
1690
+ >>> return self.__execute(output_style)
1691
+ """
1692
+ # Generate STDOUT table name and add it to the output table list.
1693
+ if output_style == OutputStyle.OUTPUT_TABLE.value:
1694
+ table_type = TeradataConstants.TERADATA_TABLE
1695
+ else:
1696
+ table_type = TeradataConstants.TERADATA_VIEW
1697
+
1698
+ tblop_stdout_temp_tablename = \
1699
+ UtilFuncs._generate_temp_table_name(prefix="td_tblop_out_",
1700
+ use_default_database=True,
1701
+ gc_on_quit=True, quote=False,
1702
+ table_type=table_type)
1703
+ try:
1704
+ if output_style == OutputStyle.OUTPUT_TABLE.value:
1705
+ UtilFuncs._create_table(tblop_stdout_temp_tablename, self._tblop_query)
1706
+ else:
1707
+ UtilFuncs._create_view(tblop_stdout_temp_tablename, self._tblop_query)
1708
+ except Exception as emsg:
1709
+ raise TeradataMlException(
1710
+ Messages.get_message(MessageCodes.TDMLDF_EXEC_SQL_FAILED, str(emsg)),
1711
+ MessageCodes.TDMLDF_EXEC_SQL_FAILED)
1712
+
1713
+ self.result = self.__awu._create_data_set_object(
1714
+ df_input=UtilFuncs._extract_table_name(tblop_stdout_temp_tablename),
1715
+ source_type="table",
1716
+ database_name=UtilFuncs._extract_db_name(tblop_stdout_temp_tablename))
1717
+
1718
+ return self.result