teradataml 20.0.0.8__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (1208) hide show
  1. teradataml/LICENSE-3RD-PARTY.pdf +0 -0
  2. teradataml/LICENSE.pdf +0 -0
  3. teradataml/README.md +2762 -0
  4. teradataml/__init__.py +78 -0
  5. teradataml/_version.py +11 -0
  6. teradataml/analytics/Transformations.py +2996 -0
  7. teradataml/analytics/__init__.py +82 -0
  8. teradataml/analytics/analytic_function_executor.py +2416 -0
  9. teradataml/analytics/analytic_query_generator.py +1050 -0
  10. teradataml/analytics/byom/H2OPredict.py +514 -0
  11. teradataml/analytics/byom/PMMLPredict.py +437 -0
  12. teradataml/analytics/byom/__init__.py +16 -0
  13. teradataml/analytics/json_parser/__init__.py +133 -0
  14. teradataml/analytics/json_parser/analytic_functions_argument.py +1805 -0
  15. teradataml/analytics/json_parser/json_store.py +191 -0
  16. teradataml/analytics/json_parser/metadata.py +1666 -0
  17. teradataml/analytics/json_parser/utils.py +805 -0
  18. teradataml/analytics/meta_class.py +236 -0
  19. teradataml/analytics/sqle/DecisionTreePredict.py +456 -0
  20. teradataml/analytics/sqle/NaiveBayesPredict.py +420 -0
  21. teradataml/analytics/sqle/__init__.py +128 -0
  22. teradataml/analytics/sqle/json/decisiontreepredict_sqle.json +78 -0
  23. teradataml/analytics/sqle/json/naivebayespredict_sqle.json +62 -0
  24. teradataml/analytics/table_operator/__init__.py +11 -0
  25. teradataml/analytics/uaf/__init__.py +82 -0
  26. teradataml/analytics/utils.py +828 -0
  27. teradataml/analytics/valib.py +1617 -0
  28. teradataml/automl/__init__.py +5835 -0
  29. teradataml/automl/autodataprep/__init__.py +493 -0
  30. teradataml/automl/custom_json_utils.py +1625 -0
  31. teradataml/automl/data_preparation.py +1384 -0
  32. teradataml/automl/data_transformation.py +1254 -0
  33. teradataml/automl/feature_engineering.py +2273 -0
  34. teradataml/automl/feature_exploration.py +1873 -0
  35. teradataml/automl/model_evaluation.py +488 -0
  36. teradataml/automl/model_training.py +1407 -0
  37. teradataml/catalog/__init__.py +2 -0
  38. teradataml/catalog/byom.py +1759 -0
  39. teradataml/catalog/function_argument_mapper.py +859 -0
  40. teradataml/catalog/model_cataloging_utils.py +491 -0
  41. teradataml/clients/__init__.py +0 -0
  42. teradataml/clients/auth_client.py +137 -0
  43. teradataml/clients/keycloak_client.py +165 -0
  44. teradataml/clients/pkce_client.py +481 -0
  45. teradataml/common/__init__.py +1 -0
  46. teradataml/common/aed_utils.py +2078 -0
  47. teradataml/common/bulk_exposed_utils.py +113 -0
  48. teradataml/common/constants.py +1669 -0
  49. teradataml/common/deprecations.py +166 -0
  50. teradataml/common/exceptions.py +147 -0
  51. teradataml/common/formula.py +743 -0
  52. teradataml/common/garbagecollector.py +666 -0
  53. teradataml/common/logger.py +1261 -0
  54. teradataml/common/messagecodes.py +518 -0
  55. teradataml/common/messages.py +262 -0
  56. teradataml/common/pylogger.py +67 -0
  57. teradataml/common/sqlbundle.py +764 -0
  58. teradataml/common/td_coltype_code_to_tdtype.py +48 -0
  59. teradataml/common/utils.py +3166 -0
  60. teradataml/common/warnings.py +36 -0
  61. teradataml/common/wrapper_utils.py +625 -0
  62. teradataml/config/__init__.py +0 -0
  63. teradataml/config/dummy_file1.cfg +5 -0
  64. teradataml/config/dummy_file2.cfg +3 -0
  65. teradataml/config/sqlengine_alias_definitions_v1.0 +14 -0
  66. teradataml/config/sqlengine_alias_definitions_v1.1 +20 -0
  67. teradataml/config/sqlengine_alias_definitions_v1.3 +19 -0
  68. teradataml/context/__init__.py +0 -0
  69. teradataml/context/aed_context.py +223 -0
  70. teradataml/context/context.py +1462 -0
  71. teradataml/data/A_loan.csv +19 -0
  72. teradataml/data/BINARY_REALS_LEFT.csv +11 -0
  73. teradataml/data/BINARY_REALS_RIGHT.csv +11 -0
  74. teradataml/data/B_loan.csv +49 -0
  75. teradataml/data/BuoyData2.csv +17 -0
  76. teradataml/data/CONVOLVE2_COMPLEX_LEFT.csv +5 -0
  77. teradataml/data/CONVOLVE2_COMPLEX_RIGHT.csv +5 -0
  78. teradataml/data/Convolve2RealsLeft.csv +5 -0
  79. teradataml/data/Convolve2RealsRight.csv +5 -0
  80. teradataml/data/Convolve2ValidLeft.csv +11 -0
  81. teradataml/data/Convolve2ValidRight.csv +11 -0
  82. teradataml/data/DFFTConv_Real_8_8.csv +65 -0
  83. teradataml/data/Employee.csv +5 -0
  84. teradataml/data/Employee_Address.csv +4 -0
  85. teradataml/data/Employee_roles.csv +5 -0
  86. teradataml/data/JulesBelvezeDummyData.csv +100 -0
  87. teradataml/data/Mall_customer_data.csv +201 -0
  88. teradataml/data/Orders1_12mf.csv +25 -0
  89. teradataml/data/Pi_loan.csv +7 -0
  90. teradataml/data/SMOOTHED_DATA.csv +7 -0
  91. teradataml/data/TestDFFT8.csv +9 -0
  92. teradataml/data/TestRiver.csv +109 -0
  93. teradataml/data/Traindata.csv +28 -0
  94. teradataml/data/__init__.py +0 -0
  95. teradataml/data/acf.csv +17 -0
  96. teradataml/data/adaboost_example.json +34 -0
  97. teradataml/data/adaboostpredict_example.json +24 -0
  98. teradataml/data/additional_table.csv +11 -0
  99. teradataml/data/admissions_test.csv +21 -0
  100. teradataml/data/admissions_train.csv +41 -0
  101. teradataml/data/admissions_train_nulls.csv +41 -0
  102. teradataml/data/advertising.csv +201 -0
  103. teradataml/data/ageandheight.csv +13 -0
  104. teradataml/data/ageandpressure.csv +31 -0
  105. teradataml/data/amazon_reviews_25.csv +26 -0
  106. teradataml/data/antiselect_example.json +36 -0
  107. teradataml/data/antiselect_input.csv +8 -0
  108. teradataml/data/antiselect_input_mixed_case.csv +8 -0
  109. teradataml/data/applicant_external.csv +7 -0
  110. teradataml/data/applicant_reference.csv +7 -0
  111. teradataml/data/apriori_example.json +22 -0
  112. teradataml/data/arima_example.json +9 -0
  113. teradataml/data/assortedtext_input.csv +8 -0
  114. teradataml/data/attribution_example.json +34 -0
  115. teradataml/data/attribution_sample_table.csv +27 -0
  116. teradataml/data/attribution_sample_table1.csv +6 -0
  117. teradataml/data/attribution_sample_table2.csv +11 -0
  118. teradataml/data/bank_churn.csv +10001 -0
  119. teradataml/data/bank_marketing.csv +11163 -0
  120. teradataml/data/bank_web_clicks1.csv +43 -0
  121. teradataml/data/bank_web_clicks2.csv +91 -0
  122. teradataml/data/bank_web_url.csv +85 -0
  123. teradataml/data/barrier.csv +2 -0
  124. teradataml/data/barrier_new.csv +3 -0
  125. teradataml/data/betweenness_example.json +14 -0
  126. teradataml/data/bike_sharing.csv +732 -0
  127. teradataml/data/bin_breaks.csv +8 -0
  128. teradataml/data/bin_fit_ip.csv +4 -0
  129. teradataml/data/binary_complex_left.csv +11 -0
  130. teradataml/data/binary_complex_right.csv +11 -0
  131. teradataml/data/binary_matrix_complex_left.csv +21 -0
  132. teradataml/data/binary_matrix_complex_right.csv +21 -0
  133. teradataml/data/binary_matrix_real_left.csv +21 -0
  134. teradataml/data/binary_matrix_real_right.csv +21 -0
  135. teradataml/data/blood2ageandweight.csv +26 -0
  136. teradataml/data/bmi.csv +501 -0
  137. teradataml/data/boston.csv +507 -0
  138. teradataml/data/boston2cols.csv +721 -0
  139. teradataml/data/breast_cancer.csv +570 -0
  140. teradataml/data/buoydata_mix.csv +11 -0
  141. teradataml/data/burst_data.csv +5 -0
  142. teradataml/data/burst_example.json +21 -0
  143. teradataml/data/byom_example.json +34 -0
  144. teradataml/data/bytes_table.csv +4 -0
  145. teradataml/data/cal_housing_ex_raw.csv +70 -0
  146. teradataml/data/callers.csv +7 -0
  147. teradataml/data/calls.csv +10 -0
  148. teradataml/data/cars_hist.csv +33 -0
  149. teradataml/data/cat_table.csv +25 -0
  150. teradataml/data/ccm_example.json +32 -0
  151. teradataml/data/ccm_input.csv +91 -0
  152. teradataml/data/ccm_input2.csv +13 -0
  153. teradataml/data/ccmexample.csv +101 -0
  154. teradataml/data/ccmprepare_example.json +9 -0
  155. teradataml/data/ccmprepare_input.csv +91 -0
  156. teradataml/data/cfilter_example.json +12 -0
  157. teradataml/data/changepointdetection_example.json +18 -0
  158. teradataml/data/changepointdetectionrt_example.json +8 -0
  159. teradataml/data/chi_sq.csv +3 -0
  160. teradataml/data/churn_data.csv +14 -0
  161. teradataml/data/churn_emission.csv +35 -0
  162. teradataml/data/churn_initial.csv +3 -0
  163. teradataml/data/churn_state_transition.csv +5 -0
  164. teradataml/data/citedges_2.csv +745 -0
  165. teradataml/data/citvertices_2.csv +1210 -0
  166. teradataml/data/clicks2.csv +16 -0
  167. teradataml/data/clickstream.csv +13 -0
  168. teradataml/data/clickstream1.csv +11 -0
  169. teradataml/data/closeness_example.json +16 -0
  170. teradataml/data/complaints.csv +21 -0
  171. teradataml/data/complaints_mini.csv +3 -0
  172. teradataml/data/complaints_test_tokenized.csv +353 -0
  173. teradataml/data/complaints_testtoken.csv +224 -0
  174. teradataml/data/complaints_tokens_model.csv +348 -0
  175. teradataml/data/complaints_tokens_test.csv +353 -0
  176. teradataml/data/complaints_traintoken.csv +472 -0
  177. teradataml/data/computers_category.csv +1001 -0
  178. teradataml/data/computers_test1.csv +1252 -0
  179. teradataml/data/computers_train1.csv +5009 -0
  180. teradataml/data/computers_train1_clustered.csv +5009 -0
  181. teradataml/data/confusionmatrix_example.json +9 -0
  182. teradataml/data/conversion_event_table.csv +3 -0
  183. teradataml/data/corr_input.csv +17 -0
  184. teradataml/data/correlation_example.json +11 -0
  185. teradataml/data/covid_confirm_sd.csv +83 -0
  186. teradataml/data/coxhazardratio_example.json +39 -0
  187. teradataml/data/coxph_example.json +15 -0
  188. teradataml/data/coxsurvival_example.json +28 -0
  189. teradataml/data/cpt.csv +41 -0
  190. teradataml/data/credit_ex_merged.csv +45 -0
  191. teradataml/data/creditcard_data.csv +1001 -0
  192. teradataml/data/customer_loyalty.csv +301 -0
  193. teradataml/data/customer_loyalty_newseq.csv +31 -0
  194. teradataml/data/customer_segmentation_test.csv +2628 -0
  195. teradataml/data/customer_segmentation_train.csv +8069 -0
  196. teradataml/data/dataframe_example.json +173 -0
  197. teradataml/data/decisionforest_example.json +37 -0
  198. teradataml/data/decisionforestpredict_example.json +38 -0
  199. teradataml/data/decisiontree_example.json +21 -0
  200. teradataml/data/decisiontreepredict_example.json +45 -0
  201. teradataml/data/dfft2_size4_real.csv +17 -0
  202. teradataml/data/dfft2_test_matrix16.csv +17 -0
  203. teradataml/data/dfft2conv_real_4_4.csv +65 -0
  204. teradataml/data/diabetes.csv +443 -0
  205. teradataml/data/diabetes_test.csv +89 -0
  206. teradataml/data/dict_table.csv +5 -0
  207. teradataml/data/docperterm_table.csv +4 -0
  208. teradataml/data/docs/__init__.py +1 -0
  209. teradataml/data/docs/byom/__init__.py +0 -0
  210. teradataml/data/docs/byom/docs/DataRobotPredict.py +180 -0
  211. teradataml/data/docs/byom/docs/DataikuPredict.py +217 -0
  212. teradataml/data/docs/byom/docs/H2OPredict.py +325 -0
  213. teradataml/data/docs/byom/docs/ONNXEmbeddings.py +242 -0
  214. teradataml/data/docs/byom/docs/ONNXPredict.py +283 -0
  215. teradataml/data/docs/byom/docs/ONNXSeq2Seq.py +255 -0
  216. teradataml/data/docs/byom/docs/PMMLPredict.py +278 -0
  217. teradataml/data/docs/byom/docs/__init__.py +0 -0
  218. teradataml/data/docs/sqle/__init__.py +0 -0
  219. teradataml/data/docs/sqle/docs_17_10/Antiselect.py +83 -0
  220. teradataml/data/docs/sqle/docs_17_10/Attribution.py +200 -0
  221. teradataml/data/docs/sqle/docs_17_10/BincodeFit.py +172 -0
  222. teradataml/data/docs/sqle/docs_17_10/BincodeTransform.py +131 -0
  223. teradataml/data/docs/sqle/docs_17_10/CategoricalSummary.py +86 -0
  224. teradataml/data/docs/sqle/docs_17_10/ChiSq.py +90 -0
  225. teradataml/data/docs/sqle/docs_17_10/ColumnSummary.py +86 -0
  226. teradataml/data/docs/sqle/docs_17_10/ConvertTo.py +96 -0
  227. teradataml/data/docs/sqle/docs_17_10/DecisionForestPredict.py +139 -0
  228. teradataml/data/docs/sqle/docs_17_10/DecisionTreePredict.py +152 -0
  229. teradataml/data/docs/sqle/docs_17_10/FTest.py +161 -0
  230. teradataml/data/docs/sqle/docs_17_10/FillRowId.py +83 -0
  231. teradataml/data/docs/sqle/docs_17_10/Fit.py +88 -0
  232. teradataml/data/docs/sqle/docs_17_10/GLMPredict.py +144 -0
  233. teradataml/data/docs/sqle/docs_17_10/GetRowsWithMissingValues.py +85 -0
  234. teradataml/data/docs/sqle/docs_17_10/GetRowsWithoutMissingValues.py +82 -0
  235. teradataml/data/docs/sqle/docs_17_10/Histogram.py +165 -0
  236. teradataml/data/docs/sqle/docs_17_10/MovingAverage.py +134 -0
  237. teradataml/data/docs/sqle/docs_17_10/NGramSplitter.py +209 -0
  238. teradataml/data/docs/sqle/docs_17_10/NPath.py +266 -0
  239. teradataml/data/docs/sqle/docs_17_10/NaiveBayesPredict.py +116 -0
  240. teradataml/data/docs/sqle/docs_17_10/NaiveBayesTextClassifierPredict.py +176 -0
  241. teradataml/data/docs/sqle/docs_17_10/NumApply.py +147 -0
  242. teradataml/data/docs/sqle/docs_17_10/OneHotEncodingFit.py +135 -0
  243. teradataml/data/docs/sqle/docs_17_10/OneHotEncodingTransform.py +109 -0
  244. teradataml/data/docs/sqle/docs_17_10/OutlierFilterFit.py +166 -0
  245. teradataml/data/docs/sqle/docs_17_10/OutlierFilterTransform.py +105 -0
  246. teradataml/data/docs/sqle/docs_17_10/Pack.py +128 -0
  247. teradataml/data/docs/sqle/docs_17_10/PolynomialFeaturesFit.py +112 -0
  248. teradataml/data/docs/sqle/docs_17_10/PolynomialFeaturesTransform.py +102 -0
  249. teradataml/data/docs/sqle/docs_17_10/QQNorm.py +105 -0
  250. teradataml/data/docs/sqle/docs_17_10/RoundColumns.py +110 -0
  251. teradataml/data/docs/sqle/docs_17_10/RowNormalizeFit.py +118 -0
  252. teradataml/data/docs/sqle/docs_17_10/RowNormalizeTransform.py +99 -0
  253. teradataml/data/docs/sqle/docs_17_10/SVMSparsePredict.py +153 -0
  254. teradataml/data/docs/sqle/docs_17_10/ScaleFit.py +197 -0
  255. teradataml/data/docs/sqle/docs_17_10/ScaleTransform.py +99 -0
  256. teradataml/data/docs/sqle/docs_17_10/Sessionize.py +114 -0
  257. teradataml/data/docs/sqle/docs_17_10/SimpleImputeFit.py +116 -0
  258. teradataml/data/docs/sqle/docs_17_10/SimpleImputeTransform.py +98 -0
  259. teradataml/data/docs/sqle/docs_17_10/StrApply.py +187 -0
  260. teradataml/data/docs/sqle/docs_17_10/StringSimilarity.py +146 -0
  261. teradataml/data/docs/sqle/docs_17_10/Transform.py +105 -0
  262. teradataml/data/docs/sqle/docs_17_10/UnivariateStatistics.py +142 -0
  263. teradataml/data/docs/sqle/docs_17_10/Unpack.py +214 -0
  264. teradataml/data/docs/sqle/docs_17_10/WhichMax.py +83 -0
  265. teradataml/data/docs/sqle/docs_17_10/WhichMin.py +83 -0
  266. teradataml/data/docs/sqle/docs_17_10/ZTest.py +155 -0
  267. teradataml/data/docs/sqle/docs_17_10/__init__.py +0 -0
  268. teradataml/data/docs/sqle/docs_17_20/ANOVA.py +186 -0
  269. teradataml/data/docs/sqle/docs_17_20/Antiselect.py +83 -0
  270. teradataml/data/docs/sqle/docs_17_20/Apriori.py +138 -0
  271. teradataml/data/docs/sqle/docs_17_20/Attribution.py +201 -0
  272. teradataml/data/docs/sqle/docs_17_20/BincodeFit.py +172 -0
  273. teradataml/data/docs/sqle/docs_17_20/BincodeTransform.py +139 -0
  274. teradataml/data/docs/sqle/docs_17_20/CFilter.py +132 -0
  275. teradataml/data/docs/sqle/docs_17_20/CategoricalSummary.py +86 -0
  276. teradataml/data/docs/sqle/docs_17_20/ChiSq.py +90 -0
  277. teradataml/data/docs/sqle/docs_17_20/ClassificationEvaluator.py +166 -0
  278. teradataml/data/docs/sqle/docs_17_20/ColumnSummary.py +86 -0
  279. teradataml/data/docs/sqle/docs_17_20/ColumnTransformer.py +246 -0
  280. teradataml/data/docs/sqle/docs_17_20/ConvertTo.py +113 -0
  281. teradataml/data/docs/sqle/docs_17_20/DecisionForest.py +280 -0
  282. teradataml/data/docs/sqle/docs_17_20/DecisionForestPredict.py +144 -0
  283. teradataml/data/docs/sqle/docs_17_20/DecisionTreePredict.py +136 -0
  284. teradataml/data/docs/sqle/docs_17_20/FTest.py +240 -0
  285. teradataml/data/docs/sqle/docs_17_20/FillRowId.py +83 -0
  286. teradataml/data/docs/sqle/docs_17_20/Fit.py +88 -0
  287. teradataml/data/docs/sqle/docs_17_20/GLM.py +541 -0
  288. teradataml/data/docs/sqle/docs_17_20/GLMPerSegment.py +415 -0
  289. teradataml/data/docs/sqle/docs_17_20/GLMPredict.py +144 -0
  290. teradataml/data/docs/sqle/docs_17_20/GLMPredictPerSegment.py +233 -0
  291. teradataml/data/docs/sqle/docs_17_20/GetFutileColumns.py +125 -0
  292. teradataml/data/docs/sqle/docs_17_20/GetRowsWithMissingValues.py +109 -0
  293. teradataml/data/docs/sqle/docs_17_20/GetRowsWithoutMissingValues.py +106 -0
  294. teradataml/data/docs/sqle/docs_17_20/Histogram.py +224 -0
  295. teradataml/data/docs/sqle/docs_17_20/KMeans.py +251 -0
  296. teradataml/data/docs/sqle/docs_17_20/KMeansPredict.py +144 -0
  297. teradataml/data/docs/sqle/docs_17_20/KNN.py +215 -0
  298. teradataml/data/docs/sqle/docs_17_20/MovingAverage.py +134 -0
  299. teradataml/data/docs/sqle/docs_17_20/NERExtractor.py +121 -0
  300. teradataml/data/docs/sqle/docs_17_20/NGramSplitter.py +209 -0
  301. teradataml/data/docs/sqle/docs_17_20/NPath.py +266 -0
  302. teradataml/data/docs/sqle/docs_17_20/NaiveBayes.py +162 -0
  303. teradataml/data/docs/sqle/docs_17_20/NaiveBayesPredict.py +116 -0
  304. teradataml/data/docs/sqle/docs_17_20/NaiveBayesTextClassifierPredict.py +177 -0
  305. teradataml/data/docs/sqle/docs_17_20/NaiveBayesTextClassifierTrainer.py +127 -0
  306. teradataml/data/docs/sqle/docs_17_20/NonLinearCombineFit.py +119 -0
  307. teradataml/data/docs/sqle/docs_17_20/NonLinearCombineTransform.py +112 -0
  308. teradataml/data/docs/sqle/docs_17_20/NumApply.py +147 -0
  309. teradataml/data/docs/sqle/docs_17_20/OneClassSVM.py +307 -0
  310. teradataml/data/docs/sqle/docs_17_20/OneClassSVMPredict.py +185 -0
  311. teradataml/data/docs/sqle/docs_17_20/OneHotEncodingFit.py +231 -0
  312. teradataml/data/docs/sqle/docs_17_20/OneHotEncodingTransform.py +121 -0
  313. teradataml/data/docs/sqle/docs_17_20/OrdinalEncodingFit.py +220 -0
  314. teradataml/data/docs/sqle/docs_17_20/OrdinalEncodingTransform.py +127 -0
  315. teradataml/data/docs/sqle/docs_17_20/OutlierFilterFit.py +191 -0
  316. teradataml/data/docs/sqle/docs_17_20/OutlierFilterTransform.py +117 -0
  317. teradataml/data/docs/sqle/docs_17_20/Pack.py +128 -0
  318. teradataml/data/docs/sqle/docs_17_20/Pivoting.py +279 -0
  319. teradataml/data/docs/sqle/docs_17_20/PolynomialFeaturesFit.py +112 -0
  320. teradataml/data/docs/sqle/docs_17_20/PolynomialFeaturesTransform.py +112 -0
  321. teradataml/data/docs/sqle/docs_17_20/QQNorm.py +105 -0
  322. teradataml/data/docs/sqle/docs_17_20/ROC.py +164 -0
  323. teradataml/data/docs/sqle/docs_17_20/RandomProjectionFit.py +155 -0
  324. teradataml/data/docs/sqle/docs_17_20/RandomProjectionMinComponents.py +106 -0
  325. teradataml/data/docs/sqle/docs_17_20/RandomProjectionTransform.py +120 -0
  326. teradataml/data/docs/sqle/docs_17_20/RegressionEvaluator.py +211 -0
  327. teradataml/data/docs/sqle/docs_17_20/RoundColumns.py +109 -0
  328. teradataml/data/docs/sqle/docs_17_20/RowNormalizeFit.py +118 -0
  329. teradataml/data/docs/sqle/docs_17_20/RowNormalizeTransform.py +111 -0
  330. teradataml/data/docs/sqle/docs_17_20/SMOTE.py +212 -0
  331. teradataml/data/docs/sqle/docs_17_20/SVM.py +414 -0
  332. teradataml/data/docs/sqle/docs_17_20/SVMPredict.py +213 -0
  333. teradataml/data/docs/sqle/docs_17_20/SVMSparsePredict.py +153 -0
  334. teradataml/data/docs/sqle/docs_17_20/ScaleFit.py +315 -0
  335. teradataml/data/docs/sqle/docs_17_20/ScaleTransform.py +202 -0
  336. teradataml/data/docs/sqle/docs_17_20/SentimentExtractor.py +206 -0
  337. teradataml/data/docs/sqle/docs_17_20/Sessionize.py +114 -0
  338. teradataml/data/docs/sqle/docs_17_20/Shap.py +225 -0
  339. teradataml/data/docs/sqle/docs_17_20/Silhouette.py +153 -0
  340. teradataml/data/docs/sqle/docs_17_20/SimpleImputeFit.py +116 -0
  341. teradataml/data/docs/sqle/docs_17_20/SimpleImputeTransform.py +109 -0
  342. teradataml/data/docs/sqle/docs_17_20/StrApply.py +187 -0
  343. teradataml/data/docs/sqle/docs_17_20/StringSimilarity.py +146 -0
  344. teradataml/data/docs/sqle/docs_17_20/TDDecisionForestPredict.py +207 -0
  345. teradataml/data/docs/sqle/docs_17_20/TDGLMPredict.py +333 -0
  346. teradataml/data/docs/sqle/docs_17_20/TDNaiveBayesPredict.py +189 -0
  347. teradataml/data/docs/sqle/docs_17_20/TFIDF.py +142 -0
  348. teradataml/data/docs/sqle/docs_17_20/TargetEncodingFit.py +267 -0
  349. teradataml/data/docs/sqle/docs_17_20/TargetEncodingTransform.py +141 -0
  350. teradataml/data/docs/sqle/docs_17_20/TextMorph.py +119 -0
  351. teradataml/data/docs/sqle/docs_17_20/TextParser.py +224 -0
  352. teradataml/data/docs/sqle/docs_17_20/TrainTestSplit.py +160 -0
  353. teradataml/data/docs/sqle/docs_17_20/Transform.py +123 -0
  354. teradataml/data/docs/sqle/docs_17_20/UnivariateStatistics.py +142 -0
  355. teradataml/data/docs/sqle/docs_17_20/Unpack.py +214 -0
  356. teradataml/data/docs/sqle/docs_17_20/Unpivoting.py +216 -0
  357. teradataml/data/docs/sqle/docs_17_20/VectorDistance.py +169 -0
  358. teradataml/data/docs/sqle/docs_17_20/WhichMax.py +83 -0
  359. teradataml/data/docs/sqle/docs_17_20/WhichMin.py +83 -0
  360. teradataml/data/docs/sqle/docs_17_20/WordEmbeddings.py +237 -0
  361. teradataml/data/docs/sqle/docs_17_20/XGBoost.py +362 -0
  362. teradataml/data/docs/sqle/docs_17_20/XGBoostPredict.py +281 -0
  363. teradataml/data/docs/sqle/docs_17_20/ZTest.py +220 -0
  364. teradataml/data/docs/sqle/docs_17_20/__init__.py +0 -0
  365. teradataml/data/docs/tableoperator/__init__.py +0 -0
  366. teradataml/data/docs/tableoperator/docs_17_00/ReadNOS.py +430 -0
  367. teradataml/data/docs/tableoperator/docs_17_00/__init__.py +0 -0
  368. teradataml/data/docs/tableoperator/docs_17_05/ReadNOS.py +430 -0
  369. teradataml/data/docs/tableoperator/docs_17_05/WriteNOS.py +348 -0
  370. teradataml/data/docs/tableoperator/docs_17_05/__init__.py +0 -0
  371. teradataml/data/docs/tableoperator/docs_17_10/ReadNOS.py +429 -0
  372. teradataml/data/docs/tableoperator/docs_17_10/WriteNOS.py +348 -0
  373. teradataml/data/docs/tableoperator/docs_17_10/__init__.py +0 -0
  374. teradataml/data/docs/tableoperator/docs_17_20/Image2Matrix.py +118 -0
  375. teradataml/data/docs/tableoperator/docs_17_20/ReadNOS.py +440 -0
  376. teradataml/data/docs/tableoperator/docs_17_20/WriteNOS.py +387 -0
  377. teradataml/data/docs/tableoperator/docs_17_20/__init__.py +0 -0
  378. teradataml/data/docs/uaf/__init__.py +0 -0
  379. teradataml/data/docs/uaf/docs_17_20/ACF.py +186 -0
  380. teradataml/data/docs/uaf/docs_17_20/ArimaEstimate.py +370 -0
  381. teradataml/data/docs/uaf/docs_17_20/ArimaForecast.py +172 -0
  382. teradataml/data/docs/uaf/docs_17_20/ArimaValidate.py +161 -0
  383. teradataml/data/docs/uaf/docs_17_20/ArimaXEstimate.py +293 -0
  384. teradataml/data/docs/uaf/docs_17_20/AutoArima.py +354 -0
  385. teradataml/data/docs/uaf/docs_17_20/BinaryMatrixOp.py +248 -0
  386. teradataml/data/docs/uaf/docs_17_20/BinarySeriesOp.py +252 -0
  387. teradataml/data/docs/uaf/docs_17_20/BreuschGodfrey.py +178 -0
  388. teradataml/data/docs/uaf/docs_17_20/BreuschPaganGodfrey.py +175 -0
  389. teradataml/data/docs/uaf/docs_17_20/Convolve.py +230 -0
  390. teradataml/data/docs/uaf/docs_17_20/Convolve2.py +218 -0
  391. teradataml/data/docs/uaf/docs_17_20/CopyArt.py +145 -0
  392. teradataml/data/docs/uaf/docs_17_20/CumulPeriodogram.py +185 -0
  393. teradataml/data/docs/uaf/docs_17_20/DFFT.py +204 -0
  394. teradataml/data/docs/uaf/docs_17_20/DFFT2.py +216 -0
  395. teradataml/data/docs/uaf/docs_17_20/DFFT2Conv.py +216 -0
  396. teradataml/data/docs/uaf/docs_17_20/DFFTConv.py +192 -0
  397. teradataml/data/docs/uaf/docs_17_20/DIFF.py +175 -0
  398. teradataml/data/docs/uaf/docs_17_20/DTW.py +180 -0
  399. teradataml/data/docs/uaf/docs_17_20/DWT.py +235 -0
  400. teradataml/data/docs/uaf/docs_17_20/DWT2D.py +217 -0
  401. teradataml/data/docs/uaf/docs_17_20/DickeyFuller.py +142 -0
  402. teradataml/data/docs/uaf/docs_17_20/DurbinWatson.py +184 -0
  403. teradataml/data/docs/uaf/docs_17_20/ExtractResults.py +185 -0
  404. teradataml/data/docs/uaf/docs_17_20/FilterFactory1d.py +160 -0
  405. teradataml/data/docs/uaf/docs_17_20/FitMetrics.py +172 -0
  406. teradataml/data/docs/uaf/docs_17_20/GenseriesFormula.py +206 -0
  407. teradataml/data/docs/uaf/docs_17_20/GenseriesSinusoids.py +143 -0
  408. teradataml/data/docs/uaf/docs_17_20/GoldfeldQuandt.py +198 -0
  409. teradataml/data/docs/uaf/docs_17_20/HoltWintersForecaster.py +260 -0
  410. teradataml/data/docs/uaf/docs_17_20/IDFFT.py +165 -0
  411. teradataml/data/docs/uaf/docs_17_20/IDFFT2.py +191 -0
  412. teradataml/data/docs/uaf/docs_17_20/IDWT.py +236 -0
  413. teradataml/data/docs/uaf/docs_17_20/IDWT2D.py +226 -0
  414. teradataml/data/docs/uaf/docs_17_20/IQR.py +134 -0
  415. teradataml/data/docs/uaf/docs_17_20/InputValidator.py +121 -0
  416. teradataml/data/docs/uaf/docs_17_20/LineSpec.py +156 -0
  417. teradataml/data/docs/uaf/docs_17_20/LinearRegr.py +215 -0
  418. teradataml/data/docs/uaf/docs_17_20/MAMean.py +174 -0
  419. teradataml/data/docs/uaf/docs_17_20/MInfo.py +134 -0
  420. teradataml/data/docs/uaf/docs_17_20/Matrix2Image.py +297 -0
  421. teradataml/data/docs/uaf/docs_17_20/MatrixMultiply.py +145 -0
  422. teradataml/data/docs/uaf/docs_17_20/MultivarRegr.py +191 -0
  423. teradataml/data/docs/uaf/docs_17_20/PACF.py +157 -0
  424. teradataml/data/docs/uaf/docs_17_20/Portman.py +217 -0
  425. teradataml/data/docs/uaf/docs_17_20/PowerSpec.py +203 -0
  426. teradataml/data/docs/uaf/docs_17_20/PowerTransform.py +155 -0
  427. teradataml/data/docs/uaf/docs_17_20/Resample.py +237 -0
  428. teradataml/data/docs/uaf/docs_17_20/SAX.py +246 -0
  429. teradataml/data/docs/uaf/docs_17_20/SInfo.py +123 -0
  430. teradataml/data/docs/uaf/docs_17_20/SeasonalNormalize.py +173 -0
  431. teradataml/data/docs/uaf/docs_17_20/SelectionCriteria.py +174 -0
  432. teradataml/data/docs/uaf/docs_17_20/SignifPeriodicities.py +171 -0
  433. teradataml/data/docs/uaf/docs_17_20/SignifResidmean.py +164 -0
  434. teradataml/data/docs/uaf/docs_17_20/SimpleExp.py +180 -0
  435. teradataml/data/docs/uaf/docs_17_20/Smoothma.py +208 -0
  436. teradataml/data/docs/uaf/docs_17_20/TrackingOp.py +151 -0
  437. teradataml/data/docs/uaf/docs_17_20/UNDIFF.py +171 -0
  438. teradataml/data/docs/uaf/docs_17_20/Unnormalize.py +202 -0
  439. teradataml/data/docs/uaf/docs_17_20/WhitesGeneral.py +171 -0
  440. teradataml/data/docs/uaf/docs_17_20/WindowDFFT.py +368 -0
  441. teradataml/data/docs/uaf/docs_17_20/__init__.py +0 -0
  442. teradataml/data/dtw_example.json +18 -0
  443. teradataml/data/dtw_t1.csv +11 -0
  444. teradataml/data/dtw_t2.csv +4 -0
  445. teradataml/data/dwt2d_dataTable.csv +65 -0
  446. teradataml/data/dwt2d_example.json +16 -0
  447. teradataml/data/dwt_dataTable.csv +8 -0
  448. teradataml/data/dwt_example.json +15 -0
  449. teradataml/data/dwt_filterTable.csv +3 -0
  450. teradataml/data/dwt_filter_dim.csv +5 -0
  451. teradataml/data/emission.csv +9 -0
  452. teradataml/data/emp_table_by_dept.csv +19 -0
  453. teradataml/data/employee_info.csv +4 -0
  454. teradataml/data/employee_table.csv +6 -0
  455. teradataml/data/excluding_event_table.csv +2 -0
  456. teradataml/data/finance_data.csv +6 -0
  457. teradataml/data/finance_data2.csv +61 -0
  458. teradataml/data/finance_data3.csv +93 -0
  459. teradataml/data/finance_data4.csv +13 -0
  460. teradataml/data/fish.csv +160 -0
  461. teradataml/data/fm_blood2ageandweight.csv +26 -0
  462. teradataml/data/fmeasure_example.json +12 -0
  463. teradataml/data/followers_leaders.csv +10 -0
  464. teradataml/data/fpgrowth_example.json +12 -0
  465. teradataml/data/frequentpaths_example.json +29 -0
  466. teradataml/data/friends.csv +9 -0
  467. teradataml/data/fs_input.csv +33 -0
  468. teradataml/data/fs_input1.csv +33 -0
  469. teradataml/data/genData.csv +513 -0
  470. teradataml/data/geodataframe_example.json +40 -0
  471. teradataml/data/glass_types.csv +215 -0
  472. teradataml/data/glm_admissions_model.csv +12 -0
  473. teradataml/data/glm_example.json +56 -0
  474. teradataml/data/glml1l2_example.json +28 -0
  475. teradataml/data/glml1l2predict_example.json +54 -0
  476. teradataml/data/glmpredict_example.json +54 -0
  477. teradataml/data/gq_t1.csv +21 -0
  478. teradataml/data/grocery_transaction.csv +19 -0
  479. teradataml/data/hconvolve_complex_right.csv +5 -0
  480. teradataml/data/hconvolve_complex_rightmulti.csv +5 -0
  481. teradataml/data/histogram_example.json +12 -0
  482. teradataml/data/hmmdecoder_example.json +79 -0
  483. teradataml/data/hmmevaluator_example.json +25 -0
  484. teradataml/data/hmmsupervised_example.json +10 -0
  485. teradataml/data/hmmunsupervised_example.json +8 -0
  486. teradataml/data/hnsw_alter_data.csv +5 -0
  487. teradataml/data/hnsw_data.csv +10 -0
  488. teradataml/data/house_values.csv +12 -0
  489. teradataml/data/house_values2.csv +13 -0
  490. teradataml/data/housing_cat.csv +7 -0
  491. teradataml/data/housing_data.csv +9 -0
  492. teradataml/data/housing_test.csv +47 -0
  493. teradataml/data/housing_test_binary.csv +47 -0
  494. teradataml/data/housing_train.csv +493 -0
  495. teradataml/data/housing_train_attribute.csv +5 -0
  496. teradataml/data/housing_train_binary.csv +437 -0
  497. teradataml/data/housing_train_parameter.csv +2 -0
  498. teradataml/data/housing_train_response.csv +493 -0
  499. teradataml/data/housing_train_segment.csv +201 -0
  500. teradataml/data/ibm_stock.csv +370 -0
  501. teradataml/data/ibm_stock1.csv +370 -0
  502. teradataml/data/identitymatch_example.json +22 -0
  503. teradataml/data/idf_table.csv +4 -0
  504. teradataml/data/idwt2d_dataTable.csv +5 -0
  505. teradataml/data/idwt_dataTable.csv +8 -0
  506. teradataml/data/idwt_filterTable.csv +3 -0
  507. teradataml/data/impressions.csv +101 -0
  508. teradataml/data/inflation.csv +21 -0
  509. teradataml/data/initial.csv +3 -0
  510. teradataml/data/insect2Cols.csv +61 -0
  511. teradataml/data/insect_sprays.csv +13 -0
  512. teradataml/data/insurance.csv +1339 -0
  513. teradataml/data/interpolator_example.json +13 -0
  514. teradataml/data/interval_data.csv +5 -0
  515. teradataml/data/iris_altinput.csv +481 -0
  516. teradataml/data/iris_attribute_output.csv +8 -0
  517. teradataml/data/iris_attribute_test.csv +121 -0
  518. teradataml/data/iris_attribute_train.csv +481 -0
  519. teradataml/data/iris_category_expect_predict.csv +31 -0
  520. teradataml/data/iris_data.csv +151 -0
  521. teradataml/data/iris_input.csv +151 -0
  522. teradataml/data/iris_response_train.csv +121 -0
  523. teradataml/data/iris_test.csv +31 -0
  524. teradataml/data/iris_train.csv +121 -0
  525. teradataml/data/join_table1.csv +4 -0
  526. teradataml/data/join_table2.csv +4 -0
  527. teradataml/data/jsons/anly_function_name.json +7 -0
  528. teradataml/data/jsons/byom/ONNXSeq2Seq.json +287 -0
  529. teradataml/data/jsons/byom/dataikupredict.json +148 -0
  530. teradataml/data/jsons/byom/datarobotpredict.json +147 -0
  531. teradataml/data/jsons/byom/h2opredict.json +195 -0
  532. teradataml/data/jsons/byom/onnxembeddings.json +267 -0
  533. teradataml/data/jsons/byom/onnxpredict.json +187 -0
  534. teradataml/data/jsons/byom/pmmlpredict.json +147 -0
  535. teradataml/data/jsons/paired_functions.json +450 -0
  536. teradataml/data/jsons/sqle/16.20/Antiselect.json +56 -0
  537. teradataml/data/jsons/sqle/16.20/Attribution.json +249 -0
  538. teradataml/data/jsons/sqle/16.20/DecisionForestPredict.json +156 -0
  539. teradataml/data/jsons/sqle/16.20/DecisionTreePredict.json +170 -0
  540. teradataml/data/jsons/sqle/16.20/GLMPredict.json +122 -0
  541. teradataml/data/jsons/sqle/16.20/MovingAverage.json +367 -0
  542. teradataml/data/jsons/sqle/16.20/NGramSplitter.json +239 -0
  543. teradataml/data/jsons/sqle/16.20/NaiveBayesPredict.json +136 -0
  544. teradataml/data/jsons/sqle/16.20/NaiveBayesTextClassifierPredict.json +235 -0
  545. teradataml/data/jsons/sqle/16.20/Pack.json +98 -0
  546. teradataml/data/jsons/sqle/16.20/SVMSparsePredict.json +162 -0
  547. teradataml/data/jsons/sqle/16.20/Sessionize.json +105 -0
  548. teradataml/data/jsons/sqle/16.20/StringSimilarity.json +86 -0
  549. teradataml/data/jsons/sqle/16.20/Unpack.json +166 -0
  550. teradataml/data/jsons/sqle/16.20/nPath.json +269 -0
  551. teradataml/data/jsons/sqle/17.00/Antiselect.json +56 -0
  552. teradataml/data/jsons/sqle/17.00/Attribution.json +249 -0
  553. teradataml/data/jsons/sqle/17.00/DecisionForestPredict.json +156 -0
  554. teradataml/data/jsons/sqle/17.00/DecisionTreePredict.json +170 -0
  555. teradataml/data/jsons/sqle/17.00/GLMPredict.json +122 -0
  556. teradataml/data/jsons/sqle/17.00/MovingAverage.json +367 -0
  557. teradataml/data/jsons/sqle/17.00/NGramSplitter.json +239 -0
  558. teradataml/data/jsons/sqle/17.00/NaiveBayesPredict.json +136 -0
  559. teradataml/data/jsons/sqle/17.00/NaiveBayesTextClassifierPredict.json +235 -0
  560. teradataml/data/jsons/sqle/17.00/Pack.json +98 -0
  561. teradataml/data/jsons/sqle/17.00/SVMSparsePredict.json +162 -0
  562. teradataml/data/jsons/sqle/17.00/Sessionize.json +105 -0
  563. teradataml/data/jsons/sqle/17.00/StringSimilarity.json +86 -0
  564. teradataml/data/jsons/sqle/17.00/Unpack.json +166 -0
  565. teradataml/data/jsons/sqle/17.00/nPath.json +269 -0
  566. teradataml/data/jsons/sqle/17.05/Antiselect.json +56 -0
  567. teradataml/data/jsons/sqle/17.05/Attribution.json +249 -0
  568. teradataml/data/jsons/sqle/17.05/DecisionForestPredict.json +156 -0
  569. teradataml/data/jsons/sqle/17.05/DecisionTreePredict.json +170 -0
  570. teradataml/data/jsons/sqle/17.05/GLMPredict.json +122 -0
  571. teradataml/data/jsons/sqle/17.05/MovingAverage.json +367 -0
  572. teradataml/data/jsons/sqle/17.05/NGramSplitter.json +239 -0
  573. teradataml/data/jsons/sqle/17.05/NaiveBayesPredict.json +136 -0
  574. teradataml/data/jsons/sqle/17.05/NaiveBayesTextClassifierPredict.json +235 -0
  575. teradataml/data/jsons/sqle/17.05/Pack.json +98 -0
  576. teradataml/data/jsons/sqle/17.05/SVMSparsePredict.json +162 -0
  577. teradataml/data/jsons/sqle/17.05/Sessionize.json +105 -0
  578. teradataml/data/jsons/sqle/17.05/StringSimilarity.json +86 -0
  579. teradataml/data/jsons/sqle/17.05/Unpack.json +166 -0
  580. teradataml/data/jsons/sqle/17.05/nPath.json +269 -0
  581. teradataml/data/jsons/sqle/17.10/Antiselect.json +56 -0
  582. teradataml/data/jsons/sqle/17.10/Attribution.json +249 -0
  583. teradataml/data/jsons/sqle/17.10/DecisionForestPredict.json +185 -0
  584. teradataml/data/jsons/sqle/17.10/DecisionTreePredict.json +172 -0
  585. teradataml/data/jsons/sqle/17.10/GLMPredict.json +151 -0
  586. teradataml/data/jsons/sqle/17.10/MovingAverage.json +368 -0
  587. teradataml/data/jsons/sqle/17.10/NGramSplitter.json +239 -0
  588. teradataml/data/jsons/sqle/17.10/NaiveBayesPredict.json +149 -0
  589. teradataml/data/jsons/sqle/17.10/NaiveBayesTextClassifierPredict.json +288 -0
  590. teradataml/data/jsons/sqle/17.10/Pack.json +133 -0
  591. teradataml/data/jsons/sqle/17.10/SVMSparsePredict.json +193 -0
  592. teradataml/data/jsons/sqle/17.10/Sessionize.json +105 -0
  593. teradataml/data/jsons/sqle/17.10/StringSimilarity.json +86 -0
  594. teradataml/data/jsons/sqle/17.10/TD_BinCodeFit.json +239 -0
  595. teradataml/data/jsons/sqle/17.10/TD_BinCodeTransform.json +70 -0
  596. teradataml/data/jsons/sqle/17.10/TD_CategoricalSummary.json +54 -0
  597. teradataml/data/jsons/sqle/17.10/TD_Chisq.json +68 -0
  598. teradataml/data/jsons/sqle/17.10/TD_ColumnSummary.json +54 -0
  599. teradataml/data/jsons/sqle/17.10/TD_ConvertTo.json +69 -0
  600. teradataml/data/jsons/sqle/17.10/TD_FTest.json +187 -0
  601. teradataml/data/jsons/sqle/17.10/TD_FillRowID.json +52 -0
  602. teradataml/data/jsons/sqle/17.10/TD_FunctionFit.json +46 -0
  603. teradataml/data/jsons/sqle/17.10/TD_FunctionTransform.json +72 -0
  604. teradataml/data/jsons/sqle/17.10/TD_GetRowsWithMissingValues.json +53 -0
  605. teradataml/data/jsons/sqle/17.10/TD_GetRowsWithoutMissingValues.json +53 -0
  606. teradataml/data/jsons/sqle/17.10/TD_Histogram.json +133 -0
  607. teradataml/data/jsons/sqle/17.10/TD_NumApply.json +147 -0
  608. teradataml/data/jsons/sqle/17.10/TD_OneHotEncodingFit.json +183 -0
  609. teradataml/data/jsons/sqle/17.10/TD_OneHotEncodingTransform.json +66 -0
  610. teradataml/data/jsons/sqle/17.10/TD_OutlierFilterFit.json +197 -0
  611. teradataml/data/jsons/sqle/17.10/TD_OutlierFilterTransform.json +48 -0
  612. teradataml/data/jsons/sqle/17.10/TD_PolynomialFeaturesFit.json +114 -0
  613. teradataml/data/jsons/sqle/17.10/TD_PolynomialFeaturesTransform.json +72 -0
  614. teradataml/data/jsons/sqle/17.10/TD_QQNorm.json +112 -0
  615. teradataml/data/jsons/sqle/17.10/TD_RoundColumns.json +93 -0
  616. teradataml/data/jsons/sqle/17.10/TD_RowNormalizeFit.json +128 -0
  617. teradataml/data/jsons/sqle/17.10/TD_RowNormalizeTransform.json +71 -0
  618. teradataml/data/jsons/sqle/17.10/TD_ScaleFit.json +157 -0
  619. teradataml/data/jsons/sqle/17.10/TD_ScaleTransform.json +71 -0
  620. teradataml/data/jsons/sqle/17.10/TD_SimpleImputeFit.json +148 -0
  621. teradataml/data/jsons/sqle/17.10/TD_SimpleImputeTransform.json +48 -0
  622. teradataml/data/jsons/sqle/17.10/TD_StrApply.json +240 -0
  623. teradataml/data/jsons/sqle/17.10/TD_UnivariateStatistics.json +119 -0
  624. teradataml/data/jsons/sqle/17.10/TD_WhichMax.json +53 -0
  625. teradataml/data/jsons/sqle/17.10/TD_WhichMin.json +53 -0
  626. teradataml/data/jsons/sqle/17.10/TD_ZTest.json +171 -0
  627. teradataml/data/jsons/sqle/17.10/Unpack.json +188 -0
  628. teradataml/data/jsons/sqle/17.10/nPath.json +269 -0
  629. teradataml/data/jsons/sqle/17.20/Antiselect.json +56 -0
  630. teradataml/data/jsons/sqle/17.20/Attribution.json +249 -0
  631. teradataml/data/jsons/sqle/17.20/DecisionForestPredict.json +185 -0
  632. teradataml/data/jsons/sqle/17.20/DecisionTreePredict.json +172 -0
  633. teradataml/data/jsons/sqle/17.20/GLMPredict.json +151 -0
  634. teradataml/data/jsons/sqle/17.20/MovingAverage.json +367 -0
  635. teradataml/data/jsons/sqle/17.20/NGramSplitter.json +239 -0
  636. teradataml/data/jsons/sqle/17.20/NaiveBayesPredict.json +149 -0
  637. teradataml/data/jsons/sqle/17.20/NaiveBayesTextClassifierPredict.json +287 -0
  638. teradataml/data/jsons/sqle/17.20/Pack.json +133 -0
  639. teradataml/data/jsons/sqle/17.20/SVMSparsePredict.json +192 -0
  640. teradataml/data/jsons/sqle/17.20/Sessionize.json +105 -0
  641. teradataml/data/jsons/sqle/17.20/StringSimilarity.json +86 -0
  642. teradataml/data/jsons/sqle/17.20/TD_ANOVA.json +149 -0
  643. teradataml/data/jsons/sqle/17.20/TD_Apriori.json +181 -0
  644. teradataml/data/jsons/sqle/17.20/TD_BinCodeFit.json +239 -0
  645. teradataml/data/jsons/sqle/17.20/TD_BinCodeTransform.json +71 -0
  646. teradataml/data/jsons/sqle/17.20/TD_CFilter.json +118 -0
  647. teradataml/data/jsons/sqle/17.20/TD_CategoricalSummary.json +53 -0
  648. teradataml/data/jsons/sqle/17.20/TD_Chisq.json +68 -0
  649. teradataml/data/jsons/sqle/17.20/TD_ClassificationEvaluator.json +146 -0
  650. teradataml/data/jsons/sqle/17.20/TD_ColumnSummary.json +53 -0
  651. teradataml/data/jsons/sqle/17.20/TD_ColumnTransformer.json +218 -0
  652. teradataml/data/jsons/sqle/17.20/TD_ConvertTo.json +92 -0
  653. teradataml/data/jsons/sqle/17.20/TD_DecisionForest.json +260 -0
  654. teradataml/data/jsons/sqle/17.20/TD_DecisionForestPredict.json +139 -0
  655. teradataml/data/jsons/sqle/17.20/TD_FTest.json +269 -0
  656. teradataml/data/jsons/sqle/17.20/TD_FillRowID.json +52 -0
  657. teradataml/data/jsons/sqle/17.20/TD_FunctionFit.json +46 -0
  658. teradataml/data/jsons/sqle/17.20/TD_FunctionTransform.json +72 -0
  659. teradataml/data/jsons/sqle/17.20/TD_GLM.json +507 -0
  660. teradataml/data/jsons/sqle/17.20/TD_GLMPREDICT.json +168 -0
  661. teradataml/data/jsons/sqle/17.20/TD_GLMPerSegment.json +411 -0
  662. teradataml/data/jsons/sqle/17.20/TD_GLMPredictPerSegment.json +146 -0
  663. teradataml/data/jsons/sqle/17.20/TD_GetFutileColumns.json +93 -0
  664. teradataml/data/jsons/sqle/17.20/TD_GetRowsWithMissingValues.json +76 -0
  665. teradataml/data/jsons/sqle/17.20/TD_GetRowsWithoutMissingValues.json +76 -0
  666. teradataml/data/jsons/sqle/17.20/TD_Histogram.json +152 -0
  667. teradataml/data/jsons/sqle/17.20/TD_KMeans.json +232 -0
  668. teradataml/data/jsons/sqle/17.20/TD_KMeansPredict.json +87 -0
  669. teradataml/data/jsons/sqle/17.20/TD_KNN.json +262 -0
  670. teradataml/data/jsons/sqle/17.20/TD_NERExtractor.json +145 -0
  671. teradataml/data/jsons/sqle/17.20/TD_NaiveBayes.json +193 -0
  672. teradataml/data/jsons/sqle/17.20/TD_NaiveBayesPredict.json +212 -0
  673. teradataml/data/jsons/sqle/17.20/TD_NaiveBayesTextClassifierTrainer.json +137 -0
  674. teradataml/data/jsons/sqle/17.20/TD_NonLinearCombineFit.json +102 -0
  675. teradataml/data/jsons/sqle/17.20/TD_NonLinearCombineTransform.json +71 -0
  676. teradataml/data/jsons/sqle/17.20/TD_NumApply.json +147 -0
  677. teradataml/data/jsons/sqle/17.20/TD_OneClassSVM.json +316 -0
  678. teradataml/data/jsons/sqle/17.20/TD_OneClassSVMPredict.json +124 -0
  679. teradataml/data/jsons/sqle/17.20/TD_OneHotEncodingFit.json +271 -0
  680. teradataml/data/jsons/sqle/17.20/TD_OneHotEncodingTransform.json +65 -0
  681. teradataml/data/jsons/sqle/17.20/TD_OrdinalEncodingFit.json +229 -0
  682. teradataml/data/jsons/sqle/17.20/TD_OrdinalEncodingTransform.json +75 -0
  683. teradataml/data/jsons/sqle/17.20/TD_OutlierFilterFit.json +217 -0
  684. teradataml/data/jsons/sqle/17.20/TD_OutlierFilterTransform.json +48 -0
  685. teradataml/data/jsons/sqle/17.20/TD_Pivoting.json +280 -0
  686. teradataml/data/jsons/sqle/17.20/TD_PolynomialFeaturesFit.json +114 -0
  687. teradataml/data/jsons/sqle/17.20/TD_PolynomialFeaturesTransform.json +72 -0
  688. teradataml/data/jsons/sqle/17.20/TD_QQNorm.json +111 -0
  689. teradataml/data/jsons/sqle/17.20/TD_ROC.json +179 -0
  690. teradataml/data/jsons/sqle/17.20/TD_RandomProjectionFit.json +179 -0
  691. teradataml/data/jsons/sqle/17.20/TD_RandomProjectionMinComponents.json +74 -0
  692. teradataml/data/jsons/sqle/17.20/TD_RandomProjectionTransform.json +74 -0
  693. teradataml/data/jsons/sqle/17.20/TD_RegressionEvaluator.json +138 -0
  694. teradataml/data/jsons/sqle/17.20/TD_RoundColumns.json +93 -0
  695. teradataml/data/jsons/sqle/17.20/TD_RowNormalizeFit.json +128 -0
  696. teradataml/data/jsons/sqle/17.20/TD_RowNormalizeTransform.json +71 -0
  697. teradataml/data/jsons/sqle/17.20/TD_SMOTE.json +267 -0
  698. teradataml/data/jsons/sqle/17.20/TD_SVM.json +389 -0
  699. teradataml/data/jsons/sqle/17.20/TD_SVMPredict.json +142 -0
  700. teradataml/data/jsons/sqle/17.20/TD_ScaleFit.json +310 -0
  701. teradataml/data/jsons/sqle/17.20/TD_ScaleTransform.json +120 -0
  702. teradataml/data/jsons/sqle/17.20/TD_SentimentExtractor.json +194 -0
  703. teradataml/data/jsons/sqle/17.20/TD_Shap.json +221 -0
  704. teradataml/data/jsons/sqle/17.20/TD_Silhouette.json +143 -0
  705. teradataml/data/jsons/sqle/17.20/TD_SimpleImputeFit.json +147 -0
  706. teradataml/data/jsons/sqle/17.20/TD_SimpleImputeTransform.json +48 -0
  707. teradataml/data/jsons/sqle/17.20/TD_StrApply.json +240 -0
  708. teradataml/data/jsons/sqle/17.20/TD_TFIDF.json +162 -0
  709. teradataml/data/jsons/sqle/17.20/TD_TargetEncodingFit.json +248 -0
  710. teradataml/data/jsons/sqle/17.20/TD_TargetEncodingTransform.json +75 -0
  711. teradataml/data/jsons/sqle/17.20/TD_TextMorph.json +134 -0
  712. teradataml/data/jsons/sqle/17.20/TD_TextParser.json +297 -0
  713. teradataml/data/jsons/sqle/17.20/TD_TrainTestSplit.json +142 -0
  714. teradataml/data/jsons/sqle/17.20/TD_UnivariateStatistics.json +117 -0
  715. teradataml/data/jsons/sqle/17.20/TD_Unpivoting.json +235 -0
  716. teradataml/data/jsons/sqle/17.20/TD_VectorDistance.json +183 -0
  717. teradataml/data/jsons/sqle/17.20/TD_WhichMax.json +53 -0
  718. teradataml/data/jsons/sqle/17.20/TD_WhichMin.json +53 -0
  719. teradataml/data/jsons/sqle/17.20/TD_WordEmbeddings.json +241 -0
  720. teradataml/data/jsons/sqle/17.20/TD_XGBoost.json +330 -0
  721. teradataml/data/jsons/sqle/17.20/TD_XGBoostPredict.json +195 -0
  722. teradataml/data/jsons/sqle/17.20/TD_ZTest.json +247 -0
  723. teradataml/data/jsons/sqle/17.20/Unpack.json +188 -0
  724. teradataml/data/jsons/sqle/17.20/nPath.json +269 -0
  725. teradataml/data/jsons/sqle/20.00/AI_AnalyzeSentiment.json +370 -0
  726. teradataml/data/jsons/sqle/20.00/AI_AskLLM.json +460 -0
  727. teradataml/data/jsons/sqle/20.00/AI_DetectLanguage.json +385 -0
  728. teradataml/data/jsons/sqle/20.00/AI_ExtractKeyPhrases.json +369 -0
  729. teradataml/data/jsons/sqle/20.00/AI_MaskPII.json +369 -0
  730. teradataml/data/jsons/sqle/20.00/AI_RecognizeEntities.json +369 -0
  731. teradataml/data/jsons/sqle/20.00/AI_RecognizePIIEntities.json +369 -0
  732. teradataml/data/jsons/sqle/20.00/AI_TextClassifier.json +400 -0
  733. teradataml/data/jsons/sqle/20.00/AI_TextEmbeddings.json +401 -0
  734. teradataml/data/jsons/sqle/20.00/AI_TextSummarize.json +384 -0
  735. teradataml/data/jsons/sqle/20.00/AI_TextTranslate.json +384 -0
  736. teradataml/data/jsons/sqle/20.00/TD_API_AzureML.json +151 -0
  737. teradataml/data/jsons/sqle/20.00/TD_API_Sagemaker.json +182 -0
  738. teradataml/data/jsons/sqle/20.00/TD_API_VertexAI.json +183 -0
  739. teradataml/data/jsons/sqle/20.00/TD_HNSW.json +296 -0
  740. teradataml/data/jsons/sqle/20.00/TD_HNSWPredict.json +206 -0
  741. teradataml/data/jsons/sqle/20.00/TD_HNSWSummary.json +32 -0
  742. teradataml/data/jsons/sqle/20.00/TD_KMeans.json +250 -0
  743. teradataml/data/jsons/sqle/20.00/TD_SMOTE.json +266 -0
  744. teradataml/data/jsons/sqle/20.00/TD_VectorDistance.json +278 -0
  745. teradataml/data/jsons/storedprocedure/17.20/TD_COPYART.json +71 -0
  746. teradataml/data/jsons/storedprocedure/17.20/TD_FILTERFACTORY1D.json +150 -0
  747. teradataml/data/jsons/tableoperator/17.00/read_nos.json +198 -0
  748. teradataml/data/jsons/tableoperator/17.05/read_nos.json +198 -0
  749. teradataml/data/jsons/tableoperator/17.05/write_nos.json +195 -0
  750. teradataml/data/jsons/tableoperator/17.10/read_nos.json +184 -0
  751. teradataml/data/jsons/tableoperator/17.10/write_nos.json +195 -0
  752. teradataml/data/jsons/tableoperator/17.20/IMAGE2MATRIX.json +53 -0
  753. teradataml/data/jsons/tableoperator/17.20/read_nos.json +183 -0
  754. teradataml/data/jsons/tableoperator/17.20/write_nos.json +224 -0
  755. teradataml/data/jsons/uaf/17.20/TD_ACF.json +132 -0
  756. teradataml/data/jsons/uaf/17.20/TD_ARIMAESTIMATE.json +396 -0
  757. teradataml/data/jsons/uaf/17.20/TD_ARIMAFORECAST.json +77 -0
  758. teradataml/data/jsons/uaf/17.20/TD_ARIMAVALIDATE.json +153 -0
  759. teradataml/data/jsons/uaf/17.20/TD_ARIMAXESTIMATE.json +362 -0
  760. teradataml/data/jsons/uaf/17.20/TD_AUTOARIMA.json +469 -0
  761. teradataml/data/jsons/uaf/17.20/TD_BINARYMATRIXOP.json +107 -0
  762. teradataml/data/jsons/uaf/17.20/TD_BINARYSERIESOP.json +106 -0
  763. teradataml/data/jsons/uaf/17.20/TD_BREUSCH_GODFREY.json +89 -0
  764. teradataml/data/jsons/uaf/17.20/TD_BREUSCH_PAGAN_GODFREY.json +104 -0
  765. teradataml/data/jsons/uaf/17.20/TD_CONVOLVE.json +78 -0
  766. teradataml/data/jsons/uaf/17.20/TD_CONVOLVE2.json +66 -0
  767. teradataml/data/jsons/uaf/17.20/TD_CUMUL_PERIODOGRAM.json +87 -0
  768. teradataml/data/jsons/uaf/17.20/TD_DFFT.json +134 -0
  769. teradataml/data/jsons/uaf/17.20/TD_DFFT2.json +144 -0
  770. teradataml/data/jsons/uaf/17.20/TD_DFFT2CONV.json +108 -0
  771. teradataml/data/jsons/uaf/17.20/TD_DFFTCONV.json +108 -0
  772. teradataml/data/jsons/uaf/17.20/TD_DICKEY_FULLER.json +78 -0
  773. teradataml/data/jsons/uaf/17.20/TD_DIFF.json +92 -0
  774. teradataml/data/jsons/uaf/17.20/TD_DTW.json +114 -0
  775. teradataml/data/jsons/uaf/17.20/TD_DURBIN_WATSON.json +101 -0
  776. teradataml/data/jsons/uaf/17.20/TD_DWT.json +173 -0
  777. teradataml/data/jsons/uaf/17.20/TD_DWT2D.json +160 -0
  778. teradataml/data/jsons/uaf/17.20/TD_EXTRACT_RESULTS.json +39 -0
  779. teradataml/data/jsons/uaf/17.20/TD_FITMETRICS.json +101 -0
  780. teradataml/data/jsons/uaf/17.20/TD_GENSERIES4FORMULA.json +85 -0
  781. teradataml/data/jsons/uaf/17.20/TD_GENSERIES4SINUSOIDS.json +71 -0
  782. teradataml/data/jsons/uaf/17.20/TD_GOLDFELD_QUANDT.json +139 -0
  783. teradataml/data/jsons/uaf/17.20/TD_HOLT_WINTERS_FORECASTER.json +313 -0
  784. teradataml/data/jsons/uaf/17.20/TD_IDFFT.json +58 -0
  785. teradataml/data/jsons/uaf/17.20/TD_IDFFT2.json +81 -0
  786. teradataml/data/jsons/uaf/17.20/TD_IDWT.json +162 -0
  787. teradataml/data/jsons/uaf/17.20/TD_IDWT2D.json +149 -0
  788. teradataml/data/jsons/uaf/17.20/TD_INPUTVALIDATOR.json +64 -0
  789. teradataml/data/jsons/uaf/17.20/TD_IQR.json +117 -0
  790. teradataml/data/jsons/uaf/17.20/TD_LINEAR_REGR.json +182 -0
  791. teradataml/data/jsons/uaf/17.20/TD_LINESPEC.json +103 -0
  792. teradataml/data/jsons/uaf/17.20/TD_MAMEAN.json +181 -0
  793. teradataml/data/jsons/uaf/17.20/TD_MATRIX2IMAGE.json +209 -0
  794. teradataml/data/jsons/uaf/17.20/TD_MATRIXMULTIPLY.json +68 -0
  795. teradataml/data/jsons/uaf/17.20/TD_MINFO.json +67 -0
  796. teradataml/data/jsons/uaf/17.20/TD_MULTIVAR_REGR.json +179 -0
  797. teradataml/data/jsons/uaf/17.20/TD_PACF.json +114 -0
  798. teradataml/data/jsons/uaf/17.20/TD_PORTMAN.json +119 -0
  799. teradataml/data/jsons/uaf/17.20/TD_POWERSPEC.json +175 -0
  800. teradataml/data/jsons/uaf/17.20/TD_POWERTRANSFORM.json +98 -0
  801. teradataml/data/jsons/uaf/17.20/TD_RESAMPLE.json +194 -0
  802. teradataml/data/jsons/uaf/17.20/TD_SAX.json +210 -0
  803. teradataml/data/jsons/uaf/17.20/TD_SEASONALNORMALIZE.json +143 -0
  804. teradataml/data/jsons/uaf/17.20/TD_SELECTION_CRITERIA.json +90 -0
  805. teradataml/data/jsons/uaf/17.20/TD_SIGNIF_PERIODICITIES.json +80 -0
  806. teradataml/data/jsons/uaf/17.20/TD_SIGNIF_RESIDMEAN.json +68 -0
  807. teradataml/data/jsons/uaf/17.20/TD_SIMPLEEXP.json +184 -0
  808. teradataml/data/jsons/uaf/17.20/TD_SINFO.json +58 -0
  809. teradataml/data/jsons/uaf/17.20/TD_SMOOTHMA.json +163 -0
  810. teradataml/data/jsons/uaf/17.20/TD_TRACKINGOP.json +101 -0
  811. teradataml/data/jsons/uaf/17.20/TD_UNDIFF.json +112 -0
  812. teradataml/data/jsons/uaf/17.20/TD_UNNORMALIZE.json +95 -0
  813. teradataml/data/jsons/uaf/17.20/TD_WHITES_GENERAL.json +78 -0
  814. teradataml/data/jsons/uaf/17.20/TD_WINDOWDFFT.json +410 -0
  815. teradataml/data/kmeans_example.json +23 -0
  816. teradataml/data/kmeans_table.csv +10 -0
  817. teradataml/data/kmeans_us_arrests_data.csv +51 -0
  818. teradataml/data/knn_example.json +19 -0
  819. teradataml/data/knnrecommender_example.json +7 -0
  820. teradataml/data/knnrecommenderpredict_example.json +12 -0
  821. teradataml/data/lar_example.json +17 -0
  822. teradataml/data/larpredict_example.json +30 -0
  823. teradataml/data/lc_new_predictors.csv +5 -0
  824. teradataml/data/lc_new_reference.csv +9 -0
  825. teradataml/data/lda_example.json +9 -0
  826. teradataml/data/ldainference_example.json +15 -0
  827. teradataml/data/ldatopicsummary_example.json +9 -0
  828. teradataml/data/levendist_input.csv +13 -0
  829. teradataml/data/levenshteindistance_example.json +10 -0
  830. teradataml/data/linreg_example.json +10 -0
  831. teradataml/data/load_example_data.py +350 -0
  832. teradataml/data/loan_prediction.csv +295 -0
  833. teradataml/data/lungcancer.csv +138 -0
  834. teradataml/data/mappingdata.csv +12 -0
  835. teradataml/data/medical_readings.csv +101 -0
  836. teradataml/data/milk_timeseries.csv +157 -0
  837. teradataml/data/min_max_titanic.csv +4 -0
  838. teradataml/data/minhash_example.json +6 -0
  839. teradataml/data/ml_ratings.csv +7547 -0
  840. teradataml/data/ml_ratings_10.csv +2445 -0
  841. teradataml/data/mobile_data.csv +13 -0
  842. teradataml/data/model1_table.csv +5 -0
  843. teradataml/data/model2_table.csv +5 -0
  844. teradataml/data/models/License_file.txt +1 -0
  845. teradataml/data/models/License_file_empty.txt +0 -0
  846. teradataml/data/models/dataiku_iris_data_ann_thin +0 -0
  847. teradataml/data/models/dr_iris_rf +0 -0
  848. teradataml/data/models/iris_db_dt_model_sklearn.onnx +0 -0
  849. teradataml/data/models/iris_db_dt_model_sklearn_floattensor.onnx +0 -0
  850. teradataml/data/models/iris_db_glm_model.pmml +57 -0
  851. teradataml/data/models/iris_db_xgb_model.pmml +4471 -0
  852. teradataml/data/models/iris_kmeans_model +0 -0
  853. teradataml/data/models/iris_mojo_glm_h2o_model +0 -0
  854. teradataml/data/models/iris_mojo_xgb_h2o_model +0 -0
  855. teradataml/data/modularity_example.json +12 -0
  856. teradataml/data/movavg_example.json +8 -0
  857. teradataml/data/mtx1.csv +7 -0
  858. teradataml/data/mtx2.csv +13 -0
  859. teradataml/data/multi_model_classification.csv +401 -0
  860. teradataml/data/multi_model_regression.csv +401 -0
  861. teradataml/data/mvdfft8.csv +9 -0
  862. teradataml/data/naivebayes_example.json +10 -0
  863. teradataml/data/naivebayespredict_example.json +19 -0
  864. teradataml/data/naivebayestextclassifier2_example.json +7 -0
  865. teradataml/data/naivebayestextclassifier_example.json +8 -0
  866. teradataml/data/naivebayestextclassifierpredict_example.json +32 -0
  867. teradataml/data/name_Find_configure.csv +10 -0
  868. teradataml/data/namedentityfinder_example.json +14 -0
  869. teradataml/data/namedentityfinderevaluator_example.json +10 -0
  870. teradataml/data/namedentityfindertrainer_example.json +6 -0
  871. teradataml/data/nb_iris_input_test.csv +31 -0
  872. teradataml/data/nb_iris_input_train.csv +121 -0
  873. teradataml/data/nbp_iris_model.csv +13 -0
  874. teradataml/data/ner_dict.csv +8 -0
  875. teradataml/data/ner_extractor_text.csv +2 -0
  876. teradataml/data/ner_input_eng.csv +7 -0
  877. teradataml/data/ner_rule.csv +5 -0
  878. teradataml/data/ner_sports_test2.csv +29 -0
  879. teradataml/data/ner_sports_train.csv +501 -0
  880. teradataml/data/nerevaluator_example.json +6 -0
  881. teradataml/data/nerextractor_example.json +18 -0
  882. teradataml/data/nermem_sports_test.csv +18 -0
  883. teradataml/data/nermem_sports_train.csv +51 -0
  884. teradataml/data/nertrainer_example.json +7 -0
  885. teradataml/data/ngrams_example.json +7 -0
  886. teradataml/data/notebooks/__init__.py +0 -0
  887. teradataml/data/notebooks/sqlalchemy/Teradata Vantage Aggregate Functions using SQLAlchemy.ipynb +1455 -0
  888. teradataml/data/notebooks/sqlalchemy/Teradata Vantage Arithmetic Functions Using SQLAlchemy.ipynb +1993 -0
  889. teradataml/data/notebooks/sqlalchemy/Teradata Vantage Bit-Byte Manipulation Functions using SQLAlchemy.ipynb +1492 -0
  890. teradataml/data/notebooks/sqlalchemy/Teradata Vantage Built-in functions using SQLAlchemy.ipynb +536 -0
  891. teradataml/data/notebooks/sqlalchemy/Teradata Vantage Regular Expressions Using SQLAlchemy.ipynb +570 -0
  892. teradataml/data/notebooks/sqlalchemy/Teradata Vantage String Functions Using SQLAlchemy.ipynb +2559 -0
  893. teradataml/data/notebooks/sqlalchemy/Teradata Vantage Window Aggregate Functions using SQLAlchemy.ipynb +2911 -0
  894. teradataml/data/notebooks/sqlalchemy/Using Generic SQLAlchemy ClauseElements teradataml DataFrame assign method.ipynb +698 -0
  895. teradataml/data/notebooks/sqlalchemy/__init__.py +0 -0
  896. teradataml/data/notebooks/sqlalchemy/teradataml filtering using SQLAlchemy ClauseElements.ipynb +784 -0
  897. teradataml/data/npath_example.json +23 -0
  898. teradataml/data/ntree_example.json +14 -0
  899. teradataml/data/numeric_strings.csv +5 -0
  900. teradataml/data/numerics.csv +4 -0
  901. teradataml/data/ocean_buoy.csv +17 -0
  902. teradataml/data/ocean_buoy2.csv +17 -0
  903. teradataml/data/ocean_buoys.csv +28 -0
  904. teradataml/data/ocean_buoys2.csv +10 -0
  905. teradataml/data/ocean_buoys_nonpti.csv +28 -0
  906. teradataml/data/ocean_buoys_seq.csv +29 -0
  907. teradataml/data/onehot_encoder_train.csv +4 -0
  908. teradataml/data/openml_example.json +92 -0
  909. teradataml/data/optional_event_table.csv +4 -0
  910. teradataml/data/orders1.csv +11 -0
  911. teradataml/data/orders1_12.csv +13 -0
  912. teradataml/data/orders_ex.csv +4 -0
  913. teradataml/data/pack_example.json +9 -0
  914. teradataml/data/package_tracking.csv +19 -0
  915. teradataml/data/package_tracking_pti.csv +19 -0
  916. teradataml/data/pagerank_example.json +13 -0
  917. teradataml/data/paragraphs_input.csv +6 -0
  918. teradataml/data/pathanalyzer_example.json +8 -0
  919. teradataml/data/pathgenerator_example.json +8 -0
  920. teradataml/data/patient_profile.csv +101 -0
  921. teradataml/data/pattern_matching_data.csv +11 -0
  922. teradataml/data/payment_fraud_dataset.csv +10001 -0
  923. teradataml/data/peppers.png +0 -0
  924. teradataml/data/phrases.csv +7 -0
  925. teradataml/data/pivot_example.json +9 -0
  926. teradataml/data/pivot_input.csv +22 -0
  927. teradataml/data/playerRating.csv +31 -0
  928. teradataml/data/pos_input.csv +40 -0
  929. teradataml/data/postagger_example.json +7 -0
  930. teradataml/data/posttagger_output.csv +44 -0
  931. teradataml/data/production_data.csv +17 -0
  932. teradataml/data/production_data2.csv +7 -0
  933. teradataml/data/randomsample_example.json +32 -0
  934. teradataml/data/randomwalksample_example.json +9 -0
  935. teradataml/data/rank_table.csv +6 -0
  936. teradataml/data/real_values.csv +14 -0
  937. teradataml/data/ref_mobile_data.csv +4 -0
  938. teradataml/data/ref_mobile_data_dense.csv +2 -0
  939. teradataml/data/ref_url.csv +17 -0
  940. teradataml/data/restaurant_reviews.csv +7 -0
  941. teradataml/data/retail_churn_table.csv +27772 -0
  942. teradataml/data/river_data.csv +145 -0
  943. teradataml/data/roc_example.json +8 -0
  944. teradataml/data/roc_input.csv +101 -0
  945. teradataml/data/rule_inputs.csv +6 -0
  946. teradataml/data/rule_table.csv +2 -0
  947. teradataml/data/sales.csv +7 -0
  948. teradataml/data/sales_transaction.csv +501 -0
  949. teradataml/data/salesdata.csv +342 -0
  950. teradataml/data/sample_cities.csv +3 -0
  951. teradataml/data/sample_shapes.csv +11 -0
  952. teradataml/data/sample_streets.csv +3 -0
  953. teradataml/data/sampling_example.json +16 -0
  954. teradataml/data/sax_example.json +17 -0
  955. teradataml/data/scale_attributes.csv +3 -0
  956. teradataml/data/scale_example.json +74 -0
  957. teradataml/data/scale_housing.csv +11 -0
  958. teradataml/data/scale_housing_test.csv +6 -0
  959. teradataml/data/scale_input_part_sparse.csv +31 -0
  960. teradataml/data/scale_input_partitioned.csv +16 -0
  961. teradataml/data/scale_input_sparse.csv +11 -0
  962. teradataml/data/scale_parameters.csv +3 -0
  963. teradataml/data/scale_stat.csv +11 -0
  964. teradataml/data/scalebypartition_example.json +13 -0
  965. teradataml/data/scalemap_example.json +13 -0
  966. teradataml/data/scalesummary_example.json +12 -0
  967. teradataml/data/score_category.csv +101 -0
  968. teradataml/data/score_summary.csv +4 -0
  969. teradataml/data/script_example.json +10 -0
  970. teradataml/data/scripts/deploy_script.py +84 -0
  971. teradataml/data/scripts/lightgbm/dataset.template +175 -0
  972. teradataml/data/scripts/lightgbm/lightgbm_class_functions.template +264 -0
  973. teradataml/data/scripts/lightgbm/lightgbm_function.template +234 -0
  974. teradataml/data/scripts/lightgbm/lightgbm_sklearn.template +177 -0
  975. teradataml/data/scripts/mapper.R +20 -0
  976. teradataml/data/scripts/mapper.py +16 -0
  977. teradataml/data/scripts/mapper_replace.py +16 -0
  978. teradataml/data/scripts/sklearn/__init__.py +0 -0
  979. teradataml/data/scripts/sklearn/sklearn_fit.py +205 -0
  980. teradataml/data/scripts/sklearn/sklearn_fit_predict.py +148 -0
  981. teradataml/data/scripts/sklearn/sklearn_function.template +144 -0
  982. teradataml/data/scripts/sklearn/sklearn_model_selection_split.py +166 -0
  983. teradataml/data/scripts/sklearn/sklearn_neighbors.py +161 -0
  984. teradataml/data/scripts/sklearn/sklearn_score.py +145 -0
  985. teradataml/data/scripts/sklearn/sklearn_transform.py +327 -0
  986. teradataml/data/sdk/modelops/modelops_spec.json +101737 -0
  987. teradataml/data/seeds.csv +10 -0
  988. teradataml/data/sentenceextractor_example.json +7 -0
  989. teradataml/data/sentiment_extract_input.csv +11 -0
  990. teradataml/data/sentiment_train.csv +16 -0
  991. teradataml/data/sentiment_word.csv +20 -0
  992. teradataml/data/sentiment_word_input.csv +20 -0
  993. teradataml/data/sentimentextractor_example.json +24 -0
  994. teradataml/data/sentimenttrainer_example.json +8 -0
  995. teradataml/data/sequence_table.csv +10 -0
  996. teradataml/data/seriessplitter_example.json +8 -0
  997. teradataml/data/sessionize_example.json +17 -0
  998. teradataml/data/sessionize_table.csv +116 -0
  999. teradataml/data/setop_test1.csv +24 -0
  1000. teradataml/data/setop_test2.csv +22 -0
  1001. teradataml/data/soc_nw_edges.csv +11 -0
  1002. teradataml/data/soc_nw_vertices.csv +8 -0
  1003. teradataml/data/souvenir_timeseries.csv +168 -0
  1004. teradataml/data/sparse_iris_attribute.csv +5 -0
  1005. teradataml/data/sparse_iris_test.csv +121 -0
  1006. teradataml/data/sparse_iris_train.csv +601 -0
  1007. teradataml/data/star1.csv +6 -0
  1008. teradataml/data/star_pivot.csv +8 -0
  1009. teradataml/data/state_transition.csv +5 -0
  1010. teradataml/data/stock_data.csv +53 -0
  1011. teradataml/data/stock_movement.csv +11 -0
  1012. teradataml/data/stock_vol.csv +76 -0
  1013. teradataml/data/stop_words.csv +8 -0
  1014. teradataml/data/store_sales.csv +37 -0
  1015. teradataml/data/stringsimilarity_example.json +8 -0
  1016. teradataml/data/strsimilarity_input.csv +13 -0
  1017. teradataml/data/students.csv +101 -0
  1018. teradataml/data/svm_iris_input_test.csv +121 -0
  1019. teradataml/data/svm_iris_input_train.csv +481 -0
  1020. teradataml/data/svm_iris_model.csv +7 -0
  1021. teradataml/data/svmdense_example.json +10 -0
  1022. teradataml/data/svmdensepredict_example.json +19 -0
  1023. teradataml/data/svmsparse_example.json +8 -0
  1024. teradataml/data/svmsparsepredict_example.json +14 -0
  1025. teradataml/data/svmsparsesummary_example.json +8 -0
  1026. teradataml/data/target_mobile_data.csv +13 -0
  1027. teradataml/data/target_mobile_data_dense.csv +5 -0
  1028. teradataml/data/target_udt_data.csv +8 -0
  1029. teradataml/data/tdnerextractor_example.json +14 -0
  1030. teradataml/data/templatedata.csv +1201 -0
  1031. teradataml/data/templates/open_source_ml.json +11 -0
  1032. teradataml/data/teradata_icon.ico +0 -0
  1033. teradataml/data/teradataml_example.json +1473 -0
  1034. teradataml/data/test_classification.csv +101 -0
  1035. teradataml/data/test_loan_prediction.csv +53 -0
  1036. teradataml/data/test_pacf_12.csv +37 -0
  1037. teradataml/data/test_prediction.csv +101 -0
  1038. teradataml/data/test_regression.csv +101 -0
  1039. teradataml/data/test_river2.csv +109 -0
  1040. teradataml/data/text_inputs.csv +6 -0
  1041. teradataml/data/textchunker_example.json +8 -0
  1042. teradataml/data/textclassifier_example.json +7 -0
  1043. teradataml/data/textclassifier_input.csv +7 -0
  1044. teradataml/data/textclassifiertrainer_example.json +7 -0
  1045. teradataml/data/textmorph_example.json +11 -0
  1046. teradataml/data/textparser_example.json +15 -0
  1047. teradataml/data/texttagger_example.json +12 -0
  1048. teradataml/data/texttokenizer_example.json +7 -0
  1049. teradataml/data/texttrainer_input.csv +11 -0
  1050. teradataml/data/tf_example.json +7 -0
  1051. teradataml/data/tfidf_example.json +14 -0
  1052. teradataml/data/tfidf_input1.csv +201 -0
  1053. teradataml/data/tfidf_train.csv +6 -0
  1054. teradataml/data/time_table1.csv +535 -0
  1055. teradataml/data/time_table2.csv +14 -0
  1056. teradataml/data/timeseriesdata.csv +1601 -0
  1057. teradataml/data/timeseriesdatasetsd4.csv +105 -0
  1058. teradataml/data/timestamp_data.csv +4 -0
  1059. teradataml/data/titanic.csv +892 -0
  1060. teradataml/data/titanic_dataset_unpivoted.csv +19 -0
  1061. teradataml/data/to_num_data.csv +4 -0
  1062. teradataml/data/tochar_data.csv +5 -0
  1063. teradataml/data/token_table.csv +696 -0
  1064. teradataml/data/train_multiclass.csv +101 -0
  1065. teradataml/data/train_regression.csv +101 -0
  1066. teradataml/data/train_regression_multiple_labels.csv +101 -0
  1067. teradataml/data/train_tracking.csv +28 -0
  1068. teradataml/data/trans_dense.csv +16 -0
  1069. teradataml/data/trans_sparse.csv +55 -0
  1070. teradataml/data/transformation_table.csv +6 -0
  1071. teradataml/data/transformation_table_new.csv +2 -0
  1072. teradataml/data/tv_spots.csv +16 -0
  1073. teradataml/data/twod_climate_data.csv +117 -0
  1074. teradataml/data/uaf_example.json +529 -0
  1075. teradataml/data/univariatestatistics_example.json +9 -0
  1076. teradataml/data/unpack_example.json +10 -0
  1077. teradataml/data/unpivot_example.json +25 -0
  1078. teradataml/data/unpivot_input.csv +8 -0
  1079. teradataml/data/url_data.csv +10 -0
  1080. teradataml/data/us_air_pass.csv +37 -0
  1081. teradataml/data/us_population.csv +624 -0
  1082. teradataml/data/us_states_shapes.csv +52 -0
  1083. teradataml/data/varmax_example.json +18 -0
  1084. teradataml/data/vectordistance_example.json +30 -0
  1085. teradataml/data/ville_climatedata.csv +121 -0
  1086. teradataml/data/ville_tempdata.csv +12 -0
  1087. teradataml/data/ville_tempdata1.csv +12 -0
  1088. teradataml/data/ville_temperature.csv +11 -0
  1089. teradataml/data/waveletTable.csv +1605 -0
  1090. teradataml/data/waveletTable2.csv +1605 -0
  1091. teradataml/data/weightedmovavg_example.json +9 -0
  1092. teradataml/data/wft_testing.csv +5 -0
  1093. teradataml/data/windowdfft.csv +16 -0
  1094. teradataml/data/wine_data.csv +1600 -0
  1095. teradataml/data/word_embed_input_table1.csv +6 -0
  1096. teradataml/data/word_embed_input_table2.csv +5 -0
  1097. teradataml/data/word_embed_model.csv +23 -0
  1098. teradataml/data/words_input.csv +13 -0
  1099. teradataml/data/xconvolve_complex_left.csv +6 -0
  1100. teradataml/data/xconvolve_complex_leftmulti.csv +6 -0
  1101. teradataml/data/xgboost_example.json +36 -0
  1102. teradataml/data/xgboostpredict_example.json +32 -0
  1103. teradataml/data/ztest_example.json +16 -0
  1104. teradataml/dataframe/__init__.py +0 -0
  1105. teradataml/dataframe/copy_to.py +2446 -0
  1106. teradataml/dataframe/data_transfer.py +2840 -0
  1107. teradataml/dataframe/dataframe.py +20908 -0
  1108. teradataml/dataframe/dataframe_utils.py +2114 -0
  1109. teradataml/dataframe/fastload.py +794 -0
  1110. teradataml/dataframe/functions.py +2110 -0
  1111. teradataml/dataframe/indexer.py +424 -0
  1112. teradataml/dataframe/row.py +160 -0
  1113. teradataml/dataframe/setop.py +1171 -0
  1114. teradataml/dataframe/sql.py +10904 -0
  1115. teradataml/dataframe/sql_function_parameters.py +440 -0
  1116. teradataml/dataframe/sql_functions.py +652 -0
  1117. teradataml/dataframe/sql_interfaces.py +220 -0
  1118. teradataml/dataframe/vantage_function_types.py +675 -0
  1119. teradataml/dataframe/window.py +694 -0
  1120. teradataml/dbutils/__init__.py +3 -0
  1121. teradataml/dbutils/dbutils.py +2871 -0
  1122. teradataml/dbutils/filemgr.py +318 -0
  1123. teradataml/gen_ai/__init__.py +2 -0
  1124. teradataml/gen_ai/convAI.py +473 -0
  1125. teradataml/geospatial/__init__.py +4 -0
  1126. teradataml/geospatial/geodataframe.py +1105 -0
  1127. teradataml/geospatial/geodataframecolumn.py +392 -0
  1128. teradataml/geospatial/geometry_types.py +926 -0
  1129. teradataml/hyperparameter_tuner/__init__.py +1 -0
  1130. teradataml/hyperparameter_tuner/optimizer.py +4115 -0
  1131. teradataml/hyperparameter_tuner/utils.py +303 -0
  1132. teradataml/lib/__init__.py +0 -0
  1133. teradataml/lib/aed_0_1.dll +0 -0
  1134. teradataml/lib/libaed_0_1.dylib +0 -0
  1135. teradataml/lib/libaed_0_1.so +0 -0
  1136. teradataml/lib/libaed_0_1_aarch64.so +0 -0
  1137. teradataml/lib/libaed_0_1_ppc64le.so +0 -0
  1138. teradataml/opensource/__init__.py +1 -0
  1139. teradataml/opensource/_base.py +1321 -0
  1140. teradataml/opensource/_class.py +464 -0
  1141. teradataml/opensource/_constants.py +61 -0
  1142. teradataml/opensource/_lightgbm.py +949 -0
  1143. teradataml/opensource/_sklearn.py +1008 -0
  1144. teradataml/opensource/_wrapper_utils.py +267 -0
  1145. teradataml/options/__init__.py +148 -0
  1146. teradataml/options/configure.py +489 -0
  1147. teradataml/options/display.py +187 -0
  1148. teradataml/plot/__init__.py +3 -0
  1149. teradataml/plot/axis.py +1427 -0
  1150. teradataml/plot/constants.py +15 -0
  1151. teradataml/plot/figure.py +431 -0
  1152. teradataml/plot/plot.py +810 -0
  1153. teradataml/plot/query_generator.py +83 -0
  1154. teradataml/plot/subplot.py +216 -0
  1155. teradataml/scriptmgmt/UserEnv.py +4273 -0
  1156. teradataml/scriptmgmt/__init__.py +3 -0
  1157. teradataml/scriptmgmt/lls_utils.py +2157 -0
  1158. teradataml/sdk/README.md +79 -0
  1159. teradataml/sdk/__init__.py +4 -0
  1160. teradataml/sdk/_auth_modes.py +422 -0
  1161. teradataml/sdk/_func_params.py +487 -0
  1162. teradataml/sdk/_json_parser.py +453 -0
  1163. teradataml/sdk/_openapi_spec_constants.py +249 -0
  1164. teradataml/sdk/_utils.py +236 -0
  1165. teradataml/sdk/api_client.py +900 -0
  1166. teradataml/sdk/constants.py +62 -0
  1167. teradataml/sdk/modelops/__init__.py +98 -0
  1168. teradataml/sdk/modelops/_client.py +409 -0
  1169. teradataml/sdk/modelops/_constants.py +304 -0
  1170. teradataml/sdk/modelops/models.py +2308 -0
  1171. teradataml/sdk/spinner.py +107 -0
  1172. teradataml/series/__init__.py +0 -0
  1173. teradataml/series/series.py +537 -0
  1174. teradataml/series/series_utils.py +71 -0
  1175. teradataml/store/__init__.py +12 -0
  1176. teradataml/store/feature_store/__init__.py +0 -0
  1177. teradataml/store/feature_store/constants.py +658 -0
  1178. teradataml/store/feature_store/feature_store.py +4814 -0
  1179. teradataml/store/feature_store/mind_map.py +639 -0
  1180. teradataml/store/feature_store/models.py +7330 -0
  1181. teradataml/store/feature_store/utils.py +390 -0
  1182. teradataml/table_operators/Apply.py +979 -0
  1183. teradataml/table_operators/Script.py +1739 -0
  1184. teradataml/table_operators/TableOperator.py +1343 -0
  1185. teradataml/table_operators/__init__.py +2 -0
  1186. teradataml/table_operators/apply_query_generator.py +262 -0
  1187. teradataml/table_operators/query_generator.py +493 -0
  1188. teradataml/table_operators/table_operator_query_generator.py +462 -0
  1189. teradataml/table_operators/table_operator_util.py +726 -0
  1190. teradataml/table_operators/templates/dataframe_apply.template +184 -0
  1191. teradataml/table_operators/templates/dataframe_map.template +176 -0
  1192. teradataml/table_operators/templates/dataframe_register.template +73 -0
  1193. teradataml/table_operators/templates/dataframe_udf.template +67 -0
  1194. teradataml/table_operators/templates/script_executor.template +170 -0
  1195. teradataml/telemetry_utils/__init__.py +0 -0
  1196. teradataml/telemetry_utils/queryband.py +53 -0
  1197. teradataml/utils/__init__.py +0 -0
  1198. teradataml/utils/docstring.py +527 -0
  1199. teradataml/utils/dtypes.py +943 -0
  1200. teradataml/utils/internal_buffer.py +122 -0
  1201. teradataml/utils/print_versions.py +206 -0
  1202. teradataml/utils/utils.py +451 -0
  1203. teradataml/utils/validators.py +3305 -0
  1204. teradataml-20.0.0.8.dist-info/METADATA +2804 -0
  1205. teradataml-20.0.0.8.dist-info/RECORD +1208 -0
  1206. teradataml-20.0.0.8.dist-info/WHEEL +5 -0
  1207. teradataml-20.0.0.8.dist-info/top_level.txt +1 -0
  1208. teradataml-20.0.0.8.dist-info/zip-safe +1 -0
@@ -0,0 +1,2840 @@
1
+ #!/usr/bin/python
2
+ # ##################################################################
3
+ #
4
+ # Copyright 2021 Teradata. All rights reserved.
5
+ # TERADATA CONFIDENTIAL AND TRADE SECRET
6
+ #
7
+ # Primary Owner: Sanath Vobilisetty (Sanath.Vobilisetty@teradata.com)
8
+ # Secondary Owner: Pankaj Vinod Purandare (PankajVinod.Purandare@Teradata.com)
9
+ #
10
+ # ##################################################################
11
+ import pandas as pd
12
+ from collections import OrderedDict
13
+ from sqlalchemy.exc import OperationalError as sqlachemyOperationalError
14
+ from teradataml.common.constants import DriverEscapeFunctions, TeradataConstants
15
+ from teradataml.common.exceptions import TeradataMlException
16
+ from teradataml.common.messages import Messages
17
+ from teradataml.common.messagecodes import MessageCodes
18
+ from teradataml.common.sqlbundle import SQLBundle
19
+ from teradataml.common.utils import UtilFuncs
20
+ from teradataml.common.constants import CopyToConstants
21
+ from teradataml.context.context import get_context, get_connection, \
22
+ _get_context_temp_databasename, _get_current_databasename
23
+ from teradataml.dataframe import dataframe as tdmldf
24
+ from teradataml.dataframe.copy_to import copy_to_sql, _create_table_object, \
25
+ _get_pd_df_column_names, _extract_column_info, \
26
+ _check_columns_insertion_compatible, _get_index_labels
27
+ from teradataml.dataframe.dataframe_utils import DataFrameUtils as df_utils
28
+ from teradataml.dbutils.dbutils import _create_table, _execute_query_and_generate_pandas_df
29
+ from teradataml.utils.validators import _Validators
30
+ from teradataml.telemetry_utils.queryband import collect_queryband
31
+
32
+
33
+ @collect_queryband(queryband="fstExprt")
34
+ def fastexport(df, export_to="pandas", index_column=None,
35
+ catch_errors_warnings=False, csv_file=None,
36
+ **kwargs):
37
+ """
38
+ DESCRIPTION:
39
+ The fastexport() API exports teradataml DataFrame to Pandas DataFrame
40
+ or CSV file using FastExport data transfer protocol.
41
+ Note:
42
+ 1. Teradata recommends to use FastExport when number of rows in
43
+ teradataml DataFrame are at least 100,000. To extract lesser rows
44
+ ignore this function and go with regular to_pandas() or to_csv()
45
+ function. FastExport opens multiple data transfer connections to the
46
+ database.
47
+ 2. FastExport does not support all Teradata Database data types.
48
+ For example, tables with BLOB and CLOB type columns cannot
49
+ be extracted.
50
+ 3. FastExport cannot be used to extract data from a volatile or
51
+ temporary table.
52
+ 4. For best efficiency, do not use DataFrame.groupby() and
53
+ DataFrame.sort() with FastExport.
54
+
55
+ For additional information about FastExport protocol through
56
+ teradatasql driver, please refer to FASTEXPORT section of
57
+ https://pypi.org/project/teradatasql/#FastExport driver documentation.
58
+
59
+ PARAMETERS:
60
+ df:
61
+ Required Argument.
62
+ Specifies teradataml DataFrame that needs to be exported.
63
+ Types: teradataml DataFrame
64
+
65
+ export_to:
66
+ Optional Argument.
67
+ Specifies a value that notifies where to export the data.
68
+ Permitted Values:
69
+ * "pandas": Export data to a Pandas DataFrame.
70
+ * "csv": Export data to a given CSV file.
71
+ Default Value: "pandas"
72
+ Types: String
73
+
74
+ index_column:
75
+ Optional Argument.
76
+ Specifies column(s) to be used as index column for the converted object.
77
+ Note:
78
+ Applicable only when 'export_to' is set to "pandas".
79
+ Default Value: None.
80
+ Types: String OR list of Strings (str)
81
+
82
+ catch_errors_warnings:
83
+ Optional Argument.
84
+ Specifies whether to catch errors/warnings(if any) raised by
85
+ fastexport protocol while converting teradataml DataFrame.
86
+ Notes :
87
+ 1. When 'export_to' is set to "pandas" and 'catch_errors_warnings' is set to True,
88
+ fastexport() returns a tuple containing:
89
+ a. Pandas DataFrame.
90
+ b. Errors(if any) in a list thrown by fastexport.
91
+ c. Warnings(if any) in a list thrown by fastexport.
92
+ When set to False, prints the fastexport errors/warnings to the
93
+ standard output, if there are any.
94
+
95
+
96
+ 2. When 'export_to' is set to "csv" and 'catch_errors_warnings' is set to True,
97
+ fastexport() returns a tuple containing:
98
+ a. Errors(if any) in a list thrown by fastexport.
99
+ b. Warnings(if any) in a list thrown by fastexport.
100
+ Default Value: False
101
+ Types: bool
102
+
103
+ csv_file:
104
+ Optional Argument.
105
+ Specifies the name of CSV file to which data is to be exported.
106
+ Note:
107
+ This is required argument when 'export_to' is set to "csv".
108
+ Types: String
109
+
110
+ kwargs:
111
+ Optional Argument.
112
+ Specifies keyword arguments. Accepts following keyword arguments:
113
+
114
+ sep:
115
+ Optional Argument.
116
+ Specifies a single character string used to separate fields in a CSV file.
117
+ Default Value: ","
118
+ Notes:
119
+ 1. "sep" cannot be line feed ('\\n') or carriage return ('\\r').
120
+ 2. "sep" should not be same as "quotechar".
121
+ 3. Length of "sep" argument should be 1.
122
+ Types: String
123
+
124
+ quotechar:
125
+ Optional Argument.
126
+ Specifies a single character string used to quote fields in a CSV file.
127
+ Default Value: "\""
128
+ Notes:
129
+ 1. "quotechar" cannot be line feed ('\\n') or carriage return ('\\r').
130
+ 2. "quotechar" should not be same as "sep".
131
+ 3. Length of "quotechar" argument should be 1.
132
+ Types: String
133
+
134
+ coerce_float:
135
+ Optional Argument.
136
+ Specifies whether to convert non-string, non-numeric objects to floating point.
137
+ Note:
138
+ For additional information about "coerce_float" please refer to:
139
+ https://pandas.pydata.org/docs/reference/api/pandas.read_sql.html
140
+
141
+ parse_dates:
142
+ Optional Argument.
143
+ Specifies columns to parse as dates.
144
+ Note:
145
+ For additional information about "parse_date" please refer to:
146
+ https://pandas.pydata.org/docs/reference/api/pandas.read_sql.html
147
+
148
+ open_sessions:
149
+ Optional Argument.
150
+ specifies the number of Teradata sessions to be opened for fastexport.
151
+ Note:
152
+ If "open_sessions" argument is not provided, the default value
153
+ is the smaller of 8 or the number of AMPs avaialble.
154
+ For additional information about number of Teradata data-transfer
155
+ sessions opened during fastexport, please refer to:
156
+ https://pypi.org/project/teradatasql/#FastExport
157
+ Types: Integer
158
+
159
+
160
+ RETURNS:
161
+ 1. When 'export_to' is set to "pandas" and "catch_errors_warnings" is set to True,
162
+ then the function returns a tuple containing:
163
+ a. Pandas DataFrame.
164
+ b. Errors, if any, thrown by fastexport in a list of strings.
165
+ c. Warnings, if any, thrown by fastexport in a list of strings.
166
+ When 'export_to' is set to "pandas" and "catch_errors_warnings" is set to False,
167
+ then the function returns a Pandas DataFrame.
168
+ 2. When 'export_to' is set to "csv" and "catch_errors_warnings" is set to True,
169
+ then the function returns a tuple containing:
170
+ a. Errors, if any, thrown by fastexport in a list of strings.
171
+ b. Warnings, if any, thrown by fastexport in a list of strings.
172
+
173
+ EXAMPLES:
174
+ >>> from teradataml import *
175
+ >>> load_example_data("dataframe", "admissions_train")
176
+ >>> df = DataFrame("admissions_train")
177
+
178
+ # Print dataframe.
179
+ >>> df
180
+ masters gpa stats programming admitted
181
+ id
182
+ 13 no 4.00 Advanced Novice 1
183
+ 26 yes 3.57 Advanced Advanced 1
184
+ 5 no 3.44 Novice Novice 0
185
+ 19 yes 1.98 Advanced Advanced 0
186
+ 15 yes 4.00 Advanced Advanced 1
187
+ 40 yes 3.95 Novice Beginner 0
188
+ 7 yes 2.33 Novice Novice 1
189
+ 22 yes 3.46 Novice Beginner 0
190
+ 36 no 3.00 Advanced Novice 0
191
+ 38 yes 2.65 Advanced Beginner 1
192
+
193
+ # Example 1: Export teradataml DataFrame df to Pandas DataFrame using
194
+ # fastexport().
195
+ >>> fastexport(df)
196
+ Errors: []
197
+ Warnings: []
198
+ masters gpa stats programming admitted
199
+ id
200
+ 38 yes 2.65 Advanced Beginner 1
201
+ 26 yes 3.57 Advanced Advanced 1
202
+ 5 no 3.44 Novice Novice 0
203
+ 24 no 1.87 Advanced Novice 1
204
+ 3 no 3.70 Novice Beginner 1
205
+ 1 yes 3.95 Beginner Beginner 0
206
+ 20 yes 3.90 Advanced Advanced 1
207
+ 18 yes 3.81 Advanced Advanced 1
208
+ 8 no 3.60 Beginner Advanced 1
209
+ 25 no 3.96 Advanced Advanced 1
210
+ ...
211
+
212
+ # Example 2: Export teradataml DataFrame df to Pandas DataFrame,
213
+ # set index column, coerce_float and catch errors/warnings thrown by
214
+ # fastexport.
215
+ >>> pandas_df, err, warn = fastexport(df, index_column="gpa",
216
+ catch_errors_warnings=True,
217
+ coerce_float=True)
218
+ # Print pandas DataFrame.
219
+ >>> pandas_df
220
+ id masters stats programming admitted
221
+ gpa
222
+ 2.65 38 yes Advanced Beginner 1
223
+ 3.57 26 yes Advanced Advanced 1
224
+ 3.44 5 no Novice Novice 0
225
+ 1.87 24 no Advanced Novice 1
226
+ 3.70 3 no Novice Beginner 1
227
+ 3.95 1 yes Beginner Beginner 0
228
+ 3.90 20 yes Advanced Advanced 1
229
+ 3.81 18 yes Advanced Advanced 1
230
+ 3.60 8 no Beginner Advanced 1
231
+ 3.96 25 no Advanced Advanced 1
232
+ 3.76 2 yes Beginner Beginner 0
233
+ 3.83 17 no Advanced Advanced 1
234
+
235
+ ...
236
+ # Print errors list.
237
+ >>> err
238
+ []
239
+ # Print warnings list.
240
+ >>> warn
241
+ []
242
+
243
+ # Example 3: Following example exports teradataml DataFrame df
244
+ # to Pandas DataFrame using fastexport() by opening specified
245
+ # number of Teradata data-transfer sessions.
246
+ >>> fastexport(df, open_sessions=2)
247
+ Errors: []
248
+ Warnings: []
249
+ masters gpa stats programming admitted
250
+ id
251
+ 38 yes 2.65 Advanced Beginner 1
252
+ 26 yes 3.57 Advanced Advanced 1
253
+ 5 no 3.44 Novice Novice 0
254
+ 24 no 1.87 Advanced Novice 1
255
+ 3 no 3.70 Novice Beginner 1
256
+ 1 yes 3.95 Beginner Beginner 0
257
+ 20 yes 3.90 Advanced Advanced 1
258
+ 18 yes 3.81 Advanced Advanced 1
259
+ 8 no 3.60 Beginner Advanced 1
260
+ 25 no 3.96 Advanced Advanced 1
261
+ ...
262
+
263
+ # Example 4: Following example exports teradataml DataFrame df
264
+ # to a given CSV file using fastexport().
265
+ >>> fastexport(df, export_to="csv", csv_file="Test.csv")
266
+ Data is successfully exported into Test.csv
267
+
268
+ # Example 5: Following example exports teradataml DataFrame df
269
+ # to a given CSV file using fastexport() by opening specified
270
+ # number of Teradata data-transfer sessions.
271
+ >>> fastexport(df, export_to="csv", csv_file="Test_1.csv", open_sessions=2)
272
+ Data is successfully exported into Test_1.csv
273
+
274
+ # Example 6: Following example exports teradataml DataFrame df
275
+ # to a given CSV file using fastexport() and catch errors/warnings
276
+ # thrown by fastexport.
277
+ >>> err, warn = fastexport(df, export_to="csv", catch_errors_warnings=True,
278
+ csv_file="Test_3.csv")
279
+ Data is successfully exported into Test_3.csv
280
+ # Print errors list.
281
+ >>> err
282
+ []
283
+ # Print warnings list.
284
+ >>> warn
285
+ []
286
+
287
+ # Example 7: Export teradataml DataFrame to CSV file with '|' as field separator
288
+ # and single quote(') as field quote character.
289
+ >>> fastexport(df, export_to="csv", csv_file="Test_4.csv", sep = "|", quotechar="'")
290
+ Data is successfully exported into Test_4.csv
291
+
292
+ """
293
+ try:
294
+ # Deriving global connection using context.get_context()
295
+ con = get_context()
296
+ if con is None:
297
+ raise TeradataMlException(
298
+ Messages.get_message(MessageCodes.CONNECTION_FAILURE),
299
+ MessageCodes.CONNECTION_FAILURE)
300
+
301
+ awu_matrix = []
302
+ # Add new exports once supported.
303
+ permitted_exports = ["pandas", "csv"]
304
+ from teradataml.dataframe.dataframe import DataFrame
305
+ awu_matrix.append(["df", df, False, DataFrame, True])
306
+ awu_matrix.append(["export_to", export_to, True, str, False,
307
+ permitted_exports])
308
+ awu_matrix.append(["csv_file", csv_file, True, str, True])
309
+
310
+ # Get open_sessions argument.
311
+ open_sessions = kwargs.get("open_sessions", None)
312
+ awu_matrix.append(["open_sessions", open_sessions, True, int, False])
313
+
314
+ # Validate arguments unique to fastexport() function.
315
+ _Validators._validate_function_arguments(awu_matrix)
316
+
317
+ if open_sessions is not None:
318
+ _Validators._validate_positive_int(open_sessions, "open_sessions")
319
+
320
+ # Convert teradataml DataFrame to pandas DataFrame.
321
+ if export_to.lower() == "pandas":
322
+ # Initialize and validate DataTransferUtils object.
323
+ dt_obj = _DataTransferUtils(df, index_column=index_column,
324
+ all_rows=True,
325
+ catch_errors_warnings=catch_errors_warnings)
326
+
327
+ # Call fastexport_get_pandas_df function to get pandas dataframe
328
+ # using fastexport datatransfer protocol.
329
+ # "require" is always True, because with this function user requires
330
+ # fastexport.
331
+
332
+ return dt_obj._fastexport_get_pandas_df(require=True, **kwargs)
333
+
334
+ # Convert teradataml DataFrame to CSV file.
335
+ if export_to.lower() == "csv":
336
+ if not csv_file:
337
+ raise TeradataMlException(
338
+ Messages.get_message(MessageCodes.DEPENDENT_ARG_MISSING, "csv_file",
339
+ "{0}='{1}'".format("export_to", "csv")),
340
+ MessageCodes.DEPENDENT_ARG_MISSING)
341
+
342
+ if not csv_file.lower().endswith(".csv"):
343
+ raise TeradataMlException(
344
+ Messages.get_message(MessageCodes.INVALID_ARG_VALUE, csv_file,
345
+ "csv_file", "file with csv format"),
346
+ MessageCodes.INVALID_ARG_VALUE)
347
+
348
+ # Get "sep" and "quotechar" argument.
349
+ sep = kwargs.pop("sep", ",")
350
+ quotechar = kwargs.pop("quotechar", "\"")
351
+
352
+ dt_obj = _DataTransferUtils(df, all_rows=True,
353
+ sep=sep, quotechar=quotechar,
354
+ catch_errors_warnings=catch_errors_warnings)
355
+ return dt_obj._get_csv(require_fastexport=True, csv_file_name=csv_file, **kwargs)
356
+
357
+ except TeradataMlException:
358
+ raise
359
+ except TypeError:
360
+ raise
361
+ except ValueError:
362
+ raise
363
+ except Exception as err:
364
+ raise TeradataMlException(
365
+ Messages.get_message(MessageCodes.DATA_EXPORT_FAILED, "fastexport",
366
+ export_to, str(err)),
367
+ MessageCodes.DATA_EXPORT_FAILED)
368
+
369
+
370
+ @collect_queryband(queryband="rdCsv")
371
+ def read_csv(filepath,
372
+ table_name,
373
+ types=None,
374
+ sep=",",
375
+ quotechar="\"",
376
+ schema_name=None,
377
+ if_exists='replace',
378
+ primary_index=None,
379
+ set_table=False,
380
+ temporary=False,
381
+ primary_time_index_name=None,
382
+ timecode_column=None,
383
+ timebucket_duration=None,
384
+ timezero_date=None,
385
+ columns_list=None,
386
+ sequence_column=None,
387
+ seq_max=None,
388
+ catch_errors_warnings=False,
389
+ save_errors=False,
390
+ use_fastload=True,
391
+ open_sessions=None):
392
+ """
393
+ The read_csv() API loads data from CSV file into Teradata Vantage.
394
+ Function can be used to quickly load large amounts of data in a table on Vantage
395
+ using FastloadCSV protocol.
396
+
397
+ Considerations when using a CSV file:
398
+ * Each record is on a separate line of the CSV file. Records are delimited
399
+ by line breaks (CRLF). The last record in the file may or may not have an
400
+ ending line break.
401
+ * First line in the CSV must be header line. The header line lists
402
+ the column names separated by the field separator (e.g. col1,col2,col3).
403
+ * Using a CSV file with FastLoad has limitations as follows:
404
+ 1. read_csv API cannot load duplicate rows in the DataFrame if the table is a
405
+ MULTISET table having primary index.
406
+ 2. read_csv API does not support all Teradata Advanced SQL Engine data types.
407
+ For example, target table having BLOB and CLOB data type columns cannot be
408
+ loaded.
409
+ 3. If there are any incorrect rows, i.e. due to constraint violations, data type
410
+ conversion errors, etc., FastLoad protocol ignores those rows and inserts
411
+ all valid rows.
412
+ 4. Rows in the DataFrame that failed to get inserted are categorized into errors
413
+ and warnings by FastLoad protocol and these errors and warnings are stored
414
+ into respective error and warning tables by FastLoad API.
415
+ 5. Teradata recommends to use Fastload protocol when number of rows to be loaded
416
+ are at least 100,000. Fastload opens multiple data transfer connections to the
417
+ database.
418
+
419
+ For additional information about FastLoadCSV protocol through teradatasql driver,
420
+ please refer the CSV BATCH INSERTS section of https://pypi.org/project/teradatasql/#CSVBatchInserts
421
+ driver documentation for more information.
422
+
423
+ PARAMETERS:
424
+ filepath:
425
+ Required Argument.
426
+ Specifies the CSV filepath including name of the file to load the data from.
427
+ Types: String
428
+
429
+ table_name:
430
+ Required Argument.
431
+ Specifies the table name to load the data into.
432
+ Types: String
433
+
434
+ types:
435
+ Optional Argument when if_exists=append and non-PTI table already exists, Required otherwise.
436
+ Specifies the data types for requested columns to be saved in Vantage.
437
+ Keys of this dictionary should be the name of the columns and values should be
438
+ teradatasqlalchemy.types.
439
+ Default Value: None
440
+ Note:
441
+ * If specified when "if_exists" is set to append and table exists, then argument is ignored.
442
+ Types: OrderedDict
443
+
444
+ sep:
445
+ Optional Argument.
446
+ Specifies a single character string used to separate fields in a CSV file.
447
+ Default Value: ","
448
+ Notes:
449
+ * "sep" cannot be line feed ('\\n') or carriage return ('\\r').
450
+ * "sep" should not be same as "quotechar".
451
+ * Length of "sep" argument should be 1.
452
+ Types: String
453
+
454
+ quotechar:
455
+ Optional Argument.
456
+ Specifies a single character string used to quote fields in a CSV file.
457
+ Default Value: "\""
458
+ Notes:
459
+ * "quotechar" cannot be line feed ('\\n') or carriage return ('\\r').
460
+ * "quotechar" should not be same as "sep".
461
+ * Length of "quotechar" argument should be 1.
462
+ Types: String
463
+
464
+ schema_name:
465
+ Optional Argument.
466
+ Specifies the name of the database/schema in Vantage to write to.
467
+ Default Value: None (Uses default database/schema).
468
+ Types: String
469
+
470
+ if_exists:
471
+ Optional Argument.
472
+ Specifies the action to take when table already exists in Vantage.
473
+ Permitted Values: 'fail', 'replace', 'append'
474
+ - fail: If table exists, raise TeradataMlException.
475
+ - replace: If table exists, drop it, recreate it, and insert data.
476
+ - append: If table exists, append the existing table.
477
+ Default Value: replace
478
+ Types: String
479
+
480
+ primary_index:
481
+ Optional Argument.
482
+ Specifies which column(s) to use as primary index while creating table
483
+ in Vantage. When set to None, No Primary Index (NoPI) tables are created.
484
+ Default Value: None
485
+ Types: String or list of strings
486
+ Example:
487
+ primary_index = 'my_primary_index'
488
+ primary_index = ['my_primary_index1', 'my_primary_index2', 'my_primary_index3']
489
+
490
+ set_table:
491
+ Optional Argument.
492
+ Specifies a flag to determine whether to create a SET or a MULTISET table.
493
+ When set to True, a SET table is created, otherwise MULTISET table is created.
494
+ Default Value: False
495
+ Notes:
496
+ 1. Specifying set_table=True also requires specifying primary_index.
497
+ 2. Creating SET table (set_table=True) results in
498
+ a. loss of duplicate rows, if CSV contains any duplicate.
499
+ 3. This argument has no effect if the table already exists and if_exists='append'.
500
+ Types: Boolean
501
+
502
+ temporary:
503
+ Optional Argument.
504
+ Specifies whether to create table as volatile.
505
+ Default Value: False
506
+ Notes:
507
+ When set to True
508
+ 1. FastloadCSV protocol is not used for loading the data.
509
+ 2. "schema_name" is ignored.
510
+ Types : Boolean
511
+
512
+ primary_time_index_name:
513
+ Optional Argument.
514
+ Specifies the name for the Primary Time Index (PTI) when the table
515
+ is to be created as PTI table.
516
+ Note:
517
+ This argument is not required or used when the table to be created
518
+ is not a PTI table. It will be ignored if specified without the "timecode_column".
519
+ Types: String
520
+
521
+ timecode_column:
522
+ Optional argument.
523
+ Required when the CSV data must be saved as a PTI table.
524
+ Specifies the column in the csv that reflects the form
525
+ of the timestamp data in the time series.
526
+ This column will be the TD_TIMECODE column in the table created.
527
+ It should be of SQL type TIMESTAMP(n), TIMESTAMP(n) WITH TIMEZONE, or DATE,
528
+ corresponding to Python types datetime.datetime or datetime.date.
529
+ Note:
530
+ When "timecode_column" argument is specified, an attempt to create a PTI table
531
+ will be made. This argument is not required when the table to be created
532
+ is not a PTI table. If this argument is specified, "primary_index" will be ignored.
533
+ Types: String
534
+
535
+ timezero_date:
536
+ Optional Argument.
537
+ Used when the CSV data must be saved as a PTI table.
538
+ Specifies the earliest time series data that the PTI table will accept,
539
+ a date that precedes the earliest date in the time series data.
540
+ Value specified must be of the following format: DATE 'YYYY-MM-DD'
541
+ Default Value: DATE '1970-01-01'.
542
+ Note:
543
+ This argument is not required or used when the table to be created
544
+ is not a PTI table. It will be ignored if specified without the "timecode_column".
545
+ Types: String
546
+
547
+ timebucket_duration:
548
+ Optional Argument.
549
+ Required if "columns_list" is not specified or is None.
550
+ Used when the CSV data must be saved as a PTI table.
551
+ Specifies a duration that serves to break up the time continum in
552
+ the time series data into discrete groups or buckets.
553
+ Specified using the formal form time_unit(n), where n is a positive
554
+ integer, and time_unit can be any of the following:
555
+ CAL_YEARS, CAL_MONTHS, CAL_DAYS, WEEKS, DAYS, HOURS, MINUTES,
556
+ SECONDS, MILLISECONDS, or MICROSECONDS.
557
+ Note:
558
+ This argument is not required or used when the table to be created
559
+ is not a PTI table. It will be ignored if specified without the "timecode_column".
560
+ Types: String
561
+
562
+ columns_list:
563
+ Optional Argument.
564
+ Used when the CSV data must be saved as a PTI table.
565
+ Required if "timebucket_duration" is not specified.
566
+ A list of one or more PTI table column names.
567
+ Note:
568
+ This argument is not required or used when the table to be created
569
+ is not a PTI table. It will be ignored if specified without the "timecode_column".
570
+ Types: String or list of Strings
571
+
572
+ sequence_column:
573
+ Optional Argument.
574
+ Used when the CSV data must be saved as a PTI table.
575
+ Specifies the column of type Integer containing the unique identifier for
576
+ time series data readings when they are not unique in time.
577
+ * When specified, implies SEQUENCED, meaning more than one reading from the same
578
+ sensor may have the same timestamp.
579
+ This column will be the TD_SEQNO column in the table created.
580
+ * When not specified, implies NONSEQUENCED, meaning there is only one sensor reading
581
+ per timestamp.
582
+ This is the default.
583
+ Note:
584
+ This argument is not required or used when the table to be created
585
+ is not a PTI table. It will be ignored if specified without the "timecode_column".
586
+ Types: String
587
+
588
+ seq_max:
589
+ Optional Argument.
590
+ Used when the CSV data must be saved as a PTI table.
591
+ Specifies the maximum number of data rows that can have the
592
+ same timestamp. Can be used when 'sequenced' is True.
593
+ Permitted range: 1 - 2147483647.
594
+ Default Value: 20000.
595
+ Note:
596
+ This argument is not required or used when the table to be created
597
+ is not a PTI table. It will be ignored if specified without the "timecode_column".
598
+ Types: Integer
599
+
600
+ save_errors:
601
+ Optional Argument.
602
+ Specifies whether to persist the errors/warnings(if any) information in Vantage
603
+ or not.
604
+ If "save_errors" is set to False:
605
+ 1. Errors or warnings (if any) are not persisted into tables.
606
+ 2. Errors table genarated by FastloadCSV are not persisted.
607
+ If "save_errors" is set to True:
608
+ 1. The errors or warnings information is persisted and names of error and
609
+ warning tables are returned. Otherwise, the function returns None for
610
+ the names of the tables.
611
+ 2. The errors tables generated by FastloadCSV are persisted and name of
612
+ error tables are returned. Otherwise, the function returns None for
613
+ the names of the tables.
614
+ Default Value: False
615
+ Types: Boolean
616
+
617
+ catch_errors_warnings:
618
+ Optional Argument.
619
+ Specifies whether to catch errors/warnings(if any) raised by fastload
620
+ protocol while loading data into the Vantage table.
621
+ When set to False, function does not catch any errors and warnings,
622
+ otherwise catches errors and warnings, if any, and returns
623
+ as a dictionary along with teradataml DataFrame.
624
+ Please see 'RETURNS' section for more details.
625
+ Default Value: False
626
+ Types: Boolean
627
+
628
+ use_fastload:
629
+ Optional Argument.
630
+ Specifies whether to use Fastload CSV protocol or not.
631
+ Default Value: True
632
+ Notes:
633
+ 1. Teradata recommends to use Fastload when number of rows to be loaded
634
+ are atleast 100,000. To load lesser rows set this argument to 'False'.
635
+ Fastload opens multiple data transfer connections to the database.
636
+ 2. When "use_fastload" is set to True, one can load the data into table
637
+ using FastloadCSV protocol:
638
+ a. Set table
639
+ b. Multiset table
640
+ 3. When "use_fastload" is set to False, one can load the data in following
641
+ types of tables:
642
+ a. Set table
643
+ b. Multiset table
644
+ c. PTI table
645
+ d. Volatile table
646
+ Types: Boolean
647
+
648
+ open_sessions:
649
+ Optional Argument.
650
+ Specifies the number of Teradata data transfer sessions to be opened for fastload operation.
651
+ Note : If "open_sessions" argument is not provided, the default value is the smaller of 8 or the
652
+ number of AMPs available.
653
+ For additional information about number of Teradata data-transfer
654
+ sessions opened during fastload, please refer to:
655
+ https://pypi.org/project/teradatasql/#FastLoad
656
+ Default Value: None
657
+ Types: int
658
+
659
+ RETURNS:
660
+ When "use_fastload" is set to False, returns teradataml dataframe.
661
+ When "use_fastload" is set to True, read_csv() returns below:
662
+ When "catch_errors_warnings" is set to False, returns only teradataml dataframe.
663
+ When "catch_errors_warnings" is set to True, read_csv() returns a tuple containing:
664
+ a. teradataml DataFrame.
665
+ b. a dict containing the following attributes:
666
+ a. errors_dataframe: It is a Pandas DataFrame containing error messages
667
+ thrown by fastload. DataFrame is empty if there are no errors.
668
+ b. warnings_dataframe: It is a Pandas DataFrame containing warning messages
669
+ thrown by fastload. DataFrame is empty if there are no warnings.
670
+ c. errors_table: Name of the table containing errors. It is None, if
671
+ argument save_errors is False.
672
+ d. warnings_table: Name of the table containing warnings. It is None, if
673
+ argument save_errors is False.
674
+ e. fastloadcsv_error_tables: Name of the tables containing errors generated
675
+ by FastloadCSV. It is empty list, if argument "save_errors" is False.
676
+
677
+ RAISES:
678
+ TeradataMlException
679
+
680
+ EXAMPLES:
681
+ >>> from teradataml.dataframe.data_transfer import read_csv
682
+ >>> from teradatasqlalchemy.types import *
683
+ >>> from collections import OrderedDict
684
+
685
+ # Example 1: Default execution with types argument is passed as OrderedDict.
686
+ >>> types = OrderedDict(id=BIGINT, fname=VARCHAR, lname=VARCHAR, marks=FLOAT)
687
+ >>> read_csv('test_file.csv', 'my_first_table', types)
688
+
689
+ # Example 2: Load the data from CSV file into a table using fastload CSV protocol,
690
+ # while doing so catch all errors and warnings as well as store those in the table.
691
+ >>> types = OrderedDict(id=BIGINT, fname=VARCHAR, lname=VARCHAR, marks=FLOAT)
692
+ >>> read_csv(filepath='test_file.csv',
693
+ ... table_name='my_first_table1', types=types,
694
+ ... save_errors=True, catch_errors_warnings=True)
695
+
696
+ # Example 3: Load the data from CSV file into a table using fastload CSV protocol.
697
+ # If table exists, then replace the same. Catch all errors and warnings as well as
698
+ # store those in the table.
699
+ >>> types = OrderedDict(id=BIGINT, fname=VARCHAR, lname=VARCHAR, marks=FLOAT)
700
+ >>> read_csv(filepath='test_file.csv',
701
+ ... table_name='my_first_table',
702
+ ... types=types, if_exists='replace',
703
+ ... save_errors=True, catch_errors_warnings=True)
704
+
705
+ # Example 4: Load the data from CSV file into a table using fastload CSV protocol.
706
+ # If table exists in specified schema, then append the same. Catch all
707
+ # errors and warnings as well as store those in the table.
708
+ >>> types = OrderedDict(id=BIGINT, fname=VARCHAR, lname=VARCHAR, marks=FLOAT)
709
+ >>> read_csv(filepath='test_file.csv',
710
+ ... table_name='my_first_table',
711
+ ... types=types, if_exists='fail',
712
+ ... save_errors=True, catch_errors_warnings=True)
713
+ >>> read_csv(filepath='test_file.csv',
714
+ ... table_name='my_first_table',
715
+ ... if_exists='append',
716
+ ... save_errors=True, catch_errors_warnings=True)
717
+
718
+ # Example 5: Load the data from CSV file into a SET table using fastload CSV protocol.
719
+ # Catch all errors and warnings as well as store those in the table.
720
+ >>> types = OrderedDict(id=BIGINT, fname=VARCHAR, lname=VARCHAR, marks=FLOAT)
721
+ >>> read_csv(filepath='test_file.csv',
722
+ ... table_name='my_first_table',
723
+ ... types=types, if_exists='replace',
724
+ ... set_table=True, primary_index='id',
725
+ ... save_errors=True, catch_errors_warnings=True)
726
+
727
+ # Example 6: Load the data from CSV file into a temporary table without fastloadCSV protocol.
728
+ # If table exists, then append to the same.
729
+ >>> types = OrderedDict(id=BIGINT, fname=VARCHAR, lname=VARCHAR, marks=FLOAT)
730
+ >>> read_csv(filepath='test_file.csv',
731
+ ... table_name='my_first_table',
732
+ ... types=types, if_exists='replace',
733
+ ... temporary=True)
734
+ >>> read_csv(filepath='test_file.csv',
735
+ ... table_name='my_first_table',
736
+ ... if_exists='append',
737
+ ... temporary=True)
738
+
739
+ # Example 7: Load the data from CSV file with DATE and TIMESTAMP columns into
740
+ # a table without Fastload protocol. If table exists in specified
741
+ # schema, then append to the table.
742
+ >>> types = OrderedDict(id=BIGINT, fname=VARCHAR, lname=VARCHAR, marks=FLOAT,
743
+ ... admission_date=DATE, admission_time=TIMESTAMP)
744
+ >>> read_csv(filepath='test_file.csv',
745
+ ... table_name='my_first_table',
746
+ ... types=types, if_exists='fail',
747
+ ... use_fastload=False)
748
+ >>> read_csv(filepath='test_file.csv',
749
+ ... table_name='my_first_table',
750
+ ... if_exists='append',
751
+ ... use_fastload=False)
752
+
753
+ # Example 8: Load the data from CSV file with TIMESTAMP columns into
754
+ # a PTI table. If specified table exists then append to the table,
755
+ # otherwise creates new table.
756
+ >>> types = OrderedDict(partition_id=INTEGER, adid=INTEGER, productid=INTEGER,
757
+ ... event=VARCHAR, clicktime=TIMESTAMP)
758
+ >>> read_csv(filepath='test_file.csv',
759
+ ... table_name='my_first_read_csv_pti_table',
760
+ ... types=types, if_exists='append',
761
+ ... timecode_column='clicktime',
762
+ ... columns_list='event',
763
+ ... use_fastload=False)
764
+
765
+ # Example 9: Load the data from CSV file with TIMESTAMP columns into
766
+ # a SET PTI table. If specified table exists then append to the table,
767
+ # otherwise creates new table.
768
+ >>> types = OrderedDict(partition_id=INTEGER, adid=INTEGER, productid=INTEGER,
769
+ event=VARCHAR, clicktime=TIMESTAMP)
770
+ >>> read_csv(filepath='test_file.csv',
771
+ ... table_name='my_first_read_csv_pti_table',
772
+ ... types=types, if_exists='append',
773
+ ... timecode_column='clicktime',
774
+ ... columns_list='event',
775
+ ... set_table=True)
776
+
777
+ # Example 10: Load the data from CSV file with TIMESTAMP columns into
778
+ # a temporary PTI table. If specified table exists then append to the table,
779
+ # otherwise creates new table.
780
+ >>> types = OrderedDict(partition_id=INTEGER, adid=INTEGER, productid=INTEGER,
781
+ event=VARCHAR, clicktime=TIMESTAMP)
782
+ >>> read_csv(filepath='test_file.csv',
783
+ ... table_name='my_first_read_csv_pti_table',
784
+ ... types=types, if_exists='append',
785
+ ... timecode_column='clicktime',
786
+ ... columns_list='event',
787
+ ... temporary=True)
788
+
789
+ # Example 11: Load the data from CSV file into Vantage table by opening specified
790
+ # number of Teradata data transfer sesions.
791
+ >>> types = OrderedDict(id=BIGINT, fname=VARCHAR, lname=VARCHAR, marks=FLOAT)
792
+ >>> read_csv(filepath='test_file.csv', table_name='my_first_table_with_open_sessions',
793
+ types=types, open_sessions=2)
794
+
795
+ # Example 12: Load the data from CSV file into Vantage table and set primary index provided
796
+ # through primary_index argument.
797
+ >>> types = OrderedDict(id=BIGINT, fname=VARCHAR, lname=VARCHAR, marks=FLOAT)
798
+ >>> read_csv(filepath='test_file.csv', table_name='my_first_table_with_primary_index',
799
+ ... types=types, primary_index = ['fname'])
800
+
801
+ # Example 13: Load the data from CSV file into VECTOR datatype in Vantage table.
802
+ >>> from teradatasqlalchemy import VECTOR
803
+ >>> from pathlib import Path
804
+ >>> types = OrderedDict(id=BIGINT, array_col=VECTOR)
805
+
806
+ # Get the absolute path of the teradataml module
807
+ >>> import teradataml
808
+ >>> base_path = Path(teradataml.__path__[0])
809
+
810
+ # Append the relative path to the CSV file
811
+ >>> csv_path = os.path.join(base_path, "data", "hnsw_alter_data.csv")
812
+
813
+ >>> read_csv(filepath=csv_path,
814
+ ... table_name='my_first_table_with_vector',
815
+ ... types=types,
816
+ ... use_fastload=False)
817
+ """
818
+ # Deriving global connection using context.get_context()
819
+ con = get_context()
820
+
821
+ try:
822
+ if con is None:
823
+ raise TeradataMlException(Messages.get_message(MessageCodes.CONNECTION_FAILURE),
824
+ MessageCodes.CONNECTION_FAILURE)
825
+
826
+ schema_name = _get_context_temp_databasename() if schema_name is None else schema_name
827
+
828
+ # Create _DataTransferUtils object.
829
+ dt_obj = _DataTransferUtils(filepath, table_name=table_name, types=types,
830
+ sep=sep, quotechar=quotechar, schema_name=schema_name,
831
+ if_exists=if_exists, primary_index=primary_index,
832
+ set_table=set_table, temporary=temporary,
833
+ primary_time_index_name=primary_time_index_name,
834
+ timecode_column=timecode_column,
835
+ timebucket_duration=timebucket_duration,
836
+ timezero_date=timezero_date, columns_list=columns_list,
837
+ sequence_column=sequence_column, seq_max=seq_max,
838
+ save_errors=save_errors,
839
+ catch_errors_warnings=catch_errors_warnings,
840
+ use_fastload=use_fastload,
841
+ api_name='read_csv',
842
+ open_sessions=open_sessions)
843
+
844
+ # Validate read_csv api argument
845
+ dt_obj._validate_read_csv_api_args()
846
+
847
+ # Check if CSV file exists
848
+ _Validators._validate_file_exists(filepath)
849
+
850
+ # Ignore open_sessions argument when use_fastload is set to False
851
+ if not use_fastload and open_sessions is not None:
852
+ UtilFuncs._get_warnings('open_sessions', open_sessions, 'use_fastload', use_fastload)
853
+ dt_obj.open_sessions = None
854
+
855
+ # If temporary=True, set use_fastload=False and schema name=None.
856
+ if temporary:
857
+ # Setting fastload related arguments to False.
858
+ dt_obj.use_fastload = False
859
+ use_fastload = False
860
+
861
+ if schema_name is not None:
862
+ UtilFuncs._get_warnings("schema_name", schema_name, "temporary", "True")
863
+ schema_name = None
864
+
865
+ # A table cannot be a SET table and have NO PRIMARY INDEX
866
+ if set_table and primary_index is None:
867
+ raise TeradataMlException(Messages.get_message(MessageCodes.SET_TABLE_NO_PI),
868
+ MessageCodes.SET_TABLE_NO_PI)
869
+
870
+ # Check for PTI table.
871
+ if timecode_column is not None:
872
+ rc_dict = dt_obj._process_pti_load_csv_data(con=con)
873
+ return dt_obj._get_result(rc_dict)
874
+
875
+ # Check if table exists.
876
+ table_exists = dt_obj._table_exists(con)
877
+
878
+ # Raise exception when table not exists and if_exists='fail'.
879
+ dt_obj._check_table_exists(is_table_exists=table_exists)
880
+
881
+ # Let's create the SQLAlchemy table object to recreate the table.
882
+ if not table_exists or if_exists.lower() == 'replace':
883
+ # Validate required argument "types" when table doesn't exist
884
+ _Validators._validate_argument_is_not_None(types, "types")
885
+
886
+ dt_obj._create_or_replace_table(con, table_exists=table_exists)
887
+
888
+ # Load the CSV data into newly created table.
889
+ if not use_fastload:
890
+ rc_dict = dt_obj._insert_from_csv_without_fastload(table_name=table_name,
891
+ column_names=types.keys())
892
+ else:
893
+ rc_dict = dt_obj._insert_from_csv_with_fastload(table_name=table_name,
894
+ column_names=types.keys())
895
+
896
+ # Check column compatibility for insertion when table exists and if_exists = 'append'.
897
+ if table_exists and if_exists.lower() == 'append':
898
+ if set_table:
899
+ UtilFuncs._get_warnings('set_table', set_table, 'if_exists', 'append')
900
+
901
+ # Create SQLAlchemy table object from existing table.
902
+ existing_table = UtilFuncs._get_sqlalchemy_table(table_name,
903
+ schema_name=schema_name)
904
+
905
+ # Check compatibility of CSV columns with existing table columns.
906
+ if types is not None:
907
+ cols = _extract_column_info(filepath, types=types)
908
+ dt_obj._check_columns_compatibility(table_obj=existing_table, cols=cols)
909
+
910
+ # Validate user provided primary index against primary index of existing table .
911
+ existing_table_primary_index = UtilFuncs._extract_table_object_index_info(existing_table)
912
+ if primary_index is not None:
913
+ dt_obj._check_index_compatibility(primary_index_1=existing_table_primary_index,
914
+ primary_index_2=primary_index)
915
+
916
+ cols_name, cols_type = UtilFuncs._extract_table_object_column_info(existing_table.c)
917
+ column_info = dict(zip(cols_name, cols_type))
918
+
919
+ if use_fastload:
920
+ rc_dict = dt_obj._create_staging_table_and_load_csv_data(column_info=column_info,
921
+ primary_index=existing_table_primary_index)
922
+ else:
923
+ rc_dict = dt_obj._insert_from_csv_without_fastload(table_name=table_name,
924
+ column_names=cols_name)
925
+ # Return the read_csv result.
926
+ return dt_obj._get_result(rc_dict)
927
+
928
+ except (TeradataMlException, sqlachemyOperationalError, ValueError, TypeError):
929
+ raise
930
+ except Exception as err:
931
+ error_code = MessageCodes.EXECUTION_FAILED
932
+ error_msg = Messages.get_message(
933
+ error_code, "execute read_csv()", '{}'.format(str(err)))
934
+ raise TeradataMlException(error_msg, error_code)
935
+
936
+
937
+ class _DataTransferUtils():
938
+ """
939
+ This class provides utility functions which enable Data Transfer from
940
+ Teradata Vantage to outside world, for example Data Transfer using
941
+ FastExport Protocol.
942
+ """
943
+
944
+ def __init__(self, df, index_column=None, num_rows=99999, all_rows=False,
945
+ catch_errors_warnings=False, table_name=None,
946
+ schema_name=None, if_exists='append', index=False,
947
+ index_label=None, primary_index=None, temporary=False,
948
+ types=None, batch_size=None, save_errors=False, sep=",",
949
+ quotechar="\"", set_table=False,
950
+ primary_time_index_name=None, timecode_column=None,
951
+ timebucket_duration=None, timezero_date=None,
952
+ columns_list=None, sequence_column=None, seq_max=None,
953
+ use_fastload=True, api_name='fastexport',
954
+ open_sessions=None, chunksize=CopyToConstants.DBAPI_BATCHSIZE.value,
955
+ match_column_order=True, err_tbl_1_suffix=None,
956
+ err_tbl_2_suffix=None, err_tbl_name=None, warn_tbl_name=None,
957
+ err_staging_db=None):
958
+ """
959
+ DESCRIPTION:
960
+ Constructor for the _DataTransferUtils class. It initialises
961
+ arguments that are required for data transfer using FastExport
962
+ protocol or non-fastexport based data transfer using to_pandas()
963
+ API.
964
+
965
+ PARAMETERS:
966
+ df:
967
+ Required Argument.
968
+ Specifies the teradataml DataFrame from which data is to be
969
+ extracted. OR
970
+ Specifies the CSV filepath including name of the file to
971
+ load the data from.
972
+ Types: teradataml DataFrame OR str
973
+
974
+ index_column:
975
+ Optional Argument.
976
+ Specifies column(s) to be used as index column for the converted
977
+ object.
978
+ Types: str OR list of Strings (str)
979
+
980
+ num_rows:
981
+ Optional Argument.
982
+ Specifies the number of rows to be retrieved from teradataml
983
+ DataFrame.
984
+ Default Value: 99999
985
+ Types: int
986
+
987
+ all_rows:
988
+ Optional Argument.
989
+ Specifies whether all rows from teradataml DataFrame should be
990
+ retrieved.
991
+ Default Value: False
992
+ Types: bool
993
+
994
+ catch_errors_warnings:
995
+ Optional Argument.
996
+ Specifies whether to catch errors/warnings(if any) raised by
997
+ fastexport protocol while converting teradataml DataFrame.
998
+ Default Value: False
999
+ Types: bool
1000
+
1001
+ table_name:
1002
+ Optional Argument.
1003
+ Specifies the table name to load the data into.
1004
+ Types: String
1005
+
1006
+ types:
1007
+ Optional Argument.
1008
+ Specifies the data types for requested columns to be saved in Vantage.
1009
+ Keys of this dictionary should be the name of the columns and values should be
1010
+ teradatasqlalchemy.types.
1011
+ Default Value: None
1012
+ Note:
1013
+ This should be OrderedDict, if CSV file does not contain header.
1014
+ Types: OrderedDict
1015
+
1016
+ sep:
1017
+ Optional Argument.
1018
+ Specifies a single character string used to separate fields in a CSV file.
1019
+ Default Value: ","
1020
+ Notes:
1021
+ * "sep" cannot be line feed ('\\n') or carriage return ('\\r').
1022
+ * "sep" should not be same as "quotechar".
1023
+ * Length of "sep" argument should be 1.
1024
+ Types: String
1025
+
1026
+ quotechar:
1027
+ Optional Argument.
1028
+ Specifies a single character string used to quote fields in a CSV file.
1029
+ Default Value: "\""
1030
+ Notes:
1031
+ * "quotechar" cannot be line feed ('\\n') or carriage return ('\\r').
1032
+ * "quotechar" should not be same as "sep".
1033
+ * Length of "quotechar" argument should be 1.
1034
+ Types: String
1035
+
1036
+ schema_name:
1037
+ Optional Argument.
1038
+ Specifies the name of the database schema in Vantage to write to.
1039
+ Default Value: None (Uses default database schema).
1040
+ Types: String
1041
+
1042
+ if_exists:
1043
+ Optional Argument.
1044
+ Specifies the action to take when table already exists in Vantage.
1045
+ Permitted Values: 'fail', 'replace', 'append'
1046
+ - fail: If table exists, raise TeradataMlException.
1047
+ - replace: If table exists, drop it, recreate it, and insert data.
1048
+ - append: If table exists, insert data. Create if does not exist.
1049
+ Default Value: fail
1050
+ Types: String
1051
+
1052
+ primary_index:
1053
+ Optional Argument.
1054
+ Specifies which column(s) to use as primary index while creating table
1055
+ in Vantage. When set to None, No Primary Index (NoPI) tables are created.
1056
+ Default Value: None
1057
+ Types: String or list of strings
1058
+ Example:
1059
+ primary_index = 'my_primary_index'
1060
+ primary_index = ['my_primary_index1', 'my_primary_index2', 'my_primary_index3']
1061
+
1062
+ temporary:
1063
+ Optional Argument.
1064
+ Specifies whether to creates Vantage tables as permanent or volatile.
1065
+ When set to True:
1066
+ 1. volatile Tables are created, and
1067
+ 2. schema_name is ignored.
1068
+ When set to False, permanent tables are created.
1069
+ Default Value: False
1070
+ Types : Boolean (True or False)
1071
+
1072
+ set_table:
1073
+ Optional Argument.
1074
+ Specifies a flag to determine whether to create a SET or a MULTISET table.
1075
+ When set to True, a SET table is created, otherwise MULTISET table is created.
1076
+ Default Value: False
1077
+ Notes:
1078
+ 1. Specifying set_table=True also requires specifying primary_index.
1079
+ 2. Creating SET table (set_table=True) results in
1080
+ a. loss of duplicate rows.
1081
+ 3. This argument has no effect if the table already exists and if_exists='append'.
1082
+ Types: Boolean
1083
+
1084
+ save_errors:
1085
+ Optional Argument.
1086
+ Specifies whether to persist the errors/warnings(if any) information in Vantage
1087
+ or not. If save_errors is set to False, errors/warnings(if any) are not persisted
1088
+ as tables. If argument is set to True, the error and warnings information
1089
+ are persisted and names of error and warning tables are returned. Otherwise,
1090
+ the function returns None for the names of the tables.
1091
+ Default Value: False
1092
+ Types: bool
1093
+
1094
+ open_sessions:
1095
+ Optional Argument.
1096
+ Specifies the number of Teradata data transfer sessions to be opened for fastload operation.
1097
+ Note :
1098
+ If "open_sessions" argument is not provided, the default value is
1099
+ the one smaller out of 8 and the number of AMPs available.
1100
+ Default Value: None
1101
+ Types: int
1102
+
1103
+ chunksize:
1104
+ Optional Argument.
1105
+ Specifies the number of rows to be loaded in a batch.
1106
+ Note:
1107
+ This is argument is used only when argument "df" is pandas DataFrame.
1108
+ Default Value: 16383
1109
+ Types: int
1110
+
1111
+ err_tbl_1_suffix:
1112
+ Optional Argument.
1113
+ Specifies the suffix for error table 1 created by fastload job.
1114
+ Types: String
1115
+
1116
+ err_tbl_2_suffix:
1117
+ Optional Argument.
1118
+ Specifies the suffix for error table 2 created by fastload job.
1119
+ Types: String
1120
+
1121
+ err_tbl_name:
1122
+ Optional Argument.
1123
+ Specifies the name for error table.
1124
+ Types: String
1125
+
1126
+ warn_tbl_name:
1127
+ Optional Argument.
1128
+ Specifies the name for warning table.
1129
+ Types: String
1130
+
1131
+ err_staging_db:
1132
+ Optional Argument.
1133
+ Specifies the name of the database to be used for creating staging
1134
+ table and error tables.
1135
+ Note:
1136
+ Current session user must have CREATE, DELETE and INSERT table
1137
+ rights on err_staging_db database.
1138
+ Types: String
1139
+
1140
+ PARAMETERS:
1141
+ None.
1142
+
1143
+ RETURNS:
1144
+ None.
1145
+
1146
+ RAISES:
1147
+ None.
1148
+
1149
+ EXAMPLES:
1150
+ dt_obj = _DataTransferUtils(df)
1151
+ dt_obj = _DataTransferUtils(df, index_column='gpa')
1152
+ dt_obj = _DataTransferUtils(df, num_rows=10)
1153
+ dt_obj = _DataTransferUtils(df, all_rows=True, num_rows=5)
1154
+ dt_obj = _DataTransferUtils(df, catch_errors_warnings=True,
1155
+ num_rows=200)
1156
+ dt_obj = _DataTransferUtils(df, table_name='my_table',
1157
+ types={'id': BIGINT, 'fname': VARCHAR,
1158
+ 'lname': VARCHAR, 'marks': FLOAT})
1159
+ """
1160
+
1161
+ self.df = df
1162
+ self.index_column = index_column
1163
+ self.num_rows = num_rows
1164
+ self.all_rows = all_rows
1165
+ self.catch_errors_warnings = catch_errors_warnings
1166
+ self.table_name = table_name
1167
+ self.schema_name = schema_name
1168
+ self.if_exists = if_exists
1169
+ self.index = index
1170
+ self.index_label = index_label
1171
+ self.primary_index = primary_index
1172
+ self.temporary = temporary
1173
+ self.types = types
1174
+ self.batch_size = batch_size
1175
+ self.save_errors = save_errors
1176
+ self.sep = sep
1177
+ self.quotechar = quotechar
1178
+ self.primary_time_index_name = primary_time_index_name
1179
+ self.timecode_column = timecode_column
1180
+ self.timebucket_duration = timebucket_duration
1181
+ self.timezero_date = timezero_date
1182
+ self.columns_list = columns_list
1183
+ self.sequence_column = sequence_column
1184
+ self.seq_max = seq_max
1185
+ self.set_table = set_table
1186
+ self.use_fastload = use_fastload
1187
+ self.api_name = api_name
1188
+ self.open_sessions = open_sessions
1189
+ self.chunksize = chunksize
1190
+ self.match_column_order = match_column_order
1191
+ self.err_tbl_1_suffix = err_tbl_1_suffix
1192
+ self.err_tbl_2_suffix = err_tbl_2_suffix
1193
+ self.err_tbl_name = err_tbl_name
1194
+ self.warn_tbl_name = warn_tbl_name
1195
+ self.err_staging_db = err_staging_db
1196
+
1197
+ # Validate arguments.
1198
+ if self.api_name == 'fastexport':
1199
+ self._validate_data_export_api_args()
1200
+
1201
+ # Functions specific to validation of arguments of
1202
+ # Datatransfer APIs.
1203
+ def _validate_data_export_api_args(self):
1204
+ """
1205
+ DESCRIPTION:
1206
+ Function to validate common arguments used in data export API's
1207
+ such as:
1208
+ 1. DataFrame.to_pandas()
1209
+ 2. DataFrame.to_csv()
1210
+ 2. fastexport()
1211
+
1212
+ PARAMETERS:
1213
+ None.
1214
+
1215
+ RETURNS:
1216
+ None.
1217
+
1218
+ RAISES:
1219
+ TeradataMlException,
1220
+ TypeError,
1221
+ ValueError.
1222
+
1223
+ EXAMPLES:
1224
+ dt_obj = _DataTransferUtils(df)
1225
+ dt_obj._validate_data_export_api_args()
1226
+ """
1227
+ awu_matrix = []
1228
+ awu_matrix.append(
1229
+ ["index_column", self.index_column, True, (str, list), True])
1230
+ awu_matrix.append(["num_rows", self.num_rows, True, (int)])
1231
+ awu_matrix.append(["all_rows", self.all_rows, True, (bool)])
1232
+ awu_matrix.append(
1233
+ ["catch_errors_warnings", self.catch_errors_warnings, True, (bool)])
1234
+ awu_matrix.append(["open_sessions", self.open_sessions, True, (int), False])
1235
+ awu_matrix.append(["sep", self.sep, True, str, False])
1236
+ awu_matrix.append(["quotechar", self.quotechar, True, str, False])
1237
+
1238
+ # Validate argument types.
1239
+ _Validators._validate_function_arguments(awu_matrix)
1240
+ # Validate if 'num_rows' is a positive int.
1241
+ _Validators._validate_positive_int(self.num_rows, "num_rows")
1242
+ # Validate if 'open_sessions' is a positive int.
1243
+ _Validators._validate_positive_int(self.open_sessions, "open_sessions")
1244
+
1245
+ # Validate "sep" and "quotechar" arguments related to export to CSV.
1246
+ self._validate_csv_sep_quotechar()
1247
+
1248
+ # Checking if meta expression exists for given dataframe.
1249
+ if self.df._metaexpr is None:
1250
+ raise TeradataMlException(
1251
+ Messages.get_message(MessageCodes.TDMLDF_INFO_ERROR),
1252
+ MessageCodes.TDMLDF_INFO_ERROR)
1253
+
1254
+ # Checking each element in passed columns to be valid column in
1255
+ # dataframe.
1256
+ _Validators._validate_column_exists_in_dataframe(self.index_column,
1257
+ self.df._metaexpr)
1258
+
1259
+ def _validate_read_csv_api_args(self):
1260
+ """
1261
+ Internal function to validate read_csv api arguments.
1262
+
1263
+ PARAMETERS:
1264
+ None
1265
+
1266
+ RETURNS:
1267
+ None
1268
+
1269
+ RAISES:
1270
+ None
1271
+
1272
+ EXAMPLES:
1273
+ dt_obj = _DataTransferUtils()
1274
+ dt_obj._validate_read_csv_api_args()
1275
+ """
1276
+ # Validate read_csv api arguments.
1277
+ self._validate()
1278
+
1279
+ # Validate "sep" and "quotechar" arguments.
1280
+ self._validate_csv_sep_quotechar()
1281
+
1282
+ def _validate_csv_sep_quotechar(self):
1283
+ """
1284
+ Internal function to validate field separator and field quote character
1285
+ used in data transfer APIs which involve CSV.
1286
+
1287
+ PARAMETERS:
1288
+ None
1289
+
1290
+ RETURNS:
1291
+ None
1292
+
1293
+ RAISES:
1294
+ TeradataMlException
1295
+
1296
+ EXAMPLES:
1297
+ dt_obj = _DataTransferUtils()
1298
+ dt_obj._validate_csv_sep_quotechar()
1299
+ """
1300
+
1301
+ # Function to validate char value for length and allowed characters.
1302
+ def validate_char_arg_csv(arg_name, arg):
1303
+
1304
+ if arg is not None:
1305
+ _Validators._validate_str_arg_length(arg_name, arg, 'EQ', 1)
1306
+
1307
+ not_allowed_values = ["\n", "\r"]
1308
+ if arg in not_allowed_values:
1309
+ message = Messages.get_message(MessageCodes.NOT_ALLOWED_VALUES,
1310
+ "{}".format(not_allowed_values), arg_name)
1311
+ raise TeradataMlException(message, MessageCodes.NOT_ALLOWED_VALUES)
1312
+
1313
+ # Validate the 'sep' and 'quotechar' arguments.
1314
+ validate_char_arg_csv("sep", self.sep)
1315
+ validate_char_arg_csv("quotechar", self.quotechar)
1316
+
1317
+ # Validate 'quotechar' and 'sep' arguments not set to same value
1318
+ if self.quotechar == self.sep:
1319
+ message = Messages.get_message(MessageCodes.ARGUMENT_VALUE_SAME, "sep",
1320
+ "quotechar")
1321
+ raise TeradataMlException(message, MessageCodes.ARGUMENT_VALUE_SAME)
1322
+
1323
+ # End of functions specific to validation of arguments of
1324
+ # Datatransfer APIs.
1325
+
1326
+ # Functions specific to fastexport().
1327
+ def _validate_df_index_column(self):
1328
+ """
1329
+ DESCRIPTION:
1330
+ Function to validate dataframe index label and throw exception if
1331
+ there is any mismatch in the index label and columns present in the
1332
+ teradataml DataFrame.
1333
+
1334
+ PARAMETERS:
1335
+ None.
1336
+
1337
+ RETURNS:
1338
+ None.
1339
+
1340
+ RAISES:
1341
+ TeradataMLException.
1342
+
1343
+ EXAMPLES:
1344
+ dt_obj = _DataTransferUtils(df)
1345
+ dt_obj._validate_df_index_column()
1346
+ """
1347
+ # Get list of columns in teradatml DataFrame.
1348
+ df_column_list = [col.name.lower() for col in self.df._metaexpr.c]
1349
+
1350
+ # Check if TDML DF has appropriate index_label set when required
1351
+ if self.df._index_label is not None:
1352
+ for index_label in UtilFuncs._as_list(self.df._index_label):
1353
+ if index_label.lower() not in df_column_list:
1354
+ raise TeradataMlException(
1355
+ Messages.get_message(MessageCodes.DF_LABEL_MISMATCH),
1356
+ MessageCodes.DF_LABEL_MISMATCH)
1357
+
1358
+ def _get_pandas_df_index(self):
1359
+ """
1360
+ DESCRIPTION:
1361
+ Function returns the final index column to be used in the resultant
1362
+ DataFrame after converting teradataml DataFrame to Pandas DataFrame.
1363
+
1364
+ PARAMETERS:
1365
+ None.
1366
+
1367
+ RETURNS:
1368
+ Final Valid index as str or list of Strings.
1369
+
1370
+ RAISES:
1371
+ None.
1372
+
1373
+ EXAMPLES:
1374
+ dt_obj = _DataTransferUtils(df)
1375
+ dt_obj._get_pandas_df_index()
1376
+ """
1377
+ index_col = None
1378
+ # Index Order: 1) User specified 2) TDMLDF index 3) DB PI
1379
+ # 4)Else default integer index
1380
+ if self.index_column:
1381
+ index_col = self.index_column
1382
+ elif self.df._index_label:
1383
+ index_col = self.df._index_label
1384
+ else:
1385
+ try:
1386
+ from teradataml.dataframe.dataframe_utils import DataFrameUtils
1387
+ index_col = DataFrameUtils._get_primary_index_from_table(
1388
+ self.df._table_name)
1389
+ except Exception as err:
1390
+ index_col = None
1391
+
1392
+ return index_col
1393
+
1394
+ def _generate_to_pandas_base_query(self):
1395
+ """
1396
+ DESCRIPTION:
1397
+ Function to generate base query for to_pandas() function. This query
1398
+ is further used to generate pandas dataframe.
1399
+
1400
+ PARAMETERS:
1401
+ None.
1402
+
1403
+ RETURNS:
1404
+ str.
1405
+
1406
+ RAISES:
1407
+ None.
1408
+
1409
+ EXAMPLES:
1410
+ dt_obj = _DataTransferUtils(df)
1411
+ base_query = dt_obj._generate_to_pandas_base_query()
1412
+ """
1413
+ # Generate SQL Query using Table name & number of rows required.
1414
+ if self.all_rows:
1415
+ # Get read query for the whole data.
1416
+ return SQLBundle._build_base_query(self.df._table_name,
1417
+ self.df._orderby)
1418
+ else:
1419
+ # Get read query using SAMPLE.
1420
+ return SQLBundle._build_sample_rows_from_table(self.df._table_name,
1421
+ self.num_rows,
1422
+ self.df._orderby)
1423
+
1424
+ def _generate_select_query(self):
1425
+ """
1426
+ DESCRIPTION:
1427
+ Function to generate SELECT query.
1428
+
1429
+ PARAMETERS:
1430
+ None.
1431
+
1432
+ RETURNS:
1433
+ str.
1434
+
1435
+ RAISES:
1436
+ None.
1437
+
1438
+ EXAMPLES:
1439
+ dt_obj = _DataTransferUtils(df)
1440
+ base_query = dt_obj._generate_select_query()
1441
+ """
1442
+ # Generate SQL Query using Table name & number of rows required.
1443
+ if self.all_rows:
1444
+ # Get read query for the whole data.
1445
+ return SQLBundle._build_base_query(self.df._table_name,
1446
+ self.df._orderby)
1447
+ else:
1448
+ # Get read query using SAMPLE.
1449
+ return SQLBundle._build_top_n_print_query(self.df._table_name,
1450
+ self.num_rows,
1451
+ self.df._orderby)
1452
+
1453
+ def _generate_fastexport_query(self, base_query, require=False, open_sessions=None,
1454
+ csv_file_name=None):
1455
+ """
1456
+ DESCRIPTION:
1457
+ Function to generate fastexport compatible query.
1458
+
1459
+ PARAMETERS:
1460
+ base_query:
1461
+ Required Argument.
1462
+ Specifies the base query to be used for forming the fastexport
1463
+ query.
1464
+ Types: str
1465
+
1466
+ require:
1467
+ Optional Argument.
1468
+ Specifies whether fastexport protocol is required for data
1469
+ transfer.
1470
+ Default Value: False
1471
+ Types: bool
1472
+
1473
+ open_sessions:
1474
+ Optional Argument.
1475
+ Specifies the number of Teradata sessions to be opened for fastexport.
1476
+ Default value: None
1477
+ Types: int
1478
+
1479
+ csv_file_name:
1480
+ Optional Argument.
1481
+ Specifies the name of CSV file to which data is to be exported.
1482
+ Types: Str
1483
+
1484
+ RETURNS:
1485
+ str.
1486
+
1487
+ RAISES:
1488
+ None.
1489
+
1490
+ EXAMPLES:
1491
+ dt_obj = _DataTransferUtils(df)
1492
+ base_query = "select * from my_table SAMPLE 200"
1493
+ target_csv = "Test.csv"
1494
+ dt_obj._generate_fastexport_query(base_query)
1495
+ dt_obj._generate_fastexport_query(base_query, require=True)
1496
+ dt_obj._generate_fastexport_query(base_query, require=True, open_sessions=5)
1497
+ dt_obj._generate_fastexport_query(base_query, require=True, csv_file_name=target_csv)
1498
+ dt_obj._generate_fastexport_query(base_query, require=True, open_sessions = 2,
1499
+ csv_file_name=target_csv)
1500
+
1501
+ """
1502
+ fastexport_esc_func = ""
1503
+ open_session_esc_func = ""
1504
+ if require is not None:
1505
+ if require:
1506
+ # If require is set to True, we are using
1507
+ # 'teradata_require_fastexport' escape sequence as this will run
1508
+ # query using fastexport only if the given query is compatible with
1509
+ # fastexport else raises error.
1510
+ fastexport_esc_func = DriverEscapeFunctions.REQUIRE_FASTEXPORT.value
1511
+ else:
1512
+ # If require is False, we are using 'teradata_try_fastexport'
1513
+ # escape sequence as this will run query using fastexport if the
1514
+ # given query is compatible with fastexport else runs it as
1515
+ # regular query.
1516
+ fastexport_esc_func = DriverEscapeFunctions.TRY_FASTEXPORT.value
1517
+
1518
+ if open_sessions is not None:
1519
+ open_session_esc_func = DriverEscapeFunctions.OPEN_SESSIONS.value.format(open_sessions)
1520
+
1521
+ write_csv_escape_func = ""
1522
+ field_sep_esc_func = ""
1523
+ field_quote_esc_func = ""
1524
+
1525
+ if csv_file_name:
1526
+ # The teradata_field_sep and teradata_field_quote escape functions have a
1527
+ # single-character string argument. The string argument must follow SQL literal
1528
+ # syntax. The string argument may be enclosed in single-quote (') characters or
1529
+ # double-quote (") characters.
1530
+ field_sep = "'{0}'".format(self.sep)
1531
+ if self.sep == "'":
1532
+ field_sep = "''''"
1533
+ elif self.sep == "\"":
1534
+ field_sep = "\"\"\"\""
1535
+ # To represent a single-quote character in a string enclosed in single-quote
1536
+ # characters, you must repeat the single-quote character.
1537
+ # {fn teradata_field_quote('''')}
1538
+ # To represent a double-quote character in a string enclosed in double-quote
1539
+ # characters, you must repeat the double-quote character.
1540
+ # {fn teradata_field_quote("""")}
1541
+ field_quote = "'{0}'".format(self.quotechar)
1542
+ if self.quotechar == "'":
1543
+ field_quote = "''''"
1544
+ elif self.quotechar == "\"":
1545
+ field_quote = "\"\"\"\""
1546
+
1547
+ write_csv_escape_func = DriverEscapeFunctions.WRITE_TO_CSV.value.format(csv_file_name)
1548
+ field_sep_esc_func = DriverEscapeFunctions.FIELD_SEP.value.format(field_sep)
1549
+ field_quote_esc_func = DriverEscapeFunctions.FIELD_QUOTE.value.format(field_quote)
1550
+
1551
+ query = "{0}{1}{2}{3}{4}{5}".format(fastexport_esc_func,
1552
+ open_session_esc_func,
1553
+ field_sep_esc_func,
1554
+ field_quote_esc_func,
1555
+ write_csv_escape_func,
1556
+ base_query)
1557
+
1558
+ return query
1559
+
1560
+ def _process_fastexport_errors_warnings(self, query):
1561
+ """
1562
+ DESCRIPTION:
1563
+ Function to process errors/warnings(if any) raised while executing
1564
+ the fastexport protocol.
1565
+
1566
+ PARAMETERS:
1567
+ query:
1568
+ Required Argument.
1569
+ Specifies the query with fastexport escape sequences that is
1570
+ used to convert teradataml DataFrame to Pandas DataFrame.
1571
+ Type: str
1572
+
1573
+ RETURNS:
1574
+ A tuple with two lists for errors, warnings each containing err/warn
1575
+ messages in string format.
1576
+
1577
+ RAISES:
1578
+ None.
1579
+
1580
+ EXAMPLES:
1581
+ dt_obj = _DataTransferUtils(df)
1582
+ query = "{fn teradata_try_fastexport}select * from my_table SAMPLE
1583
+ 200"
1584
+ err, warn = dt_obj._process_fastexport_errors_warnings(query)
1585
+ """
1586
+ err = None
1587
+ warn = None
1588
+ conn = get_connection().connection
1589
+ # Create a cursor from connection object.
1590
+ cur = conn.cursor()
1591
+ # Get err/warn
1592
+ err = self._get_errors_warnings(cur, query,
1593
+ DriverEscapeFunctions.GET_ERRORS)
1594
+ warn = self._get_errors_warnings(cur, query,
1595
+ DriverEscapeFunctions.GET_WARNINGS)
1596
+ return err, warn
1597
+
1598
+ # Functions specific to exporting table data in Vantage into pandas DataFrame.
1599
+ def _get_pandas_dataframe(self, **kwargs):
1600
+ """
1601
+ DESCRIPTION:
1602
+ Function that converts teradataml DataFrame to Pandas DataFrame
1603
+ using regular approach.
1604
+
1605
+ PARAMETERS:
1606
+ kwargs:
1607
+ Specifies keyword arguments.
1608
+
1609
+ RETURNS:
1610
+ Pandas DataFrame.
1611
+
1612
+ RAISES:
1613
+ None.
1614
+
1615
+ EXAMPLES:
1616
+ dt_obj = _DataTransferUtils(df)
1617
+ dt_obj._get_pandas_dataframe()
1618
+
1619
+ """
1620
+ # Get the final index column.
1621
+ final_index_column = self._get_pandas_df_index()
1622
+ # Get the base query.
1623
+ base_query = self._generate_to_pandas_base_query()
1624
+ # Generate pandas dataframe using base query.
1625
+ pandas_df = _execute_query_and_generate_pandas_df(base_query,
1626
+ final_index_column,
1627
+ **kwargs)
1628
+ return pandas_df
1629
+
1630
+ def _fastexport_get_pandas_df(self, require=False, **kwargs):
1631
+ """
1632
+ DESCRIPTION:
1633
+ Internal function to convert teradataml DataFrame to Pandas
1634
+ DataFrame using FastExport protocol. This internal function can be
1635
+ directly used in to_pandas() and fastexport API's if either of
1636
+ the functions has to use fastexport.
1637
+
1638
+ PARAMETERS:
1639
+ require:
1640
+ Optional Argument.
1641
+ Specifies whether fastexport protocol is required for data
1642
+ transfer.
1643
+ Default Value: False
1644
+ Types: bool
1645
+
1646
+ kwargs:
1647
+ Specifies keyword arguments. Argument "open_sessions"
1648
+ can be passed as keyword arguments.
1649
+ * "open_sessions" specifies the number of Teradata sessions to
1650
+ be opened for fastexport.
1651
+
1652
+ RETURNS:
1653
+ When "catch_errors_warnings" is set to True, the function returns
1654
+ a tuple containing:
1655
+ * Pandas DataFrame.
1656
+ * Errors, if any, thrown by fastexport in a list of strings.
1657
+ * Warnings, if any, thrown by fastexport in a list of strings.
1658
+ Only Pandas DataFrame otherwise.
1659
+
1660
+ RAISES:
1661
+ TeradataMlException.
1662
+
1663
+ EXAMPLES:
1664
+ dt_obj = _DataTransferUtils(df)
1665
+ dt_obj._fastexport_get_pandas_df(require=False)
1666
+
1667
+ """
1668
+
1669
+ try:
1670
+ self._validate_df_index_column()
1671
+ final_index_col = self._get_pandas_df_index()
1672
+ self.df._DataFrame__execute_node_and_set_table_name(self.df._nodeid,
1673
+ self.df._metaexpr)
1674
+ base_query = self._generate_to_pandas_base_query()
1675
+
1676
+ # Get open_sessions argument.
1677
+ open_sessions = kwargs.pop("open_sessions", None)
1678
+ fastexport_query = self._generate_fastexport_query(base_query,
1679
+ require=require,
1680
+ open_sessions=open_sessions)
1681
+ pandas_df = _execute_query_and_generate_pandas_df(fastexport_query,
1682
+ final_index_col,
1683
+ **kwargs)
1684
+ err, warn = \
1685
+ self._process_fastexport_errors_warnings(fastexport_query)
1686
+ if self.catch_errors_warnings:
1687
+ return pandas_df, err, warn
1688
+ else:
1689
+ print("Errors: {0}".format(err))
1690
+ print("Warnings: {0}".format(warn))
1691
+ return pandas_df
1692
+ except TeradataMlException:
1693
+ raise
1694
+
1695
+ # End of functions specific to exporting table data in Vantage into pandas DataFrame.
1696
+
1697
+ # General functions to get warrnings and errors.
1698
+ def _get_errors_warnings(self, cur, insert_stmt, escape_function):
1699
+ """
1700
+ Internal function executes teradatasql provided escape functions
1701
+ to get the errors and warnings.
1702
+
1703
+ PARAMETERS:
1704
+ cur:
1705
+ Required Argument.
1706
+ The cursor of connection type which will be used to execute query.
1707
+ Types: teradatasql cursor object
1708
+
1709
+ insert_stmt:
1710
+ Required Argument.
1711
+ Statement to be executed along with escape method.
1712
+ Types: String
1713
+
1714
+ escape_function:
1715
+ Required Argument.
1716
+ Type of escape method to be passed.
1717
+ Types: String
1718
+
1719
+ RETURNS:
1720
+ A list containing error/warning information.
1721
+
1722
+ RAISES:
1723
+ None
1724
+
1725
+ EXAMPLES:
1726
+ dt_obj = _DataTransferUtils(df, table_name, types)
1727
+ dt_obj._get_errors_warnings(cur, insert_stmt, escape_function)
1728
+ """
1729
+ errorwarninglist = self._process_escape_functions(cur,
1730
+ escape_function=escape_function,
1731
+ insert_query=insert_stmt)
1732
+
1733
+ from teradatasql import vernumber
1734
+ msg = []
1735
+ if errorwarninglist:
1736
+ if (errorwarninglist[0][0] != ""):
1737
+ msg = errorwarninglist[0][0].split('[Version ' + vernumber.sVersionNumber + ']')[1:]
1738
+
1739
+ return [err_msg.split("\n")[0] for err_msg in msg]
1740
+
1741
+ def _get_pandas_df_from_errors_warnings(self, err_or_warn_dict):
1742
+ """
1743
+ DESCRIPTION:
1744
+ Internal function creates Pandas dataframe.
1745
+
1746
+ PARAMETERS:
1747
+ err_or_warn_dict:
1748
+ Required Argument.
1749
+ Specifies the error or warning dictionary.
1750
+ Types: Python dictionary
1751
+
1752
+ RETURNS:
1753
+ Pandas Dataframe.
1754
+
1755
+ RAISES:
1756
+ None.
1757
+
1758
+ EXAMPLES:
1759
+ dt_obj = _DataTransferUtils(df)
1760
+ dt_obj._get_pandas_df_from_errors_warnings(err_or_warn_dict)
1761
+ """
1762
+ # For fastload API, we check the 'batch_no' key is present in dictionary and
1763
+ # whether it has non-zero value or not.
1764
+ # For read_csv API, we check whether the length of 'error_message' key's
1765
+ # value is greater than 0 or not.
1766
+ if ('batch_no' in err_or_warn_dict and bool(err_or_warn_dict.get('batch_no'))) or \
1767
+ len(err_or_warn_dict['error_message']) != 0:
1768
+ return pd.DataFrame(err_or_warn_dict)
1769
+
1770
+ return pd.DataFrame()
1771
+
1772
+ def _create_error_warnings_table(self, pdf, msg_type, logon_seq_number, table_name=None):
1773
+ """
1774
+ DESCRIPTION:
1775
+ Internal function creates the errors and warnings table in Vantage.
1776
+
1777
+ PARAMETERS:
1778
+ pdf:
1779
+ Required Argument.
1780
+ Specifies pandas dataframe containing errors and warnings.
1781
+ Types: Pandas DataFrame
1782
+
1783
+ msg_type:
1784
+ Required Argument.
1785
+ Specifies the type of message.
1786
+ Possible Values : 'err', 'warn'
1787
+ Types: String
1788
+
1789
+ logon_seq_number:
1790
+ Required Argument.
1791
+ Specifies logon sequence number of the session.
1792
+ Types: Integer
1793
+
1794
+ RETURNS:
1795
+ str
1796
+
1797
+ RAISES:
1798
+ None
1799
+
1800
+ EXAMPLES:
1801
+ dt_obj = _DataTransferUtils(df, table_name, types)
1802
+ dt_obj._create_error_warnings_table(pdf, msg_type, logon_seq_number)
1803
+ """
1804
+ if not table_name:
1805
+ table_name = "td_fl_{0}_{1}_{2}".format(self.table_name, msg_type, logon_seq_number)
1806
+ copy_to_sql(pdf, table_name, schema_name=self.err_staging_db,
1807
+ if_exists='replace')
1808
+ return "{}.{}".format(self.err_staging_db if self.err_staging_db
1809
+ else _get_current_databasename(),
1810
+ table_name)
1811
+
1812
+ def _process_escape_functions(self, cur, escape_function, insert_query=None):
1813
+ """
1814
+ DESCRIPTION:
1815
+ Internal function executes the autocommit to manage the transactions.
1816
+
1817
+ PARAMETERS:
1818
+ cur:
1819
+ Required Argument.
1820
+ The cursor of connection type which will be used to execute query.
1821
+ Types: teradatasql cursor object
1822
+
1823
+ escape_function:
1824
+ Required Argument.
1825
+ Specifies the escape function to use.
1826
+ Permitted Values : 'autocommit_off', 'autocommit_on', 'logon_sequence_number'
1827
+ Types: String
1828
+
1829
+ insert_query:
1830
+ Optional Argument.
1831
+ Specifies the insert query.
1832
+ Types: String
1833
+
1834
+ RETURNS:
1835
+ If escape_function is 'logon_sequence_number' then returns
1836
+ a list containing logon sequence number.
1837
+ If escape_function is 'teradata_get_errors' then returns
1838
+ a list containing errors observed by FastLoad.
1839
+ If escape_function is 'teradata_get_warnings' then returns
1840
+ a list containing warnings observed by FastLoad.
1841
+ Otherwise returns None.
1842
+
1843
+ RAISES:
1844
+ None
1845
+
1846
+ EXAMPLES:
1847
+ dt_obj = _DataTransferUtils(df)
1848
+ dt_obj._process_escape_functions(cur, escape_function)
1849
+ """
1850
+ # All escape functions follow same syntax hence, the query formation at the start.
1851
+ query = "{}{}".format(DriverEscapeFunctions.NATIVE_SQL.value, escape_function.value)
1852
+
1853
+ # If escape function requires to be executed along with the query.
1854
+ if escape_function in [DriverEscapeFunctions.LOGON_SEQ_NUM,
1855
+ DriverEscapeFunctions.GET_ERRORS,
1856
+ DriverEscapeFunctions.GET_WARNINGS]:
1857
+ cur.execute(query + insert_query)
1858
+ return [row for row in cur.fetchall()]
1859
+ else:
1860
+ cur.execute(query)
1861
+
1862
+ # End of general functions to get warnings and errors.
1863
+
1864
+ # Functions specific to read_csv().
1865
+ def _form_insert_query(self, table, column_names=None):
1866
+ """
1867
+ DESCRIPTION:
1868
+ Internal function forms the INSERT query using escape function.
1869
+
1870
+ PARAMETERS:
1871
+ table:
1872
+ Required Argument.
1873
+ Specifies the table name.
1874
+ Types: str
1875
+
1876
+ column_names:
1877
+ Optional Argument.
1878
+ Specifies the list of column names.
1879
+ Types: list of Strings (str)
1880
+
1881
+ RETURNS:
1882
+ str.
1883
+
1884
+ RAISES:
1885
+ None.
1886
+
1887
+ EXAMPLES:
1888
+ dt_obj = _DataTransferUtils(df, api_name='read_csv')
1889
+ ins_query = dt_obj._form_insert_query()
1890
+ """
1891
+
1892
+ escape_funcs = ""
1893
+
1894
+ # Get the fastload escape function.
1895
+ if self.use_fastload:
1896
+ escape_funcs = escape_funcs + DriverEscapeFunctions.REQUIRE_FASTLOAD.value
1897
+
1898
+ # Get the escape function clause for open_sessions.
1899
+ if self.open_sessions is not None:
1900
+ escape_funcs = escape_funcs + DriverEscapeFunctions.OPEN_SESSIONS.value.format(self.open_sessions)
1901
+
1902
+ # Create the list of values to be inserted.
1903
+ if self.api_name == "fastload":
1904
+ col_names = _get_pd_df_column_names(self.df)
1905
+ insert_values = ", ".join(['?' for i in range(len(col_names) + len(self.df.index.names)
1906
+ if self.index is True else len(col_names))])
1907
+
1908
+ # Get escape functions related to read_csv.
1909
+ if self.api_name == "read_csv":
1910
+ # Get the column names.
1911
+ if self.if_exists == 'append' and column_names is not None:
1912
+ col_names = column_names
1913
+ else:
1914
+ col_names, _ = _extract_column_info(self.df, self.types)
1915
+
1916
+ # Get read_csv escape function.
1917
+ escape_funcs = escape_funcs + DriverEscapeFunctions.READ_CSV.value.format(self.df)
1918
+ insert_values = ", ".join(['?' for i in range(len(col_names))])
1919
+
1920
+ # Create escape function for sep.
1921
+ field_sep = "'{0}'".format(self.sep)
1922
+ if self.sep == "'":
1923
+ field_sep = "''''"
1924
+ elif self.sep == "\"":
1925
+ field_sep = "\"\"\"\""
1926
+ escape_funcs = escape_funcs + DriverEscapeFunctions.FIELD_SEP.value.format(field_sep)
1927
+
1928
+ # Create escape function for quotechar.
1929
+ field_quote = "'{0}'".format(self.quotechar)
1930
+ if self.quotechar == "'":
1931
+ field_quote = "''''"
1932
+ elif self.quotechar == "\"":
1933
+ field_quote = "\"\"\"\""
1934
+ escape_funcs = escape_funcs + DriverEscapeFunctions.FIELD_QUOTE.value.format(field_quote)
1935
+
1936
+ # Create base insert query.
1937
+ base_insert_query = "INSERT INTO {0} VALUES ({1});".format(table, insert_values)
1938
+
1939
+ # Get the escape function clauses for error table and DB related escape functions.
1940
+ # TODO: This condition will be optimized with ELE-6743.
1941
+ if self.api_name == "fastload" and self.save_errors and not self.err_tbl_name:
1942
+ escape_funcs = escape_funcs + DriverEscapeFunctions.ERR_TBL_MNG_FLAG.value.format("off")
1943
+
1944
+ if self.err_tbl_1_suffix:
1945
+ escape_funcs = escape_funcs + DriverEscapeFunctions.ERR_TBL_1.value.format(self.err_tbl_1_suffix)
1946
+
1947
+ if self.err_tbl_2_suffix:
1948
+ escape_funcs = escape_funcs + DriverEscapeFunctions.ERR_TBL_2.value.format(self.err_tbl_2_suffix)
1949
+
1950
+ if self.err_staging_db:
1951
+ escape_funcs = escape_funcs + DriverEscapeFunctions.ERR_STAGING_DB.value.format(self.err_staging_db)
1952
+
1953
+ # Generate final insert query by appending all escape functions.
1954
+ query = "{0}{1}".format(escape_funcs, base_insert_query)
1955
+ return query
1956
+
1957
+ def _table_exists(self, con):
1958
+ """
1959
+ DESCRIPTION:
1960
+ Internal function validates whether table exists or not.
1961
+
1962
+ PARAMETERS:
1963
+ con:
1964
+ Required argument.
1965
+ Specifies a SQLAlchemy connectable (engine/connection) object
1966
+ Types: Teradata connection object
1967
+
1968
+ RETURNS:
1969
+ boolean.
1970
+
1971
+ RAISES:
1972
+ None.
1973
+
1974
+ EXAMPLES:
1975
+ dt_obj = _DataTransferUtils(df)
1976
+ ins_query = dt_obj._table_exists()
1977
+ """
1978
+ return con.dialect.has_table(get_connection(), self.table_name, self.schema_name,
1979
+ table_only=True)
1980
+
1981
+ def _get_fully_qualified_table_name(self, table_name=None, schema_name=None):
1982
+ """
1983
+ DESCRIPTION:
1984
+ Function returns schema qualified table name
1985
+ Such as:
1986
+ "alice"."my_table" OR
1987
+ "mldb"."my_table"
1988
+
1989
+ PARAMETERS:
1990
+ table_name:
1991
+ Optional Argument.
1992
+ Specifies the table name.
1993
+ Types: str
1994
+
1995
+ schema_name:
1996
+ Optional Argument.
1997
+ Specifies the schema name.
1998
+ Types: str
1999
+
2000
+ RETURNS:
2001
+ str.
2002
+
2003
+ RAISES:
2004
+ None
2005
+
2006
+ EXAMPLES:
2007
+ dt_obj = _DataTransferUtils(df, table_name, types)
2008
+ dt_obj._get_fully_qualified_table_name()
2009
+ """
2010
+ table_name = table_name if table_name else self.table_name
2011
+
2012
+ table = '"{}"'.format(table_name)
2013
+ if schema_name is not None:
2014
+ table = '"{}"."{}"'.format(schema_name, table_name)
2015
+ elif self.schema_name is not None:
2016
+ table = '"{}"."{}"'.format(self.schema_name, table_name)
2017
+
2018
+ return table
2019
+
2020
+ def _create_table(self, con, table_name=None, schema_name=None):
2021
+ """
2022
+ DESCRIPTION:
2023
+ Internal function creates table in the Vantage.
2024
+
2025
+ PARAMETERS:
2026
+ con:
2027
+ Required Argument.
2028
+ A SQLAlchemy connectable (engine/connection) object
2029
+ Types: Teradata connection object
2030
+
2031
+ table_name:
2032
+ Optional Argument.
2033
+ Specifies the table name.
2034
+ Types: str
2035
+
2036
+ schema_name:
2037
+ Optional Argument.
2038
+ Specifies the schema name where table needs to be created.
2039
+ Types: str
2040
+
2041
+ RETURNS:
2042
+ None.
2043
+
2044
+ RAISES:
2045
+ None
2046
+
2047
+ EXAMPLES:
2048
+ dt_obj = _DataTransferUtils(df, table_name, types)
2049
+ dt_obj._create_table(con)
2050
+ """
2051
+ table_name = table_name if table_name else self.table_name
2052
+ schema_name = schema_name if schema_name else self.schema_name
2053
+ table = _create_table_object(df=self.df, table_name=table_name, types=self.types, con=con,
2054
+ schema_name=schema_name, primary_index=self.primary_index,
2055
+ temporary=self.temporary, set_table=self.set_table, index=self.index,
2056
+ index_label=self.index_label)
2057
+
2058
+ UtilFuncs._create_table_using_table_object(table)
2059
+
2060
+ def _insert_from_csv_with_fastload(self, table_name=None, column_names=None):
2061
+ """
2062
+ DESCRIPTION:
2063
+ This insert function loads the data from csv file to Vantage table.
2064
+
2065
+ PARAMETERS:
2066
+ table_name:
2067
+ Optional Argument.
2068
+ Specifies the table name.
2069
+ Types: str
2070
+
2071
+ column_names:
2072
+ Optional Argument.
2073
+ Specifies the list column names.
2074
+ Types: list of Strings (str).
2075
+
2076
+ RETURNS:
2077
+ A dict containing the following attributes:
2078
+ 1. errors_dataframe: It is a Pandas DataFrame containing error messages
2079
+ thrown by fastload. DataFrame is empty if there are no errors.
2080
+ 2. warnings_dataframe: It is a Pandas DataFrame containing warning messages
2081
+ thrown by fastload. DataFrame is empty if there are no warnings.
2082
+ 3. errors_table: Name of the table containing errors. It is None, if
2083
+ argument save_errors is False.
2084
+ 4. warnings_table: Name of the table containing warnings. It is None, if
2085
+ argument save_errors is False.
2086
+
2087
+ RAISES:
2088
+ None
2089
+
2090
+ EXAMPLES:
2091
+ dt_obj = _DataTransferUtils(df, table_name, types, api_name='read_csv')
2092
+ dt_obj._insert_from_csv()
2093
+ """
2094
+ conn = get_connection()
2095
+ conn1 = conn.connection
2096
+ cur = conn1.cursor()
2097
+
2098
+ error_tablename = ""
2099
+ warn_tablename = ""
2100
+
2101
+ try:
2102
+ # Quoted, schema-qualified table name.
2103
+ table = self._get_fully_qualified_table_name(table_name)
2104
+
2105
+ # Form the INSERT query for read_csv.
2106
+ ins = self._form_insert_query(table, column_names=column_names)
2107
+
2108
+ # Turn off autocommit before the Fastload insertion.
2109
+ self._process_escape_functions(cur, escape_function= \
2110
+ DriverEscapeFunctions.AUTOCOMMIT_OFF)
2111
+
2112
+ # Initialize dict template for saving error/warning information.
2113
+ err_dict = {}
2114
+ warn_dict = {}
2115
+ err_dict['error_message'] = []
2116
+ warn_dict['error_message'] = []
2117
+
2118
+ # Empty queryband buffer before SQL call.
2119
+ UtilFuncs._set_queryband()
2120
+ # Execute insert statement
2121
+ cur.execute(ins)
2122
+
2123
+ # Get error and warning information
2124
+ err, _ = self._process_fastexport_errors_warnings(ins)
2125
+ if len(err) != 0:
2126
+ err_dict['error_message'].extend(err)
2127
+
2128
+ # Get logon sequence number to be used for error/warning table names
2129
+ logon_seq_number = self._process_escape_functions(cur, escape_function= \
2130
+ DriverEscapeFunctions.LOGON_SEQ_NUM,
2131
+ insert_query=ins)
2132
+
2133
+ # Commit the rows
2134
+ conn1.commit()
2135
+
2136
+ # Get error and warning information, if any.
2137
+ # Errors/Warnings like duplicate rows are added here.
2138
+ _, warn = self._process_fastexport_errors_warnings(ins)
2139
+ if len(warn) != 0:
2140
+ warn_dict['error_message'].extend(warn)
2141
+
2142
+ # Get error and warning information for error and warning tables, persist
2143
+ # error and warning tables to Vantage if user has specified save_error as True
2144
+ # else show it as pandas dataframe on console.
2145
+ pd_err_df = self._get_pandas_df_from_errors_warnings(err_dict)
2146
+ if not pd_err_df.empty and self.save_errors:
2147
+ msg_type = "err"
2148
+ error_tablename = self._create_error_warnings_table(pd_err_df, msg_type, logon_seq_number[0][0])
2149
+
2150
+ pd_warn_df = self._get_pandas_df_from_errors_warnings(warn_dict)
2151
+ if not pd_warn_df.empty and self.save_errors:
2152
+ msg_type = "warn"
2153
+ warn_tablename = self._create_error_warnings_table(pd_warn_df, msg_type, logon_seq_number[0][0])
2154
+
2155
+ # These tables are created by FastloadCSV.
2156
+ fastloadcsv_err_tables = [table_name + "_ERR_1", table_name + "_ERR_2"]
2157
+ err_warn_dict = {"errors_dataframe": pd_err_df, "warnings_dataframe": pd_warn_df,
2158
+ "errors_table": error_tablename, "warnings_table": warn_tablename,
2159
+ "fastloadcsv_error_tables": fastloadcsv_err_tables}
2160
+
2161
+ # If user don't want to persist the error tables,
2162
+ # drop the tables created by FastloadCSV.
2163
+ if not self.save_errors:
2164
+ for table in fastloadcsv_err_tables:
2165
+ if conn.dialect.has_table(conn, table_name=table, schema=self.schema_name,
2166
+ table_only=True):
2167
+ UtilFuncs._drop_table(self._get_fully_qualified_table_name(table))
2168
+ err_warn_dict.update({"fastloadcsv_error_tables": []})
2169
+ return err_warn_dict
2170
+
2171
+ return err_warn_dict
2172
+
2173
+ except Exception:
2174
+ conn1.rollback()
2175
+ raise
2176
+ finally:
2177
+ # Turn on autocommit.
2178
+ self._process_escape_functions(cur, escape_function= \
2179
+ DriverEscapeFunctions.AUTOCOMMIT_ON)
2180
+ cur.close()
2181
+
2182
+ def _get_result(self, result_dict=None):
2183
+ """
2184
+ DESCRIPTION:
2185
+ Internal function loads data from csv to table.
2186
+ And created the teradataml dataframe of the table.
2187
+
2188
+ PARAMETERS:
2189
+ result_dict:
2190
+ Optional Argument.
2191
+ Specifies the dictionary containg values for error_dataframe,
2192
+ warning_dataframe, error_table, warning_table.
2193
+ Types: dict
2194
+
2195
+ RETURNS:
2196
+ When "catch_errors_warnings" is set to True, then the
2197
+ function returns a tuple containing:
2198
+ a. Teradataml DataFrame.
2199
+ b. A dict containing the following attributes:
2200
+ a. errors_dataframe: It is a Pandas DataFrame containing error messages
2201
+ thrown by fastload. DataFrame is empty if there are no errors.
2202
+ b. warnings_dataframe: It is a Pandas DataFrame containing warning messages
2203
+ thrown by fastload. DataFrame is empty if there are no warnings.
2204
+ d. errors_table: Name of the table containing errors. It is None, if
2205
+ argument save_errors is False.
2206
+ e. warnings_table: Name of the table containing warnings. It is None, if
2207
+ argument save_errors is False.
2208
+ When "catch_errors_warnings" is False, then the function
2209
+ returns a Teradataml DataFrame.
2210
+
2211
+ RAISES:
2212
+ None.
2213
+
2214
+ EXAMPLES:
2215
+ dt_obj = _DataTransferUtils(df, table_name, types)
2216
+ ins_query = dt_obj._get_result()
2217
+ """
2218
+ tdml_df = tdmldf.DataFrame(self._get_fully_qualified_table_name())
2219
+
2220
+ if not self.use_fastload:
2221
+ return tdml_df
2222
+
2223
+ if self.catch_errors_warnings:
2224
+ return tdml_df, result_dict
2225
+ return tdml_df
2226
+
2227
+ def _get_dataframe_columns(self):
2228
+ """
2229
+ Internal function used to get dataframe columns name.
2230
+
2231
+ PARAMETERS:
2232
+ None
2233
+
2234
+ Returns
2235
+ list
2236
+
2237
+ RAISES:
2238
+ None
2239
+
2240
+ EXAMPLES:
2241
+ _get_dataframe_columns()
2242
+ """
2243
+ if isinstance(self.df, pd.DataFrame):
2244
+ return _get_pd_df_column_names(self.df)
2245
+ elif isinstance(self.df, tdmldf.DataFrame):
2246
+ return [col.name for col in self.df._metaexpr.c]
2247
+
2248
+ def _get_sqlalc_table_columns(self):
2249
+ """
2250
+ Internal function to get column info if types is not None otherwise get
2251
+ the column info from existing table.
2252
+ If types is None and if_exists is append then it will first check for table
2253
+ existence, if table does not exists on system, it will raise an exception.
2254
+
2255
+ PARAMETERS:
2256
+ None
2257
+
2258
+ RETURNS:
2259
+ list
2260
+
2261
+ RAISES:
2262
+ ValueError
2263
+
2264
+ EXAMPLES:
2265
+ _get_sqlalc_table_columns()
2266
+ """
2267
+ table = None
2268
+ if self.types is None and self.if_exists == 'append':
2269
+ # Raise an exception when the table does not exists, if_exists='append'
2270
+ # and types=None.
2271
+ if not self._table_exists(get_connection()):
2272
+ err = "Table '{}' does not exists in the system and if_exists " \
2273
+ "is set to 'append'.".format(self.table_name)
2274
+ _Validators._validate_argument_is_not_None(self.types, "types",
2275
+ additional_error=err)
2276
+
2277
+ table = UtilFuncs._get_sqlalchemy_table(self.table_name,
2278
+ schema_name=self.schema_name)
2279
+
2280
+ df_columns, _ = UtilFuncs._extract_table_object_column_info(table.c) \
2281
+ if table is not None else \
2282
+ _extract_column_info(self.df, types=self.types)
2283
+
2284
+ return df_columns
2285
+
2286
+ def _validate(self):
2287
+ """
2288
+ This is an internal function used to validate the api parameters.
2289
+ Dataframe, connection & related parameters are checked.
2290
+ Saving to Vantage is proceeded to only when validation returns True.
2291
+
2292
+ PARAMETERS:
2293
+ None
2294
+
2295
+ RETURNS:
2296
+ True, when all parameters are valid.
2297
+
2298
+ RAISES:
2299
+ TeradataMlException, when parameter validation fails.
2300
+
2301
+ EXAMPLES:
2302
+ dt_obj = _DataTransferUtils()
2303
+ dt_obj._validate()
2304
+ """
2305
+ awu = _Validators()
2306
+ awu_matrix = []
2307
+
2308
+ # The arguments added to awu_martix are:
2309
+ # arg_name, arg, is_optional, acceptable types
2310
+ # The value for is_optional is set to False when the argument
2311
+ # a) is a required argument
2312
+ # b) is not allowed to be None, even if it is optional
2313
+ types_t = (dict)
2314
+ is_optional = True
2315
+ if self.api_name == 'read_csv':
2316
+ awu_matrix.append(['filepath', self.df, False, (str)])
2317
+
2318
+ types_t = (OrderedDict)
2319
+ is_optional = False if self.if_exists in ['replace', 'fail'] else True
2320
+ else:
2321
+ # Validates the dataframe passed to copyto and fastload apis.
2322
+ # We are validating dataframe with separate function because
2323
+ # current implementation of __getTypeAsStr() always returns
2324
+ # 'teradataml DataFrame' if argument type.__name__ is DataFrame.
2325
+ _Validators._validate_dataframe(self.df)
2326
+
2327
+ awu_matrix.append(['types', self.types, is_optional, types_t])
2328
+ awu_matrix.append(['table_name', self.table_name, False, (str), True])
2329
+ awu_matrix.append(['schema_name', self.schema_name, True, (str), True])
2330
+ awu_matrix.append(['index', self.index, False, (bool)])
2331
+ awu_matrix.append(['temporary', self.temporary, False, (bool)])
2332
+ awu_matrix.append(['if_exists', self.if_exists, False, (str),
2333
+ True, ['APPEND', 'REPLACE', 'FAIL']])
2334
+ awu_matrix.append(['primary_index', self.primary_index, True, (str, list), True])
2335
+ awu_matrix.append(['set_table', self.set_table, False, (bool)])
2336
+ awu_matrix.append(['save_errors', self.save_errors, False, (bool)])
2337
+ awu_matrix.append(['sep', self.sep, False, (str)])
2338
+ awu_matrix.append(['quotechar', self.quotechar, True, (str)])
2339
+ awu_matrix.append(['catch_errors_warnings', self.catch_errors_warnings, False, (bool)])
2340
+ awu_matrix.append(['use_fastload', self.use_fastload, False, (bool)])
2341
+ awu_matrix.append(['open_sessions', self.open_sessions, True, (int), False])
2342
+ awu_matrix.append(['chunksize', self.chunksize, False, (int)])
2343
+ awu_matrix.append(['match_column_order', self.match_column_order, True, (bool)])
2344
+ if isinstance(self.df, pd.DataFrame):
2345
+ awu_matrix.append(['index_label', self.index_label, True, (str, list), True])
2346
+
2347
+ # Validate types
2348
+ awu._validate_function_arguments(awu_matrix)
2349
+
2350
+ # Validate if 'open_sessions' is a positive int.
2351
+ if self.open_sessions is not None:
2352
+ _Validators._validate_positive_int(self.open_sessions, "open_sessions")
2353
+
2354
+ # Validate 'chunksize' is a positive int.
2355
+ _Validators._validate_positive_int(self.chunksize, "chunksize")
2356
+
2357
+ # Get columns name
2358
+ if self.api_name == 'read_csv':
2359
+ df_columns = self._get_sqlalc_table_columns()
2360
+ else:
2361
+ df_columns = self._get_dataframe_columns()
2362
+
2363
+ eng = get_context()
2364
+ current_user = eng.url.username
2365
+
2366
+ allowed_schemas = df_utils._get_database_names(get_connection(), self.schema_name)
2367
+ allowed_schemas.append(current_user)
2368
+
2369
+ if self.schema_name is not None and self.schema_name.lower() not in allowed_schemas:
2370
+ raise TeradataMlException(Messages.get_message(MessageCodes.INVALID_ARG_VALUE,
2371
+ str(self.schema_name), 'schema_name',
2372
+ 'A valid database/schema name.'),
2373
+ MessageCodes.INVALID_ARG_VALUE)
2374
+
2375
+ if isinstance(self.df, pd.DataFrame):
2376
+ if self.index:
2377
+ is_multi_index = isinstance(self.df.index, pd.MultiIndex)
2378
+ if self.index_label:
2379
+ if hasattr(self.df.index, 'levels'):
2380
+ index_levels = len(self.df.index.levels)
2381
+ num_index = len(self.index_label)
2382
+ is_index_list = isinstance(self.index_label, list)
2383
+
2384
+ if (is_multi_index and ((isinstance(self.index_label, str) and index_levels != 1) or
2385
+ (is_index_list and index_levels != len(self.index_label)))) or \
2386
+ (not is_multi_index and is_index_list and
2387
+ (is_index_list and num_index > 1)):
2388
+ valid_arg_msg = 'String or list of Strings with the number of ' \
2389
+ 'Strings matching the number of levels' \
2390
+ ' in the index'
2391
+ raise TeradataMlException(Messages.get_message(MessageCodes.INVALID_ARG_VALUE,
2392
+ self.index_label, 'index_label',
2393
+ valid_arg_msg),
2394
+ MessageCodes.INVALID_ARG_VALUE)
2395
+
2396
+ # When Pandas DF's used and Pandas Index is saved, get list of levels to add as columns
2397
+ index_names_to_add = _get_index_labels(self.df, self.index_label)[0]
2398
+ for label in index_names_to_add:
2399
+ if label in df_columns:
2400
+ raise TeradataMlException(Messages.get_message(MessageCodes.INDEX_ALREADY_EXISTS, label),
2401
+ MessageCodes.INDEX_ALREADY_EXISTS)
2402
+
2403
+ df_columns = df_columns + index_names_to_add
2404
+
2405
+ if self.index_label is not None and self.index is False:
2406
+ raise TeradataMlException(Messages.get_message(MessageCodes.INVALID_INDEX_LABEL),
2407
+ MessageCodes.INVALID_INDEX_LABEL)
2408
+
2409
+ elif isinstance(self.df, tdmldf.DataFrame):
2410
+ # teradataml DataFrame's do not support saving pandas index/index_label
2411
+ if self.index_label is not None:
2412
+ raise TeradataMlException(Messages.get_message(MessageCodes.INVALID_ARG_VALUE,
2413
+ str(self.index_label), 'index_label', 'None'),
2414
+ MessageCodes.INVALID_ARG_VALUE)
2415
+
2416
+ if self.index is not False:
2417
+ raise TeradataMlException(Messages.get_message(MessageCodes.INVALID_ARG_VALUE,
2418
+ str(self.index), 'index', 'False'),
2419
+ MessageCodes.INVALID_ARG_VALUE)
2420
+
2421
+ # Check for number of columns
2422
+ if len(df_columns) > TeradataConstants.TABLE_COLUMN_LIMIT.value:
2423
+ raise TeradataMlException(Messages.get_message(MessageCodes.TD_MAX_COL_MESSAGE),
2424
+ MessageCodes.TD_MAX_COL_MESSAGE)
2425
+
2426
+ # Check for existence of Primary Index Columns
2427
+ pindex = self.primary_index
2428
+ if self.primary_index is not None:
2429
+ if isinstance(self.primary_index, str):
2430
+ pindex = [self.primary_index]
2431
+
2432
+ for column in pindex:
2433
+ if column not in df_columns:
2434
+ raise TeradataMlException(Messages.get_message(MessageCodes.INVALID_PRIMARY_INDEX),
2435
+ MessageCodes.INVALID_PRIMARY_INDEX)
2436
+
2437
+ # Verify types argument is a dictionary, non-empty, and contains appropriate columns
2438
+ if self.types is not None:
2439
+ # Verify types argument is non-empty when specified
2440
+ if not (self.types):
2441
+ raise TeradataMlException(Messages.get_message(MessageCodes.ARG_EMPTY, 'types'),
2442
+ MessageCodes.ARG_EMPTY)
2443
+
2444
+ # Check if all column names provided in types are valid DataFrame columns
2445
+ if any(key not in df_columns for key in self.types):
2446
+ # Only iterate entire types dictionary if an invalid column value passed
2447
+ for key in self.types:
2448
+ if key not in df_columns:
2449
+ raise TeradataMlException(Messages.get_message(MessageCodes.INVALID_ARG_VALUE,
2450
+ str(key), 'types', ', '.join(df_columns)),
2451
+ MessageCodes.INVALID_ARG_VALUE)
2452
+
2453
+ def _check_table_exists(self, is_table_exists):
2454
+ """
2455
+ Internal function to raise an exception when table does not exists
2456
+ and if_exists is set to 'fail'.
2457
+
2458
+ PARAMETERS:
2459
+ is_table_exists:
2460
+ Required Argument.
2461
+ Specifies whether table exists or not.
2462
+ Types: bool
2463
+
2464
+ RETURNS:
2465
+ None
2466
+
2467
+ RAISES:
2468
+ TeradataMlException
2469
+
2470
+ EXAMPLES:
2471
+ _check_table_exists(True)
2472
+ """
2473
+ # Raise an exception when the table exists and if_exists = 'fail'
2474
+ if is_table_exists and self.if_exists.lower() == 'fail':
2475
+ raise TeradataMlException(Messages.get_message(MessageCodes.TABLE_ALREADY_EXISTS, self.table_name),
2476
+ MessageCodes.TABLE_ALREADY_EXISTS)
2477
+
2478
+ def _create_or_replace_table(self, con, table_exists):
2479
+ """
2480
+ Internal function to create the table if table does not exists on
2481
+ system. If table exists then drop the existing table and create
2482
+ new table.
2483
+
2484
+ PARAMETERS:
2485
+ con:
2486
+ Required Argument.
2487
+ A SQLAlchemy connectable (engine/connection) object
2488
+ Types: Teradata connection object
2489
+
2490
+ table_exists:
2491
+ Required Argument.
2492
+ Specifies whether table exists or not.
2493
+ Types: bool
2494
+
2495
+ Returns:
2496
+ None
2497
+
2498
+ RAISES:
2499
+ None
2500
+
2501
+ EXAMPLES:
2502
+ _create_or_replace_table(con, True)
2503
+ """
2504
+ # If the table need to be replaced, let's drop the existing table first.
2505
+ if table_exists:
2506
+ UtilFuncs._drop_table(self._get_fully_qualified_table_name())
2507
+
2508
+ # Create target table for FastLoad
2509
+ self._create_table(con)
2510
+
2511
+ def _check_columns_compatibility(self, table_obj, cols):
2512
+ """
2513
+ Internal function checks the columns' compatibility with already
2514
+ existing table.
2515
+
2516
+ PARAMETERS:
2517
+ table_obj:
2518
+ Required Argument.
2519
+ Specifies the table object.
2520
+ Types: sqlalchemy.sql.schema.Table
2521
+
2522
+ cols:
2523
+ Required Argument.
2524
+ Specifies list of column names.
2525
+ Types: list
2526
+
2527
+ RETURNS:
2528
+ None
2529
+
2530
+ RAISES:
2531
+ TeradataMlException
2532
+
2533
+ EXAMPLES:
2534
+ _check_columns_compatibility(table_obj, ['c1', 'c2'])
2535
+ """
2536
+ cols_compatible = _check_columns_insertion_compatible(table_obj.c, cols,
2537
+ True)
2538
+ if not cols_compatible:
2539
+ raise TeradataMlException(Messages.get_message(MessageCodes.INSERTION_INCOMPATIBLE),
2540
+ MessageCodes.INSERTION_INCOMPATIBLE)
2541
+
2542
+ def _create_staging_table_and_load_csv_data(self, column_info=None, primary_index=None):
2543
+ """
2544
+ Internal function to create the staging table and load the csv
2545
+ data into it.
2546
+
2547
+ PARAMETERS:
2548
+ column_info:
2549
+ Required Argument.
2550
+ Specifies the data types for columns to be saved in Vantage.
2551
+ Keys of this dictionary should be the name of the columns and values should be
2552
+ teradatasqlalchemy.types.
2553
+ Types: dict (key should be the name of column,
2554
+ value should be the data type of column)
2555
+
2556
+ primary_index:
2557
+ Optional Argument.
2558
+ Specifies primary index.
2559
+ Default Value: None
2560
+ Types: list of Strings (str)
2561
+
2562
+ RETURNS:
2563
+ dict
2564
+
2565
+ RAISES:
2566
+ None
2567
+
2568
+ EXAMPLES:
2569
+ _create_staging_table_and_load_csv_data(column_info={"id": INTEGER})
2570
+ _create_staging_table_and_load_csv_data(column_info={"id": INTEGER}, primary_index = ['id'])
2571
+
2572
+ """
2573
+ stage_table_created = False
2574
+ try:
2575
+ # Generate the temporary table.
2576
+ stag_table_name = UtilFuncs._generate_temp_table_name(prefix="fl_staging",
2577
+ gc_on_quit=False,
2578
+ quote=False,
2579
+ table_type=TeradataConstants.TERADATA_TABLE)
2580
+
2581
+ # If configure.temp_object_type="VT", _generate_temp_table_name() retruns the
2582
+ # table name in fully qualified format. Because of this , test cases started
2583
+ # failing with Blank name in quotation mark. Hence, extracted only the table name.
2584
+ stag_table_name = UtilFuncs._extract_table_name(stag_table_name)
2585
+
2586
+ # Information about uniqueness of primary index and
2587
+ # SET/MULTISET property of existing table is not available,
2588
+ # so over-assuming to be False.
2589
+ unique = False
2590
+ set_table = False
2591
+
2592
+ if not primary_index:
2593
+ primary_index = None
2594
+
2595
+ # Create staging table where column info deduced from the existing table.
2596
+ _create_table(table_name=stag_table_name, columns=column_info,
2597
+ primary_index=primary_index, unique=unique,
2598
+ temporary=False, schema_name=self.schema_name,
2599
+ set_table=set_table)
2600
+ stage_table_created = True
2601
+ column_names = list(column_info.keys())
2602
+
2603
+ # Load the data from CSV to staging table.
2604
+ rc_dict = self._insert_from_csv_with_fastload(table_name=stag_table_name,
2605
+ column_names=column_names)
2606
+
2607
+ # Insert all rows from staging table to already existing table.
2608
+ df_utils._insert_all_from_table(self.table_name,
2609
+ stag_table_name,
2610
+ column_names,
2611
+ to_schema_name=self.schema_name,
2612
+ from_schema_name=self.schema_name)
2613
+
2614
+ return rc_dict
2615
+ finally:
2616
+ # Drop the staging table.
2617
+ if stage_table_created:
2618
+ UtilFuncs._drop_table(self._get_fully_qualified_table_name(stag_table_name))
2619
+
2620
+ #
2621
+ # These functions are specific to read_csv() without using FastloadCSV.
2622
+ #
2623
+ def _insert_from_csv_without_fastload(self, table_name=None, column_names=None):
2624
+ """
2625
+ Internal function to read the csv file and load the data into
2626
+ a table in Vantage.
2627
+
2628
+ PARAMETERS:
2629
+ table_name:
2630
+ Optional Argument.
2631
+ Specifies the table name to load the data into.
2632
+ Types: String
2633
+
2634
+ column_names:
2635
+ Optional Argument.
2636
+ Specifies the list of column names.
2637
+ Types: list of Strings (str)
2638
+
2639
+ RETURNS:
2640
+ None
2641
+
2642
+ RAISES:
2643
+ None
2644
+
2645
+ EXAMPLES:
2646
+ _insert_from_csv_without_fastload(con, "my_table", [VARCHAR, DATE])
2647
+
2648
+ """
2649
+ # Form the INSERT query.
2650
+ insert_stmt = self._form_insert_query(table=table_name,
2651
+ column_names=column_names)
2652
+
2653
+ # Empty queryband buffer before SQL call.
2654
+ UtilFuncs._set_queryband()
2655
+ try:
2656
+ conn = get_connection().connection
2657
+ cur = conn.cursor()
2658
+
2659
+ # Execute INSERT query.
2660
+ cur.execute(insert_stmt)
2661
+ except Exception:
2662
+ raise
2663
+
2664
+ def _check_index_compatibility(self, primary_index_1, primary_index_2):
2665
+ """
2666
+ Internal function to check the compatibility of user provided primary index
2667
+ with primary index of already existing table.
2668
+
2669
+ PARAMETERS:
2670
+ primary_index_1:
2671
+ Required Argument.
2672
+ Specifies the primary index columns of existing table.
2673
+ Types: list of Strings (str)
2674
+
2675
+ primary_index_2:
2676
+ Required Argument.
2677
+ Specifies the primary index columns provided by user.
2678
+ Types: str OR list of Strings (str)
2679
+
2680
+ RETURNS:
2681
+ None
2682
+
2683
+ RAISES:
2684
+ TeradataMlException
2685
+
2686
+ EXAMPLES:
2687
+ _check_index_compatibility(['c1'], ['c1', 'c2'])
2688
+ _check_index_compatibility(['c1'], 'c1')
2689
+ """
2690
+ if isinstance(primary_index_2, str):
2691
+ primary_index_2 = [primary_index_2]
2692
+
2693
+ if not (sorted(primary_index_1) == (sorted(primary_index_2))):
2694
+ raise TeradataMlException("Unable to perform insertion to existing table; Indexes do not match.",
2695
+ MessageCodes.INSERTION_INCOMPATIBLE)
2696
+
2697
+ def _process_pti_load_csv_data(self, con):
2698
+ """
2699
+ Function processes the CSV data so that it can be loaded in a PTI table.
2700
+ To do so, it first creates a staging table and loads the data into same
2701
+ and the creates/loads into the PTI table using copy_to_sql().
2702
+
2703
+ PARAMETERS:
2704
+ con:
2705
+ Required Argument.
2706
+ Specifies a SQLAlchemy connectable (engine/connection) object.
2707
+ Types: sqlalchemy.engine.base.Engine
2708
+
2709
+ RETURNS:
2710
+ If catch_errors_warnings is set to True returns dict, otherwise None.
2711
+
2712
+ RAISES:
2713
+ ValueError
2714
+
2715
+ EXAMPLES:
2716
+ _process_pti_load_csv_data(con)
2717
+
2718
+ """
2719
+ stag_table_name = ""
2720
+ try:
2721
+ # Types argument is required while appending the PTI table.
2722
+ if self._table_exists(con) and self.if_exists == "append" and self.types is None:
2723
+ _Validators._validate_argument_is_not_None(self.types, "types")
2724
+
2725
+ # Generate the staging table name.
2726
+ stag_table_name = UtilFuncs._generate_temp_table_name(prefix="rc_pti_staging",
2727
+ gc_on_quit=False,
2728
+ quote=False,
2729
+ table_type=TeradataConstants.TERADATA_TABLE)
2730
+
2731
+ # If configure.temp_object_type="VT", _generate_temp_table_name() retruns the
2732
+ # table name in fully qualified format. Because of this , test cases started
2733
+ # failing with Blank name in quotation mark. Hence, extracted only the table name.
2734
+ stag_table_name = UtilFuncs._extract_table_name(stag_table_name)
2735
+
2736
+ # Get the teradataml dataframe from staging table using read_csv()
2737
+ read_csv_output = read_csv(filepath=self.df, table_name=stag_table_name,
2738
+ types=self.types, sep=self.sep,
2739
+ quotechar=self.quotechar,
2740
+ schema_name=self.schema_name,
2741
+ save_errors=self.save_errors,
2742
+ catch_errors_warnings=self.catch_errors_warnings,
2743
+ use_fastload=self.use_fastload)
2744
+
2745
+ rc_dict = None
2746
+ if self.catch_errors_warnings:
2747
+ stage_table_tdml_df, rc_dict = read_csv_output
2748
+ else:
2749
+ stage_table_tdml_df = read_csv_output
2750
+
2751
+ # To create the PTI table use copy_to_sql().
2752
+ copy_to_sql(df=stage_table_tdml_df, table_name=self.table_name,
2753
+ schema_name=self.schema_name, if_exists=self.if_exists,
2754
+ primary_index=self.primary_index,
2755
+ temporary=self.temporary,
2756
+ primary_time_index_name=self.primary_time_index_name,
2757
+ timecode_column=self.timecode_column,
2758
+ timebucket_duration=self.timebucket_duration,
2759
+ timezero_date=self.timezero_date,
2760
+ columns_list=self.columns_list,
2761
+ sequence_column=self.sequence_column,
2762
+ seq_max=self.seq_max, set_table=self.set_table)
2763
+
2764
+ return rc_dict
2765
+ finally:
2766
+ if stag_table_name:
2767
+ UtilFuncs._drop_table(self._get_fully_qualified_table_name(stag_table_name))
2768
+
2769
+ # End of functions specific to read_csv().
2770
+
2771
+ # Functions specific to exporting Vantage table data into CSV
2772
+ # which can be done with teradatasql driver escape functions with or without
2773
+ # fastexport protocol.
2774
+ def _get_csv(self, require_fastexport=True, csv_file_name=None, **kwargs):
2775
+ """
2776
+ DESCRIPTION:
2777
+ Internal function to export the data present in teradataml DataFrame into a CSV file
2778
+ using teradatasql driver's escape functions meant for writing to CSV. This can be done with or
2779
+ without Fastexport protocol.
2780
+
2781
+ PARAMETERS:
2782
+ require_fastexport:
2783
+ Optional Argument.
2784
+ Specifies whether fastexport protocol is required for data
2785
+ transfer.
2786
+ Default Value: True
2787
+ Types: boolean
2788
+
2789
+ csv_file_name:
2790
+ Required Argument.
2791
+ Specifies the name of CSV file to which data is to be exported.
2792
+ Default Value: None
2793
+ Types: Str
2794
+
2795
+ kwargs:
2796
+ Specifies keyword arguments. Argument "open_sessions"
2797
+ can be passed as keyword arguments.
2798
+ * "open_sessions" specifies the number of Teradata sessions to
2799
+ be opened for fastexport.
2800
+
2801
+ RETURNS:
2802
+ When "catch_errors_warnings" is set to True, the function returns
2803
+ a tuple containing:
2804
+ * Errors, if any, thrown by fastexport in a list of strings.
2805
+ * Warnings, if any, thrown by fastexport in a list of strings.
2806
+
2807
+ RAISES:
2808
+ TeradataMlException.
2809
+
2810
+ EXAMPLES:
2811
+ dt_obj = _DataTransferUtils(df)
2812
+ dt_obj._get_csv(require_fastexport = True, csv_file_name = "Test.csv")
2813
+ dt_obj._get_csv(csv_file_name = "Test.csv")
2814
+ dt_obj._get_csv(csv_file_name = "Test.csv", open_sessions = 2)
2815
+
2816
+ """
2817
+ try:
2818
+ base_query = self._generate_to_pandas_base_query()
2819
+ base_query = base_query.lstrip()
2820
+ # Get open_sessions argument.
2821
+ open_sessions = kwargs.pop("open_sessions", None)
2822
+ if not require_fastexport and open_sessions is not None:
2823
+ raise TeradataMlException("'{0}' can only be used when '{1}' is set to True." \
2824
+ .format("open_sessions", "fastexport or require"),
2825
+ MessageCodes.DEPENDENT_ARGUMENT)
2826
+
2827
+ csv_export_query = self._generate_fastexport_query(base_query,
2828
+ require=require_fastexport,
2829
+ open_sessions=open_sessions,
2830
+ csv_file_name=csv_file_name)
2831
+
2832
+ UtilFuncs._execute_query(csv_export_query)
2833
+ err, warn = \
2834
+ self._process_fastexport_errors_warnings(csv_export_query)
2835
+
2836
+ print('\nData is successfully exported into {}'.format(csv_file_name))
2837
+ if self.catch_errors_warnings:
2838
+ return err, warn
2839
+ except TeradataMlException:
2840
+ raise