teradataml 20.0.0.8__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- teradataml/LICENSE-3RD-PARTY.pdf +0 -0
- teradataml/LICENSE.pdf +0 -0
- teradataml/README.md +2762 -0
- teradataml/__init__.py +78 -0
- teradataml/_version.py +11 -0
- teradataml/analytics/Transformations.py +2996 -0
- teradataml/analytics/__init__.py +82 -0
- teradataml/analytics/analytic_function_executor.py +2416 -0
- teradataml/analytics/analytic_query_generator.py +1050 -0
- teradataml/analytics/byom/H2OPredict.py +514 -0
- teradataml/analytics/byom/PMMLPredict.py +437 -0
- teradataml/analytics/byom/__init__.py +16 -0
- teradataml/analytics/json_parser/__init__.py +133 -0
- teradataml/analytics/json_parser/analytic_functions_argument.py +1805 -0
- teradataml/analytics/json_parser/json_store.py +191 -0
- teradataml/analytics/json_parser/metadata.py +1666 -0
- teradataml/analytics/json_parser/utils.py +805 -0
- teradataml/analytics/meta_class.py +236 -0
- teradataml/analytics/sqle/DecisionTreePredict.py +456 -0
- teradataml/analytics/sqle/NaiveBayesPredict.py +420 -0
- teradataml/analytics/sqle/__init__.py +128 -0
- teradataml/analytics/sqle/json/decisiontreepredict_sqle.json +78 -0
- teradataml/analytics/sqle/json/naivebayespredict_sqle.json +62 -0
- teradataml/analytics/table_operator/__init__.py +11 -0
- teradataml/analytics/uaf/__init__.py +82 -0
- teradataml/analytics/utils.py +828 -0
- teradataml/analytics/valib.py +1617 -0
- teradataml/automl/__init__.py +5835 -0
- teradataml/automl/autodataprep/__init__.py +493 -0
- teradataml/automl/custom_json_utils.py +1625 -0
- teradataml/automl/data_preparation.py +1384 -0
- teradataml/automl/data_transformation.py +1254 -0
- teradataml/automl/feature_engineering.py +2273 -0
- teradataml/automl/feature_exploration.py +1873 -0
- teradataml/automl/model_evaluation.py +488 -0
- teradataml/automl/model_training.py +1407 -0
- teradataml/catalog/__init__.py +2 -0
- teradataml/catalog/byom.py +1759 -0
- teradataml/catalog/function_argument_mapper.py +859 -0
- teradataml/catalog/model_cataloging_utils.py +491 -0
- teradataml/clients/__init__.py +0 -0
- teradataml/clients/auth_client.py +137 -0
- teradataml/clients/keycloak_client.py +165 -0
- teradataml/clients/pkce_client.py +481 -0
- teradataml/common/__init__.py +1 -0
- teradataml/common/aed_utils.py +2078 -0
- teradataml/common/bulk_exposed_utils.py +113 -0
- teradataml/common/constants.py +1669 -0
- teradataml/common/deprecations.py +166 -0
- teradataml/common/exceptions.py +147 -0
- teradataml/common/formula.py +743 -0
- teradataml/common/garbagecollector.py +666 -0
- teradataml/common/logger.py +1261 -0
- teradataml/common/messagecodes.py +518 -0
- teradataml/common/messages.py +262 -0
- teradataml/common/pylogger.py +67 -0
- teradataml/common/sqlbundle.py +764 -0
- teradataml/common/td_coltype_code_to_tdtype.py +48 -0
- teradataml/common/utils.py +3166 -0
- teradataml/common/warnings.py +36 -0
- teradataml/common/wrapper_utils.py +625 -0
- teradataml/config/__init__.py +0 -0
- teradataml/config/dummy_file1.cfg +5 -0
- teradataml/config/dummy_file2.cfg +3 -0
- teradataml/config/sqlengine_alias_definitions_v1.0 +14 -0
- teradataml/config/sqlengine_alias_definitions_v1.1 +20 -0
- teradataml/config/sqlengine_alias_definitions_v1.3 +19 -0
- teradataml/context/__init__.py +0 -0
- teradataml/context/aed_context.py +223 -0
- teradataml/context/context.py +1462 -0
- teradataml/data/A_loan.csv +19 -0
- teradataml/data/BINARY_REALS_LEFT.csv +11 -0
- teradataml/data/BINARY_REALS_RIGHT.csv +11 -0
- teradataml/data/B_loan.csv +49 -0
- teradataml/data/BuoyData2.csv +17 -0
- teradataml/data/CONVOLVE2_COMPLEX_LEFT.csv +5 -0
- teradataml/data/CONVOLVE2_COMPLEX_RIGHT.csv +5 -0
- teradataml/data/Convolve2RealsLeft.csv +5 -0
- teradataml/data/Convolve2RealsRight.csv +5 -0
- teradataml/data/Convolve2ValidLeft.csv +11 -0
- teradataml/data/Convolve2ValidRight.csv +11 -0
- teradataml/data/DFFTConv_Real_8_8.csv +65 -0
- teradataml/data/Employee.csv +5 -0
- teradataml/data/Employee_Address.csv +4 -0
- teradataml/data/Employee_roles.csv +5 -0
- teradataml/data/JulesBelvezeDummyData.csv +100 -0
- teradataml/data/Mall_customer_data.csv +201 -0
- teradataml/data/Orders1_12mf.csv +25 -0
- teradataml/data/Pi_loan.csv +7 -0
- teradataml/data/SMOOTHED_DATA.csv +7 -0
- teradataml/data/TestDFFT8.csv +9 -0
- teradataml/data/TestRiver.csv +109 -0
- teradataml/data/Traindata.csv +28 -0
- teradataml/data/__init__.py +0 -0
- teradataml/data/acf.csv +17 -0
- teradataml/data/adaboost_example.json +34 -0
- teradataml/data/adaboostpredict_example.json +24 -0
- teradataml/data/additional_table.csv +11 -0
- teradataml/data/admissions_test.csv +21 -0
- teradataml/data/admissions_train.csv +41 -0
- teradataml/data/admissions_train_nulls.csv +41 -0
- teradataml/data/advertising.csv +201 -0
- teradataml/data/ageandheight.csv +13 -0
- teradataml/data/ageandpressure.csv +31 -0
- teradataml/data/amazon_reviews_25.csv +26 -0
- teradataml/data/antiselect_example.json +36 -0
- teradataml/data/antiselect_input.csv +8 -0
- teradataml/data/antiselect_input_mixed_case.csv +8 -0
- teradataml/data/applicant_external.csv +7 -0
- teradataml/data/applicant_reference.csv +7 -0
- teradataml/data/apriori_example.json +22 -0
- teradataml/data/arima_example.json +9 -0
- teradataml/data/assortedtext_input.csv +8 -0
- teradataml/data/attribution_example.json +34 -0
- teradataml/data/attribution_sample_table.csv +27 -0
- teradataml/data/attribution_sample_table1.csv +6 -0
- teradataml/data/attribution_sample_table2.csv +11 -0
- teradataml/data/bank_churn.csv +10001 -0
- teradataml/data/bank_marketing.csv +11163 -0
- teradataml/data/bank_web_clicks1.csv +43 -0
- teradataml/data/bank_web_clicks2.csv +91 -0
- teradataml/data/bank_web_url.csv +85 -0
- teradataml/data/barrier.csv +2 -0
- teradataml/data/barrier_new.csv +3 -0
- teradataml/data/betweenness_example.json +14 -0
- teradataml/data/bike_sharing.csv +732 -0
- teradataml/data/bin_breaks.csv +8 -0
- teradataml/data/bin_fit_ip.csv +4 -0
- teradataml/data/binary_complex_left.csv +11 -0
- teradataml/data/binary_complex_right.csv +11 -0
- teradataml/data/binary_matrix_complex_left.csv +21 -0
- teradataml/data/binary_matrix_complex_right.csv +21 -0
- teradataml/data/binary_matrix_real_left.csv +21 -0
- teradataml/data/binary_matrix_real_right.csv +21 -0
- teradataml/data/blood2ageandweight.csv +26 -0
- teradataml/data/bmi.csv +501 -0
- teradataml/data/boston.csv +507 -0
- teradataml/data/boston2cols.csv +721 -0
- teradataml/data/breast_cancer.csv +570 -0
- teradataml/data/buoydata_mix.csv +11 -0
- teradataml/data/burst_data.csv +5 -0
- teradataml/data/burst_example.json +21 -0
- teradataml/data/byom_example.json +34 -0
- teradataml/data/bytes_table.csv +4 -0
- teradataml/data/cal_housing_ex_raw.csv +70 -0
- teradataml/data/callers.csv +7 -0
- teradataml/data/calls.csv +10 -0
- teradataml/data/cars_hist.csv +33 -0
- teradataml/data/cat_table.csv +25 -0
- teradataml/data/ccm_example.json +32 -0
- teradataml/data/ccm_input.csv +91 -0
- teradataml/data/ccm_input2.csv +13 -0
- teradataml/data/ccmexample.csv +101 -0
- teradataml/data/ccmprepare_example.json +9 -0
- teradataml/data/ccmprepare_input.csv +91 -0
- teradataml/data/cfilter_example.json +12 -0
- teradataml/data/changepointdetection_example.json +18 -0
- teradataml/data/changepointdetectionrt_example.json +8 -0
- teradataml/data/chi_sq.csv +3 -0
- teradataml/data/churn_data.csv +14 -0
- teradataml/data/churn_emission.csv +35 -0
- teradataml/data/churn_initial.csv +3 -0
- teradataml/data/churn_state_transition.csv +5 -0
- teradataml/data/citedges_2.csv +745 -0
- teradataml/data/citvertices_2.csv +1210 -0
- teradataml/data/clicks2.csv +16 -0
- teradataml/data/clickstream.csv +13 -0
- teradataml/data/clickstream1.csv +11 -0
- teradataml/data/closeness_example.json +16 -0
- teradataml/data/complaints.csv +21 -0
- teradataml/data/complaints_mini.csv +3 -0
- teradataml/data/complaints_test_tokenized.csv +353 -0
- teradataml/data/complaints_testtoken.csv +224 -0
- teradataml/data/complaints_tokens_model.csv +348 -0
- teradataml/data/complaints_tokens_test.csv +353 -0
- teradataml/data/complaints_traintoken.csv +472 -0
- teradataml/data/computers_category.csv +1001 -0
- teradataml/data/computers_test1.csv +1252 -0
- teradataml/data/computers_train1.csv +5009 -0
- teradataml/data/computers_train1_clustered.csv +5009 -0
- teradataml/data/confusionmatrix_example.json +9 -0
- teradataml/data/conversion_event_table.csv +3 -0
- teradataml/data/corr_input.csv +17 -0
- teradataml/data/correlation_example.json +11 -0
- teradataml/data/covid_confirm_sd.csv +83 -0
- teradataml/data/coxhazardratio_example.json +39 -0
- teradataml/data/coxph_example.json +15 -0
- teradataml/data/coxsurvival_example.json +28 -0
- teradataml/data/cpt.csv +41 -0
- teradataml/data/credit_ex_merged.csv +45 -0
- teradataml/data/creditcard_data.csv +1001 -0
- teradataml/data/customer_loyalty.csv +301 -0
- teradataml/data/customer_loyalty_newseq.csv +31 -0
- teradataml/data/customer_segmentation_test.csv +2628 -0
- teradataml/data/customer_segmentation_train.csv +8069 -0
- teradataml/data/dataframe_example.json +173 -0
- teradataml/data/decisionforest_example.json +37 -0
- teradataml/data/decisionforestpredict_example.json +38 -0
- teradataml/data/decisiontree_example.json +21 -0
- teradataml/data/decisiontreepredict_example.json +45 -0
- teradataml/data/dfft2_size4_real.csv +17 -0
- teradataml/data/dfft2_test_matrix16.csv +17 -0
- teradataml/data/dfft2conv_real_4_4.csv +65 -0
- teradataml/data/diabetes.csv +443 -0
- teradataml/data/diabetes_test.csv +89 -0
- teradataml/data/dict_table.csv +5 -0
- teradataml/data/docperterm_table.csv +4 -0
- teradataml/data/docs/__init__.py +1 -0
- teradataml/data/docs/byom/__init__.py +0 -0
- teradataml/data/docs/byom/docs/DataRobotPredict.py +180 -0
- teradataml/data/docs/byom/docs/DataikuPredict.py +217 -0
- teradataml/data/docs/byom/docs/H2OPredict.py +325 -0
- teradataml/data/docs/byom/docs/ONNXEmbeddings.py +242 -0
- teradataml/data/docs/byom/docs/ONNXPredict.py +283 -0
- teradataml/data/docs/byom/docs/ONNXSeq2Seq.py +255 -0
- teradataml/data/docs/byom/docs/PMMLPredict.py +278 -0
- teradataml/data/docs/byom/docs/__init__.py +0 -0
- teradataml/data/docs/sqle/__init__.py +0 -0
- teradataml/data/docs/sqle/docs_17_10/Antiselect.py +83 -0
- teradataml/data/docs/sqle/docs_17_10/Attribution.py +200 -0
- teradataml/data/docs/sqle/docs_17_10/BincodeFit.py +172 -0
- teradataml/data/docs/sqle/docs_17_10/BincodeTransform.py +131 -0
- teradataml/data/docs/sqle/docs_17_10/CategoricalSummary.py +86 -0
- teradataml/data/docs/sqle/docs_17_10/ChiSq.py +90 -0
- teradataml/data/docs/sqle/docs_17_10/ColumnSummary.py +86 -0
- teradataml/data/docs/sqle/docs_17_10/ConvertTo.py +96 -0
- teradataml/data/docs/sqle/docs_17_10/DecisionForestPredict.py +139 -0
- teradataml/data/docs/sqle/docs_17_10/DecisionTreePredict.py +152 -0
- teradataml/data/docs/sqle/docs_17_10/FTest.py +161 -0
- teradataml/data/docs/sqle/docs_17_10/FillRowId.py +83 -0
- teradataml/data/docs/sqle/docs_17_10/Fit.py +88 -0
- teradataml/data/docs/sqle/docs_17_10/GLMPredict.py +144 -0
- teradataml/data/docs/sqle/docs_17_10/GetRowsWithMissingValues.py +85 -0
- teradataml/data/docs/sqle/docs_17_10/GetRowsWithoutMissingValues.py +82 -0
- teradataml/data/docs/sqle/docs_17_10/Histogram.py +165 -0
- teradataml/data/docs/sqle/docs_17_10/MovingAverage.py +134 -0
- teradataml/data/docs/sqle/docs_17_10/NGramSplitter.py +209 -0
- teradataml/data/docs/sqle/docs_17_10/NPath.py +266 -0
- teradataml/data/docs/sqle/docs_17_10/NaiveBayesPredict.py +116 -0
- teradataml/data/docs/sqle/docs_17_10/NaiveBayesTextClassifierPredict.py +176 -0
- teradataml/data/docs/sqle/docs_17_10/NumApply.py +147 -0
- teradataml/data/docs/sqle/docs_17_10/OneHotEncodingFit.py +135 -0
- teradataml/data/docs/sqle/docs_17_10/OneHotEncodingTransform.py +109 -0
- teradataml/data/docs/sqle/docs_17_10/OutlierFilterFit.py +166 -0
- teradataml/data/docs/sqle/docs_17_10/OutlierFilterTransform.py +105 -0
- teradataml/data/docs/sqle/docs_17_10/Pack.py +128 -0
- teradataml/data/docs/sqle/docs_17_10/PolynomialFeaturesFit.py +112 -0
- teradataml/data/docs/sqle/docs_17_10/PolynomialFeaturesTransform.py +102 -0
- teradataml/data/docs/sqle/docs_17_10/QQNorm.py +105 -0
- teradataml/data/docs/sqle/docs_17_10/RoundColumns.py +110 -0
- teradataml/data/docs/sqle/docs_17_10/RowNormalizeFit.py +118 -0
- teradataml/data/docs/sqle/docs_17_10/RowNormalizeTransform.py +99 -0
- teradataml/data/docs/sqle/docs_17_10/SVMSparsePredict.py +153 -0
- teradataml/data/docs/sqle/docs_17_10/ScaleFit.py +197 -0
- teradataml/data/docs/sqle/docs_17_10/ScaleTransform.py +99 -0
- teradataml/data/docs/sqle/docs_17_10/Sessionize.py +114 -0
- teradataml/data/docs/sqle/docs_17_10/SimpleImputeFit.py +116 -0
- teradataml/data/docs/sqle/docs_17_10/SimpleImputeTransform.py +98 -0
- teradataml/data/docs/sqle/docs_17_10/StrApply.py +187 -0
- teradataml/data/docs/sqle/docs_17_10/StringSimilarity.py +146 -0
- teradataml/data/docs/sqle/docs_17_10/Transform.py +105 -0
- teradataml/data/docs/sqle/docs_17_10/UnivariateStatistics.py +142 -0
- teradataml/data/docs/sqle/docs_17_10/Unpack.py +214 -0
- teradataml/data/docs/sqle/docs_17_10/WhichMax.py +83 -0
- teradataml/data/docs/sqle/docs_17_10/WhichMin.py +83 -0
- teradataml/data/docs/sqle/docs_17_10/ZTest.py +155 -0
- teradataml/data/docs/sqle/docs_17_10/__init__.py +0 -0
- teradataml/data/docs/sqle/docs_17_20/ANOVA.py +186 -0
- teradataml/data/docs/sqle/docs_17_20/Antiselect.py +83 -0
- teradataml/data/docs/sqle/docs_17_20/Apriori.py +138 -0
- teradataml/data/docs/sqle/docs_17_20/Attribution.py +201 -0
- teradataml/data/docs/sqle/docs_17_20/BincodeFit.py +172 -0
- teradataml/data/docs/sqle/docs_17_20/BincodeTransform.py +139 -0
- teradataml/data/docs/sqle/docs_17_20/CFilter.py +132 -0
- teradataml/data/docs/sqle/docs_17_20/CategoricalSummary.py +86 -0
- teradataml/data/docs/sqle/docs_17_20/ChiSq.py +90 -0
- teradataml/data/docs/sqle/docs_17_20/ClassificationEvaluator.py +166 -0
- teradataml/data/docs/sqle/docs_17_20/ColumnSummary.py +86 -0
- teradataml/data/docs/sqle/docs_17_20/ColumnTransformer.py +246 -0
- teradataml/data/docs/sqle/docs_17_20/ConvertTo.py +113 -0
- teradataml/data/docs/sqle/docs_17_20/DecisionForest.py +280 -0
- teradataml/data/docs/sqle/docs_17_20/DecisionForestPredict.py +144 -0
- teradataml/data/docs/sqle/docs_17_20/DecisionTreePredict.py +136 -0
- teradataml/data/docs/sqle/docs_17_20/FTest.py +240 -0
- teradataml/data/docs/sqle/docs_17_20/FillRowId.py +83 -0
- teradataml/data/docs/sqle/docs_17_20/Fit.py +88 -0
- teradataml/data/docs/sqle/docs_17_20/GLM.py +541 -0
- teradataml/data/docs/sqle/docs_17_20/GLMPerSegment.py +415 -0
- teradataml/data/docs/sqle/docs_17_20/GLMPredict.py +144 -0
- teradataml/data/docs/sqle/docs_17_20/GLMPredictPerSegment.py +233 -0
- teradataml/data/docs/sqle/docs_17_20/GetFutileColumns.py +125 -0
- teradataml/data/docs/sqle/docs_17_20/GetRowsWithMissingValues.py +109 -0
- teradataml/data/docs/sqle/docs_17_20/GetRowsWithoutMissingValues.py +106 -0
- teradataml/data/docs/sqle/docs_17_20/Histogram.py +224 -0
- teradataml/data/docs/sqle/docs_17_20/KMeans.py +251 -0
- teradataml/data/docs/sqle/docs_17_20/KMeansPredict.py +144 -0
- teradataml/data/docs/sqle/docs_17_20/KNN.py +215 -0
- teradataml/data/docs/sqle/docs_17_20/MovingAverage.py +134 -0
- teradataml/data/docs/sqle/docs_17_20/NERExtractor.py +121 -0
- teradataml/data/docs/sqle/docs_17_20/NGramSplitter.py +209 -0
- teradataml/data/docs/sqle/docs_17_20/NPath.py +266 -0
- teradataml/data/docs/sqle/docs_17_20/NaiveBayes.py +162 -0
- teradataml/data/docs/sqle/docs_17_20/NaiveBayesPredict.py +116 -0
- teradataml/data/docs/sqle/docs_17_20/NaiveBayesTextClassifierPredict.py +177 -0
- teradataml/data/docs/sqle/docs_17_20/NaiveBayesTextClassifierTrainer.py +127 -0
- teradataml/data/docs/sqle/docs_17_20/NonLinearCombineFit.py +119 -0
- teradataml/data/docs/sqle/docs_17_20/NonLinearCombineTransform.py +112 -0
- teradataml/data/docs/sqle/docs_17_20/NumApply.py +147 -0
- teradataml/data/docs/sqle/docs_17_20/OneClassSVM.py +307 -0
- teradataml/data/docs/sqle/docs_17_20/OneClassSVMPredict.py +185 -0
- teradataml/data/docs/sqle/docs_17_20/OneHotEncodingFit.py +231 -0
- teradataml/data/docs/sqle/docs_17_20/OneHotEncodingTransform.py +121 -0
- teradataml/data/docs/sqle/docs_17_20/OrdinalEncodingFit.py +220 -0
- teradataml/data/docs/sqle/docs_17_20/OrdinalEncodingTransform.py +127 -0
- teradataml/data/docs/sqle/docs_17_20/OutlierFilterFit.py +191 -0
- teradataml/data/docs/sqle/docs_17_20/OutlierFilterTransform.py +117 -0
- teradataml/data/docs/sqle/docs_17_20/Pack.py +128 -0
- teradataml/data/docs/sqle/docs_17_20/Pivoting.py +279 -0
- teradataml/data/docs/sqle/docs_17_20/PolynomialFeaturesFit.py +112 -0
- teradataml/data/docs/sqle/docs_17_20/PolynomialFeaturesTransform.py +112 -0
- teradataml/data/docs/sqle/docs_17_20/QQNorm.py +105 -0
- teradataml/data/docs/sqle/docs_17_20/ROC.py +164 -0
- teradataml/data/docs/sqle/docs_17_20/RandomProjectionFit.py +155 -0
- teradataml/data/docs/sqle/docs_17_20/RandomProjectionMinComponents.py +106 -0
- teradataml/data/docs/sqle/docs_17_20/RandomProjectionTransform.py +120 -0
- teradataml/data/docs/sqle/docs_17_20/RegressionEvaluator.py +211 -0
- teradataml/data/docs/sqle/docs_17_20/RoundColumns.py +109 -0
- teradataml/data/docs/sqle/docs_17_20/RowNormalizeFit.py +118 -0
- teradataml/data/docs/sqle/docs_17_20/RowNormalizeTransform.py +111 -0
- teradataml/data/docs/sqle/docs_17_20/SMOTE.py +212 -0
- teradataml/data/docs/sqle/docs_17_20/SVM.py +414 -0
- teradataml/data/docs/sqle/docs_17_20/SVMPredict.py +213 -0
- teradataml/data/docs/sqle/docs_17_20/SVMSparsePredict.py +153 -0
- teradataml/data/docs/sqle/docs_17_20/ScaleFit.py +315 -0
- teradataml/data/docs/sqle/docs_17_20/ScaleTransform.py +202 -0
- teradataml/data/docs/sqle/docs_17_20/SentimentExtractor.py +206 -0
- teradataml/data/docs/sqle/docs_17_20/Sessionize.py +114 -0
- teradataml/data/docs/sqle/docs_17_20/Shap.py +225 -0
- teradataml/data/docs/sqle/docs_17_20/Silhouette.py +153 -0
- teradataml/data/docs/sqle/docs_17_20/SimpleImputeFit.py +116 -0
- teradataml/data/docs/sqle/docs_17_20/SimpleImputeTransform.py +109 -0
- teradataml/data/docs/sqle/docs_17_20/StrApply.py +187 -0
- teradataml/data/docs/sqle/docs_17_20/StringSimilarity.py +146 -0
- teradataml/data/docs/sqle/docs_17_20/TDDecisionForestPredict.py +207 -0
- teradataml/data/docs/sqle/docs_17_20/TDGLMPredict.py +333 -0
- teradataml/data/docs/sqle/docs_17_20/TDNaiveBayesPredict.py +189 -0
- teradataml/data/docs/sqle/docs_17_20/TFIDF.py +142 -0
- teradataml/data/docs/sqle/docs_17_20/TargetEncodingFit.py +267 -0
- teradataml/data/docs/sqle/docs_17_20/TargetEncodingTransform.py +141 -0
- teradataml/data/docs/sqle/docs_17_20/TextMorph.py +119 -0
- teradataml/data/docs/sqle/docs_17_20/TextParser.py +224 -0
- teradataml/data/docs/sqle/docs_17_20/TrainTestSplit.py +160 -0
- teradataml/data/docs/sqle/docs_17_20/Transform.py +123 -0
- teradataml/data/docs/sqle/docs_17_20/UnivariateStatistics.py +142 -0
- teradataml/data/docs/sqle/docs_17_20/Unpack.py +214 -0
- teradataml/data/docs/sqle/docs_17_20/Unpivoting.py +216 -0
- teradataml/data/docs/sqle/docs_17_20/VectorDistance.py +169 -0
- teradataml/data/docs/sqle/docs_17_20/WhichMax.py +83 -0
- teradataml/data/docs/sqle/docs_17_20/WhichMin.py +83 -0
- teradataml/data/docs/sqle/docs_17_20/WordEmbeddings.py +237 -0
- teradataml/data/docs/sqle/docs_17_20/XGBoost.py +362 -0
- teradataml/data/docs/sqle/docs_17_20/XGBoostPredict.py +281 -0
- teradataml/data/docs/sqle/docs_17_20/ZTest.py +220 -0
- teradataml/data/docs/sqle/docs_17_20/__init__.py +0 -0
- teradataml/data/docs/tableoperator/__init__.py +0 -0
- teradataml/data/docs/tableoperator/docs_17_00/ReadNOS.py +430 -0
- teradataml/data/docs/tableoperator/docs_17_00/__init__.py +0 -0
- teradataml/data/docs/tableoperator/docs_17_05/ReadNOS.py +430 -0
- teradataml/data/docs/tableoperator/docs_17_05/WriteNOS.py +348 -0
- teradataml/data/docs/tableoperator/docs_17_05/__init__.py +0 -0
- teradataml/data/docs/tableoperator/docs_17_10/ReadNOS.py +429 -0
- teradataml/data/docs/tableoperator/docs_17_10/WriteNOS.py +348 -0
- teradataml/data/docs/tableoperator/docs_17_10/__init__.py +0 -0
- teradataml/data/docs/tableoperator/docs_17_20/Image2Matrix.py +118 -0
- teradataml/data/docs/tableoperator/docs_17_20/ReadNOS.py +440 -0
- teradataml/data/docs/tableoperator/docs_17_20/WriteNOS.py +387 -0
- teradataml/data/docs/tableoperator/docs_17_20/__init__.py +0 -0
- teradataml/data/docs/uaf/__init__.py +0 -0
- teradataml/data/docs/uaf/docs_17_20/ACF.py +186 -0
- teradataml/data/docs/uaf/docs_17_20/ArimaEstimate.py +370 -0
- teradataml/data/docs/uaf/docs_17_20/ArimaForecast.py +172 -0
- teradataml/data/docs/uaf/docs_17_20/ArimaValidate.py +161 -0
- teradataml/data/docs/uaf/docs_17_20/ArimaXEstimate.py +293 -0
- teradataml/data/docs/uaf/docs_17_20/AutoArima.py +354 -0
- teradataml/data/docs/uaf/docs_17_20/BinaryMatrixOp.py +248 -0
- teradataml/data/docs/uaf/docs_17_20/BinarySeriesOp.py +252 -0
- teradataml/data/docs/uaf/docs_17_20/BreuschGodfrey.py +178 -0
- teradataml/data/docs/uaf/docs_17_20/BreuschPaganGodfrey.py +175 -0
- teradataml/data/docs/uaf/docs_17_20/Convolve.py +230 -0
- teradataml/data/docs/uaf/docs_17_20/Convolve2.py +218 -0
- teradataml/data/docs/uaf/docs_17_20/CopyArt.py +145 -0
- teradataml/data/docs/uaf/docs_17_20/CumulPeriodogram.py +185 -0
- teradataml/data/docs/uaf/docs_17_20/DFFT.py +204 -0
- teradataml/data/docs/uaf/docs_17_20/DFFT2.py +216 -0
- teradataml/data/docs/uaf/docs_17_20/DFFT2Conv.py +216 -0
- teradataml/data/docs/uaf/docs_17_20/DFFTConv.py +192 -0
- teradataml/data/docs/uaf/docs_17_20/DIFF.py +175 -0
- teradataml/data/docs/uaf/docs_17_20/DTW.py +180 -0
- teradataml/data/docs/uaf/docs_17_20/DWT.py +235 -0
- teradataml/data/docs/uaf/docs_17_20/DWT2D.py +217 -0
- teradataml/data/docs/uaf/docs_17_20/DickeyFuller.py +142 -0
- teradataml/data/docs/uaf/docs_17_20/DurbinWatson.py +184 -0
- teradataml/data/docs/uaf/docs_17_20/ExtractResults.py +185 -0
- teradataml/data/docs/uaf/docs_17_20/FilterFactory1d.py +160 -0
- teradataml/data/docs/uaf/docs_17_20/FitMetrics.py +172 -0
- teradataml/data/docs/uaf/docs_17_20/GenseriesFormula.py +206 -0
- teradataml/data/docs/uaf/docs_17_20/GenseriesSinusoids.py +143 -0
- teradataml/data/docs/uaf/docs_17_20/GoldfeldQuandt.py +198 -0
- teradataml/data/docs/uaf/docs_17_20/HoltWintersForecaster.py +260 -0
- teradataml/data/docs/uaf/docs_17_20/IDFFT.py +165 -0
- teradataml/data/docs/uaf/docs_17_20/IDFFT2.py +191 -0
- teradataml/data/docs/uaf/docs_17_20/IDWT.py +236 -0
- teradataml/data/docs/uaf/docs_17_20/IDWT2D.py +226 -0
- teradataml/data/docs/uaf/docs_17_20/IQR.py +134 -0
- teradataml/data/docs/uaf/docs_17_20/InputValidator.py +121 -0
- teradataml/data/docs/uaf/docs_17_20/LineSpec.py +156 -0
- teradataml/data/docs/uaf/docs_17_20/LinearRegr.py +215 -0
- teradataml/data/docs/uaf/docs_17_20/MAMean.py +174 -0
- teradataml/data/docs/uaf/docs_17_20/MInfo.py +134 -0
- teradataml/data/docs/uaf/docs_17_20/Matrix2Image.py +297 -0
- teradataml/data/docs/uaf/docs_17_20/MatrixMultiply.py +145 -0
- teradataml/data/docs/uaf/docs_17_20/MultivarRegr.py +191 -0
- teradataml/data/docs/uaf/docs_17_20/PACF.py +157 -0
- teradataml/data/docs/uaf/docs_17_20/Portman.py +217 -0
- teradataml/data/docs/uaf/docs_17_20/PowerSpec.py +203 -0
- teradataml/data/docs/uaf/docs_17_20/PowerTransform.py +155 -0
- teradataml/data/docs/uaf/docs_17_20/Resample.py +237 -0
- teradataml/data/docs/uaf/docs_17_20/SAX.py +246 -0
- teradataml/data/docs/uaf/docs_17_20/SInfo.py +123 -0
- teradataml/data/docs/uaf/docs_17_20/SeasonalNormalize.py +173 -0
- teradataml/data/docs/uaf/docs_17_20/SelectionCriteria.py +174 -0
- teradataml/data/docs/uaf/docs_17_20/SignifPeriodicities.py +171 -0
- teradataml/data/docs/uaf/docs_17_20/SignifResidmean.py +164 -0
- teradataml/data/docs/uaf/docs_17_20/SimpleExp.py +180 -0
- teradataml/data/docs/uaf/docs_17_20/Smoothma.py +208 -0
- teradataml/data/docs/uaf/docs_17_20/TrackingOp.py +151 -0
- teradataml/data/docs/uaf/docs_17_20/UNDIFF.py +171 -0
- teradataml/data/docs/uaf/docs_17_20/Unnormalize.py +202 -0
- teradataml/data/docs/uaf/docs_17_20/WhitesGeneral.py +171 -0
- teradataml/data/docs/uaf/docs_17_20/WindowDFFT.py +368 -0
- teradataml/data/docs/uaf/docs_17_20/__init__.py +0 -0
- teradataml/data/dtw_example.json +18 -0
- teradataml/data/dtw_t1.csv +11 -0
- teradataml/data/dtw_t2.csv +4 -0
- teradataml/data/dwt2d_dataTable.csv +65 -0
- teradataml/data/dwt2d_example.json +16 -0
- teradataml/data/dwt_dataTable.csv +8 -0
- teradataml/data/dwt_example.json +15 -0
- teradataml/data/dwt_filterTable.csv +3 -0
- teradataml/data/dwt_filter_dim.csv +5 -0
- teradataml/data/emission.csv +9 -0
- teradataml/data/emp_table_by_dept.csv +19 -0
- teradataml/data/employee_info.csv +4 -0
- teradataml/data/employee_table.csv +6 -0
- teradataml/data/excluding_event_table.csv +2 -0
- teradataml/data/finance_data.csv +6 -0
- teradataml/data/finance_data2.csv +61 -0
- teradataml/data/finance_data3.csv +93 -0
- teradataml/data/finance_data4.csv +13 -0
- teradataml/data/fish.csv +160 -0
- teradataml/data/fm_blood2ageandweight.csv +26 -0
- teradataml/data/fmeasure_example.json +12 -0
- teradataml/data/followers_leaders.csv +10 -0
- teradataml/data/fpgrowth_example.json +12 -0
- teradataml/data/frequentpaths_example.json +29 -0
- teradataml/data/friends.csv +9 -0
- teradataml/data/fs_input.csv +33 -0
- teradataml/data/fs_input1.csv +33 -0
- teradataml/data/genData.csv +513 -0
- teradataml/data/geodataframe_example.json +40 -0
- teradataml/data/glass_types.csv +215 -0
- teradataml/data/glm_admissions_model.csv +12 -0
- teradataml/data/glm_example.json +56 -0
- teradataml/data/glml1l2_example.json +28 -0
- teradataml/data/glml1l2predict_example.json +54 -0
- teradataml/data/glmpredict_example.json +54 -0
- teradataml/data/gq_t1.csv +21 -0
- teradataml/data/grocery_transaction.csv +19 -0
- teradataml/data/hconvolve_complex_right.csv +5 -0
- teradataml/data/hconvolve_complex_rightmulti.csv +5 -0
- teradataml/data/histogram_example.json +12 -0
- teradataml/data/hmmdecoder_example.json +79 -0
- teradataml/data/hmmevaluator_example.json +25 -0
- teradataml/data/hmmsupervised_example.json +10 -0
- teradataml/data/hmmunsupervised_example.json +8 -0
- teradataml/data/hnsw_alter_data.csv +5 -0
- teradataml/data/hnsw_data.csv +10 -0
- teradataml/data/house_values.csv +12 -0
- teradataml/data/house_values2.csv +13 -0
- teradataml/data/housing_cat.csv +7 -0
- teradataml/data/housing_data.csv +9 -0
- teradataml/data/housing_test.csv +47 -0
- teradataml/data/housing_test_binary.csv +47 -0
- teradataml/data/housing_train.csv +493 -0
- teradataml/data/housing_train_attribute.csv +5 -0
- teradataml/data/housing_train_binary.csv +437 -0
- teradataml/data/housing_train_parameter.csv +2 -0
- teradataml/data/housing_train_response.csv +493 -0
- teradataml/data/housing_train_segment.csv +201 -0
- teradataml/data/ibm_stock.csv +370 -0
- teradataml/data/ibm_stock1.csv +370 -0
- teradataml/data/identitymatch_example.json +22 -0
- teradataml/data/idf_table.csv +4 -0
- teradataml/data/idwt2d_dataTable.csv +5 -0
- teradataml/data/idwt_dataTable.csv +8 -0
- teradataml/data/idwt_filterTable.csv +3 -0
- teradataml/data/impressions.csv +101 -0
- teradataml/data/inflation.csv +21 -0
- teradataml/data/initial.csv +3 -0
- teradataml/data/insect2Cols.csv +61 -0
- teradataml/data/insect_sprays.csv +13 -0
- teradataml/data/insurance.csv +1339 -0
- teradataml/data/interpolator_example.json +13 -0
- teradataml/data/interval_data.csv +5 -0
- teradataml/data/iris_altinput.csv +481 -0
- teradataml/data/iris_attribute_output.csv +8 -0
- teradataml/data/iris_attribute_test.csv +121 -0
- teradataml/data/iris_attribute_train.csv +481 -0
- teradataml/data/iris_category_expect_predict.csv +31 -0
- teradataml/data/iris_data.csv +151 -0
- teradataml/data/iris_input.csv +151 -0
- teradataml/data/iris_response_train.csv +121 -0
- teradataml/data/iris_test.csv +31 -0
- teradataml/data/iris_train.csv +121 -0
- teradataml/data/join_table1.csv +4 -0
- teradataml/data/join_table2.csv +4 -0
- teradataml/data/jsons/anly_function_name.json +7 -0
- teradataml/data/jsons/byom/ONNXSeq2Seq.json +287 -0
- teradataml/data/jsons/byom/dataikupredict.json +148 -0
- teradataml/data/jsons/byom/datarobotpredict.json +147 -0
- teradataml/data/jsons/byom/h2opredict.json +195 -0
- teradataml/data/jsons/byom/onnxembeddings.json +267 -0
- teradataml/data/jsons/byom/onnxpredict.json +187 -0
- teradataml/data/jsons/byom/pmmlpredict.json +147 -0
- teradataml/data/jsons/paired_functions.json +450 -0
- teradataml/data/jsons/sqle/16.20/Antiselect.json +56 -0
- teradataml/data/jsons/sqle/16.20/Attribution.json +249 -0
- teradataml/data/jsons/sqle/16.20/DecisionForestPredict.json +156 -0
- teradataml/data/jsons/sqle/16.20/DecisionTreePredict.json +170 -0
- teradataml/data/jsons/sqle/16.20/GLMPredict.json +122 -0
- teradataml/data/jsons/sqle/16.20/MovingAverage.json +367 -0
- teradataml/data/jsons/sqle/16.20/NGramSplitter.json +239 -0
- teradataml/data/jsons/sqle/16.20/NaiveBayesPredict.json +136 -0
- teradataml/data/jsons/sqle/16.20/NaiveBayesTextClassifierPredict.json +235 -0
- teradataml/data/jsons/sqle/16.20/Pack.json +98 -0
- teradataml/data/jsons/sqle/16.20/SVMSparsePredict.json +162 -0
- teradataml/data/jsons/sqle/16.20/Sessionize.json +105 -0
- teradataml/data/jsons/sqle/16.20/StringSimilarity.json +86 -0
- teradataml/data/jsons/sqle/16.20/Unpack.json +166 -0
- teradataml/data/jsons/sqle/16.20/nPath.json +269 -0
- teradataml/data/jsons/sqle/17.00/Antiselect.json +56 -0
- teradataml/data/jsons/sqle/17.00/Attribution.json +249 -0
- teradataml/data/jsons/sqle/17.00/DecisionForestPredict.json +156 -0
- teradataml/data/jsons/sqle/17.00/DecisionTreePredict.json +170 -0
- teradataml/data/jsons/sqle/17.00/GLMPredict.json +122 -0
- teradataml/data/jsons/sqle/17.00/MovingAverage.json +367 -0
- teradataml/data/jsons/sqle/17.00/NGramSplitter.json +239 -0
- teradataml/data/jsons/sqle/17.00/NaiveBayesPredict.json +136 -0
- teradataml/data/jsons/sqle/17.00/NaiveBayesTextClassifierPredict.json +235 -0
- teradataml/data/jsons/sqle/17.00/Pack.json +98 -0
- teradataml/data/jsons/sqle/17.00/SVMSparsePredict.json +162 -0
- teradataml/data/jsons/sqle/17.00/Sessionize.json +105 -0
- teradataml/data/jsons/sqle/17.00/StringSimilarity.json +86 -0
- teradataml/data/jsons/sqle/17.00/Unpack.json +166 -0
- teradataml/data/jsons/sqle/17.00/nPath.json +269 -0
- teradataml/data/jsons/sqle/17.05/Antiselect.json +56 -0
- teradataml/data/jsons/sqle/17.05/Attribution.json +249 -0
- teradataml/data/jsons/sqle/17.05/DecisionForestPredict.json +156 -0
- teradataml/data/jsons/sqle/17.05/DecisionTreePredict.json +170 -0
- teradataml/data/jsons/sqle/17.05/GLMPredict.json +122 -0
- teradataml/data/jsons/sqle/17.05/MovingAverage.json +367 -0
- teradataml/data/jsons/sqle/17.05/NGramSplitter.json +239 -0
- teradataml/data/jsons/sqle/17.05/NaiveBayesPredict.json +136 -0
- teradataml/data/jsons/sqle/17.05/NaiveBayesTextClassifierPredict.json +235 -0
- teradataml/data/jsons/sqle/17.05/Pack.json +98 -0
- teradataml/data/jsons/sqle/17.05/SVMSparsePredict.json +162 -0
- teradataml/data/jsons/sqle/17.05/Sessionize.json +105 -0
- teradataml/data/jsons/sqle/17.05/StringSimilarity.json +86 -0
- teradataml/data/jsons/sqle/17.05/Unpack.json +166 -0
- teradataml/data/jsons/sqle/17.05/nPath.json +269 -0
- teradataml/data/jsons/sqle/17.10/Antiselect.json +56 -0
- teradataml/data/jsons/sqle/17.10/Attribution.json +249 -0
- teradataml/data/jsons/sqle/17.10/DecisionForestPredict.json +185 -0
- teradataml/data/jsons/sqle/17.10/DecisionTreePredict.json +172 -0
- teradataml/data/jsons/sqle/17.10/GLMPredict.json +151 -0
- teradataml/data/jsons/sqle/17.10/MovingAverage.json +368 -0
- teradataml/data/jsons/sqle/17.10/NGramSplitter.json +239 -0
- teradataml/data/jsons/sqle/17.10/NaiveBayesPredict.json +149 -0
- teradataml/data/jsons/sqle/17.10/NaiveBayesTextClassifierPredict.json +288 -0
- teradataml/data/jsons/sqle/17.10/Pack.json +133 -0
- teradataml/data/jsons/sqle/17.10/SVMSparsePredict.json +193 -0
- teradataml/data/jsons/sqle/17.10/Sessionize.json +105 -0
- teradataml/data/jsons/sqle/17.10/StringSimilarity.json +86 -0
- teradataml/data/jsons/sqle/17.10/TD_BinCodeFit.json +239 -0
- teradataml/data/jsons/sqle/17.10/TD_BinCodeTransform.json +70 -0
- teradataml/data/jsons/sqle/17.10/TD_CategoricalSummary.json +54 -0
- teradataml/data/jsons/sqle/17.10/TD_Chisq.json +68 -0
- teradataml/data/jsons/sqle/17.10/TD_ColumnSummary.json +54 -0
- teradataml/data/jsons/sqle/17.10/TD_ConvertTo.json +69 -0
- teradataml/data/jsons/sqle/17.10/TD_FTest.json +187 -0
- teradataml/data/jsons/sqle/17.10/TD_FillRowID.json +52 -0
- teradataml/data/jsons/sqle/17.10/TD_FunctionFit.json +46 -0
- teradataml/data/jsons/sqle/17.10/TD_FunctionTransform.json +72 -0
- teradataml/data/jsons/sqle/17.10/TD_GetRowsWithMissingValues.json +53 -0
- teradataml/data/jsons/sqle/17.10/TD_GetRowsWithoutMissingValues.json +53 -0
- teradataml/data/jsons/sqle/17.10/TD_Histogram.json +133 -0
- teradataml/data/jsons/sqle/17.10/TD_NumApply.json +147 -0
- teradataml/data/jsons/sqle/17.10/TD_OneHotEncodingFit.json +183 -0
- teradataml/data/jsons/sqle/17.10/TD_OneHotEncodingTransform.json +66 -0
- teradataml/data/jsons/sqle/17.10/TD_OutlierFilterFit.json +197 -0
- teradataml/data/jsons/sqle/17.10/TD_OutlierFilterTransform.json +48 -0
- teradataml/data/jsons/sqle/17.10/TD_PolynomialFeaturesFit.json +114 -0
- teradataml/data/jsons/sqle/17.10/TD_PolynomialFeaturesTransform.json +72 -0
- teradataml/data/jsons/sqle/17.10/TD_QQNorm.json +112 -0
- teradataml/data/jsons/sqle/17.10/TD_RoundColumns.json +93 -0
- teradataml/data/jsons/sqle/17.10/TD_RowNormalizeFit.json +128 -0
- teradataml/data/jsons/sqle/17.10/TD_RowNormalizeTransform.json +71 -0
- teradataml/data/jsons/sqle/17.10/TD_ScaleFit.json +157 -0
- teradataml/data/jsons/sqle/17.10/TD_ScaleTransform.json +71 -0
- teradataml/data/jsons/sqle/17.10/TD_SimpleImputeFit.json +148 -0
- teradataml/data/jsons/sqle/17.10/TD_SimpleImputeTransform.json +48 -0
- teradataml/data/jsons/sqle/17.10/TD_StrApply.json +240 -0
- teradataml/data/jsons/sqle/17.10/TD_UnivariateStatistics.json +119 -0
- teradataml/data/jsons/sqle/17.10/TD_WhichMax.json +53 -0
- teradataml/data/jsons/sqle/17.10/TD_WhichMin.json +53 -0
- teradataml/data/jsons/sqle/17.10/TD_ZTest.json +171 -0
- teradataml/data/jsons/sqle/17.10/Unpack.json +188 -0
- teradataml/data/jsons/sqle/17.10/nPath.json +269 -0
- teradataml/data/jsons/sqle/17.20/Antiselect.json +56 -0
- teradataml/data/jsons/sqle/17.20/Attribution.json +249 -0
- teradataml/data/jsons/sqle/17.20/DecisionForestPredict.json +185 -0
- teradataml/data/jsons/sqle/17.20/DecisionTreePredict.json +172 -0
- teradataml/data/jsons/sqle/17.20/GLMPredict.json +151 -0
- teradataml/data/jsons/sqle/17.20/MovingAverage.json +367 -0
- teradataml/data/jsons/sqle/17.20/NGramSplitter.json +239 -0
- teradataml/data/jsons/sqle/17.20/NaiveBayesPredict.json +149 -0
- teradataml/data/jsons/sqle/17.20/NaiveBayesTextClassifierPredict.json +287 -0
- teradataml/data/jsons/sqle/17.20/Pack.json +133 -0
- teradataml/data/jsons/sqle/17.20/SVMSparsePredict.json +192 -0
- teradataml/data/jsons/sqle/17.20/Sessionize.json +105 -0
- teradataml/data/jsons/sqle/17.20/StringSimilarity.json +86 -0
- teradataml/data/jsons/sqle/17.20/TD_ANOVA.json +149 -0
- teradataml/data/jsons/sqle/17.20/TD_Apriori.json +181 -0
- teradataml/data/jsons/sqle/17.20/TD_BinCodeFit.json +239 -0
- teradataml/data/jsons/sqle/17.20/TD_BinCodeTransform.json +71 -0
- teradataml/data/jsons/sqle/17.20/TD_CFilter.json +118 -0
- teradataml/data/jsons/sqle/17.20/TD_CategoricalSummary.json +53 -0
- teradataml/data/jsons/sqle/17.20/TD_Chisq.json +68 -0
- teradataml/data/jsons/sqle/17.20/TD_ClassificationEvaluator.json +146 -0
- teradataml/data/jsons/sqle/17.20/TD_ColumnSummary.json +53 -0
- teradataml/data/jsons/sqle/17.20/TD_ColumnTransformer.json +218 -0
- teradataml/data/jsons/sqle/17.20/TD_ConvertTo.json +92 -0
- teradataml/data/jsons/sqle/17.20/TD_DecisionForest.json +260 -0
- teradataml/data/jsons/sqle/17.20/TD_DecisionForestPredict.json +139 -0
- teradataml/data/jsons/sqle/17.20/TD_FTest.json +269 -0
- teradataml/data/jsons/sqle/17.20/TD_FillRowID.json +52 -0
- teradataml/data/jsons/sqle/17.20/TD_FunctionFit.json +46 -0
- teradataml/data/jsons/sqle/17.20/TD_FunctionTransform.json +72 -0
- teradataml/data/jsons/sqle/17.20/TD_GLM.json +507 -0
- teradataml/data/jsons/sqle/17.20/TD_GLMPREDICT.json +168 -0
- teradataml/data/jsons/sqle/17.20/TD_GLMPerSegment.json +411 -0
- teradataml/data/jsons/sqle/17.20/TD_GLMPredictPerSegment.json +146 -0
- teradataml/data/jsons/sqle/17.20/TD_GetFutileColumns.json +93 -0
- teradataml/data/jsons/sqle/17.20/TD_GetRowsWithMissingValues.json +76 -0
- teradataml/data/jsons/sqle/17.20/TD_GetRowsWithoutMissingValues.json +76 -0
- teradataml/data/jsons/sqle/17.20/TD_Histogram.json +152 -0
- teradataml/data/jsons/sqle/17.20/TD_KMeans.json +232 -0
- teradataml/data/jsons/sqle/17.20/TD_KMeansPredict.json +87 -0
- teradataml/data/jsons/sqle/17.20/TD_KNN.json +262 -0
- teradataml/data/jsons/sqle/17.20/TD_NERExtractor.json +145 -0
- teradataml/data/jsons/sqle/17.20/TD_NaiveBayes.json +193 -0
- teradataml/data/jsons/sqle/17.20/TD_NaiveBayesPredict.json +212 -0
- teradataml/data/jsons/sqle/17.20/TD_NaiveBayesTextClassifierTrainer.json +137 -0
- teradataml/data/jsons/sqle/17.20/TD_NonLinearCombineFit.json +102 -0
- teradataml/data/jsons/sqle/17.20/TD_NonLinearCombineTransform.json +71 -0
- teradataml/data/jsons/sqle/17.20/TD_NumApply.json +147 -0
- teradataml/data/jsons/sqle/17.20/TD_OneClassSVM.json +316 -0
- teradataml/data/jsons/sqle/17.20/TD_OneClassSVMPredict.json +124 -0
- teradataml/data/jsons/sqle/17.20/TD_OneHotEncodingFit.json +271 -0
- teradataml/data/jsons/sqle/17.20/TD_OneHotEncodingTransform.json +65 -0
- teradataml/data/jsons/sqle/17.20/TD_OrdinalEncodingFit.json +229 -0
- teradataml/data/jsons/sqle/17.20/TD_OrdinalEncodingTransform.json +75 -0
- teradataml/data/jsons/sqle/17.20/TD_OutlierFilterFit.json +217 -0
- teradataml/data/jsons/sqle/17.20/TD_OutlierFilterTransform.json +48 -0
- teradataml/data/jsons/sqle/17.20/TD_Pivoting.json +280 -0
- teradataml/data/jsons/sqle/17.20/TD_PolynomialFeaturesFit.json +114 -0
- teradataml/data/jsons/sqle/17.20/TD_PolynomialFeaturesTransform.json +72 -0
- teradataml/data/jsons/sqle/17.20/TD_QQNorm.json +111 -0
- teradataml/data/jsons/sqle/17.20/TD_ROC.json +179 -0
- teradataml/data/jsons/sqle/17.20/TD_RandomProjectionFit.json +179 -0
- teradataml/data/jsons/sqle/17.20/TD_RandomProjectionMinComponents.json +74 -0
- teradataml/data/jsons/sqle/17.20/TD_RandomProjectionTransform.json +74 -0
- teradataml/data/jsons/sqle/17.20/TD_RegressionEvaluator.json +138 -0
- teradataml/data/jsons/sqle/17.20/TD_RoundColumns.json +93 -0
- teradataml/data/jsons/sqle/17.20/TD_RowNormalizeFit.json +128 -0
- teradataml/data/jsons/sqle/17.20/TD_RowNormalizeTransform.json +71 -0
- teradataml/data/jsons/sqle/17.20/TD_SMOTE.json +267 -0
- teradataml/data/jsons/sqle/17.20/TD_SVM.json +389 -0
- teradataml/data/jsons/sqle/17.20/TD_SVMPredict.json +142 -0
- teradataml/data/jsons/sqle/17.20/TD_ScaleFit.json +310 -0
- teradataml/data/jsons/sqle/17.20/TD_ScaleTransform.json +120 -0
- teradataml/data/jsons/sqle/17.20/TD_SentimentExtractor.json +194 -0
- teradataml/data/jsons/sqle/17.20/TD_Shap.json +221 -0
- teradataml/data/jsons/sqle/17.20/TD_Silhouette.json +143 -0
- teradataml/data/jsons/sqle/17.20/TD_SimpleImputeFit.json +147 -0
- teradataml/data/jsons/sqle/17.20/TD_SimpleImputeTransform.json +48 -0
- teradataml/data/jsons/sqle/17.20/TD_StrApply.json +240 -0
- teradataml/data/jsons/sqle/17.20/TD_TFIDF.json +162 -0
- teradataml/data/jsons/sqle/17.20/TD_TargetEncodingFit.json +248 -0
- teradataml/data/jsons/sqle/17.20/TD_TargetEncodingTransform.json +75 -0
- teradataml/data/jsons/sqle/17.20/TD_TextMorph.json +134 -0
- teradataml/data/jsons/sqle/17.20/TD_TextParser.json +297 -0
- teradataml/data/jsons/sqle/17.20/TD_TrainTestSplit.json +142 -0
- teradataml/data/jsons/sqle/17.20/TD_UnivariateStatistics.json +117 -0
- teradataml/data/jsons/sqle/17.20/TD_Unpivoting.json +235 -0
- teradataml/data/jsons/sqle/17.20/TD_VectorDistance.json +183 -0
- teradataml/data/jsons/sqle/17.20/TD_WhichMax.json +53 -0
- teradataml/data/jsons/sqle/17.20/TD_WhichMin.json +53 -0
- teradataml/data/jsons/sqle/17.20/TD_WordEmbeddings.json +241 -0
- teradataml/data/jsons/sqle/17.20/TD_XGBoost.json +330 -0
- teradataml/data/jsons/sqle/17.20/TD_XGBoostPredict.json +195 -0
- teradataml/data/jsons/sqle/17.20/TD_ZTest.json +247 -0
- teradataml/data/jsons/sqle/17.20/Unpack.json +188 -0
- teradataml/data/jsons/sqle/17.20/nPath.json +269 -0
- teradataml/data/jsons/sqle/20.00/AI_AnalyzeSentiment.json +370 -0
- teradataml/data/jsons/sqle/20.00/AI_AskLLM.json +460 -0
- teradataml/data/jsons/sqle/20.00/AI_DetectLanguage.json +385 -0
- teradataml/data/jsons/sqle/20.00/AI_ExtractKeyPhrases.json +369 -0
- teradataml/data/jsons/sqle/20.00/AI_MaskPII.json +369 -0
- teradataml/data/jsons/sqle/20.00/AI_RecognizeEntities.json +369 -0
- teradataml/data/jsons/sqle/20.00/AI_RecognizePIIEntities.json +369 -0
- teradataml/data/jsons/sqle/20.00/AI_TextClassifier.json +400 -0
- teradataml/data/jsons/sqle/20.00/AI_TextEmbeddings.json +401 -0
- teradataml/data/jsons/sqle/20.00/AI_TextSummarize.json +384 -0
- teradataml/data/jsons/sqle/20.00/AI_TextTranslate.json +384 -0
- teradataml/data/jsons/sqle/20.00/TD_API_AzureML.json +151 -0
- teradataml/data/jsons/sqle/20.00/TD_API_Sagemaker.json +182 -0
- teradataml/data/jsons/sqle/20.00/TD_API_VertexAI.json +183 -0
- teradataml/data/jsons/sqle/20.00/TD_HNSW.json +296 -0
- teradataml/data/jsons/sqle/20.00/TD_HNSWPredict.json +206 -0
- teradataml/data/jsons/sqle/20.00/TD_HNSWSummary.json +32 -0
- teradataml/data/jsons/sqle/20.00/TD_KMeans.json +250 -0
- teradataml/data/jsons/sqle/20.00/TD_SMOTE.json +266 -0
- teradataml/data/jsons/sqle/20.00/TD_VectorDistance.json +278 -0
- teradataml/data/jsons/storedprocedure/17.20/TD_COPYART.json +71 -0
- teradataml/data/jsons/storedprocedure/17.20/TD_FILTERFACTORY1D.json +150 -0
- teradataml/data/jsons/tableoperator/17.00/read_nos.json +198 -0
- teradataml/data/jsons/tableoperator/17.05/read_nos.json +198 -0
- teradataml/data/jsons/tableoperator/17.05/write_nos.json +195 -0
- teradataml/data/jsons/tableoperator/17.10/read_nos.json +184 -0
- teradataml/data/jsons/tableoperator/17.10/write_nos.json +195 -0
- teradataml/data/jsons/tableoperator/17.20/IMAGE2MATRIX.json +53 -0
- teradataml/data/jsons/tableoperator/17.20/read_nos.json +183 -0
- teradataml/data/jsons/tableoperator/17.20/write_nos.json +224 -0
- teradataml/data/jsons/uaf/17.20/TD_ACF.json +132 -0
- teradataml/data/jsons/uaf/17.20/TD_ARIMAESTIMATE.json +396 -0
- teradataml/data/jsons/uaf/17.20/TD_ARIMAFORECAST.json +77 -0
- teradataml/data/jsons/uaf/17.20/TD_ARIMAVALIDATE.json +153 -0
- teradataml/data/jsons/uaf/17.20/TD_ARIMAXESTIMATE.json +362 -0
- teradataml/data/jsons/uaf/17.20/TD_AUTOARIMA.json +469 -0
- teradataml/data/jsons/uaf/17.20/TD_BINARYMATRIXOP.json +107 -0
- teradataml/data/jsons/uaf/17.20/TD_BINARYSERIESOP.json +106 -0
- teradataml/data/jsons/uaf/17.20/TD_BREUSCH_GODFREY.json +89 -0
- teradataml/data/jsons/uaf/17.20/TD_BREUSCH_PAGAN_GODFREY.json +104 -0
- teradataml/data/jsons/uaf/17.20/TD_CONVOLVE.json +78 -0
- teradataml/data/jsons/uaf/17.20/TD_CONVOLVE2.json +66 -0
- teradataml/data/jsons/uaf/17.20/TD_CUMUL_PERIODOGRAM.json +87 -0
- teradataml/data/jsons/uaf/17.20/TD_DFFT.json +134 -0
- teradataml/data/jsons/uaf/17.20/TD_DFFT2.json +144 -0
- teradataml/data/jsons/uaf/17.20/TD_DFFT2CONV.json +108 -0
- teradataml/data/jsons/uaf/17.20/TD_DFFTCONV.json +108 -0
- teradataml/data/jsons/uaf/17.20/TD_DICKEY_FULLER.json +78 -0
- teradataml/data/jsons/uaf/17.20/TD_DIFF.json +92 -0
- teradataml/data/jsons/uaf/17.20/TD_DTW.json +114 -0
- teradataml/data/jsons/uaf/17.20/TD_DURBIN_WATSON.json +101 -0
- teradataml/data/jsons/uaf/17.20/TD_DWT.json +173 -0
- teradataml/data/jsons/uaf/17.20/TD_DWT2D.json +160 -0
- teradataml/data/jsons/uaf/17.20/TD_EXTRACT_RESULTS.json +39 -0
- teradataml/data/jsons/uaf/17.20/TD_FITMETRICS.json +101 -0
- teradataml/data/jsons/uaf/17.20/TD_GENSERIES4FORMULA.json +85 -0
- teradataml/data/jsons/uaf/17.20/TD_GENSERIES4SINUSOIDS.json +71 -0
- teradataml/data/jsons/uaf/17.20/TD_GOLDFELD_QUANDT.json +139 -0
- teradataml/data/jsons/uaf/17.20/TD_HOLT_WINTERS_FORECASTER.json +313 -0
- teradataml/data/jsons/uaf/17.20/TD_IDFFT.json +58 -0
- teradataml/data/jsons/uaf/17.20/TD_IDFFT2.json +81 -0
- teradataml/data/jsons/uaf/17.20/TD_IDWT.json +162 -0
- teradataml/data/jsons/uaf/17.20/TD_IDWT2D.json +149 -0
- teradataml/data/jsons/uaf/17.20/TD_INPUTVALIDATOR.json +64 -0
- teradataml/data/jsons/uaf/17.20/TD_IQR.json +117 -0
- teradataml/data/jsons/uaf/17.20/TD_LINEAR_REGR.json +182 -0
- teradataml/data/jsons/uaf/17.20/TD_LINESPEC.json +103 -0
- teradataml/data/jsons/uaf/17.20/TD_MAMEAN.json +181 -0
- teradataml/data/jsons/uaf/17.20/TD_MATRIX2IMAGE.json +209 -0
- teradataml/data/jsons/uaf/17.20/TD_MATRIXMULTIPLY.json +68 -0
- teradataml/data/jsons/uaf/17.20/TD_MINFO.json +67 -0
- teradataml/data/jsons/uaf/17.20/TD_MULTIVAR_REGR.json +179 -0
- teradataml/data/jsons/uaf/17.20/TD_PACF.json +114 -0
- teradataml/data/jsons/uaf/17.20/TD_PORTMAN.json +119 -0
- teradataml/data/jsons/uaf/17.20/TD_POWERSPEC.json +175 -0
- teradataml/data/jsons/uaf/17.20/TD_POWERTRANSFORM.json +98 -0
- teradataml/data/jsons/uaf/17.20/TD_RESAMPLE.json +194 -0
- teradataml/data/jsons/uaf/17.20/TD_SAX.json +210 -0
- teradataml/data/jsons/uaf/17.20/TD_SEASONALNORMALIZE.json +143 -0
- teradataml/data/jsons/uaf/17.20/TD_SELECTION_CRITERIA.json +90 -0
- teradataml/data/jsons/uaf/17.20/TD_SIGNIF_PERIODICITIES.json +80 -0
- teradataml/data/jsons/uaf/17.20/TD_SIGNIF_RESIDMEAN.json +68 -0
- teradataml/data/jsons/uaf/17.20/TD_SIMPLEEXP.json +184 -0
- teradataml/data/jsons/uaf/17.20/TD_SINFO.json +58 -0
- teradataml/data/jsons/uaf/17.20/TD_SMOOTHMA.json +163 -0
- teradataml/data/jsons/uaf/17.20/TD_TRACKINGOP.json +101 -0
- teradataml/data/jsons/uaf/17.20/TD_UNDIFF.json +112 -0
- teradataml/data/jsons/uaf/17.20/TD_UNNORMALIZE.json +95 -0
- teradataml/data/jsons/uaf/17.20/TD_WHITES_GENERAL.json +78 -0
- teradataml/data/jsons/uaf/17.20/TD_WINDOWDFFT.json +410 -0
- teradataml/data/kmeans_example.json +23 -0
- teradataml/data/kmeans_table.csv +10 -0
- teradataml/data/kmeans_us_arrests_data.csv +51 -0
- teradataml/data/knn_example.json +19 -0
- teradataml/data/knnrecommender_example.json +7 -0
- teradataml/data/knnrecommenderpredict_example.json +12 -0
- teradataml/data/lar_example.json +17 -0
- teradataml/data/larpredict_example.json +30 -0
- teradataml/data/lc_new_predictors.csv +5 -0
- teradataml/data/lc_new_reference.csv +9 -0
- teradataml/data/lda_example.json +9 -0
- teradataml/data/ldainference_example.json +15 -0
- teradataml/data/ldatopicsummary_example.json +9 -0
- teradataml/data/levendist_input.csv +13 -0
- teradataml/data/levenshteindistance_example.json +10 -0
- teradataml/data/linreg_example.json +10 -0
- teradataml/data/load_example_data.py +350 -0
- teradataml/data/loan_prediction.csv +295 -0
- teradataml/data/lungcancer.csv +138 -0
- teradataml/data/mappingdata.csv +12 -0
- teradataml/data/medical_readings.csv +101 -0
- teradataml/data/milk_timeseries.csv +157 -0
- teradataml/data/min_max_titanic.csv +4 -0
- teradataml/data/minhash_example.json +6 -0
- teradataml/data/ml_ratings.csv +7547 -0
- teradataml/data/ml_ratings_10.csv +2445 -0
- teradataml/data/mobile_data.csv +13 -0
- teradataml/data/model1_table.csv +5 -0
- teradataml/data/model2_table.csv +5 -0
- teradataml/data/models/License_file.txt +1 -0
- teradataml/data/models/License_file_empty.txt +0 -0
- teradataml/data/models/dataiku_iris_data_ann_thin +0 -0
- teradataml/data/models/dr_iris_rf +0 -0
- teradataml/data/models/iris_db_dt_model_sklearn.onnx +0 -0
- teradataml/data/models/iris_db_dt_model_sklearn_floattensor.onnx +0 -0
- teradataml/data/models/iris_db_glm_model.pmml +57 -0
- teradataml/data/models/iris_db_xgb_model.pmml +4471 -0
- teradataml/data/models/iris_kmeans_model +0 -0
- teradataml/data/models/iris_mojo_glm_h2o_model +0 -0
- teradataml/data/models/iris_mojo_xgb_h2o_model +0 -0
- teradataml/data/modularity_example.json +12 -0
- teradataml/data/movavg_example.json +8 -0
- teradataml/data/mtx1.csv +7 -0
- teradataml/data/mtx2.csv +13 -0
- teradataml/data/multi_model_classification.csv +401 -0
- teradataml/data/multi_model_regression.csv +401 -0
- teradataml/data/mvdfft8.csv +9 -0
- teradataml/data/naivebayes_example.json +10 -0
- teradataml/data/naivebayespredict_example.json +19 -0
- teradataml/data/naivebayestextclassifier2_example.json +7 -0
- teradataml/data/naivebayestextclassifier_example.json +8 -0
- teradataml/data/naivebayestextclassifierpredict_example.json +32 -0
- teradataml/data/name_Find_configure.csv +10 -0
- teradataml/data/namedentityfinder_example.json +14 -0
- teradataml/data/namedentityfinderevaluator_example.json +10 -0
- teradataml/data/namedentityfindertrainer_example.json +6 -0
- teradataml/data/nb_iris_input_test.csv +31 -0
- teradataml/data/nb_iris_input_train.csv +121 -0
- teradataml/data/nbp_iris_model.csv +13 -0
- teradataml/data/ner_dict.csv +8 -0
- teradataml/data/ner_extractor_text.csv +2 -0
- teradataml/data/ner_input_eng.csv +7 -0
- teradataml/data/ner_rule.csv +5 -0
- teradataml/data/ner_sports_test2.csv +29 -0
- teradataml/data/ner_sports_train.csv +501 -0
- teradataml/data/nerevaluator_example.json +6 -0
- teradataml/data/nerextractor_example.json +18 -0
- teradataml/data/nermem_sports_test.csv +18 -0
- teradataml/data/nermem_sports_train.csv +51 -0
- teradataml/data/nertrainer_example.json +7 -0
- teradataml/data/ngrams_example.json +7 -0
- teradataml/data/notebooks/__init__.py +0 -0
- teradataml/data/notebooks/sqlalchemy/Teradata Vantage Aggregate Functions using SQLAlchemy.ipynb +1455 -0
- teradataml/data/notebooks/sqlalchemy/Teradata Vantage Arithmetic Functions Using SQLAlchemy.ipynb +1993 -0
- teradataml/data/notebooks/sqlalchemy/Teradata Vantage Bit-Byte Manipulation Functions using SQLAlchemy.ipynb +1492 -0
- teradataml/data/notebooks/sqlalchemy/Teradata Vantage Built-in functions using SQLAlchemy.ipynb +536 -0
- teradataml/data/notebooks/sqlalchemy/Teradata Vantage Regular Expressions Using SQLAlchemy.ipynb +570 -0
- teradataml/data/notebooks/sqlalchemy/Teradata Vantage String Functions Using SQLAlchemy.ipynb +2559 -0
- teradataml/data/notebooks/sqlalchemy/Teradata Vantage Window Aggregate Functions using SQLAlchemy.ipynb +2911 -0
- teradataml/data/notebooks/sqlalchemy/Using Generic SQLAlchemy ClauseElements teradataml DataFrame assign method.ipynb +698 -0
- teradataml/data/notebooks/sqlalchemy/__init__.py +0 -0
- teradataml/data/notebooks/sqlalchemy/teradataml filtering using SQLAlchemy ClauseElements.ipynb +784 -0
- teradataml/data/npath_example.json +23 -0
- teradataml/data/ntree_example.json +14 -0
- teradataml/data/numeric_strings.csv +5 -0
- teradataml/data/numerics.csv +4 -0
- teradataml/data/ocean_buoy.csv +17 -0
- teradataml/data/ocean_buoy2.csv +17 -0
- teradataml/data/ocean_buoys.csv +28 -0
- teradataml/data/ocean_buoys2.csv +10 -0
- teradataml/data/ocean_buoys_nonpti.csv +28 -0
- teradataml/data/ocean_buoys_seq.csv +29 -0
- teradataml/data/onehot_encoder_train.csv +4 -0
- teradataml/data/openml_example.json +92 -0
- teradataml/data/optional_event_table.csv +4 -0
- teradataml/data/orders1.csv +11 -0
- teradataml/data/orders1_12.csv +13 -0
- teradataml/data/orders_ex.csv +4 -0
- teradataml/data/pack_example.json +9 -0
- teradataml/data/package_tracking.csv +19 -0
- teradataml/data/package_tracking_pti.csv +19 -0
- teradataml/data/pagerank_example.json +13 -0
- teradataml/data/paragraphs_input.csv +6 -0
- teradataml/data/pathanalyzer_example.json +8 -0
- teradataml/data/pathgenerator_example.json +8 -0
- teradataml/data/patient_profile.csv +101 -0
- teradataml/data/pattern_matching_data.csv +11 -0
- teradataml/data/payment_fraud_dataset.csv +10001 -0
- teradataml/data/peppers.png +0 -0
- teradataml/data/phrases.csv +7 -0
- teradataml/data/pivot_example.json +9 -0
- teradataml/data/pivot_input.csv +22 -0
- teradataml/data/playerRating.csv +31 -0
- teradataml/data/pos_input.csv +40 -0
- teradataml/data/postagger_example.json +7 -0
- teradataml/data/posttagger_output.csv +44 -0
- teradataml/data/production_data.csv +17 -0
- teradataml/data/production_data2.csv +7 -0
- teradataml/data/randomsample_example.json +32 -0
- teradataml/data/randomwalksample_example.json +9 -0
- teradataml/data/rank_table.csv +6 -0
- teradataml/data/real_values.csv +14 -0
- teradataml/data/ref_mobile_data.csv +4 -0
- teradataml/data/ref_mobile_data_dense.csv +2 -0
- teradataml/data/ref_url.csv +17 -0
- teradataml/data/restaurant_reviews.csv +7 -0
- teradataml/data/retail_churn_table.csv +27772 -0
- teradataml/data/river_data.csv +145 -0
- teradataml/data/roc_example.json +8 -0
- teradataml/data/roc_input.csv +101 -0
- teradataml/data/rule_inputs.csv +6 -0
- teradataml/data/rule_table.csv +2 -0
- teradataml/data/sales.csv +7 -0
- teradataml/data/sales_transaction.csv +501 -0
- teradataml/data/salesdata.csv +342 -0
- teradataml/data/sample_cities.csv +3 -0
- teradataml/data/sample_shapes.csv +11 -0
- teradataml/data/sample_streets.csv +3 -0
- teradataml/data/sampling_example.json +16 -0
- teradataml/data/sax_example.json +17 -0
- teradataml/data/scale_attributes.csv +3 -0
- teradataml/data/scale_example.json +74 -0
- teradataml/data/scale_housing.csv +11 -0
- teradataml/data/scale_housing_test.csv +6 -0
- teradataml/data/scale_input_part_sparse.csv +31 -0
- teradataml/data/scale_input_partitioned.csv +16 -0
- teradataml/data/scale_input_sparse.csv +11 -0
- teradataml/data/scale_parameters.csv +3 -0
- teradataml/data/scale_stat.csv +11 -0
- teradataml/data/scalebypartition_example.json +13 -0
- teradataml/data/scalemap_example.json +13 -0
- teradataml/data/scalesummary_example.json +12 -0
- teradataml/data/score_category.csv +101 -0
- teradataml/data/score_summary.csv +4 -0
- teradataml/data/script_example.json +10 -0
- teradataml/data/scripts/deploy_script.py +84 -0
- teradataml/data/scripts/lightgbm/dataset.template +175 -0
- teradataml/data/scripts/lightgbm/lightgbm_class_functions.template +264 -0
- teradataml/data/scripts/lightgbm/lightgbm_function.template +234 -0
- teradataml/data/scripts/lightgbm/lightgbm_sklearn.template +177 -0
- teradataml/data/scripts/mapper.R +20 -0
- teradataml/data/scripts/mapper.py +16 -0
- teradataml/data/scripts/mapper_replace.py +16 -0
- teradataml/data/scripts/sklearn/__init__.py +0 -0
- teradataml/data/scripts/sklearn/sklearn_fit.py +205 -0
- teradataml/data/scripts/sklearn/sklearn_fit_predict.py +148 -0
- teradataml/data/scripts/sklearn/sklearn_function.template +144 -0
- teradataml/data/scripts/sklearn/sklearn_model_selection_split.py +166 -0
- teradataml/data/scripts/sklearn/sklearn_neighbors.py +161 -0
- teradataml/data/scripts/sklearn/sklearn_score.py +145 -0
- teradataml/data/scripts/sklearn/sklearn_transform.py +327 -0
- teradataml/data/sdk/modelops/modelops_spec.json +101737 -0
- teradataml/data/seeds.csv +10 -0
- teradataml/data/sentenceextractor_example.json +7 -0
- teradataml/data/sentiment_extract_input.csv +11 -0
- teradataml/data/sentiment_train.csv +16 -0
- teradataml/data/sentiment_word.csv +20 -0
- teradataml/data/sentiment_word_input.csv +20 -0
- teradataml/data/sentimentextractor_example.json +24 -0
- teradataml/data/sentimenttrainer_example.json +8 -0
- teradataml/data/sequence_table.csv +10 -0
- teradataml/data/seriessplitter_example.json +8 -0
- teradataml/data/sessionize_example.json +17 -0
- teradataml/data/sessionize_table.csv +116 -0
- teradataml/data/setop_test1.csv +24 -0
- teradataml/data/setop_test2.csv +22 -0
- teradataml/data/soc_nw_edges.csv +11 -0
- teradataml/data/soc_nw_vertices.csv +8 -0
- teradataml/data/souvenir_timeseries.csv +168 -0
- teradataml/data/sparse_iris_attribute.csv +5 -0
- teradataml/data/sparse_iris_test.csv +121 -0
- teradataml/data/sparse_iris_train.csv +601 -0
- teradataml/data/star1.csv +6 -0
- teradataml/data/star_pivot.csv +8 -0
- teradataml/data/state_transition.csv +5 -0
- teradataml/data/stock_data.csv +53 -0
- teradataml/data/stock_movement.csv +11 -0
- teradataml/data/stock_vol.csv +76 -0
- teradataml/data/stop_words.csv +8 -0
- teradataml/data/store_sales.csv +37 -0
- teradataml/data/stringsimilarity_example.json +8 -0
- teradataml/data/strsimilarity_input.csv +13 -0
- teradataml/data/students.csv +101 -0
- teradataml/data/svm_iris_input_test.csv +121 -0
- teradataml/data/svm_iris_input_train.csv +481 -0
- teradataml/data/svm_iris_model.csv +7 -0
- teradataml/data/svmdense_example.json +10 -0
- teradataml/data/svmdensepredict_example.json +19 -0
- teradataml/data/svmsparse_example.json +8 -0
- teradataml/data/svmsparsepredict_example.json +14 -0
- teradataml/data/svmsparsesummary_example.json +8 -0
- teradataml/data/target_mobile_data.csv +13 -0
- teradataml/data/target_mobile_data_dense.csv +5 -0
- teradataml/data/target_udt_data.csv +8 -0
- teradataml/data/tdnerextractor_example.json +14 -0
- teradataml/data/templatedata.csv +1201 -0
- teradataml/data/templates/open_source_ml.json +11 -0
- teradataml/data/teradata_icon.ico +0 -0
- teradataml/data/teradataml_example.json +1473 -0
- teradataml/data/test_classification.csv +101 -0
- teradataml/data/test_loan_prediction.csv +53 -0
- teradataml/data/test_pacf_12.csv +37 -0
- teradataml/data/test_prediction.csv +101 -0
- teradataml/data/test_regression.csv +101 -0
- teradataml/data/test_river2.csv +109 -0
- teradataml/data/text_inputs.csv +6 -0
- teradataml/data/textchunker_example.json +8 -0
- teradataml/data/textclassifier_example.json +7 -0
- teradataml/data/textclassifier_input.csv +7 -0
- teradataml/data/textclassifiertrainer_example.json +7 -0
- teradataml/data/textmorph_example.json +11 -0
- teradataml/data/textparser_example.json +15 -0
- teradataml/data/texttagger_example.json +12 -0
- teradataml/data/texttokenizer_example.json +7 -0
- teradataml/data/texttrainer_input.csv +11 -0
- teradataml/data/tf_example.json +7 -0
- teradataml/data/tfidf_example.json +14 -0
- teradataml/data/tfidf_input1.csv +201 -0
- teradataml/data/tfidf_train.csv +6 -0
- teradataml/data/time_table1.csv +535 -0
- teradataml/data/time_table2.csv +14 -0
- teradataml/data/timeseriesdata.csv +1601 -0
- teradataml/data/timeseriesdatasetsd4.csv +105 -0
- teradataml/data/timestamp_data.csv +4 -0
- teradataml/data/titanic.csv +892 -0
- teradataml/data/titanic_dataset_unpivoted.csv +19 -0
- teradataml/data/to_num_data.csv +4 -0
- teradataml/data/tochar_data.csv +5 -0
- teradataml/data/token_table.csv +696 -0
- teradataml/data/train_multiclass.csv +101 -0
- teradataml/data/train_regression.csv +101 -0
- teradataml/data/train_regression_multiple_labels.csv +101 -0
- teradataml/data/train_tracking.csv +28 -0
- teradataml/data/trans_dense.csv +16 -0
- teradataml/data/trans_sparse.csv +55 -0
- teradataml/data/transformation_table.csv +6 -0
- teradataml/data/transformation_table_new.csv +2 -0
- teradataml/data/tv_spots.csv +16 -0
- teradataml/data/twod_climate_data.csv +117 -0
- teradataml/data/uaf_example.json +529 -0
- teradataml/data/univariatestatistics_example.json +9 -0
- teradataml/data/unpack_example.json +10 -0
- teradataml/data/unpivot_example.json +25 -0
- teradataml/data/unpivot_input.csv +8 -0
- teradataml/data/url_data.csv +10 -0
- teradataml/data/us_air_pass.csv +37 -0
- teradataml/data/us_population.csv +624 -0
- teradataml/data/us_states_shapes.csv +52 -0
- teradataml/data/varmax_example.json +18 -0
- teradataml/data/vectordistance_example.json +30 -0
- teradataml/data/ville_climatedata.csv +121 -0
- teradataml/data/ville_tempdata.csv +12 -0
- teradataml/data/ville_tempdata1.csv +12 -0
- teradataml/data/ville_temperature.csv +11 -0
- teradataml/data/waveletTable.csv +1605 -0
- teradataml/data/waveletTable2.csv +1605 -0
- teradataml/data/weightedmovavg_example.json +9 -0
- teradataml/data/wft_testing.csv +5 -0
- teradataml/data/windowdfft.csv +16 -0
- teradataml/data/wine_data.csv +1600 -0
- teradataml/data/word_embed_input_table1.csv +6 -0
- teradataml/data/word_embed_input_table2.csv +5 -0
- teradataml/data/word_embed_model.csv +23 -0
- teradataml/data/words_input.csv +13 -0
- teradataml/data/xconvolve_complex_left.csv +6 -0
- teradataml/data/xconvolve_complex_leftmulti.csv +6 -0
- teradataml/data/xgboost_example.json +36 -0
- teradataml/data/xgboostpredict_example.json +32 -0
- teradataml/data/ztest_example.json +16 -0
- teradataml/dataframe/__init__.py +0 -0
- teradataml/dataframe/copy_to.py +2446 -0
- teradataml/dataframe/data_transfer.py +2840 -0
- teradataml/dataframe/dataframe.py +20908 -0
- teradataml/dataframe/dataframe_utils.py +2114 -0
- teradataml/dataframe/fastload.py +794 -0
- teradataml/dataframe/functions.py +2110 -0
- teradataml/dataframe/indexer.py +424 -0
- teradataml/dataframe/row.py +160 -0
- teradataml/dataframe/setop.py +1171 -0
- teradataml/dataframe/sql.py +10904 -0
- teradataml/dataframe/sql_function_parameters.py +440 -0
- teradataml/dataframe/sql_functions.py +652 -0
- teradataml/dataframe/sql_interfaces.py +220 -0
- teradataml/dataframe/vantage_function_types.py +675 -0
- teradataml/dataframe/window.py +694 -0
- teradataml/dbutils/__init__.py +3 -0
- teradataml/dbutils/dbutils.py +2871 -0
- teradataml/dbutils/filemgr.py +318 -0
- teradataml/gen_ai/__init__.py +2 -0
- teradataml/gen_ai/convAI.py +473 -0
- teradataml/geospatial/__init__.py +4 -0
- teradataml/geospatial/geodataframe.py +1105 -0
- teradataml/geospatial/geodataframecolumn.py +392 -0
- teradataml/geospatial/geometry_types.py +926 -0
- teradataml/hyperparameter_tuner/__init__.py +1 -0
- teradataml/hyperparameter_tuner/optimizer.py +4115 -0
- teradataml/hyperparameter_tuner/utils.py +303 -0
- teradataml/lib/__init__.py +0 -0
- teradataml/lib/aed_0_1.dll +0 -0
- teradataml/lib/libaed_0_1.dylib +0 -0
- teradataml/lib/libaed_0_1.so +0 -0
- teradataml/lib/libaed_0_1_aarch64.so +0 -0
- teradataml/lib/libaed_0_1_ppc64le.so +0 -0
- teradataml/opensource/__init__.py +1 -0
- teradataml/opensource/_base.py +1321 -0
- teradataml/opensource/_class.py +464 -0
- teradataml/opensource/_constants.py +61 -0
- teradataml/opensource/_lightgbm.py +949 -0
- teradataml/opensource/_sklearn.py +1008 -0
- teradataml/opensource/_wrapper_utils.py +267 -0
- teradataml/options/__init__.py +148 -0
- teradataml/options/configure.py +489 -0
- teradataml/options/display.py +187 -0
- teradataml/plot/__init__.py +3 -0
- teradataml/plot/axis.py +1427 -0
- teradataml/plot/constants.py +15 -0
- teradataml/plot/figure.py +431 -0
- teradataml/plot/plot.py +810 -0
- teradataml/plot/query_generator.py +83 -0
- teradataml/plot/subplot.py +216 -0
- teradataml/scriptmgmt/UserEnv.py +4273 -0
- teradataml/scriptmgmt/__init__.py +3 -0
- teradataml/scriptmgmt/lls_utils.py +2157 -0
- teradataml/sdk/README.md +79 -0
- teradataml/sdk/__init__.py +4 -0
- teradataml/sdk/_auth_modes.py +422 -0
- teradataml/sdk/_func_params.py +487 -0
- teradataml/sdk/_json_parser.py +453 -0
- teradataml/sdk/_openapi_spec_constants.py +249 -0
- teradataml/sdk/_utils.py +236 -0
- teradataml/sdk/api_client.py +900 -0
- teradataml/sdk/constants.py +62 -0
- teradataml/sdk/modelops/__init__.py +98 -0
- teradataml/sdk/modelops/_client.py +409 -0
- teradataml/sdk/modelops/_constants.py +304 -0
- teradataml/sdk/modelops/models.py +2308 -0
- teradataml/sdk/spinner.py +107 -0
- teradataml/series/__init__.py +0 -0
- teradataml/series/series.py +537 -0
- teradataml/series/series_utils.py +71 -0
- teradataml/store/__init__.py +12 -0
- teradataml/store/feature_store/__init__.py +0 -0
- teradataml/store/feature_store/constants.py +658 -0
- teradataml/store/feature_store/feature_store.py +4814 -0
- teradataml/store/feature_store/mind_map.py +639 -0
- teradataml/store/feature_store/models.py +7330 -0
- teradataml/store/feature_store/utils.py +390 -0
- teradataml/table_operators/Apply.py +979 -0
- teradataml/table_operators/Script.py +1739 -0
- teradataml/table_operators/TableOperator.py +1343 -0
- teradataml/table_operators/__init__.py +2 -0
- teradataml/table_operators/apply_query_generator.py +262 -0
- teradataml/table_operators/query_generator.py +493 -0
- teradataml/table_operators/table_operator_query_generator.py +462 -0
- teradataml/table_operators/table_operator_util.py +726 -0
- teradataml/table_operators/templates/dataframe_apply.template +184 -0
- teradataml/table_operators/templates/dataframe_map.template +176 -0
- teradataml/table_operators/templates/dataframe_register.template +73 -0
- teradataml/table_operators/templates/dataframe_udf.template +67 -0
- teradataml/table_operators/templates/script_executor.template +170 -0
- teradataml/telemetry_utils/__init__.py +0 -0
- teradataml/telemetry_utils/queryband.py +53 -0
- teradataml/utils/__init__.py +0 -0
- teradataml/utils/docstring.py +527 -0
- teradataml/utils/dtypes.py +943 -0
- teradataml/utils/internal_buffer.py +122 -0
- teradataml/utils/print_versions.py +206 -0
- teradataml/utils/utils.py +451 -0
- teradataml/utils/validators.py +3305 -0
- teradataml-20.0.0.8.dist-info/METADATA +2804 -0
- teradataml-20.0.0.8.dist-info/RECORD +1208 -0
- teradataml-20.0.0.8.dist-info/WHEEL +5 -0
- teradataml-20.0.0.8.dist-info/top_level.txt +1 -0
- teradataml-20.0.0.8.dist-info/zip-safe +1 -0
|
@@ -0,0 +1,2840 @@
|
|
|
1
|
+
#!/usr/bin/python
|
|
2
|
+
# ##################################################################
|
|
3
|
+
#
|
|
4
|
+
# Copyright 2021 Teradata. All rights reserved.
|
|
5
|
+
# TERADATA CONFIDENTIAL AND TRADE SECRET
|
|
6
|
+
#
|
|
7
|
+
# Primary Owner: Sanath Vobilisetty (Sanath.Vobilisetty@teradata.com)
|
|
8
|
+
# Secondary Owner: Pankaj Vinod Purandare (PankajVinod.Purandare@Teradata.com)
|
|
9
|
+
#
|
|
10
|
+
# ##################################################################
|
|
11
|
+
import pandas as pd
|
|
12
|
+
from collections import OrderedDict
|
|
13
|
+
from sqlalchemy.exc import OperationalError as sqlachemyOperationalError
|
|
14
|
+
from teradataml.common.constants import DriverEscapeFunctions, TeradataConstants
|
|
15
|
+
from teradataml.common.exceptions import TeradataMlException
|
|
16
|
+
from teradataml.common.messages import Messages
|
|
17
|
+
from teradataml.common.messagecodes import MessageCodes
|
|
18
|
+
from teradataml.common.sqlbundle import SQLBundle
|
|
19
|
+
from teradataml.common.utils import UtilFuncs
|
|
20
|
+
from teradataml.common.constants import CopyToConstants
|
|
21
|
+
from teradataml.context.context import get_context, get_connection, \
|
|
22
|
+
_get_context_temp_databasename, _get_current_databasename
|
|
23
|
+
from teradataml.dataframe import dataframe as tdmldf
|
|
24
|
+
from teradataml.dataframe.copy_to import copy_to_sql, _create_table_object, \
|
|
25
|
+
_get_pd_df_column_names, _extract_column_info, \
|
|
26
|
+
_check_columns_insertion_compatible, _get_index_labels
|
|
27
|
+
from teradataml.dataframe.dataframe_utils import DataFrameUtils as df_utils
|
|
28
|
+
from teradataml.dbutils.dbutils import _create_table, _execute_query_and_generate_pandas_df
|
|
29
|
+
from teradataml.utils.validators import _Validators
|
|
30
|
+
from teradataml.telemetry_utils.queryband import collect_queryband
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
@collect_queryband(queryband="fstExprt")
|
|
34
|
+
def fastexport(df, export_to="pandas", index_column=None,
|
|
35
|
+
catch_errors_warnings=False, csv_file=None,
|
|
36
|
+
**kwargs):
|
|
37
|
+
"""
|
|
38
|
+
DESCRIPTION:
|
|
39
|
+
The fastexport() API exports teradataml DataFrame to Pandas DataFrame
|
|
40
|
+
or CSV file using FastExport data transfer protocol.
|
|
41
|
+
Note:
|
|
42
|
+
1. Teradata recommends to use FastExport when number of rows in
|
|
43
|
+
teradataml DataFrame are at least 100,000. To extract lesser rows
|
|
44
|
+
ignore this function and go with regular to_pandas() or to_csv()
|
|
45
|
+
function. FastExport opens multiple data transfer connections to the
|
|
46
|
+
database.
|
|
47
|
+
2. FastExport does not support all Teradata Database data types.
|
|
48
|
+
For example, tables with BLOB and CLOB type columns cannot
|
|
49
|
+
be extracted.
|
|
50
|
+
3. FastExport cannot be used to extract data from a volatile or
|
|
51
|
+
temporary table.
|
|
52
|
+
4. For best efficiency, do not use DataFrame.groupby() and
|
|
53
|
+
DataFrame.sort() with FastExport.
|
|
54
|
+
|
|
55
|
+
For additional information about FastExport protocol through
|
|
56
|
+
teradatasql driver, please refer to FASTEXPORT section of
|
|
57
|
+
https://pypi.org/project/teradatasql/#FastExport driver documentation.
|
|
58
|
+
|
|
59
|
+
PARAMETERS:
|
|
60
|
+
df:
|
|
61
|
+
Required Argument.
|
|
62
|
+
Specifies teradataml DataFrame that needs to be exported.
|
|
63
|
+
Types: teradataml DataFrame
|
|
64
|
+
|
|
65
|
+
export_to:
|
|
66
|
+
Optional Argument.
|
|
67
|
+
Specifies a value that notifies where to export the data.
|
|
68
|
+
Permitted Values:
|
|
69
|
+
* "pandas": Export data to a Pandas DataFrame.
|
|
70
|
+
* "csv": Export data to a given CSV file.
|
|
71
|
+
Default Value: "pandas"
|
|
72
|
+
Types: String
|
|
73
|
+
|
|
74
|
+
index_column:
|
|
75
|
+
Optional Argument.
|
|
76
|
+
Specifies column(s) to be used as index column for the converted object.
|
|
77
|
+
Note:
|
|
78
|
+
Applicable only when 'export_to' is set to "pandas".
|
|
79
|
+
Default Value: None.
|
|
80
|
+
Types: String OR list of Strings (str)
|
|
81
|
+
|
|
82
|
+
catch_errors_warnings:
|
|
83
|
+
Optional Argument.
|
|
84
|
+
Specifies whether to catch errors/warnings(if any) raised by
|
|
85
|
+
fastexport protocol while converting teradataml DataFrame.
|
|
86
|
+
Notes :
|
|
87
|
+
1. When 'export_to' is set to "pandas" and 'catch_errors_warnings' is set to True,
|
|
88
|
+
fastexport() returns a tuple containing:
|
|
89
|
+
a. Pandas DataFrame.
|
|
90
|
+
b. Errors(if any) in a list thrown by fastexport.
|
|
91
|
+
c. Warnings(if any) in a list thrown by fastexport.
|
|
92
|
+
When set to False, prints the fastexport errors/warnings to the
|
|
93
|
+
standard output, if there are any.
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
2. When 'export_to' is set to "csv" and 'catch_errors_warnings' is set to True,
|
|
97
|
+
fastexport() returns a tuple containing:
|
|
98
|
+
a. Errors(if any) in a list thrown by fastexport.
|
|
99
|
+
b. Warnings(if any) in a list thrown by fastexport.
|
|
100
|
+
Default Value: False
|
|
101
|
+
Types: bool
|
|
102
|
+
|
|
103
|
+
csv_file:
|
|
104
|
+
Optional Argument.
|
|
105
|
+
Specifies the name of CSV file to which data is to be exported.
|
|
106
|
+
Note:
|
|
107
|
+
This is required argument when 'export_to' is set to "csv".
|
|
108
|
+
Types: String
|
|
109
|
+
|
|
110
|
+
kwargs:
|
|
111
|
+
Optional Argument.
|
|
112
|
+
Specifies keyword arguments. Accepts following keyword arguments:
|
|
113
|
+
|
|
114
|
+
sep:
|
|
115
|
+
Optional Argument.
|
|
116
|
+
Specifies a single character string used to separate fields in a CSV file.
|
|
117
|
+
Default Value: ","
|
|
118
|
+
Notes:
|
|
119
|
+
1. "sep" cannot be line feed ('\\n') or carriage return ('\\r').
|
|
120
|
+
2. "sep" should not be same as "quotechar".
|
|
121
|
+
3. Length of "sep" argument should be 1.
|
|
122
|
+
Types: String
|
|
123
|
+
|
|
124
|
+
quotechar:
|
|
125
|
+
Optional Argument.
|
|
126
|
+
Specifies a single character string used to quote fields in a CSV file.
|
|
127
|
+
Default Value: "\""
|
|
128
|
+
Notes:
|
|
129
|
+
1. "quotechar" cannot be line feed ('\\n') or carriage return ('\\r').
|
|
130
|
+
2. "quotechar" should not be same as "sep".
|
|
131
|
+
3. Length of "quotechar" argument should be 1.
|
|
132
|
+
Types: String
|
|
133
|
+
|
|
134
|
+
coerce_float:
|
|
135
|
+
Optional Argument.
|
|
136
|
+
Specifies whether to convert non-string, non-numeric objects to floating point.
|
|
137
|
+
Note:
|
|
138
|
+
For additional information about "coerce_float" please refer to:
|
|
139
|
+
https://pandas.pydata.org/docs/reference/api/pandas.read_sql.html
|
|
140
|
+
|
|
141
|
+
parse_dates:
|
|
142
|
+
Optional Argument.
|
|
143
|
+
Specifies columns to parse as dates.
|
|
144
|
+
Note:
|
|
145
|
+
For additional information about "parse_date" please refer to:
|
|
146
|
+
https://pandas.pydata.org/docs/reference/api/pandas.read_sql.html
|
|
147
|
+
|
|
148
|
+
open_sessions:
|
|
149
|
+
Optional Argument.
|
|
150
|
+
specifies the number of Teradata sessions to be opened for fastexport.
|
|
151
|
+
Note:
|
|
152
|
+
If "open_sessions" argument is not provided, the default value
|
|
153
|
+
is the smaller of 8 or the number of AMPs avaialble.
|
|
154
|
+
For additional information about number of Teradata data-transfer
|
|
155
|
+
sessions opened during fastexport, please refer to:
|
|
156
|
+
https://pypi.org/project/teradatasql/#FastExport
|
|
157
|
+
Types: Integer
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
RETURNS:
|
|
161
|
+
1. When 'export_to' is set to "pandas" and "catch_errors_warnings" is set to True,
|
|
162
|
+
then the function returns a tuple containing:
|
|
163
|
+
a. Pandas DataFrame.
|
|
164
|
+
b. Errors, if any, thrown by fastexport in a list of strings.
|
|
165
|
+
c. Warnings, if any, thrown by fastexport in a list of strings.
|
|
166
|
+
When 'export_to' is set to "pandas" and "catch_errors_warnings" is set to False,
|
|
167
|
+
then the function returns a Pandas DataFrame.
|
|
168
|
+
2. When 'export_to' is set to "csv" and "catch_errors_warnings" is set to True,
|
|
169
|
+
then the function returns a tuple containing:
|
|
170
|
+
a. Errors, if any, thrown by fastexport in a list of strings.
|
|
171
|
+
b. Warnings, if any, thrown by fastexport in a list of strings.
|
|
172
|
+
|
|
173
|
+
EXAMPLES:
|
|
174
|
+
>>> from teradataml import *
|
|
175
|
+
>>> load_example_data("dataframe", "admissions_train")
|
|
176
|
+
>>> df = DataFrame("admissions_train")
|
|
177
|
+
|
|
178
|
+
# Print dataframe.
|
|
179
|
+
>>> df
|
|
180
|
+
masters gpa stats programming admitted
|
|
181
|
+
id
|
|
182
|
+
13 no 4.00 Advanced Novice 1
|
|
183
|
+
26 yes 3.57 Advanced Advanced 1
|
|
184
|
+
5 no 3.44 Novice Novice 0
|
|
185
|
+
19 yes 1.98 Advanced Advanced 0
|
|
186
|
+
15 yes 4.00 Advanced Advanced 1
|
|
187
|
+
40 yes 3.95 Novice Beginner 0
|
|
188
|
+
7 yes 2.33 Novice Novice 1
|
|
189
|
+
22 yes 3.46 Novice Beginner 0
|
|
190
|
+
36 no 3.00 Advanced Novice 0
|
|
191
|
+
38 yes 2.65 Advanced Beginner 1
|
|
192
|
+
|
|
193
|
+
# Example 1: Export teradataml DataFrame df to Pandas DataFrame using
|
|
194
|
+
# fastexport().
|
|
195
|
+
>>> fastexport(df)
|
|
196
|
+
Errors: []
|
|
197
|
+
Warnings: []
|
|
198
|
+
masters gpa stats programming admitted
|
|
199
|
+
id
|
|
200
|
+
38 yes 2.65 Advanced Beginner 1
|
|
201
|
+
26 yes 3.57 Advanced Advanced 1
|
|
202
|
+
5 no 3.44 Novice Novice 0
|
|
203
|
+
24 no 1.87 Advanced Novice 1
|
|
204
|
+
3 no 3.70 Novice Beginner 1
|
|
205
|
+
1 yes 3.95 Beginner Beginner 0
|
|
206
|
+
20 yes 3.90 Advanced Advanced 1
|
|
207
|
+
18 yes 3.81 Advanced Advanced 1
|
|
208
|
+
8 no 3.60 Beginner Advanced 1
|
|
209
|
+
25 no 3.96 Advanced Advanced 1
|
|
210
|
+
...
|
|
211
|
+
|
|
212
|
+
# Example 2: Export teradataml DataFrame df to Pandas DataFrame,
|
|
213
|
+
# set index column, coerce_float and catch errors/warnings thrown by
|
|
214
|
+
# fastexport.
|
|
215
|
+
>>> pandas_df, err, warn = fastexport(df, index_column="gpa",
|
|
216
|
+
catch_errors_warnings=True,
|
|
217
|
+
coerce_float=True)
|
|
218
|
+
# Print pandas DataFrame.
|
|
219
|
+
>>> pandas_df
|
|
220
|
+
id masters stats programming admitted
|
|
221
|
+
gpa
|
|
222
|
+
2.65 38 yes Advanced Beginner 1
|
|
223
|
+
3.57 26 yes Advanced Advanced 1
|
|
224
|
+
3.44 5 no Novice Novice 0
|
|
225
|
+
1.87 24 no Advanced Novice 1
|
|
226
|
+
3.70 3 no Novice Beginner 1
|
|
227
|
+
3.95 1 yes Beginner Beginner 0
|
|
228
|
+
3.90 20 yes Advanced Advanced 1
|
|
229
|
+
3.81 18 yes Advanced Advanced 1
|
|
230
|
+
3.60 8 no Beginner Advanced 1
|
|
231
|
+
3.96 25 no Advanced Advanced 1
|
|
232
|
+
3.76 2 yes Beginner Beginner 0
|
|
233
|
+
3.83 17 no Advanced Advanced 1
|
|
234
|
+
|
|
235
|
+
...
|
|
236
|
+
# Print errors list.
|
|
237
|
+
>>> err
|
|
238
|
+
[]
|
|
239
|
+
# Print warnings list.
|
|
240
|
+
>>> warn
|
|
241
|
+
[]
|
|
242
|
+
|
|
243
|
+
# Example 3: Following example exports teradataml DataFrame df
|
|
244
|
+
# to Pandas DataFrame using fastexport() by opening specified
|
|
245
|
+
# number of Teradata data-transfer sessions.
|
|
246
|
+
>>> fastexport(df, open_sessions=2)
|
|
247
|
+
Errors: []
|
|
248
|
+
Warnings: []
|
|
249
|
+
masters gpa stats programming admitted
|
|
250
|
+
id
|
|
251
|
+
38 yes 2.65 Advanced Beginner 1
|
|
252
|
+
26 yes 3.57 Advanced Advanced 1
|
|
253
|
+
5 no 3.44 Novice Novice 0
|
|
254
|
+
24 no 1.87 Advanced Novice 1
|
|
255
|
+
3 no 3.70 Novice Beginner 1
|
|
256
|
+
1 yes 3.95 Beginner Beginner 0
|
|
257
|
+
20 yes 3.90 Advanced Advanced 1
|
|
258
|
+
18 yes 3.81 Advanced Advanced 1
|
|
259
|
+
8 no 3.60 Beginner Advanced 1
|
|
260
|
+
25 no 3.96 Advanced Advanced 1
|
|
261
|
+
...
|
|
262
|
+
|
|
263
|
+
# Example 4: Following example exports teradataml DataFrame df
|
|
264
|
+
# to a given CSV file using fastexport().
|
|
265
|
+
>>> fastexport(df, export_to="csv", csv_file="Test.csv")
|
|
266
|
+
Data is successfully exported into Test.csv
|
|
267
|
+
|
|
268
|
+
# Example 5: Following example exports teradataml DataFrame df
|
|
269
|
+
# to a given CSV file using fastexport() by opening specified
|
|
270
|
+
# number of Teradata data-transfer sessions.
|
|
271
|
+
>>> fastexport(df, export_to="csv", csv_file="Test_1.csv", open_sessions=2)
|
|
272
|
+
Data is successfully exported into Test_1.csv
|
|
273
|
+
|
|
274
|
+
# Example 6: Following example exports teradataml DataFrame df
|
|
275
|
+
# to a given CSV file using fastexport() and catch errors/warnings
|
|
276
|
+
# thrown by fastexport.
|
|
277
|
+
>>> err, warn = fastexport(df, export_to="csv", catch_errors_warnings=True,
|
|
278
|
+
csv_file="Test_3.csv")
|
|
279
|
+
Data is successfully exported into Test_3.csv
|
|
280
|
+
# Print errors list.
|
|
281
|
+
>>> err
|
|
282
|
+
[]
|
|
283
|
+
# Print warnings list.
|
|
284
|
+
>>> warn
|
|
285
|
+
[]
|
|
286
|
+
|
|
287
|
+
# Example 7: Export teradataml DataFrame to CSV file with '|' as field separator
|
|
288
|
+
# and single quote(') as field quote character.
|
|
289
|
+
>>> fastexport(df, export_to="csv", csv_file="Test_4.csv", sep = "|", quotechar="'")
|
|
290
|
+
Data is successfully exported into Test_4.csv
|
|
291
|
+
|
|
292
|
+
"""
|
|
293
|
+
try:
|
|
294
|
+
# Deriving global connection using context.get_context()
|
|
295
|
+
con = get_context()
|
|
296
|
+
if con is None:
|
|
297
|
+
raise TeradataMlException(
|
|
298
|
+
Messages.get_message(MessageCodes.CONNECTION_FAILURE),
|
|
299
|
+
MessageCodes.CONNECTION_FAILURE)
|
|
300
|
+
|
|
301
|
+
awu_matrix = []
|
|
302
|
+
# Add new exports once supported.
|
|
303
|
+
permitted_exports = ["pandas", "csv"]
|
|
304
|
+
from teradataml.dataframe.dataframe import DataFrame
|
|
305
|
+
awu_matrix.append(["df", df, False, DataFrame, True])
|
|
306
|
+
awu_matrix.append(["export_to", export_to, True, str, False,
|
|
307
|
+
permitted_exports])
|
|
308
|
+
awu_matrix.append(["csv_file", csv_file, True, str, True])
|
|
309
|
+
|
|
310
|
+
# Get open_sessions argument.
|
|
311
|
+
open_sessions = kwargs.get("open_sessions", None)
|
|
312
|
+
awu_matrix.append(["open_sessions", open_sessions, True, int, False])
|
|
313
|
+
|
|
314
|
+
# Validate arguments unique to fastexport() function.
|
|
315
|
+
_Validators._validate_function_arguments(awu_matrix)
|
|
316
|
+
|
|
317
|
+
if open_sessions is not None:
|
|
318
|
+
_Validators._validate_positive_int(open_sessions, "open_sessions")
|
|
319
|
+
|
|
320
|
+
# Convert teradataml DataFrame to pandas DataFrame.
|
|
321
|
+
if export_to.lower() == "pandas":
|
|
322
|
+
# Initialize and validate DataTransferUtils object.
|
|
323
|
+
dt_obj = _DataTransferUtils(df, index_column=index_column,
|
|
324
|
+
all_rows=True,
|
|
325
|
+
catch_errors_warnings=catch_errors_warnings)
|
|
326
|
+
|
|
327
|
+
# Call fastexport_get_pandas_df function to get pandas dataframe
|
|
328
|
+
# using fastexport datatransfer protocol.
|
|
329
|
+
# "require" is always True, because with this function user requires
|
|
330
|
+
# fastexport.
|
|
331
|
+
|
|
332
|
+
return dt_obj._fastexport_get_pandas_df(require=True, **kwargs)
|
|
333
|
+
|
|
334
|
+
# Convert teradataml DataFrame to CSV file.
|
|
335
|
+
if export_to.lower() == "csv":
|
|
336
|
+
if not csv_file:
|
|
337
|
+
raise TeradataMlException(
|
|
338
|
+
Messages.get_message(MessageCodes.DEPENDENT_ARG_MISSING, "csv_file",
|
|
339
|
+
"{0}='{1}'".format("export_to", "csv")),
|
|
340
|
+
MessageCodes.DEPENDENT_ARG_MISSING)
|
|
341
|
+
|
|
342
|
+
if not csv_file.lower().endswith(".csv"):
|
|
343
|
+
raise TeradataMlException(
|
|
344
|
+
Messages.get_message(MessageCodes.INVALID_ARG_VALUE, csv_file,
|
|
345
|
+
"csv_file", "file with csv format"),
|
|
346
|
+
MessageCodes.INVALID_ARG_VALUE)
|
|
347
|
+
|
|
348
|
+
# Get "sep" and "quotechar" argument.
|
|
349
|
+
sep = kwargs.pop("sep", ",")
|
|
350
|
+
quotechar = kwargs.pop("quotechar", "\"")
|
|
351
|
+
|
|
352
|
+
dt_obj = _DataTransferUtils(df, all_rows=True,
|
|
353
|
+
sep=sep, quotechar=quotechar,
|
|
354
|
+
catch_errors_warnings=catch_errors_warnings)
|
|
355
|
+
return dt_obj._get_csv(require_fastexport=True, csv_file_name=csv_file, **kwargs)
|
|
356
|
+
|
|
357
|
+
except TeradataMlException:
|
|
358
|
+
raise
|
|
359
|
+
except TypeError:
|
|
360
|
+
raise
|
|
361
|
+
except ValueError:
|
|
362
|
+
raise
|
|
363
|
+
except Exception as err:
|
|
364
|
+
raise TeradataMlException(
|
|
365
|
+
Messages.get_message(MessageCodes.DATA_EXPORT_FAILED, "fastexport",
|
|
366
|
+
export_to, str(err)),
|
|
367
|
+
MessageCodes.DATA_EXPORT_FAILED)
|
|
368
|
+
|
|
369
|
+
|
|
370
|
+
@collect_queryband(queryband="rdCsv")
|
|
371
|
+
def read_csv(filepath,
|
|
372
|
+
table_name,
|
|
373
|
+
types=None,
|
|
374
|
+
sep=",",
|
|
375
|
+
quotechar="\"",
|
|
376
|
+
schema_name=None,
|
|
377
|
+
if_exists='replace',
|
|
378
|
+
primary_index=None,
|
|
379
|
+
set_table=False,
|
|
380
|
+
temporary=False,
|
|
381
|
+
primary_time_index_name=None,
|
|
382
|
+
timecode_column=None,
|
|
383
|
+
timebucket_duration=None,
|
|
384
|
+
timezero_date=None,
|
|
385
|
+
columns_list=None,
|
|
386
|
+
sequence_column=None,
|
|
387
|
+
seq_max=None,
|
|
388
|
+
catch_errors_warnings=False,
|
|
389
|
+
save_errors=False,
|
|
390
|
+
use_fastload=True,
|
|
391
|
+
open_sessions=None):
|
|
392
|
+
"""
|
|
393
|
+
The read_csv() API loads data from CSV file into Teradata Vantage.
|
|
394
|
+
Function can be used to quickly load large amounts of data in a table on Vantage
|
|
395
|
+
using FastloadCSV protocol.
|
|
396
|
+
|
|
397
|
+
Considerations when using a CSV file:
|
|
398
|
+
* Each record is on a separate line of the CSV file. Records are delimited
|
|
399
|
+
by line breaks (CRLF). The last record in the file may or may not have an
|
|
400
|
+
ending line break.
|
|
401
|
+
* First line in the CSV must be header line. The header line lists
|
|
402
|
+
the column names separated by the field separator (e.g. col1,col2,col3).
|
|
403
|
+
* Using a CSV file with FastLoad has limitations as follows:
|
|
404
|
+
1. read_csv API cannot load duplicate rows in the DataFrame if the table is a
|
|
405
|
+
MULTISET table having primary index.
|
|
406
|
+
2. read_csv API does not support all Teradata Advanced SQL Engine data types.
|
|
407
|
+
For example, target table having BLOB and CLOB data type columns cannot be
|
|
408
|
+
loaded.
|
|
409
|
+
3. If there are any incorrect rows, i.e. due to constraint violations, data type
|
|
410
|
+
conversion errors, etc., FastLoad protocol ignores those rows and inserts
|
|
411
|
+
all valid rows.
|
|
412
|
+
4. Rows in the DataFrame that failed to get inserted are categorized into errors
|
|
413
|
+
and warnings by FastLoad protocol and these errors and warnings are stored
|
|
414
|
+
into respective error and warning tables by FastLoad API.
|
|
415
|
+
5. Teradata recommends to use Fastload protocol when number of rows to be loaded
|
|
416
|
+
are at least 100,000. Fastload opens multiple data transfer connections to the
|
|
417
|
+
database.
|
|
418
|
+
|
|
419
|
+
For additional information about FastLoadCSV protocol through teradatasql driver,
|
|
420
|
+
please refer the CSV BATCH INSERTS section of https://pypi.org/project/teradatasql/#CSVBatchInserts
|
|
421
|
+
driver documentation for more information.
|
|
422
|
+
|
|
423
|
+
PARAMETERS:
|
|
424
|
+
filepath:
|
|
425
|
+
Required Argument.
|
|
426
|
+
Specifies the CSV filepath including name of the file to load the data from.
|
|
427
|
+
Types: String
|
|
428
|
+
|
|
429
|
+
table_name:
|
|
430
|
+
Required Argument.
|
|
431
|
+
Specifies the table name to load the data into.
|
|
432
|
+
Types: String
|
|
433
|
+
|
|
434
|
+
types:
|
|
435
|
+
Optional Argument when if_exists=append and non-PTI table already exists, Required otherwise.
|
|
436
|
+
Specifies the data types for requested columns to be saved in Vantage.
|
|
437
|
+
Keys of this dictionary should be the name of the columns and values should be
|
|
438
|
+
teradatasqlalchemy.types.
|
|
439
|
+
Default Value: None
|
|
440
|
+
Note:
|
|
441
|
+
* If specified when "if_exists" is set to append and table exists, then argument is ignored.
|
|
442
|
+
Types: OrderedDict
|
|
443
|
+
|
|
444
|
+
sep:
|
|
445
|
+
Optional Argument.
|
|
446
|
+
Specifies a single character string used to separate fields in a CSV file.
|
|
447
|
+
Default Value: ","
|
|
448
|
+
Notes:
|
|
449
|
+
* "sep" cannot be line feed ('\\n') or carriage return ('\\r').
|
|
450
|
+
* "sep" should not be same as "quotechar".
|
|
451
|
+
* Length of "sep" argument should be 1.
|
|
452
|
+
Types: String
|
|
453
|
+
|
|
454
|
+
quotechar:
|
|
455
|
+
Optional Argument.
|
|
456
|
+
Specifies a single character string used to quote fields in a CSV file.
|
|
457
|
+
Default Value: "\""
|
|
458
|
+
Notes:
|
|
459
|
+
* "quotechar" cannot be line feed ('\\n') or carriage return ('\\r').
|
|
460
|
+
* "quotechar" should not be same as "sep".
|
|
461
|
+
* Length of "quotechar" argument should be 1.
|
|
462
|
+
Types: String
|
|
463
|
+
|
|
464
|
+
schema_name:
|
|
465
|
+
Optional Argument.
|
|
466
|
+
Specifies the name of the database/schema in Vantage to write to.
|
|
467
|
+
Default Value: None (Uses default database/schema).
|
|
468
|
+
Types: String
|
|
469
|
+
|
|
470
|
+
if_exists:
|
|
471
|
+
Optional Argument.
|
|
472
|
+
Specifies the action to take when table already exists in Vantage.
|
|
473
|
+
Permitted Values: 'fail', 'replace', 'append'
|
|
474
|
+
- fail: If table exists, raise TeradataMlException.
|
|
475
|
+
- replace: If table exists, drop it, recreate it, and insert data.
|
|
476
|
+
- append: If table exists, append the existing table.
|
|
477
|
+
Default Value: replace
|
|
478
|
+
Types: String
|
|
479
|
+
|
|
480
|
+
primary_index:
|
|
481
|
+
Optional Argument.
|
|
482
|
+
Specifies which column(s) to use as primary index while creating table
|
|
483
|
+
in Vantage. When set to None, No Primary Index (NoPI) tables are created.
|
|
484
|
+
Default Value: None
|
|
485
|
+
Types: String or list of strings
|
|
486
|
+
Example:
|
|
487
|
+
primary_index = 'my_primary_index'
|
|
488
|
+
primary_index = ['my_primary_index1', 'my_primary_index2', 'my_primary_index3']
|
|
489
|
+
|
|
490
|
+
set_table:
|
|
491
|
+
Optional Argument.
|
|
492
|
+
Specifies a flag to determine whether to create a SET or a MULTISET table.
|
|
493
|
+
When set to True, a SET table is created, otherwise MULTISET table is created.
|
|
494
|
+
Default Value: False
|
|
495
|
+
Notes:
|
|
496
|
+
1. Specifying set_table=True also requires specifying primary_index.
|
|
497
|
+
2. Creating SET table (set_table=True) results in
|
|
498
|
+
a. loss of duplicate rows, if CSV contains any duplicate.
|
|
499
|
+
3. This argument has no effect if the table already exists and if_exists='append'.
|
|
500
|
+
Types: Boolean
|
|
501
|
+
|
|
502
|
+
temporary:
|
|
503
|
+
Optional Argument.
|
|
504
|
+
Specifies whether to create table as volatile.
|
|
505
|
+
Default Value: False
|
|
506
|
+
Notes:
|
|
507
|
+
When set to True
|
|
508
|
+
1. FastloadCSV protocol is not used for loading the data.
|
|
509
|
+
2. "schema_name" is ignored.
|
|
510
|
+
Types : Boolean
|
|
511
|
+
|
|
512
|
+
primary_time_index_name:
|
|
513
|
+
Optional Argument.
|
|
514
|
+
Specifies the name for the Primary Time Index (PTI) when the table
|
|
515
|
+
is to be created as PTI table.
|
|
516
|
+
Note:
|
|
517
|
+
This argument is not required or used when the table to be created
|
|
518
|
+
is not a PTI table. It will be ignored if specified without the "timecode_column".
|
|
519
|
+
Types: String
|
|
520
|
+
|
|
521
|
+
timecode_column:
|
|
522
|
+
Optional argument.
|
|
523
|
+
Required when the CSV data must be saved as a PTI table.
|
|
524
|
+
Specifies the column in the csv that reflects the form
|
|
525
|
+
of the timestamp data in the time series.
|
|
526
|
+
This column will be the TD_TIMECODE column in the table created.
|
|
527
|
+
It should be of SQL type TIMESTAMP(n), TIMESTAMP(n) WITH TIMEZONE, or DATE,
|
|
528
|
+
corresponding to Python types datetime.datetime or datetime.date.
|
|
529
|
+
Note:
|
|
530
|
+
When "timecode_column" argument is specified, an attempt to create a PTI table
|
|
531
|
+
will be made. This argument is not required when the table to be created
|
|
532
|
+
is not a PTI table. If this argument is specified, "primary_index" will be ignored.
|
|
533
|
+
Types: String
|
|
534
|
+
|
|
535
|
+
timezero_date:
|
|
536
|
+
Optional Argument.
|
|
537
|
+
Used when the CSV data must be saved as a PTI table.
|
|
538
|
+
Specifies the earliest time series data that the PTI table will accept,
|
|
539
|
+
a date that precedes the earliest date in the time series data.
|
|
540
|
+
Value specified must be of the following format: DATE 'YYYY-MM-DD'
|
|
541
|
+
Default Value: DATE '1970-01-01'.
|
|
542
|
+
Note:
|
|
543
|
+
This argument is not required or used when the table to be created
|
|
544
|
+
is not a PTI table. It will be ignored if specified without the "timecode_column".
|
|
545
|
+
Types: String
|
|
546
|
+
|
|
547
|
+
timebucket_duration:
|
|
548
|
+
Optional Argument.
|
|
549
|
+
Required if "columns_list" is not specified or is None.
|
|
550
|
+
Used when the CSV data must be saved as a PTI table.
|
|
551
|
+
Specifies a duration that serves to break up the time continum in
|
|
552
|
+
the time series data into discrete groups or buckets.
|
|
553
|
+
Specified using the formal form time_unit(n), where n is a positive
|
|
554
|
+
integer, and time_unit can be any of the following:
|
|
555
|
+
CAL_YEARS, CAL_MONTHS, CAL_DAYS, WEEKS, DAYS, HOURS, MINUTES,
|
|
556
|
+
SECONDS, MILLISECONDS, or MICROSECONDS.
|
|
557
|
+
Note:
|
|
558
|
+
This argument is not required or used when the table to be created
|
|
559
|
+
is not a PTI table. It will be ignored if specified without the "timecode_column".
|
|
560
|
+
Types: String
|
|
561
|
+
|
|
562
|
+
columns_list:
|
|
563
|
+
Optional Argument.
|
|
564
|
+
Used when the CSV data must be saved as a PTI table.
|
|
565
|
+
Required if "timebucket_duration" is not specified.
|
|
566
|
+
A list of one or more PTI table column names.
|
|
567
|
+
Note:
|
|
568
|
+
This argument is not required or used when the table to be created
|
|
569
|
+
is not a PTI table. It will be ignored if specified without the "timecode_column".
|
|
570
|
+
Types: String or list of Strings
|
|
571
|
+
|
|
572
|
+
sequence_column:
|
|
573
|
+
Optional Argument.
|
|
574
|
+
Used when the CSV data must be saved as a PTI table.
|
|
575
|
+
Specifies the column of type Integer containing the unique identifier for
|
|
576
|
+
time series data readings when they are not unique in time.
|
|
577
|
+
* When specified, implies SEQUENCED, meaning more than one reading from the same
|
|
578
|
+
sensor may have the same timestamp.
|
|
579
|
+
This column will be the TD_SEQNO column in the table created.
|
|
580
|
+
* When not specified, implies NONSEQUENCED, meaning there is only one sensor reading
|
|
581
|
+
per timestamp.
|
|
582
|
+
This is the default.
|
|
583
|
+
Note:
|
|
584
|
+
This argument is not required or used when the table to be created
|
|
585
|
+
is not a PTI table. It will be ignored if specified without the "timecode_column".
|
|
586
|
+
Types: String
|
|
587
|
+
|
|
588
|
+
seq_max:
|
|
589
|
+
Optional Argument.
|
|
590
|
+
Used when the CSV data must be saved as a PTI table.
|
|
591
|
+
Specifies the maximum number of data rows that can have the
|
|
592
|
+
same timestamp. Can be used when 'sequenced' is True.
|
|
593
|
+
Permitted range: 1 - 2147483647.
|
|
594
|
+
Default Value: 20000.
|
|
595
|
+
Note:
|
|
596
|
+
This argument is not required or used when the table to be created
|
|
597
|
+
is not a PTI table. It will be ignored if specified without the "timecode_column".
|
|
598
|
+
Types: Integer
|
|
599
|
+
|
|
600
|
+
save_errors:
|
|
601
|
+
Optional Argument.
|
|
602
|
+
Specifies whether to persist the errors/warnings(if any) information in Vantage
|
|
603
|
+
or not.
|
|
604
|
+
If "save_errors" is set to False:
|
|
605
|
+
1. Errors or warnings (if any) are not persisted into tables.
|
|
606
|
+
2. Errors table genarated by FastloadCSV are not persisted.
|
|
607
|
+
If "save_errors" is set to True:
|
|
608
|
+
1. The errors or warnings information is persisted and names of error and
|
|
609
|
+
warning tables are returned. Otherwise, the function returns None for
|
|
610
|
+
the names of the tables.
|
|
611
|
+
2. The errors tables generated by FastloadCSV are persisted and name of
|
|
612
|
+
error tables are returned. Otherwise, the function returns None for
|
|
613
|
+
the names of the tables.
|
|
614
|
+
Default Value: False
|
|
615
|
+
Types: Boolean
|
|
616
|
+
|
|
617
|
+
catch_errors_warnings:
|
|
618
|
+
Optional Argument.
|
|
619
|
+
Specifies whether to catch errors/warnings(if any) raised by fastload
|
|
620
|
+
protocol while loading data into the Vantage table.
|
|
621
|
+
When set to False, function does not catch any errors and warnings,
|
|
622
|
+
otherwise catches errors and warnings, if any, and returns
|
|
623
|
+
as a dictionary along with teradataml DataFrame.
|
|
624
|
+
Please see 'RETURNS' section for more details.
|
|
625
|
+
Default Value: False
|
|
626
|
+
Types: Boolean
|
|
627
|
+
|
|
628
|
+
use_fastload:
|
|
629
|
+
Optional Argument.
|
|
630
|
+
Specifies whether to use Fastload CSV protocol or not.
|
|
631
|
+
Default Value: True
|
|
632
|
+
Notes:
|
|
633
|
+
1. Teradata recommends to use Fastload when number of rows to be loaded
|
|
634
|
+
are atleast 100,000. To load lesser rows set this argument to 'False'.
|
|
635
|
+
Fastload opens multiple data transfer connections to the database.
|
|
636
|
+
2. When "use_fastload" is set to True, one can load the data into table
|
|
637
|
+
using FastloadCSV protocol:
|
|
638
|
+
a. Set table
|
|
639
|
+
b. Multiset table
|
|
640
|
+
3. When "use_fastload" is set to False, one can load the data in following
|
|
641
|
+
types of tables:
|
|
642
|
+
a. Set table
|
|
643
|
+
b. Multiset table
|
|
644
|
+
c. PTI table
|
|
645
|
+
d. Volatile table
|
|
646
|
+
Types: Boolean
|
|
647
|
+
|
|
648
|
+
open_sessions:
|
|
649
|
+
Optional Argument.
|
|
650
|
+
Specifies the number of Teradata data transfer sessions to be opened for fastload operation.
|
|
651
|
+
Note : If "open_sessions" argument is not provided, the default value is the smaller of 8 or the
|
|
652
|
+
number of AMPs available.
|
|
653
|
+
For additional information about number of Teradata data-transfer
|
|
654
|
+
sessions opened during fastload, please refer to:
|
|
655
|
+
https://pypi.org/project/teradatasql/#FastLoad
|
|
656
|
+
Default Value: None
|
|
657
|
+
Types: int
|
|
658
|
+
|
|
659
|
+
RETURNS:
|
|
660
|
+
When "use_fastload" is set to False, returns teradataml dataframe.
|
|
661
|
+
When "use_fastload" is set to True, read_csv() returns below:
|
|
662
|
+
When "catch_errors_warnings" is set to False, returns only teradataml dataframe.
|
|
663
|
+
When "catch_errors_warnings" is set to True, read_csv() returns a tuple containing:
|
|
664
|
+
a. teradataml DataFrame.
|
|
665
|
+
b. a dict containing the following attributes:
|
|
666
|
+
a. errors_dataframe: It is a Pandas DataFrame containing error messages
|
|
667
|
+
thrown by fastload. DataFrame is empty if there are no errors.
|
|
668
|
+
b. warnings_dataframe: It is a Pandas DataFrame containing warning messages
|
|
669
|
+
thrown by fastload. DataFrame is empty if there are no warnings.
|
|
670
|
+
c. errors_table: Name of the table containing errors. It is None, if
|
|
671
|
+
argument save_errors is False.
|
|
672
|
+
d. warnings_table: Name of the table containing warnings. It is None, if
|
|
673
|
+
argument save_errors is False.
|
|
674
|
+
e. fastloadcsv_error_tables: Name of the tables containing errors generated
|
|
675
|
+
by FastloadCSV. It is empty list, if argument "save_errors" is False.
|
|
676
|
+
|
|
677
|
+
RAISES:
|
|
678
|
+
TeradataMlException
|
|
679
|
+
|
|
680
|
+
EXAMPLES:
|
|
681
|
+
>>> from teradataml.dataframe.data_transfer import read_csv
|
|
682
|
+
>>> from teradatasqlalchemy.types import *
|
|
683
|
+
>>> from collections import OrderedDict
|
|
684
|
+
|
|
685
|
+
# Example 1: Default execution with types argument is passed as OrderedDict.
|
|
686
|
+
>>> types = OrderedDict(id=BIGINT, fname=VARCHAR, lname=VARCHAR, marks=FLOAT)
|
|
687
|
+
>>> read_csv('test_file.csv', 'my_first_table', types)
|
|
688
|
+
|
|
689
|
+
# Example 2: Load the data from CSV file into a table using fastload CSV protocol,
|
|
690
|
+
# while doing so catch all errors and warnings as well as store those in the table.
|
|
691
|
+
>>> types = OrderedDict(id=BIGINT, fname=VARCHAR, lname=VARCHAR, marks=FLOAT)
|
|
692
|
+
>>> read_csv(filepath='test_file.csv',
|
|
693
|
+
... table_name='my_first_table1', types=types,
|
|
694
|
+
... save_errors=True, catch_errors_warnings=True)
|
|
695
|
+
|
|
696
|
+
# Example 3: Load the data from CSV file into a table using fastload CSV protocol.
|
|
697
|
+
# If table exists, then replace the same. Catch all errors and warnings as well as
|
|
698
|
+
# store those in the table.
|
|
699
|
+
>>> types = OrderedDict(id=BIGINT, fname=VARCHAR, lname=VARCHAR, marks=FLOAT)
|
|
700
|
+
>>> read_csv(filepath='test_file.csv',
|
|
701
|
+
... table_name='my_first_table',
|
|
702
|
+
... types=types, if_exists='replace',
|
|
703
|
+
... save_errors=True, catch_errors_warnings=True)
|
|
704
|
+
|
|
705
|
+
# Example 4: Load the data from CSV file into a table using fastload CSV protocol.
|
|
706
|
+
# If table exists in specified schema, then append the same. Catch all
|
|
707
|
+
# errors and warnings as well as store those in the table.
|
|
708
|
+
>>> types = OrderedDict(id=BIGINT, fname=VARCHAR, lname=VARCHAR, marks=FLOAT)
|
|
709
|
+
>>> read_csv(filepath='test_file.csv',
|
|
710
|
+
... table_name='my_first_table',
|
|
711
|
+
... types=types, if_exists='fail',
|
|
712
|
+
... save_errors=True, catch_errors_warnings=True)
|
|
713
|
+
>>> read_csv(filepath='test_file.csv',
|
|
714
|
+
... table_name='my_first_table',
|
|
715
|
+
... if_exists='append',
|
|
716
|
+
... save_errors=True, catch_errors_warnings=True)
|
|
717
|
+
|
|
718
|
+
# Example 5: Load the data from CSV file into a SET table using fastload CSV protocol.
|
|
719
|
+
# Catch all errors and warnings as well as store those in the table.
|
|
720
|
+
>>> types = OrderedDict(id=BIGINT, fname=VARCHAR, lname=VARCHAR, marks=FLOAT)
|
|
721
|
+
>>> read_csv(filepath='test_file.csv',
|
|
722
|
+
... table_name='my_first_table',
|
|
723
|
+
... types=types, if_exists='replace',
|
|
724
|
+
... set_table=True, primary_index='id',
|
|
725
|
+
... save_errors=True, catch_errors_warnings=True)
|
|
726
|
+
|
|
727
|
+
# Example 6: Load the data from CSV file into a temporary table without fastloadCSV protocol.
|
|
728
|
+
# If table exists, then append to the same.
|
|
729
|
+
>>> types = OrderedDict(id=BIGINT, fname=VARCHAR, lname=VARCHAR, marks=FLOAT)
|
|
730
|
+
>>> read_csv(filepath='test_file.csv',
|
|
731
|
+
... table_name='my_first_table',
|
|
732
|
+
... types=types, if_exists='replace',
|
|
733
|
+
... temporary=True)
|
|
734
|
+
>>> read_csv(filepath='test_file.csv',
|
|
735
|
+
... table_name='my_first_table',
|
|
736
|
+
... if_exists='append',
|
|
737
|
+
... temporary=True)
|
|
738
|
+
|
|
739
|
+
# Example 7: Load the data from CSV file with DATE and TIMESTAMP columns into
|
|
740
|
+
# a table without Fastload protocol. If table exists in specified
|
|
741
|
+
# schema, then append to the table.
|
|
742
|
+
>>> types = OrderedDict(id=BIGINT, fname=VARCHAR, lname=VARCHAR, marks=FLOAT,
|
|
743
|
+
... admission_date=DATE, admission_time=TIMESTAMP)
|
|
744
|
+
>>> read_csv(filepath='test_file.csv',
|
|
745
|
+
... table_name='my_first_table',
|
|
746
|
+
... types=types, if_exists='fail',
|
|
747
|
+
... use_fastload=False)
|
|
748
|
+
>>> read_csv(filepath='test_file.csv',
|
|
749
|
+
... table_name='my_first_table',
|
|
750
|
+
... if_exists='append',
|
|
751
|
+
... use_fastload=False)
|
|
752
|
+
|
|
753
|
+
# Example 8: Load the data from CSV file with TIMESTAMP columns into
|
|
754
|
+
# a PTI table. If specified table exists then append to the table,
|
|
755
|
+
# otherwise creates new table.
|
|
756
|
+
>>> types = OrderedDict(partition_id=INTEGER, adid=INTEGER, productid=INTEGER,
|
|
757
|
+
... event=VARCHAR, clicktime=TIMESTAMP)
|
|
758
|
+
>>> read_csv(filepath='test_file.csv',
|
|
759
|
+
... table_name='my_first_read_csv_pti_table',
|
|
760
|
+
... types=types, if_exists='append',
|
|
761
|
+
... timecode_column='clicktime',
|
|
762
|
+
... columns_list='event',
|
|
763
|
+
... use_fastload=False)
|
|
764
|
+
|
|
765
|
+
# Example 9: Load the data from CSV file with TIMESTAMP columns into
|
|
766
|
+
# a SET PTI table. If specified table exists then append to the table,
|
|
767
|
+
# otherwise creates new table.
|
|
768
|
+
>>> types = OrderedDict(partition_id=INTEGER, adid=INTEGER, productid=INTEGER,
|
|
769
|
+
event=VARCHAR, clicktime=TIMESTAMP)
|
|
770
|
+
>>> read_csv(filepath='test_file.csv',
|
|
771
|
+
... table_name='my_first_read_csv_pti_table',
|
|
772
|
+
... types=types, if_exists='append',
|
|
773
|
+
... timecode_column='clicktime',
|
|
774
|
+
... columns_list='event',
|
|
775
|
+
... set_table=True)
|
|
776
|
+
|
|
777
|
+
# Example 10: Load the data from CSV file with TIMESTAMP columns into
|
|
778
|
+
# a temporary PTI table. If specified table exists then append to the table,
|
|
779
|
+
# otherwise creates new table.
|
|
780
|
+
>>> types = OrderedDict(partition_id=INTEGER, adid=INTEGER, productid=INTEGER,
|
|
781
|
+
event=VARCHAR, clicktime=TIMESTAMP)
|
|
782
|
+
>>> read_csv(filepath='test_file.csv',
|
|
783
|
+
... table_name='my_first_read_csv_pti_table',
|
|
784
|
+
... types=types, if_exists='append',
|
|
785
|
+
... timecode_column='clicktime',
|
|
786
|
+
... columns_list='event',
|
|
787
|
+
... temporary=True)
|
|
788
|
+
|
|
789
|
+
# Example 11: Load the data from CSV file into Vantage table by opening specified
|
|
790
|
+
# number of Teradata data transfer sesions.
|
|
791
|
+
>>> types = OrderedDict(id=BIGINT, fname=VARCHAR, lname=VARCHAR, marks=FLOAT)
|
|
792
|
+
>>> read_csv(filepath='test_file.csv', table_name='my_first_table_with_open_sessions',
|
|
793
|
+
types=types, open_sessions=2)
|
|
794
|
+
|
|
795
|
+
# Example 12: Load the data from CSV file into Vantage table and set primary index provided
|
|
796
|
+
# through primary_index argument.
|
|
797
|
+
>>> types = OrderedDict(id=BIGINT, fname=VARCHAR, lname=VARCHAR, marks=FLOAT)
|
|
798
|
+
>>> read_csv(filepath='test_file.csv', table_name='my_first_table_with_primary_index',
|
|
799
|
+
... types=types, primary_index = ['fname'])
|
|
800
|
+
|
|
801
|
+
# Example 13: Load the data from CSV file into VECTOR datatype in Vantage table.
|
|
802
|
+
>>> from teradatasqlalchemy import VECTOR
|
|
803
|
+
>>> from pathlib import Path
|
|
804
|
+
>>> types = OrderedDict(id=BIGINT, array_col=VECTOR)
|
|
805
|
+
|
|
806
|
+
# Get the absolute path of the teradataml module
|
|
807
|
+
>>> import teradataml
|
|
808
|
+
>>> base_path = Path(teradataml.__path__[0])
|
|
809
|
+
|
|
810
|
+
# Append the relative path to the CSV file
|
|
811
|
+
>>> csv_path = os.path.join(base_path, "data", "hnsw_alter_data.csv")
|
|
812
|
+
|
|
813
|
+
>>> read_csv(filepath=csv_path,
|
|
814
|
+
... table_name='my_first_table_with_vector',
|
|
815
|
+
... types=types,
|
|
816
|
+
... use_fastload=False)
|
|
817
|
+
"""
|
|
818
|
+
# Deriving global connection using context.get_context()
|
|
819
|
+
con = get_context()
|
|
820
|
+
|
|
821
|
+
try:
|
|
822
|
+
if con is None:
|
|
823
|
+
raise TeradataMlException(Messages.get_message(MessageCodes.CONNECTION_FAILURE),
|
|
824
|
+
MessageCodes.CONNECTION_FAILURE)
|
|
825
|
+
|
|
826
|
+
schema_name = _get_context_temp_databasename() if schema_name is None else schema_name
|
|
827
|
+
|
|
828
|
+
# Create _DataTransferUtils object.
|
|
829
|
+
dt_obj = _DataTransferUtils(filepath, table_name=table_name, types=types,
|
|
830
|
+
sep=sep, quotechar=quotechar, schema_name=schema_name,
|
|
831
|
+
if_exists=if_exists, primary_index=primary_index,
|
|
832
|
+
set_table=set_table, temporary=temporary,
|
|
833
|
+
primary_time_index_name=primary_time_index_name,
|
|
834
|
+
timecode_column=timecode_column,
|
|
835
|
+
timebucket_duration=timebucket_duration,
|
|
836
|
+
timezero_date=timezero_date, columns_list=columns_list,
|
|
837
|
+
sequence_column=sequence_column, seq_max=seq_max,
|
|
838
|
+
save_errors=save_errors,
|
|
839
|
+
catch_errors_warnings=catch_errors_warnings,
|
|
840
|
+
use_fastload=use_fastload,
|
|
841
|
+
api_name='read_csv',
|
|
842
|
+
open_sessions=open_sessions)
|
|
843
|
+
|
|
844
|
+
# Validate read_csv api argument
|
|
845
|
+
dt_obj._validate_read_csv_api_args()
|
|
846
|
+
|
|
847
|
+
# Check if CSV file exists
|
|
848
|
+
_Validators._validate_file_exists(filepath)
|
|
849
|
+
|
|
850
|
+
# Ignore open_sessions argument when use_fastload is set to False
|
|
851
|
+
if not use_fastload and open_sessions is not None:
|
|
852
|
+
UtilFuncs._get_warnings('open_sessions', open_sessions, 'use_fastload', use_fastload)
|
|
853
|
+
dt_obj.open_sessions = None
|
|
854
|
+
|
|
855
|
+
# If temporary=True, set use_fastload=False and schema name=None.
|
|
856
|
+
if temporary:
|
|
857
|
+
# Setting fastload related arguments to False.
|
|
858
|
+
dt_obj.use_fastload = False
|
|
859
|
+
use_fastload = False
|
|
860
|
+
|
|
861
|
+
if schema_name is not None:
|
|
862
|
+
UtilFuncs._get_warnings("schema_name", schema_name, "temporary", "True")
|
|
863
|
+
schema_name = None
|
|
864
|
+
|
|
865
|
+
# A table cannot be a SET table and have NO PRIMARY INDEX
|
|
866
|
+
if set_table and primary_index is None:
|
|
867
|
+
raise TeradataMlException(Messages.get_message(MessageCodes.SET_TABLE_NO_PI),
|
|
868
|
+
MessageCodes.SET_TABLE_NO_PI)
|
|
869
|
+
|
|
870
|
+
# Check for PTI table.
|
|
871
|
+
if timecode_column is not None:
|
|
872
|
+
rc_dict = dt_obj._process_pti_load_csv_data(con=con)
|
|
873
|
+
return dt_obj._get_result(rc_dict)
|
|
874
|
+
|
|
875
|
+
# Check if table exists.
|
|
876
|
+
table_exists = dt_obj._table_exists(con)
|
|
877
|
+
|
|
878
|
+
# Raise exception when table not exists and if_exists='fail'.
|
|
879
|
+
dt_obj._check_table_exists(is_table_exists=table_exists)
|
|
880
|
+
|
|
881
|
+
# Let's create the SQLAlchemy table object to recreate the table.
|
|
882
|
+
if not table_exists or if_exists.lower() == 'replace':
|
|
883
|
+
# Validate required argument "types" when table doesn't exist
|
|
884
|
+
_Validators._validate_argument_is_not_None(types, "types")
|
|
885
|
+
|
|
886
|
+
dt_obj._create_or_replace_table(con, table_exists=table_exists)
|
|
887
|
+
|
|
888
|
+
# Load the CSV data into newly created table.
|
|
889
|
+
if not use_fastload:
|
|
890
|
+
rc_dict = dt_obj._insert_from_csv_without_fastload(table_name=table_name,
|
|
891
|
+
column_names=types.keys())
|
|
892
|
+
else:
|
|
893
|
+
rc_dict = dt_obj._insert_from_csv_with_fastload(table_name=table_name,
|
|
894
|
+
column_names=types.keys())
|
|
895
|
+
|
|
896
|
+
# Check column compatibility for insertion when table exists and if_exists = 'append'.
|
|
897
|
+
if table_exists and if_exists.lower() == 'append':
|
|
898
|
+
if set_table:
|
|
899
|
+
UtilFuncs._get_warnings('set_table', set_table, 'if_exists', 'append')
|
|
900
|
+
|
|
901
|
+
# Create SQLAlchemy table object from existing table.
|
|
902
|
+
existing_table = UtilFuncs._get_sqlalchemy_table(table_name,
|
|
903
|
+
schema_name=schema_name)
|
|
904
|
+
|
|
905
|
+
# Check compatibility of CSV columns with existing table columns.
|
|
906
|
+
if types is not None:
|
|
907
|
+
cols = _extract_column_info(filepath, types=types)
|
|
908
|
+
dt_obj._check_columns_compatibility(table_obj=existing_table, cols=cols)
|
|
909
|
+
|
|
910
|
+
# Validate user provided primary index against primary index of existing table .
|
|
911
|
+
existing_table_primary_index = UtilFuncs._extract_table_object_index_info(existing_table)
|
|
912
|
+
if primary_index is not None:
|
|
913
|
+
dt_obj._check_index_compatibility(primary_index_1=existing_table_primary_index,
|
|
914
|
+
primary_index_2=primary_index)
|
|
915
|
+
|
|
916
|
+
cols_name, cols_type = UtilFuncs._extract_table_object_column_info(existing_table.c)
|
|
917
|
+
column_info = dict(zip(cols_name, cols_type))
|
|
918
|
+
|
|
919
|
+
if use_fastload:
|
|
920
|
+
rc_dict = dt_obj._create_staging_table_and_load_csv_data(column_info=column_info,
|
|
921
|
+
primary_index=existing_table_primary_index)
|
|
922
|
+
else:
|
|
923
|
+
rc_dict = dt_obj._insert_from_csv_without_fastload(table_name=table_name,
|
|
924
|
+
column_names=cols_name)
|
|
925
|
+
# Return the read_csv result.
|
|
926
|
+
return dt_obj._get_result(rc_dict)
|
|
927
|
+
|
|
928
|
+
except (TeradataMlException, sqlachemyOperationalError, ValueError, TypeError):
|
|
929
|
+
raise
|
|
930
|
+
except Exception as err:
|
|
931
|
+
error_code = MessageCodes.EXECUTION_FAILED
|
|
932
|
+
error_msg = Messages.get_message(
|
|
933
|
+
error_code, "execute read_csv()", '{}'.format(str(err)))
|
|
934
|
+
raise TeradataMlException(error_msg, error_code)
|
|
935
|
+
|
|
936
|
+
|
|
937
|
+
class _DataTransferUtils():
|
|
938
|
+
"""
|
|
939
|
+
This class provides utility functions which enable Data Transfer from
|
|
940
|
+
Teradata Vantage to outside world, for example Data Transfer using
|
|
941
|
+
FastExport Protocol.
|
|
942
|
+
"""
|
|
943
|
+
|
|
944
|
+
def __init__(self, df, index_column=None, num_rows=99999, all_rows=False,
|
|
945
|
+
catch_errors_warnings=False, table_name=None,
|
|
946
|
+
schema_name=None, if_exists='append', index=False,
|
|
947
|
+
index_label=None, primary_index=None, temporary=False,
|
|
948
|
+
types=None, batch_size=None, save_errors=False, sep=",",
|
|
949
|
+
quotechar="\"", set_table=False,
|
|
950
|
+
primary_time_index_name=None, timecode_column=None,
|
|
951
|
+
timebucket_duration=None, timezero_date=None,
|
|
952
|
+
columns_list=None, sequence_column=None, seq_max=None,
|
|
953
|
+
use_fastload=True, api_name='fastexport',
|
|
954
|
+
open_sessions=None, chunksize=CopyToConstants.DBAPI_BATCHSIZE.value,
|
|
955
|
+
match_column_order=True, err_tbl_1_suffix=None,
|
|
956
|
+
err_tbl_2_suffix=None, err_tbl_name=None, warn_tbl_name=None,
|
|
957
|
+
err_staging_db=None):
|
|
958
|
+
"""
|
|
959
|
+
DESCRIPTION:
|
|
960
|
+
Constructor for the _DataTransferUtils class. It initialises
|
|
961
|
+
arguments that are required for data transfer using FastExport
|
|
962
|
+
protocol or non-fastexport based data transfer using to_pandas()
|
|
963
|
+
API.
|
|
964
|
+
|
|
965
|
+
PARAMETERS:
|
|
966
|
+
df:
|
|
967
|
+
Required Argument.
|
|
968
|
+
Specifies the teradataml DataFrame from which data is to be
|
|
969
|
+
extracted. OR
|
|
970
|
+
Specifies the CSV filepath including name of the file to
|
|
971
|
+
load the data from.
|
|
972
|
+
Types: teradataml DataFrame OR str
|
|
973
|
+
|
|
974
|
+
index_column:
|
|
975
|
+
Optional Argument.
|
|
976
|
+
Specifies column(s) to be used as index column for the converted
|
|
977
|
+
object.
|
|
978
|
+
Types: str OR list of Strings (str)
|
|
979
|
+
|
|
980
|
+
num_rows:
|
|
981
|
+
Optional Argument.
|
|
982
|
+
Specifies the number of rows to be retrieved from teradataml
|
|
983
|
+
DataFrame.
|
|
984
|
+
Default Value: 99999
|
|
985
|
+
Types: int
|
|
986
|
+
|
|
987
|
+
all_rows:
|
|
988
|
+
Optional Argument.
|
|
989
|
+
Specifies whether all rows from teradataml DataFrame should be
|
|
990
|
+
retrieved.
|
|
991
|
+
Default Value: False
|
|
992
|
+
Types: bool
|
|
993
|
+
|
|
994
|
+
catch_errors_warnings:
|
|
995
|
+
Optional Argument.
|
|
996
|
+
Specifies whether to catch errors/warnings(if any) raised by
|
|
997
|
+
fastexport protocol while converting teradataml DataFrame.
|
|
998
|
+
Default Value: False
|
|
999
|
+
Types: bool
|
|
1000
|
+
|
|
1001
|
+
table_name:
|
|
1002
|
+
Optional Argument.
|
|
1003
|
+
Specifies the table name to load the data into.
|
|
1004
|
+
Types: String
|
|
1005
|
+
|
|
1006
|
+
types:
|
|
1007
|
+
Optional Argument.
|
|
1008
|
+
Specifies the data types for requested columns to be saved in Vantage.
|
|
1009
|
+
Keys of this dictionary should be the name of the columns and values should be
|
|
1010
|
+
teradatasqlalchemy.types.
|
|
1011
|
+
Default Value: None
|
|
1012
|
+
Note:
|
|
1013
|
+
This should be OrderedDict, if CSV file does not contain header.
|
|
1014
|
+
Types: OrderedDict
|
|
1015
|
+
|
|
1016
|
+
sep:
|
|
1017
|
+
Optional Argument.
|
|
1018
|
+
Specifies a single character string used to separate fields in a CSV file.
|
|
1019
|
+
Default Value: ","
|
|
1020
|
+
Notes:
|
|
1021
|
+
* "sep" cannot be line feed ('\\n') or carriage return ('\\r').
|
|
1022
|
+
* "sep" should not be same as "quotechar".
|
|
1023
|
+
* Length of "sep" argument should be 1.
|
|
1024
|
+
Types: String
|
|
1025
|
+
|
|
1026
|
+
quotechar:
|
|
1027
|
+
Optional Argument.
|
|
1028
|
+
Specifies a single character string used to quote fields in a CSV file.
|
|
1029
|
+
Default Value: "\""
|
|
1030
|
+
Notes:
|
|
1031
|
+
* "quotechar" cannot be line feed ('\\n') or carriage return ('\\r').
|
|
1032
|
+
* "quotechar" should not be same as "sep".
|
|
1033
|
+
* Length of "quotechar" argument should be 1.
|
|
1034
|
+
Types: String
|
|
1035
|
+
|
|
1036
|
+
schema_name:
|
|
1037
|
+
Optional Argument.
|
|
1038
|
+
Specifies the name of the database schema in Vantage to write to.
|
|
1039
|
+
Default Value: None (Uses default database schema).
|
|
1040
|
+
Types: String
|
|
1041
|
+
|
|
1042
|
+
if_exists:
|
|
1043
|
+
Optional Argument.
|
|
1044
|
+
Specifies the action to take when table already exists in Vantage.
|
|
1045
|
+
Permitted Values: 'fail', 'replace', 'append'
|
|
1046
|
+
- fail: If table exists, raise TeradataMlException.
|
|
1047
|
+
- replace: If table exists, drop it, recreate it, and insert data.
|
|
1048
|
+
- append: If table exists, insert data. Create if does not exist.
|
|
1049
|
+
Default Value: fail
|
|
1050
|
+
Types: String
|
|
1051
|
+
|
|
1052
|
+
primary_index:
|
|
1053
|
+
Optional Argument.
|
|
1054
|
+
Specifies which column(s) to use as primary index while creating table
|
|
1055
|
+
in Vantage. When set to None, No Primary Index (NoPI) tables are created.
|
|
1056
|
+
Default Value: None
|
|
1057
|
+
Types: String or list of strings
|
|
1058
|
+
Example:
|
|
1059
|
+
primary_index = 'my_primary_index'
|
|
1060
|
+
primary_index = ['my_primary_index1', 'my_primary_index2', 'my_primary_index3']
|
|
1061
|
+
|
|
1062
|
+
temporary:
|
|
1063
|
+
Optional Argument.
|
|
1064
|
+
Specifies whether to creates Vantage tables as permanent or volatile.
|
|
1065
|
+
When set to True:
|
|
1066
|
+
1. volatile Tables are created, and
|
|
1067
|
+
2. schema_name is ignored.
|
|
1068
|
+
When set to False, permanent tables are created.
|
|
1069
|
+
Default Value: False
|
|
1070
|
+
Types : Boolean (True or False)
|
|
1071
|
+
|
|
1072
|
+
set_table:
|
|
1073
|
+
Optional Argument.
|
|
1074
|
+
Specifies a flag to determine whether to create a SET or a MULTISET table.
|
|
1075
|
+
When set to True, a SET table is created, otherwise MULTISET table is created.
|
|
1076
|
+
Default Value: False
|
|
1077
|
+
Notes:
|
|
1078
|
+
1. Specifying set_table=True also requires specifying primary_index.
|
|
1079
|
+
2. Creating SET table (set_table=True) results in
|
|
1080
|
+
a. loss of duplicate rows.
|
|
1081
|
+
3. This argument has no effect if the table already exists and if_exists='append'.
|
|
1082
|
+
Types: Boolean
|
|
1083
|
+
|
|
1084
|
+
save_errors:
|
|
1085
|
+
Optional Argument.
|
|
1086
|
+
Specifies whether to persist the errors/warnings(if any) information in Vantage
|
|
1087
|
+
or not. If save_errors is set to False, errors/warnings(if any) are not persisted
|
|
1088
|
+
as tables. If argument is set to True, the error and warnings information
|
|
1089
|
+
are persisted and names of error and warning tables are returned. Otherwise,
|
|
1090
|
+
the function returns None for the names of the tables.
|
|
1091
|
+
Default Value: False
|
|
1092
|
+
Types: bool
|
|
1093
|
+
|
|
1094
|
+
open_sessions:
|
|
1095
|
+
Optional Argument.
|
|
1096
|
+
Specifies the number of Teradata data transfer sessions to be opened for fastload operation.
|
|
1097
|
+
Note :
|
|
1098
|
+
If "open_sessions" argument is not provided, the default value is
|
|
1099
|
+
the one smaller out of 8 and the number of AMPs available.
|
|
1100
|
+
Default Value: None
|
|
1101
|
+
Types: int
|
|
1102
|
+
|
|
1103
|
+
chunksize:
|
|
1104
|
+
Optional Argument.
|
|
1105
|
+
Specifies the number of rows to be loaded in a batch.
|
|
1106
|
+
Note:
|
|
1107
|
+
This is argument is used only when argument "df" is pandas DataFrame.
|
|
1108
|
+
Default Value: 16383
|
|
1109
|
+
Types: int
|
|
1110
|
+
|
|
1111
|
+
err_tbl_1_suffix:
|
|
1112
|
+
Optional Argument.
|
|
1113
|
+
Specifies the suffix for error table 1 created by fastload job.
|
|
1114
|
+
Types: String
|
|
1115
|
+
|
|
1116
|
+
err_tbl_2_suffix:
|
|
1117
|
+
Optional Argument.
|
|
1118
|
+
Specifies the suffix for error table 2 created by fastload job.
|
|
1119
|
+
Types: String
|
|
1120
|
+
|
|
1121
|
+
err_tbl_name:
|
|
1122
|
+
Optional Argument.
|
|
1123
|
+
Specifies the name for error table.
|
|
1124
|
+
Types: String
|
|
1125
|
+
|
|
1126
|
+
warn_tbl_name:
|
|
1127
|
+
Optional Argument.
|
|
1128
|
+
Specifies the name for warning table.
|
|
1129
|
+
Types: String
|
|
1130
|
+
|
|
1131
|
+
err_staging_db:
|
|
1132
|
+
Optional Argument.
|
|
1133
|
+
Specifies the name of the database to be used for creating staging
|
|
1134
|
+
table and error tables.
|
|
1135
|
+
Note:
|
|
1136
|
+
Current session user must have CREATE, DELETE and INSERT table
|
|
1137
|
+
rights on err_staging_db database.
|
|
1138
|
+
Types: String
|
|
1139
|
+
|
|
1140
|
+
PARAMETERS:
|
|
1141
|
+
None.
|
|
1142
|
+
|
|
1143
|
+
RETURNS:
|
|
1144
|
+
None.
|
|
1145
|
+
|
|
1146
|
+
RAISES:
|
|
1147
|
+
None.
|
|
1148
|
+
|
|
1149
|
+
EXAMPLES:
|
|
1150
|
+
dt_obj = _DataTransferUtils(df)
|
|
1151
|
+
dt_obj = _DataTransferUtils(df, index_column='gpa')
|
|
1152
|
+
dt_obj = _DataTransferUtils(df, num_rows=10)
|
|
1153
|
+
dt_obj = _DataTransferUtils(df, all_rows=True, num_rows=5)
|
|
1154
|
+
dt_obj = _DataTransferUtils(df, catch_errors_warnings=True,
|
|
1155
|
+
num_rows=200)
|
|
1156
|
+
dt_obj = _DataTransferUtils(df, table_name='my_table',
|
|
1157
|
+
types={'id': BIGINT, 'fname': VARCHAR,
|
|
1158
|
+
'lname': VARCHAR, 'marks': FLOAT})
|
|
1159
|
+
"""
|
|
1160
|
+
|
|
1161
|
+
self.df = df
|
|
1162
|
+
self.index_column = index_column
|
|
1163
|
+
self.num_rows = num_rows
|
|
1164
|
+
self.all_rows = all_rows
|
|
1165
|
+
self.catch_errors_warnings = catch_errors_warnings
|
|
1166
|
+
self.table_name = table_name
|
|
1167
|
+
self.schema_name = schema_name
|
|
1168
|
+
self.if_exists = if_exists
|
|
1169
|
+
self.index = index
|
|
1170
|
+
self.index_label = index_label
|
|
1171
|
+
self.primary_index = primary_index
|
|
1172
|
+
self.temporary = temporary
|
|
1173
|
+
self.types = types
|
|
1174
|
+
self.batch_size = batch_size
|
|
1175
|
+
self.save_errors = save_errors
|
|
1176
|
+
self.sep = sep
|
|
1177
|
+
self.quotechar = quotechar
|
|
1178
|
+
self.primary_time_index_name = primary_time_index_name
|
|
1179
|
+
self.timecode_column = timecode_column
|
|
1180
|
+
self.timebucket_duration = timebucket_duration
|
|
1181
|
+
self.timezero_date = timezero_date
|
|
1182
|
+
self.columns_list = columns_list
|
|
1183
|
+
self.sequence_column = sequence_column
|
|
1184
|
+
self.seq_max = seq_max
|
|
1185
|
+
self.set_table = set_table
|
|
1186
|
+
self.use_fastload = use_fastload
|
|
1187
|
+
self.api_name = api_name
|
|
1188
|
+
self.open_sessions = open_sessions
|
|
1189
|
+
self.chunksize = chunksize
|
|
1190
|
+
self.match_column_order = match_column_order
|
|
1191
|
+
self.err_tbl_1_suffix = err_tbl_1_suffix
|
|
1192
|
+
self.err_tbl_2_suffix = err_tbl_2_suffix
|
|
1193
|
+
self.err_tbl_name = err_tbl_name
|
|
1194
|
+
self.warn_tbl_name = warn_tbl_name
|
|
1195
|
+
self.err_staging_db = err_staging_db
|
|
1196
|
+
|
|
1197
|
+
# Validate arguments.
|
|
1198
|
+
if self.api_name == 'fastexport':
|
|
1199
|
+
self._validate_data_export_api_args()
|
|
1200
|
+
|
|
1201
|
+
# Functions specific to validation of arguments of
|
|
1202
|
+
# Datatransfer APIs.
|
|
1203
|
+
def _validate_data_export_api_args(self):
|
|
1204
|
+
"""
|
|
1205
|
+
DESCRIPTION:
|
|
1206
|
+
Function to validate common arguments used in data export API's
|
|
1207
|
+
such as:
|
|
1208
|
+
1. DataFrame.to_pandas()
|
|
1209
|
+
2. DataFrame.to_csv()
|
|
1210
|
+
2. fastexport()
|
|
1211
|
+
|
|
1212
|
+
PARAMETERS:
|
|
1213
|
+
None.
|
|
1214
|
+
|
|
1215
|
+
RETURNS:
|
|
1216
|
+
None.
|
|
1217
|
+
|
|
1218
|
+
RAISES:
|
|
1219
|
+
TeradataMlException,
|
|
1220
|
+
TypeError,
|
|
1221
|
+
ValueError.
|
|
1222
|
+
|
|
1223
|
+
EXAMPLES:
|
|
1224
|
+
dt_obj = _DataTransferUtils(df)
|
|
1225
|
+
dt_obj._validate_data_export_api_args()
|
|
1226
|
+
"""
|
|
1227
|
+
awu_matrix = []
|
|
1228
|
+
awu_matrix.append(
|
|
1229
|
+
["index_column", self.index_column, True, (str, list), True])
|
|
1230
|
+
awu_matrix.append(["num_rows", self.num_rows, True, (int)])
|
|
1231
|
+
awu_matrix.append(["all_rows", self.all_rows, True, (bool)])
|
|
1232
|
+
awu_matrix.append(
|
|
1233
|
+
["catch_errors_warnings", self.catch_errors_warnings, True, (bool)])
|
|
1234
|
+
awu_matrix.append(["open_sessions", self.open_sessions, True, (int), False])
|
|
1235
|
+
awu_matrix.append(["sep", self.sep, True, str, False])
|
|
1236
|
+
awu_matrix.append(["quotechar", self.quotechar, True, str, False])
|
|
1237
|
+
|
|
1238
|
+
# Validate argument types.
|
|
1239
|
+
_Validators._validate_function_arguments(awu_matrix)
|
|
1240
|
+
# Validate if 'num_rows' is a positive int.
|
|
1241
|
+
_Validators._validate_positive_int(self.num_rows, "num_rows")
|
|
1242
|
+
# Validate if 'open_sessions' is a positive int.
|
|
1243
|
+
_Validators._validate_positive_int(self.open_sessions, "open_sessions")
|
|
1244
|
+
|
|
1245
|
+
# Validate "sep" and "quotechar" arguments related to export to CSV.
|
|
1246
|
+
self._validate_csv_sep_quotechar()
|
|
1247
|
+
|
|
1248
|
+
# Checking if meta expression exists for given dataframe.
|
|
1249
|
+
if self.df._metaexpr is None:
|
|
1250
|
+
raise TeradataMlException(
|
|
1251
|
+
Messages.get_message(MessageCodes.TDMLDF_INFO_ERROR),
|
|
1252
|
+
MessageCodes.TDMLDF_INFO_ERROR)
|
|
1253
|
+
|
|
1254
|
+
# Checking each element in passed columns to be valid column in
|
|
1255
|
+
# dataframe.
|
|
1256
|
+
_Validators._validate_column_exists_in_dataframe(self.index_column,
|
|
1257
|
+
self.df._metaexpr)
|
|
1258
|
+
|
|
1259
|
+
def _validate_read_csv_api_args(self):
|
|
1260
|
+
"""
|
|
1261
|
+
Internal function to validate read_csv api arguments.
|
|
1262
|
+
|
|
1263
|
+
PARAMETERS:
|
|
1264
|
+
None
|
|
1265
|
+
|
|
1266
|
+
RETURNS:
|
|
1267
|
+
None
|
|
1268
|
+
|
|
1269
|
+
RAISES:
|
|
1270
|
+
None
|
|
1271
|
+
|
|
1272
|
+
EXAMPLES:
|
|
1273
|
+
dt_obj = _DataTransferUtils()
|
|
1274
|
+
dt_obj._validate_read_csv_api_args()
|
|
1275
|
+
"""
|
|
1276
|
+
# Validate read_csv api arguments.
|
|
1277
|
+
self._validate()
|
|
1278
|
+
|
|
1279
|
+
# Validate "sep" and "quotechar" arguments.
|
|
1280
|
+
self._validate_csv_sep_quotechar()
|
|
1281
|
+
|
|
1282
|
+
def _validate_csv_sep_quotechar(self):
|
|
1283
|
+
"""
|
|
1284
|
+
Internal function to validate field separator and field quote character
|
|
1285
|
+
used in data transfer APIs which involve CSV.
|
|
1286
|
+
|
|
1287
|
+
PARAMETERS:
|
|
1288
|
+
None
|
|
1289
|
+
|
|
1290
|
+
RETURNS:
|
|
1291
|
+
None
|
|
1292
|
+
|
|
1293
|
+
RAISES:
|
|
1294
|
+
TeradataMlException
|
|
1295
|
+
|
|
1296
|
+
EXAMPLES:
|
|
1297
|
+
dt_obj = _DataTransferUtils()
|
|
1298
|
+
dt_obj._validate_csv_sep_quotechar()
|
|
1299
|
+
"""
|
|
1300
|
+
|
|
1301
|
+
# Function to validate char value for length and allowed characters.
|
|
1302
|
+
def validate_char_arg_csv(arg_name, arg):
|
|
1303
|
+
|
|
1304
|
+
if arg is not None:
|
|
1305
|
+
_Validators._validate_str_arg_length(arg_name, arg, 'EQ', 1)
|
|
1306
|
+
|
|
1307
|
+
not_allowed_values = ["\n", "\r"]
|
|
1308
|
+
if arg in not_allowed_values:
|
|
1309
|
+
message = Messages.get_message(MessageCodes.NOT_ALLOWED_VALUES,
|
|
1310
|
+
"{}".format(not_allowed_values), arg_name)
|
|
1311
|
+
raise TeradataMlException(message, MessageCodes.NOT_ALLOWED_VALUES)
|
|
1312
|
+
|
|
1313
|
+
# Validate the 'sep' and 'quotechar' arguments.
|
|
1314
|
+
validate_char_arg_csv("sep", self.sep)
|
|
1315
|
+
validate_char_arg_csv("quotechar", self.quotechar)
|
|
1316
|
+
|
|
1317
|
+
# Validate 'quotechar' and 'sep' arguments not set to same value
|
|
1318
|
+
if self.quotechar == self.sep:
|
|
1319
|
+
message = Messages.get_message(MessageCodes.ARGUMENT_VALUE_SAME, "sep",
|
|
1320
|
+
"quotechar")
|
|
1321
|
+
raise TeradataMlException(message, MessageCodes.ARGUMENT_VALUE_SAME)
|
|
1322
|
+
|
|
1323
|
+
# End of functions specific to validation of arguments of
|
|
1324
|
+
# Datatransfer APIs.
|
|
1325
|
+
|
|
1326
|
+
# Functions specific to fastexport().
|
|
1327
|
+
def _validate_df_index_column(self):
|
|
1328
|
+
"""
|
|
1329
|
+
DESCRIPTION:
|
|
1330
|
+
Function to validate dataframe index label and throw exception if
|
|
1331
|
+
there is any mismatch in the index label and columns present in the
|
|
1332
|
+
teradataml DataFrame.
|
|
1333
|
+
|
|
1334
|
+
PARAMETERS:
|
|
1335
|
+
None.
|
|
1336
|
+
|
|
1337
|
+
RETURNS:
|
|
1338
|
+
None.
|
|
1339
|
+
|
|
1340
|
+
RAISES:
|
|
1341
|
+
TeradataMLException.
|
|
1342
|
+
|
|
1343
|
+
EXAMPLES:
|
|
1344
|
+
dt_obj = _DataTransferUtils(df)
|
|
1345
|
+
dt_obj._validate_df_index_column()
|
|
1346
|
+
"""
|
|
1347
|
+
# Get list of columns in teradatml DataFrame.
|
|
1348
|
+
df_column_list = [col.name.lower() for col in self.df._metaexpr.c]
|
|
1349
|
+
|
|
1350
|
+
# Check if TDML DF has appropriate index_label set when required
|
|
1351
|
+
if self.df._index_label is not None:
|
|
1352
|
+
for index_label in UtilFuncs._as_list(self.df._index_label):
|
|
1353
|
+
if index_label.lower() not in df_column_list:
|
|
1354
|
+
raise TeradataMlException(
|
|
1355
|
+
Messages.get_message(MessageCodes.DF_LABEL_MISMATCH),
|
|
1356
|
+
MessageCodes.DF_LABEL_MISMATCH)
|
|
1357
|
+
|
|
1358
|
+
def _get_pandas_df_index(self):
|
|
1359
|
+
"""
|
|
1360
|
+
DESCRIPTION:
|
|
1361
|
+
Function returns the final index column to be used in the resultant
|
|
1362
|
+
DataFrame after converting teradataml DataFrame to Pandas DataFrame.
|
|
1363
|
+
|
|
1364
|
+
PARAMETERS:
|
|
1365
|
+
None.
|
|
1366
|
+
|
|
1367
|
+
RETURNS:
|
|
1368
|
+
Final Valid index as str or list of Strings.
|
|
1369
|
+
|
|
1370
|
+
RAISES:
|
|
1371
|
+
None.
|
|
1372
|
+
|
|
1373
|
+
EXAMPLES:
|
|
1374
|
+
dt_obj = _DataTransferUtils(df)
|
|
1375
|
+
dt_obj._get_pandas_df_index()
|
|
1376
|
+
"""
|
|
1377
|
+
index_col = None
|
|
1378
|
+
# Index Order: 1) User specified 2) TDMLDF index 3) DB PI
|
|
1379
|
+
# 4)Else default integer index
|
|
1380
|
+
if self.index_column:
|
|
1381
|
+
index_col = self.index_column
|
|
1382
|
+
elif self.df._index_label:
|
|
1383
|
+
index_col = self.df._index_label
|
|
1384
|
+
else:
|
|
1385
|
+
try:
|
|
1386
|
+
from teradataml.dataframe.dataframe_utils import DataFrameUtils
|
|
1387
|
+
index_col = DataFrameUtils._get_primary_index_from_table(
|
|
1388
|
+
self.df._table_name)
|
|
1389
|
+
except Exception as err:
|
|
1390
|
+
index_col = None
|
|
1391
|
+
|
|
1392
|
+
return index_col
|
|
1393
|
+
|
|
1394
|
+
def _generate_to_pandas_base_query(self):
|
|
1395
|
+
"""
|
|
1396
|
+
DESCRIPTION:
|
|
1397
|
+
Function to generate base query for to_pandas() function. This query
|
|
1398
|
+
is further used to generate pandas dataframe.
|
|
1399
|
+
|
|
1400
|
+
PARAMETERS:
|
|
1401
|
+
None.
|
|
1402
|
+
|
|
1403
|
+
RETURNS:
|
|
1404
|
+
str.
|
|
1405
|
+
|
|
1406
|
+
RAISES:
|
|
1407
|
+
None.
|
|
1408
|
+
|
|
1409
|
+
EXAMPLES:
|
|
1410
|
+
dt_obj = _DataTransferUtils(df)
|
|
1411
|
+
base_query = dt_obj._generate_to_pandas_base_query()
|
|
1412
|
+
"""
|
|
1413
|
+
# Generate SQL Query using Table name & number of rows required.
|
|
1414
|
+
if self.all_rows:
|
|
1415
|
+
# Get read query for the whole data.
|
|
1416
|
+
return SQLBundle._build_base_query(self.df._table_name,
|
|
1417
|
+
self.df._orderby)
|
|
1418
|
+
else:
|
|
1419
|
+
# Get read query using SAMPLE.
|
|
1420
|
+
return SQLBundle._build_sample_rows_from_table(self.df._table_name,
|
|
1421
|
+
self.num_rows,
|
|
1422
|
+
self.df._orderby)
|
|
1423
|
+
|
|
1424
|
+
def _generate_select_query(self):
|
|
1425
|
+
"""
|
|
1426
|
+
DESCRIPTION:
|
|
1427
|
+
Function to generate SELECT query.
|
|
1428
|
+
|
|
1429
|
+
PARAMETERS:
|
|
1430
|
+
None.
|
|
1431
|
+
|
|
1432
|
+
RETURNS:
|
|
1433
|
+
str.
|
|
1434
|
+
|
|
1435
|
+
RAISES:
|
|
1436
|
+
None.
|
|
1437
|
+
|
|
1438
|
+
EXAMPLES:
|
|
1439
|
+
dt_obj = _DataTransferUtils(df)
|
|
1440
|
+
base_query = dt_obj._generate_select_query()
|
|
1441
|
+
"""
|
|
1442
|
+
# Generate SQL Query using Table name & number of rows required.
|
|
1443
|
+
if self.all_rows:
|
|
1444
|
+
# Get read query for the whole data.
|
|
1445
|
+
return SQLBundle._build_base_query(self.df._table_name,
|
|
1446
|
+
self.df._orderby)
|
|
1447
|
+
else:
|
|
1448
|
+
# Get read query using SAMPLE.
|
|
1449
|
+
return SQLBundle._build_top_n_print_query(self.df._table_name,
|
|
1450
|
+
self.num_rows,
|
|
1451
|
+
self.df._orderby)
|
|
1452
|
+
|
|
1453
|
+
def _generate_fastexport_query(self, base_query, require=False, open_sessions=None,
|
|
1454
|
+
csv_file_name=None):
|
|
1455
|
+
"""
|
|
1456
|
+
DESCRIPTION:
|
|
1457
|
+
Function to generate fastexport compatible query.
|
|
1458
|
+
|
|
1459
|
+
PARAMETERS:
|
|
1460
|
+
base_query:
|
|
1461
|
+
Required Argument.
|
|
1462
|
+
Specifies the base query to be used for forming the fastexport
|
|
1463
|
+
query.
|
|
1464
|
+
Types: str
|
|
1465
|
+
|
|
1466
|
+
require:
|
|
1467
|
+
Optional Argument.
|
|
1468
|
+
Specifies whether fastexport protocol is required for data
|
|
1469
|
+
transfer.
|
|
1470
|
+
Default Value: False
|
|
1471
|
+
Types: bool
|
|
1472
|
+
|
|
1473
|
+
open_sessions:
|
|
1474
|
+
Optional Argument.
|
|
1475
|
+
Specifies the number of Teradata sessions to be opened for fastexport.
|
|
1476
|
+
Default value: None
|
|
1477
|
+
Types: int
|
|
1478
|
+
|
|
1479
|
+
csv_file_name:
|
|
1480
|
+
Optional Argument.
|
|
1481
|
+
Specifies the name of CSV file to which data is to be exported.
|
|
1482
|
+
Types: Str
|
|
1483
|
+
|
|
1484
|
+
RETURNS:
|
|
1485
|
+
str.
|
|
1486
|
+
|
|
1487
|
+
RAISES:
|
|
1488
|
+
None.
|
|
1489
|
+
|
|
1490
|
+
EXAMPLES:
|
|
1491
|
+
dt_obj = _DataTransferUtils(df)
|
|
1492
|
+
base_query = "select * from my_table SAMPLE 200"
|
|
1493
|
+
target_csv = "Test.csv"
|
|
1494
|
+
dt_obj._generate_fastexport_query(base_query)
|
|
1495
|
+
dt_obj._generate_fastexport_query(base_query, require=True)
|
|
1496
|
+
dt_obj._generate_fastexport_query(base_query, require=True, open_sessions=5)
|
|
1497
|
+
dt_obj._generate_fastexport_query(base_query, require=True, csv_file_name=target_csv)
|
|
1498
|
+
dt_obj._generate_fastexport_query(base_query, require=True, open_sessions = 2,
|
|
1499
|
+
csv_file_name=target_csv)
|
|
1500
|
+
|
|
1501
|
+
"""
|
|
1502
|
+
fastexport_esc_func = ""
|
|
1503
|
+
open_session_esc_func = ""
|
|
1504
|
+
if require is not None:
|
|
1505
|
+
if require:
|
|
1506
|
+
# If require is set to True, we are using
|
|
1507
|
+
# 'teradata_require_fastexport' escape sequence as this will run
|
|
1508
|
+
# query using fastexport only if the given query is compatible with
|
|
1509
|
+
# fastexport else raises error.
|
|
1510
|
+
fastexport_esc_func = DriverEscapeFunctions.REQUIRE_FASTEXPORT.value
|
|
1511
|
+
else:
|
|
1512
|
+
# If require is False, we are using 'teradata_try_fastexport'
|
|
1513
|
+
# escape sequence as this will run query using fastexport if the
|
|
1514
|
+
# given query is compatible with fastexport else runs it as
|
|
1515
|
+
# regular query.
|
|
1516
|
+
fastexport_esc_func = DriverEscapeFunctions.TRY_FASTEXPORT.value
|
|
1517
|
+
|
|
1518
|
+
if open_sessions is not None:
|
|
1519
|
+
open_session_esc_func = DriverEscapeFunctions.OPEN_SESSIONS.value.format(open_sessions)
|
|
1520
|
+
|
|
1521
|
+
write_csv_escape_func = ""
|
|
1522
|
+
field_sep_esc_func = ""
|
|
1523
|
+
field_quote_esc_func = ""
|
|
1524
|
+
|
|
1525
|
+
if csv_file_name:
|
|
1526
|
+
# The teradata_field_sep and teradata_field_quote escape functions have a
|
|
1527
|
+
# single-character string argument. The string argument must follow SQL literal
|
|
1528
|
+
# syntax. The string argument may be enclosed in single-quote (') characters or
|
|
1529
|
+
# double-quote (") characters.
|
|
1530
|
+
field_sep = "'{0}'".format(self.sep)
|
|
1531
|
+
if self.sep == "'":
|
|
1532
|
+
field_sep = "''''"
|
|
1533
|
+
elif self.sep == "\"":
|
|
1534
|
+
field_sep = "\"\"\"\""
|
|
1535
|
+
# To represent a single-quote character in a string enclosed in single-quote
|
|
1536
|
+
# characters, you must repeat the single-quote character.
|
|
1537
|
+
# {fn teradata_field_quote('''')}
|
|
1538
|
+
# To represent a double-quote character in a string enclosed in double-quote
|
|
1539
|
+
# characters, you must repeat the double-quote character.
|
|
1540
|
+
# {fn teradata_field_quote("""")}
|
|
1541
|
+
field_quote = "'{0}'".format(self.quotechar)
|
|
1542
|
+
if self.quotechar == "'":
|
|
1543
|
+
field_quote = "''''"
|
|
1544
|
+
elif self.quotechar == "\"":
|
|
1545
|
+
field_quote = "\"\"\"\""
|
|
1546
|
+
|
|
1547
|
+
write_csv_escape_func = DriverEscapeFunctions.WRITE_TO_CSV.value.format(csv_file_name)
|
|
1548
|
+
field_sep_esc_func = DriverEscapeFunctions.FIELD_SEP.value.format(field_sep)
|
|
1549
|
+
field_quote_esc_func = DriverEscapeFunctions.FIELD_QUOTE.value.format(field_quote)
|
|
1550
|
+
|
|
1551
|
+
query = "{0}{1}{2}{3}{4}{5}".format(fastexport_esc_func,
|
|
1552
|
+
open_session_esc_func,
|
|
1553
|
+
field_sep_esc_func,
|
|
1554
|
+
field_quote_esc_func,
|
|
1555
|
+
write_csv_escape_func,
|
|
1556
|
+
base_query)
|
|
1557
|
+
|
|
1558
|
+
return query
|
|
1559
|
+
|
|
1560
|
+
def _process_fastexport_errors_warnings(self, query):
|
|
1561
|
+
"""
|
|
1562
|
+
DESCRIPTION:
|
|
1563
|
+
Function to process errors/warnings(if any) raised while executing
|
|
1564
|
+
the fastexport protocol.
|
|
1565
|
+
|
|
1566
|
+
PARAMETERS:
|
|
1567
|
+
query:
|
|
1568
|
+
Required Argument.
|
|
1569
|
+
Specifies the query with fastexport escape sequences that is
|
|
1570
|
+
used to convert teradataml DataFrame to Pandas DataFrame.
|
|
1571
|
+
Type: str
|
|
1572
|
+
|
|
1573
|
+
RETURNS:
|
|
1574
|
+
A tuple with two lists for errors, warnings each containing err/warn
|
|
1575
|
+
messages in string format.
|
|
1576
|
+
|
|
1577
|
+
RAISES:
|
|
1578
|
+
None.
|
|
1579
|
+
|
|
1580
|
+
EXAMPLES:
|
|
1581
|
+
dt_obj = _DataTransferUtils(df)
|
|
1582
|
+
query = "{fn teradata_try_fastexport}select * from my_table SAMPLE
|
|
1583
|
+
200"
|
|
1584
|
+
err, warn = dt_obj._process_fastexport_errors_warnings(query)
|
|
1585
|
+
"""
|
|
1586
|
+
err = None
|
|
1587
|
+
warn = None
|
|
1588
|
+
conn = get_connection().connection
|
|
1589
|
+
# Create a cursor from connection object.
|
|
1590
|
+
cur = conn.cursor()
|
|
1591
|
+
# Get err/warn
|
|
1592
|
+
err = self._get_errors_warnings(cur, query,
|
|
1593
|
+
DriverEscapeFunctions.GET_ERRORS)
|
|
1594
|
+
warn = self._get_errors_warnings(cur, query,
|
|
1595
|
+
DriverEscapeFunctions.GET_WARNINGS)
|
|
1596
|
+
return err, warn
|
|
1597
|
+
|
|
1598
|
+
# Functions specific to exporting table data in Vantage into pandas DataFrame.
|
|
1599
|
+
def _get_pandas_dataframe(self, **kwargs):
|
|
1600
|
+
"""
|
|
1601
|
+
DESCRIPTION:
|
|
1602
|
+
Function that converts teradataml DataFrame to Pandas DataFrame
|
|
1603
|
+
using regular approach.
|
|
1604
|
+
|
|
1605
|
+
PARAMETERS:
|
|
1606
|
+
kwargs:
|
|
1607
|
+
Specifies keyword arguments.
|
|
1608
|
+
|
|
1609
|
+
RETURNS:
|
|
1610
|
+
Pandas DataFrame.
|
|
1611
|
+
|
|
1612
|
+
RAISES:
|
|
1613
|
+
None.
|
|
1614
|
+
|
|
1615
|
+
EXAMPLES:
|
|
1616
|
+
dt_obj = _DataTransferUtils(df)
|
|
1617
|
+
dt_obj._get_pandas_dataframe()
|
|
1618
|
+
|
|
1619
|
+
"""
|
|
1620
|
+
# Get the final index column.
|
|
1621
|
+
final_index_column = self._get_pandas_df_index()
|
|
1622
|
+
# Get the base query.
|
|
1623
|
+
base_query = self._generate_to_pandas_base_query()
|
|
1624
|
+
# Generate pandas dataframe using base query.
|
|
1625
|
+
pandas_df = _execute_query_and_generate_pandas_df(base_query,
|
|
1626
|
+
final_index_column,
|
|
1627
|
+
**kwargs)
|
|
1628
|
+
return pandas_df
|
|
1629
|
+
|
|
1630
|
+
def _fastexport_get_pandas_df(self, require=False, **kwargs):
|
|
1631
|
+
"""
|
|
1632
|
+
DESCRIPTION:
|
|
1633
|
+
Internal function to convert teradataml DataFrame to Pandas
|
|
1634
|
+
DataFrame using FastExport protocol. This internal function can be
|
|
1635
|
+
directly used in to_pandas() and fastexport API's if either of
|
|
1636
|
+
the functions has to use fastexport.
|
|
1637
|
+
|
|
1638
|
+
PARAMETERS:
|
|
1639
|
+
require:
|
|
1640
|
+
Optional Argument.
|
|
1641
|
+
Specifies whether fastexport protocol is required for data
|
|
1642
|
+
transfer.
|
|
1643
|
+
Default Value: False
|
|
1644
|
+
Types: bool
|
|
1645
|
+
|
|
1646
|
+
kwargs:
|
|
1647
|
+
Specifies keyword arguments. Argument "open_sessions"
|
|
1648
|
+
can be passed as keyword arguments.
|
|
1649
|
+
* "open_sessions" specifies the number of Teradata sessions to
|
|
1650
|
+
be opened for fastexport.
|
|
1651
|
+
|
|
1652
|
+
RETURNS:
|
|
1653
|
+
When "catch_errors_warnings" is set to True, the function returns
|
|
1654
|
+
a tuple containing:
|
|
1655
|
+
* Pandas DataFrame.
|
|
1656
|
+
* Errors, if any, thrown by fastexport in a list of strings.
|
|
1657
|
+
* Warnings, if any, thrown by fastexport in a list of strings.
|
|
1658
|
+
Only Pandas DataFrame otherwise.
|
|
1659
|
+
|
|
1660
|
+
RAISES:
|
|
1661
|
+
TeradataMlException.
|
|
1662
|
+
|
|
1663
|
+
EXAMPLES:
|
|
1664
|
+
dt_obj = _DataTransferUtils(df)
|
|
1665
|
+
dt_obj._fastexport_get_pandas_df(require=False)
|
|
1666
|
+
|
|
1667
|
+
"""
|
|
1668
|
+
|
|
1669
|
+
try:
|
|
1670
|
+
self._validate_df_index_column()
|
|
1671
|
+
final_index_col = self._get_pandas_df_index()
|
|
1672
|
+
self.df._DataFrame__execute_node_and_set_table_name(self.df._nodeid,
|
|
1673
|
+
self.df._metaexpr)
|
|
1674
|
+
base_query = self._generate_to_pandas_base_query()
|
|
1675
|
+
|
|
1676
|
+
# Get open_sessions argument.
|
|
1677
|
+
open_sessions = kwargs.pop("open_sessions", None)
|
|
1678
|
+
fastexport_query = self._generate_fastexport_query(base_query,
|
|
1679
|
+
require=require,
|
|
1680
|
+
open_sessions=open_sessions)
|
|
1681
|
+
pandas_df = _execute_query_and_generate_pandas_df(fastexport_query,
|
|
1682
|
+
final_index_col,
|
|
1683
|
+
**kwargs)
|
|
1684
|
+
err, warn = \
|
|
1685
|
+
self._process_fastexport_errors_warnings(fastexport_query)
|
|
1686
|
+
if self.catch_errors_warnings:
|
|
1687
|
+
return pandas_df, err, warn
|
|
1688
|
+
else:
|
|
1689
|
+
print("Errors: {0}".format(err))
|
|
1690
|
+
print("Warnings: {0}".format(warn))
|
|
1691
|
+
return pandas_df
|
|
1692
|
+
except TeradataMlException:
|
|
1693
|
+
raise
|
|
1694
|
+
|
|
1695
|
+
# End of functions specific to exporting table data in Vantage into pandas DataFrame.
|
|
1696
|
+
|
|
1697
|
+
# General functions to get warrnings and errors.
|
|
1698
|
+
def _get_errors_warnings(self, cur, insert_stmt, escape_function):
|
|
1699
|
+
"""
|
|
1700
|
+
Internal function executes teradatasql provided escape functions
|
|
1701
|
+
to get the errors and warnings.
|
|
1702
|
+
|
|
1703
|
+
PARAMETERS:
|
|
1704
|
+
cur:
|
|
1705
|
+
Required Argument.
|
|
1706
|
+
The cursor of connection type which will be used to execute query.
|
|
1707
|
+
Types: teradatasql cursor object
|
|
1708
|
+
|
|
1709
|
+
insert_stmt:
|
|
1710
|
+
Required Argument.
|
|
1711
|
+
Statement to be executed along with escape method.
|
|
1712
|
+
Types: String
|
|
1713
|
+
|
|
1714
|
+
escape_function:
|
|
1715
|
+
Required Argument.
|
|
1716
|
+
Type of escape method to be passed.
|
|
1717
|
+
Types: String
|
|
1718
|
+
|
|
1719
|
+
RETURNS:
|
|
1720
|
+
A list containing error/warning information.
|
|
1721
|
+
|
|
1722
|
+
RAISES:
|
|
1723
|
+
None
|
|
1724
|
+
|
|
1725
|
+
EXAMPLES:
|
|
1726
|
+
dt_obj = _DataTransferUtils(df, table_name, types)
|
|
1727
|
+
dt_obj._get_errors_warnings(cur, insert_stmt, escape_function)
|
|
1728
|
+
"""
|
|
1729
|
+
errorwarninglist = self._process_escape_functions(cur,
|
|
1730
|
+
escape_function=escape_function,
|
|
1731
|
+
insert_query=insert_stmt)
|
|
1732
|
+
|
|
1733
|
+
from teradatasql import vernumber
|
|
1734
|
+
msg = []
|
|
1735
|
+
if errorwarninglist:
|
|
1736
|
+
if (errorwarninglist[0][0] != ""):
|
|
1737
|
+
msg = errorwarninglist[0][0].split('[Version ' + vernumber.sVersionNumber + ']')[1:]
|
|
1738
|
+
|
|
1739
|
+
return [err_msg.split("\n")[0] for err_msg in msg]
|
|
1740
|
+
|
|
1741
|
+
def _get_pandas_df_from_errors_warnings(self, err_or_warn_dict):
|
|
1742
|
+
"""
|
|
1743
|
+
DESCRIPTION:
|
|
1744
|
+
Internal function creates Pandas dataframe.
|
|
1745
|
+
|
|
1746
|
+
PARAMETERS:
|
|
1747
|
+
err_or_warn_dict:
|
|
1748
|
+
Required Argument.
|
|
1749
|
+
Specifies the error or warning dictionary.
|
|
1750
|
+
Types: Python dictionary
|
|
1751
|
+
|
|
1752
|
+
RETURNS:
|
|
1753
|
+
Pandas Dataframe.
|
|
1754
|
+
|
|
1755
|
+
RAISES:
|
|
1756
|
+
None.
|
|
1757
|
+
|
|
1758
|
+
EXAMPLES:
|
|
1759
|
+
dt_obj = _DataTransferUtils(df)
|
|
1760
|
+
dt_obj._get_pandas_df_from_errors_warnings(err_or_warn_dict)
|
|
1761
|
+
"""
|
|
1762
|
+
# For fastload API, we check the 'batch_no' key is present in dictionary and
|
|
1763
|
+
# whether it has non-zero value or not.
|
|
1764
|
+
# For read_csv API, we check whether the length of 'error_message' key's
|
|
1765
|
+
# value is greater than 0 or not.
|
|
1766
|
+
if ('batch_no' in err_or_warn_dict and bool(err_or_warn_dict.get('batch_no'))) or \
|
|
1767
|
+
len(err_or_warn_dict['error_message']) != 0:
|
|
1768
|
+
return pd.DataFrame(err_or_warn_dict)
|
|
1769
|
+
|
|
1770
|
+
return pd.DataFrame()
|
|
1771
|
+
|
|
1772
|
+
def _create_error_warnings_table(self, pdf, msg_type, logon_seq_number, table_name=None):
|
|
1773
|
+
"""
|
|
1774
|
+
DESCRIPTION:
|
|
1775
|
+
Internal function creates the errors and warnings table in Vantage.
|
|
1776
|
+
|
|
1777
|
+
PARAMETERS:
|
|
1778
|
+
pdf:
|
|
1779
|
+
Required Argument.
|
|
1780
|
+
Specifies pandas dataframe containing errors and warnings.
|
|
1781
|
+
Types: Pandas DataFrame
|
|
1782
|
+
|
|
1783
|
+
msg_type:
|
|
1784
|
+
Required Argument.
|
|
1785
|
+
Specifies the type of message.
|
|
1786
|
+
Possible Values : 'err', 'warn'
|
|
1787
|
+
Types: String
|
|
1788
|
+
|
|
1789
|
+
logon_seq_number:
|
|
1790
|
+
Required Argument.
|
|
1791
|
+
Specifies logon sequence number of the session.
|
|
1792
|
+
Types: Integer
|
|
1793
|
+
|
|
1794
|
+
RETURNS:
|
|
1795
|
+
str
|
|
1796
|
+
|
|
1797
|
+
RAISES:
|
|
1798
|
+
None
|
|
1799
|
+
|
|
1800
|
+
EXAMPLES:
|
|
1801
|
+
dt_obj = _DataTransferUtils(df, table_name, types)
|
|
1802
|
+
dt_obj._create_error_warnings_table(pdf, msg_type, logon_seq_number)
|
|
1803
|
+
"""
|
|
1804
|
+
if not table_name:
|
|
1805
|
+
table_name = "td_fl_{0}_{1}_{2}".format(self.table_name, msg_type, logon_seq_number)
|
|
1806
|
+
copy_to_sql(pdf, table_name, schema_name=self.err_staging_db,
|
|
1807
|
+
if_exists='replace')
|
|
1808
|
+
return "{}.{}".format(self.err_staging_db if self.err_staging_db
|
|
1809
|
+
else _get_current_databasename(),
|
|
1810
|
+
table_name)
|
|
1811
|
+
|
|
1812
|
+
def _process_escape_functions(self, cur, escape_function, insert_query=None):
|
|
1813
|
+
"""
|
|
1814
|
+
DESCRIPTION:
|
|
1815
|
+
Internal function executes the autocommit to manage the transactions.
|
|
1816
|
+
|
|
1817
|
+
PARAMETERS:
|
|
1818
|
+
cur:
|
|
1819
|
+
Required Argument.
|
|
1820
|
+
The cursor of connection type which will be used to execute query.
|
|
1821
|
+
Types: teradatasql cursor object
|
|
1822
|
+
|
|
1823
|
+
escape_function:
|
|
1824
|
+
Required Argument.
|
|
1825
|
+
Specifies the escape function to use.
|
|
1826
|
+
Permitted Values : 'autocommit_off', 'autocommit_on', 'logon_sequence_number'
|
|
1827
|
+
Types: String
|
|
1828
|
+
|
|
1829
|
+
insert_query:
|
|
1830
|
+
Optional Argument.
|
|
1831
|
+
Specifies the insert query.
|
|
1832
|
+
Types: String
|
|
1833
|
+
|
|
1834
|
+
RETURNS:
|
|
1835
|
+
If escape_function is 'logon_sequence_number' then returns
|
|
1836
|
+
a list containing logon sequence number.
|
|
1837
|
+
If escape_function is 'teradata_get_errors' then returns
|
|
1838
|
+
a list containing errors observed by FastLoad.
|
|
1839
|
+
If escape_function is 'teradata_get_warnings' then returns
|
|
1840
|
+
a list containing warnings observed by FastLoad.
|
|
1841
|
+
Otherwise returns None.
|
|
1842
|
+
|
|
1843
|
+
RAISES:
|
|
1844
|
+
None
|
|
1845
|
+
|
|
1846
|
+
EXAMPLES:
|
|
1847
|
+
dt_obj = _DataTransferUtils(df)
|
|
1848
|
+
dt_obj._process_escape_functions(cur, escape_function)
|
|
1849
|
+
"""
|
|
1850
|
+
# All escape functions follow same syntax hence, the query formation at the start.
|
|
1851
|
+
query = "{}{}".format(DriverEscapeFunctions.NATIVE_SQL.value, escape_function.value)
|
|
1852
|
+
|
|
1853
|
+
# If escape function requires to be executed along with the query.
|
|
1854
|
+
if escape_function in [DriverEscapeFunctions.LOGON_SEQ_NUM,
|
|
1855
|
+
DriverEscapeFunctions.GET_ERRORS,
|
|
1856
|
+
DriverEscapeFunctions.GET_WARNINGS]:
|
|
1857
|
+
cur.execute(query + insert_query)
|
|
1858
|
+
return [row for row in cur.fetchall()]
|
|
1859
|
+
else:
|
|
1860
|
+
cur.execute(query)
|
|
1861
|
+
|
|
1862
|
+
# End of general functions to get warnings and errors.
|
|
1863
|
+
|
|
1864
|
+
# Functions specific to read_csv().
|
|
1865
|
+
def _form_insert_query(self, table, column_names=None):
|
|
1866
|
+
"""
|
|
1867
|
+
DESCRIPTION:
|
|
1868
|
+
Internal function forms the INSERT query using escape function.
|
|
1869
|
+
|
|
1870
|
+
PARAMETERS:
|
|
1871
|
+
table:
|
|
1872
|
+
Required Argument.
|
|
1873
|
+
Specifies the table name.
|
|
1874
|
+
Types: str
|
|
1875
|
+
|
|
1876
|
+
column_names:
|
|
1877
|
+
Optional Argument.
|
|
1878
|
+
Specifies the list of column names.
|
|
1879
|
+
Types: list of Strings (str)
|
|
1880
|
+
|
|
1881
|
+
RETURNS:
|
|
1882
|
+
str.
|
|
1883
|
+
|
|
1884
|
+
RAISES:
|
|
1885
|
+
None.
|
|
1886
|
+
|
|
1887
|
+
EXAMPLES:
|
|
1888
|
+
dt_obj = _DataTransferUtils(df, api_name='read_csv')
|
|
1889
|
+
ins_query = dt_obj._form_insert_query()
|
|
1890
|
+
"""
|
|
1891
|
+
|
|
1892
|
+
escape_funcs = ""
|
|
1893
|
+
|
|
1894
|
+
# Get the fastload escape function.
|
|
1895
|
+
if self.use_fastload:
|
|
1896
|
+
escape_funcs = escape_funcs + DriverEscapeFunctions.REQUIRE_FASTLOAD.value
|
|
1897
|
+
|
|
1898
|
+
# Get the escape function clause for open_sessions.
|
|
1899
|
+
if self.open_sessions is not None:
|
|
1900
|
+
escape_funcs = escape_funcs + DriverEscapeFunctions.OPEN_SESSIONS.value.format(self.open_sessions)
|
|
1901
|
+
|
|
1902
|
+
# Create the list of values to be inserted.
|
|
1903
|
+
if self.api_name == "fastload":
|
|
1904
|
+
col_names = _get_pd_df_column_names(self.df)
|
|
1905
|
+
insert_values = ", ".join(['?' for i in range(len(col_names) + len(self.df.index.names)
|
|
1906
|
+
if self.index is True else len(col_names))])
|
|
1907
|
+
|
|
1908
|
+
# Get escape functions related to read_csv.
|
|
1909
|
+
if self.api_name == "read_csv":
|
|
1910
|
+
# Get the column names.
|
|
1911
|
+
if self.if_exists == 'append' and column_names is not None:
|
|
1912
|
+
col_names = column_names
|
|
1913
|
+
else:
|
|
1914
|
+
col_names, _ = _extract_column_info(self.df, self.types)
|
|
1915
|
+
|
|
1916
|
+
# Get read_csv escape function.
|
|
1917
|
+
escape_funcs = escape_funcs + DriverEscapeFunctions.READ_CSV.value.format(self.df)
|
|
1918
|
+
insert_values = ", ".join(['?' for i in range(len(col_names))])
|
|
1919
|
+
|
|
1920
|
+
# Create escape function for sep.
|
|
1921
|
+
field_sep = "'{0}'".format(self.sep)
|
|
1922
|
+
if self.sep == "'":
|
|
1923
|
+
field_sep = "''''"
|
|
1924
|
+
elif self.sep == "\"":
|
|
1925
|
+
field_sep = "\"\"\"\""
|
|
1926
|
+
escape_funcs = escape_funcs + DriverEscapeFunctions.FIELD_SEP.value.format(field_sep)
|
|
1927
|
+
|
|
1928
|
+
# Create escape function for quotechar.
|
|
1929
|
+
field_quote = "'{0}'".format(self.quotechar)
|
|
1930
|
+
if self.quotechar == "'":
|
|
1931
|
+
field_quote = "''''"
|
|
1932
|
+
elif self.quotechar == "\"":
|
|
1933
|
+
field_quote = "\"\"\"\""
|
|
1934
|
+
escape_funcs = escape_funcs + DriverEscapeFunctions.FIELD_QUOTE.value.format(field_quote)
|
|
1935
|
+
|
|
1936
|
+
# Create base insert query.
|
|
1937
|
+
base_insert_query = "INSERT INTO {0} VALUES ({1});".format(table, insert_values)
|
|
1938
|
+
|
|
1939
|
+
# Get the escape function clauses for error table and DB related escape functions.
|
|
1940
|
+
# TODO: This condition will be optimized with ELE-6743.
|
|
1941
|
+
if self.api_name == "fastload" and self.save_errors and not self.err_tbl_name:
|
|
1942
|
+
escape_funcs = escape_funcs + DriverEscapeFunctions.ERR_TBL_MNG_FLAG.value.format("off")
|
|
1943
|
+
|
|
1944
|
+
if self.err_tbl_1_suffix:
|
|
1945
|
+
escape_funcs = escape_funcs + DriverEscapeFunctions.ERR_TBL_1.value.format(self.err_tbl_1_suffix)
|
|
1946
|
+
|
|
1947
|
+
if self.err_tbl_2_suffix:
|
|
1948
|
+
escape_funcs = escape_funcs + DriverEscapeFunctions.ERR_TBL_2.value.format(self.err_tbl_2_suffix)
|
|
1949
|
+
|
|
1950
|
+
if self.err_staging_db:
|
|
1951
|
+
escape_funcs = escape_funcs + DriverEscapeFunctions.ERR_STAGING_DB.value.format(self.err_staging_db)
|
|
1952
|
+
|
|
1953
|
+
# Generate final insert query by appending all escape functions.
|
|
1954
|
+
query = "{0}{1}".format(escape_funcs, base_insert_query)
|
|
1955
|
+
return query
|
|
1956
|
+
|
|
1957
|
+
def _table_exists(self, con):
|
|
1958
|
+
"""
|
|
1959
|
+
DESCRIPTION:
|
|
1960
|
+
Internal function validates whether table exists or not.
|
|
1961
|
+
|
|
1962
|
+
PARAMETERS:
|
|
1963
|
+
con:
|
|
1964
|
+
Required argument.
|
|
1965
|
+
Specifies a SQLAlchemy connectable (engine/connection) object
|
|
1966
|
+
Types: Teradata connection object
|
|
1967
|
+
|
|
1968
|
+
RETURNS:
|
|
1969
|
+
boolean.
|
|
1970
|
+
|
|
1971
|
+
RAISES:
|
|
1972
|
+
None.
|
|
1973
|
+
|
|
1974
|
+
EXAMPLES:
|
|
1975
|
+
dt_obj = _DataTransferUtils(df)
|
|
1976
|
+
ins_query = dt_obj._table_exists()
|
|
1977
|
+
"""
|
|
1978
|
+
return con.dialect.has_table(get_connection(), self.table_name, self.schema_name,
|
|
1979
|
+
table_only=True)
|
|
1980
|
+
|
|
1981
|
+
def _get_fully_qualified_table_name(self, table_name=None, schema_name=None):
|
|
1982
|
+
"""
|
|
1983
|
+
DESCRIPTION:
|
|
1984
|
+
Function returns schema qualified table name
|
|
1985
|
+
Such as:
|
|
1986
|
+
"alice"."my_table" OR
|
|
1987
|
+
"mldb"."my_table"
|
|
1988
|
+
|
|
1989
|
+
PARAMETERS:
|
|
1990
|
+
table_name:
|
|
1991
|
+
Optional Argument.
|
|
1992
|
+
Specifies the table name.
|
|
1993
|
+
Types: str
|
|
1994
|
+
|
|
1995
|
+
schema_name:
|
|
1996
|
+
Optional Argument.
|
|
1997
|
+
Specifies the schema name.
|
|
1998
|
+
Types: str
|
|
1999
|
+
|
|
2000
|
+
RETURNS:
|
|
2001
|
+
str.
|
|
2002
|
+
|
|
2003
|
+
RAISES:
|
|
2004
|
+
None
|
|
2005
|
+
|
|
2006
|
+
EXAMPLES:
|
|
2007
|
+
dt_obj = _DataTransferUtils(df, table_name, types)
|
|
2008
|
+
dt_obj._get_fully_qualified_table_name()
|
|
2009
|
+
"""
|
|
2010
|
+
table_name = table_name if table_name else self.table_name
|
|
2011
|
+
|
|
2012
|
+
table = '"{}"'.format(table_name)
|
|
2013
|
+
if schema_name is not None:
|
|
2014
|
+
table = '"{}"."{}"'.format(schema_name, table_name)
|
|
2015
|
+
elif self.schema_name is not None:
|
|
2016
|
+
table = '"{}"."{}"'.format(self.schema_name, table_name)
|
|
2017
|
+
|
|
2018
|
+
return table
|
|
2019
|
+
|
|
2020
|
+
def _create_table(self, con, table_name=None, schema_name=None):
|
|
2021
|
+
"""
|
|
2022
|
+
DESCRIPTION:
|
|
2023
|
+
Internal function creates table in the Vantage.
|
|
2024
|
+
|
|
2025
|
+
PARAMETERS:
|
|
2026
|
+
con:
|
|
2027
|
+
Required Argument.
|
|
2028
|
+
A SQLAlchemy connectable (engine/connection) object
|
|
2029
|
+
Types: Teradata connection object
|
|
2030
|
+
|
|
2031
|
+
table_name:
|
|
2032
|
+
Optional Argument.
|
|
2033
|
+
Specifies the table name.
|
|
2034
|
+
Types: str
|
|
2035
|
+
|
|
2036
|
+
schema_name:
|
|
2037
|
+
Optional Argument.
|
|
2038
|
+
Specifies the schema name where table needs to be created.
|
|
2039
|
+
Types: str
|
|
2040
|
+
|
|
2041
|
+
RETURNS:
|
|
2042
|
+
None.
|
|
2043
|
+
|
|
2044
|
+
RAISES:
|
|
2045
|
+
None
|
|
2046
|
+
|
|
2047
|
+
EXAMPLES:
|
|
2048
|
+
dt_obj = _DataTransferUtils(df, table_name, types)
|
|
2049
|
+
dt_obj._create_table(con)
|
|
2050
|
+
"""
|
|
2051
|
+
table_name = table_name if table_name else self.table_name
|
|
2052
|
+
schema_name = schema_name if schema_name else self.schema_name
|
|
2053
|
+
table = _create_table_object(df=self.df, table_name=table_name, types=self.types, con=con,
|
|
2054
|
+
schema_name=schema_name, primary_index=self.primary_index,
|
|
2055
|
+
temporary=self.temporary, set_table=self.set_table, index=self.index,
|
|
2056
|
+
index_label=self.index_label)
|
|
2057
|
+
|
|
2058
|
+
UtilFuncs._create_table_using_table_object(table)
|
|
2059
|
+
|
|
2060
|
+
def _insert_from_csv_with_fastload(self, table_name=None, column_names=None):
|
|
2061
|
+
"""
|
|
2062
|
+
DESCRIPTION:
|
|
2063
|
+
This insert function loads the data from csv file to Vantage table.
|
|
2064
|
+
|
|
2065
|
+
PARAMETERS:
|
|
2066
|
+
table_name:
|
|
2067
|
+
Optional Argument.
|
|
2068
|
+
Specifies the table name.
|
|
2069
|
+
Types: str
|
|
2070
|
+
|
|
2071
|
+
column_names:
|
|
2072
|
+
Optional Argument.
|
|
2073
|
+
Specifies the list column names.
|
|
2074
|
+
Types: list of Strings (str).
|
|
2075
|
+
|
|
2076
|
+
RETURNS:
|
|
2077
|
+
A dict containing the following attributes:
|
|
2078
|
+
1. errors_dataframe: It is a Pandas DataFrame containing error messages
|
|
2079
|
+
thrown by fastload. DataFrame is empty if there are no errors.
|
|
2080
|
+
2. warnings_dataframe: It is a Pandas DataFrame containing warning messages
|
|
2081
|
+
thrown by fastload. DataFrame is empty if there are no warnings.
|
|
2082
|
+
3. errors_table: Name of the table containing errors. It is None, if
|
|
2083
|
+
argument save_errors is False.
|
|
2084
|
+
4. warnings_table: Name of the table containing warnings. It is None, if
|
|
2085
|
+
argument save_errors is False.
|
|
2086
|
+
|
|
2087
|
+
RAISES:
|
|
2088
|
+
None
|
|
2089
|
+
|
|
2090
|
+
EXAMPLES:
|
|
2091
|
+
dt_obj = _DataTransferUtils(df, table_name, types, api_name='read_csv')
|
|
2092
|
+
dt_obj._insert_from_csv()
|
|
2093
|
+
"""
|
|
2094
|
+
conn = get_connection()
|
|
2095
|
+
conn1 = conn.connection
|
|
2096
|
+
cur = conn1.cursor()
|
|
2097
|
+
|
|
2098
|
+
error_tablename = ""
|
|
2099
|
+
warn_tablename = ""
|
|
2100
|
+
|
|
2101
|
+
try:
|
|
2102
|
+
# Quoted, schema-qualified table name.
|
|
2103
|
+
table = self._get_fully_qualified_table_name(table_name)
|
|
2104
|
+
|
|
2105
|
+
# Form the INSERT query for read_csv.
|
|
2106
|
+
ins = self._form_insert_query(table, column_names=column_names)
|
|
2107
|
+
|
|
2108
|
+
# Turn off autocommit before the Fastload insertion.
|
|
2109
|
+
self._process_escape_functions(cur, escape_function= \
|
|
2110
|
+
DriverEscapeFunctions.AUTOCOMMIT_OFF)
|
|
2111
|
+
|
|
2112
|
+
# Initialize dict template for saving error/warning information.
|
|
2113
|
+
err_dict = {}
|
|
2114
|
+
warn_dict = {}
|
|
2115
|
+
err_dict['error_message'] = []
|
|
2116
|
+
warn_dict['error_message'] = []
|
|
2117
|
+
|
|
2118
|
+
# Empty queryband buffer before SQL call.
|
|
2119
|
+
UtilFuncs._set_queryband()
|
|
2120
|
+
# Execute insert statement
|
|
2121
|
+
cur.execute(ins)
|
|
2122
|
+
|
|
2123
|
+
# Get error and warning information
|
|
2124
|
+
err, _ = self._process_fastexport_errors_warnings(ins)
|
|
2125
|
+
if len(err) != 0:
|
|
2126
|
+
err_dict['error_message'].extend(err)
|
|
2127
|
+
|
|
2128
|
+
# Get logon sequence number to be used for error/warning table names
|
|
2129
|
+
logon_seq_number = self._process_escape_functions(cur, escape_function= \
|
|
2130
|
+
DriverEscapeFunctions.LOGON_SEQ_NUM,
|
|
2131
|
+
insert_query=ins)
|
|
2132
|
+
|
|
2133
|
+
# Commit the rows
|
|
2134
|
+
conn1.commit()
|
|
2135
|
+
|
|
2136
|
+
# Get error and warning information, if any.
|
|
2137
|
+
# Errors/Warnings like duplicate rows are added here.
|
|
2138
|
+
_, warn = self._process_fastexport_errors_warnings(ins)
|
|
2139
|
+
if len(warn) != 0:
|
|
2140
|
+
warn_dict['error_message'].extend(warn)
|
|
2141
|
+
|
|
2142
|
+
# Get error and warning information for error and warning tables, persist
|
|
2143
|
+
# error and warning tables to Vantage if user has specified save_error as True
|
|
2144
|
+
# else show it as pandas dataframe on console.
|
|
2145
|
+
pd_err_df = self._get_pandas_df_from_errors_warnings(err_dict)
|
|
2146
|
+
if not pd_err_df.empty and self.save_errors:
|
|
2147
|
+
msg_type = "err"
|
|
2148
|
+
error_tablename = self._create_error_warnings_table(pd_err_df, msg_type, logon_seq_number[0][0])
|
|
2149
|
+
|
|
2150
|
+
pd_warn_df = self._get_pandas_df_from_errors_warnings(warn_dict)
|
|
2151
|
+
if not pd_warn_df.empty and self.save_errors:
|
|
2152
|
+
msg_type = "warn"
|
|
2153
|
+
warn_tablename = self._create_error_warnings_table(pd_warn_df, msg_type, logon_seq_number[0][0])
|
|
2154
|
+
|
|
2155
|
+
# These tables are created by FastloadCSV.
|
|
2156
|
+
fastloadcsv_err_tables = [table_name + "_ERR_1", table_name + "_ERR_2"]
|
|
2157
|
+
err_warn_dict = {"errors_dataframe": pd_err_df, "warnings_dataframe": pd_warn_df,
|
|
2158
|
+
"errors_table": error_tablename, "warnings_table": warn_tablename,
|
|
2159
|
+
"fastloadcsv_error_tables": fastloadcsv_err_tables}
|
|
2160
|
+
|
|
2161
|
+
# If user don't want to persist the error tables,
|
|
2162
|
+
# drop the tables created by FastloadCSV.
|
|
2163
|
+
if not self.save_errors:
|
|
2164
|
+
for table in fastloadcsv_err_tables:
|
|
2165
|
+
if conn.dialect.has_table(conn, table_name=table, schema=self.schema_name,
|
|
2166
|
+
table_only=True):
|
|
2167
|
+
UtilFuncs._drop_table(self._get_fully_qualified_table_name(table))
|
|
2168
|
+
err_warn_dict.update({"fastloadcsv_error_tables": []})
|
|
2169
|
+
return err_warn_dict
|
|
2170
|
+
|
|
2171
|
+
return err_warn_dict
|
|
2172
|
+
|
|
2173
|
+
except Exception:
|
|
2174
|
+
conn1.rollback()
|
|
2175
|
+
raise
|
|
2176
|
+
finally:
|
|
2177
|
+
# Turn on autocommit.
|
|
2178
|
+
self._process_escape_functions(cur, escape_function= \
|
|
2179
|
+
DriverEscapeFunctions.AUTOCOMMIT_ON)
|
|
2180
|
+
cur.close()
|
|
2181
|
+
|
|
2182
|
+
def _get_result(self, result_dict=None):
|
|
2183
|
+
"""
|
|
2184
|
+
DESCRIPTION:
|
|
2185
|
+
Internal function loads data from csv to table.
|
|
2186
|
+
And created the teradataml dataframe of the table.
|
|
2187
|
+
|
|
2188
|
+
PARAMETERS:
|
|
2189
|
+
result_dict:
|
|
2190
|
+
Optional Argument.
|
|
2191
|
+
Specifies the dictionary containg values for error_dataframe,
|
|
2192
|
+
warning_dataframe, error_table, warning_table.
|
|
2193
|
+
Types: dict
|
|
2194
|
+
|
|
2195
|
+
RETURNS:
|
|
2196
|
+
When "catch_errors_warnings" is set to True, then the
|
|
2197
|
+
function returns a tuple containing:
|
|
2198
|
+
a. Teradataml DataFrame.
|
|
2199
|
+
b. A dict containing the following attributes:
|
|
2200
|
+
a. errors_dataframe: It is a Pandas DataFrame containing error messages
|
|
2201
|
+
thrown by fastload. DataFrame is empty if there are no errors.
|
|
2202
|
+
b. warnings_dataframe: It is a Pandas DataFrame containing warning messages
|
|
2203
|
+
thrown by fastload. DataFrame is empty if there are no warnings.
|
|
2204
|
+
d. errors_table: Name of the table containing errors. It is None, if
|
|
2205
|
+
argument save_errors is False.
|
|
2206
|
+
e. warnings_table: Name of the table containing warnings. It is None, if
|
|
2207
|
+
argument save_errors is False.
|
|
2208
|
+
When "catch_errors_warnings" is False, then the function
|
|
2209
|
+
returns a Teradataml DataFrame.
|
|
2210
|
+
|
|
2211
|
+
RAISES:
|
|
2212
|
+
None.
|
|
2213
|
+
|
|
2214
|
+
EXAMPLES:
|
|
2215
|
+
dt_obj = _DataTransferUtils(df, table_name, types)
|
|
2216
|
+
ins_query = dt_obj._get_result()
|
|
2217
|
+
"""
|
|
2218
|
+
tdml_df = tdmldf.DataFrame(self._get_fully_qualified_table_name())
|
|
2219
|
+
|
|
2220
|
+
if not self.use_fastload:
|
|
2221
|
+
return tdml_df
|
|
2222
|
+
|
|
2223
|
+
if self.catch_errors_warnings:
|
|
2224
|
+
return tdml_df, result_dict
|
|
2225
|
+
return tdml_df
|
|
2226
|
+
|
|
2227
|
+
def _get_dataframe_columns(self):
|
|
2228
|
+
"""
|
|
2229
|
+
Internal function used to get dataframe columns name.
|
|
2230
|
+
|
|
2231
|
+
PARAMETERS:
|
|
2232
|
+
None
|
|
2233
|
+
|
|
2234
|
+
Returns
|
|
2235
|
+
list
|
|
2236
|
+
|
|
2237
|
+
RAISES:
|
|
2238
|
+
None
|
|
2239
|
+
|
|
2240
|
+
EXAMPLES:
|
|
2241
|
+
_get_dataframe_columns()
|
|
2242
|
+
"""
|
|
2243
|
+
if isinstance(self.df, pd.DataFrame):
|
|
2244
|
+
return _get_pd_df_column_names(self.df)
|
|
2245
|
+
elif isinstance(self.df, tdmldf.DataFrame):
|
|
2246
|
+
return [col.name for col in self.df._metaexpr.c]
|
|
2247
|
+
|
|
2248
|
+
def _get_sqlalc_table_columns(self):
|
|
2249
|
+
"""
|
|
2250
|
+
Internal function to get column info if types is not None otherwise get
|
|
2251
|
+
the column info from existing table.
|
|
2252
|
+
If types is None and if_exists is append then it will first check for table
|
|
2253
|
+
existence, if table does not exists on system, it will raise an exception.
|
|
2254
|
+
|
|
2255
|
+
PARAMETERS:
|
|
2256
|
+
None
|
|
2257
|
+
|
|
2258
|
+
RETURNS:
|
|
2259
|
+
list
|
|
2260
|
+
|
|
2261
|
+
RAISES:
|
|
2262
|
+
ValueError
|
|
2263
|
+
|
|
2264
|
+
EXAMPLES:
|
|
2265
|
+
_get_sqlalc_table_columns()
|
|
2266
|
+
"""
|
|
2267
|
+
table = None
|
|
2268
|
+
if self.types is None and self.if_exists == 'append':
|
|
2269
|
+
# Raise an exception when the table does not exists, if_exists='append'
|
|
2270
|
+
# and types=None.
|
|
2271
|
+
if not self._table_exists(get_connection()):
|
|
2272
|
+
err = "Table '{}' does not exists in the system and if_exists " \
|
|
2273
|
+
"is set to 'append'.".format(self.table_name)
|
|
2274
|
+
_Validators._validate_argument_is_not_None(self.types, "types",
|
|
2275
|
+
additional_error=err)
|
|
2276
|
+
|
|
2277
|
+
table = UtilFuncs._get_sqlalchemy_table(self.table_name,
|
|
2278
|
+
schema_name=self.schema_name)
|
|
2279
|
+
|
|
2280
|
+
df_columns, _ = UtilFuncs._extract_table_object_column_info(table.c) \
|
|
2281
|
+
if table is not None else \
|
|
2282
|
+
_extract_column_info(self.df, types=self.types)
|
|
2283
|
+
|
|
2284
|
+
return df_columns
|
|
2285
|
+
|
|
2286
|
+
def _validate(self):
|
|
2287
|
+
"""
|
|
2288
|
+
This is an internal function used to validate the api parameters.
|
|
2289
|
+
Dataframe, connection & related parameters are checked.
|
|
2290
|
+
Saving to Vantage is proceeded to only when validation returns True.
|
|
2291
|
+
|
|
2292
|
+
PARAMETERS:
|
|
2293
|
+
None
|
|
2294
|
+
|
|
2295
|
+
RETURNS:
|
|
2296
|
+
True, when all parameters are valid.
|
|
2297
|
+
|
|
2298
|
+
RAISES:
|
|
2299
|
+
TeradataMlException, when parameter validation fails.
|
|
2300
|
+
|
|
2301
|
+
EXAMPLES:
|
|
2302
|
+
dt_obj = _DataTransferUtils()
|
|
2303
|
+
dt_obj._validate()
|
|
2304
|
+
"""
|
|
2305
|
+
awu = _Validators()
|
|
2306
|
+
awu_matrix = []
|
|
2307
|
+
|
|
2308
|
+
# The arguments added to awu_martix are:
|
|
2309
|
+
# arg_name, arg, is_optional, acceptable types
|
|
2310
|
+
# The value for is_optional is set to False when the argument
|
|
2311
|
+
# a) is a required argument
|
|
2312
|
+
# b) is not allowed to be None, even if it is optional
|
|
2313
|
+
types_t = (dict)
|
|
2314
|
+
is_optional = True
|
|
2315
|
+
if self.api_name == 'read_csv':
|
|
2316
|
+
awu_matrix.append(['filepath', self.df, False, (str)])
|
|
2317
|
+
|
|
2318
|
+
types_t = (OrderedDict)
|
|
2319
|
+
is_optional = False if self.if_exists in ['replace', 'fail'] else True
|
|
2320
|
+
else:
|
|
2321
|
+
# Validates the dataframe passed to copyto and fastload apis.
|
|
2322
|
+
# We are validating dataframe with separate function because
|
|
2323
|
+
# current implementation of __getTypeAsStr() always returns
|
|
2324
|
+
# 'teradataml DataFrame' if argument type.__name__ is DataFrame.
|
|
2325
|
+
_Validators._validate_dataframe(self.df)
|
|
2326
|
+
|
|
2327
|
+
awu_matrix.append(['types', self.types, is_optional, types_t])
|
|
2328
|
+
awu_matrix.append(['table_name', self.table_name, False, (str), True])
|
|
2329
|
+
awu_matrix.append(['schema_name', self.schema_name, True, (str), True])
|
|
2330
|
+
awu_matrix.append(['index', self.index, False, (bool)])
|
|
2331
|
+
awu_matrix.append(['temporary', self.temporary, False, (bool)])
|
|
2332
|
+
awu_matrix.append(['if_exists', self.if_exists, False, (str),
|
|
2333
|
+
True, ['APPEND', 'REPLACE', 'FAIL']])
|
|
2334
|
+
awu_matrix.append(['primary_index', self.primary_index, True, (str, list), True])
|
|
2335
|
+
awu_matrix.append(['set_table', self.set_table, False, (bool)])
|
|
2336
|
+
awu_matrix.append(['save_errors', self.save_errors, False, (bool)])
|
|
2337
|
+
awu_matrix.append(['sep', self.sep, False, (str)])
|
|
2338
|
+
awu_matrix.append(['quotechar', self.quotechar, True, (str)])
|
|
2339
|
+
awu_matrix.append(['catch_errors_warnings', self.catch_errors_warnings, False, (bool)])
|
|
2340
|
+
awu_matrix.append(['use_fastload', self.use_fastload, False, (bool)])
|
|
2341
|
+
awu_matrix.append(['open_sessions', self.open_sessions, True, (int), False])
|
|
2342
|
+
awu_matrix.append(['chunksize', self.chunksize, False, (int)])
|
|
2343
|
+
awu_matrix.append(['match_column_order', self.match_column_order, True, (bool)])
|
|
2344
|
+
if isinstance(self.df, pd.DataFrame):
|
|
2345
|
+
awu_matrix.append(['index_label', self.index_label, True, (str, list), True])
|
|
2346
|
+
|
|
2347
|
+
# Validate types
|
|
2348
|
+
awu._validate_function_arguments(awu_matrix)
|
|
2349
|
+
|
|
2350
|
+
# Validate if 'open_sessions' is a positive int.
|
|
2351
|
+
if self.open_sessions is not None:
|
|
2352
|
+
_Validators._validate_positive_int(self.open_sessions, "open_sessions")
|
|
2353
|
+
|
|
2354
|
+
# Validate 'chunksize' is a positive int.
|
|
2355
|
+
_Validators._validate_positive_int(self.chunksize, "chunksize")
|
|
2356
|
+
|
|
2357
|
+
# Get columns name
|
|
2358
|
+
if self.api_name == 'read_csv':
|
|
2359
|
+
df_columns = self._get_sqlalc_table_columns()
|
|
2360
|
+
else:
|
|
2361
|
+
df_columns = self._get_dataframe_columns()
|
|
2362
|
+
|
|
2363
|
+
eng = get_context()
|
|
2364
|
+
current_user = eng.url.username
|
|
2365
|
+
|
|
2366
|
+
allowed_schemas = df_utils._get_database_names(get_connection(), self.schema_name)
|
|
2367
|
+
allowed_schemas.append(current_user)
|
|
2368
|
+
|
|
2369
|
+
if self.schema_name is not None and self.schema_name.lower() not in allowed_schemas:
|
|
2370
|
+
raise TeradataMlException(Messages.get_message(MessageCodes.INVALID_ARG_VALUE,
|
|
2371
|
+
str(self.schema_name), 'schema_name',
|
|
2372
|
+
'A valid database/schema name.'),
|
|
2373
|
+
MessageCodes.INVALID_ARG_VALUE)
|
|
2374
|
+
|
|
2375
|
+
if isinstance(self.df, pd.DataFrame):
|
|
2376
|
+
if self.index:
|
|
2377
|
+
is_multi_index = isinstance(self.df.index, pd.MultiIndex)
|
|
2378
|
+
if self.index_label:
|
|
2379
|
+
if hasattr(self.df.index, 'levels'):
|
|
2380
|
+
index_levels = len(self.df.index.levels)
|
|
2381
|
+
num_index = len(self.index_label)
|
|
2382
|
+
is_index_list = isinstance(self.index_label, list)
|
|
2383
|
+
|
|
2384
|
+
if (is_multi_index and ((isinstance(self.index_label, str) and index_levels != 1) or
|
|
2385
|
+
(is_index_list and index_levels != len(self.index_label)))) or \
|
|
2386
|
+
(not is_multi_index and is_index_list and
|
|
2387
|
+
(is_index_list and num_index > 1)):
|
|
2388
|
+
valid_arg_msg = 'String or list of Strings with the number of ' \
|
|
2389
|
+
'Strings matching the number of levels' \
|
|
2390
|
+
' in the index'
|
|
2391
|
+
raise TeradataMlException(Messages.get_message(MessageCodes.INVALID_ARG_VALUE,
|
|
2392
|
+
self.index_label, 'index_label',
|
|
2393
|
+
valid_arg_msg),
|
|
2394
|
+
MessageCodes.INVALID_ARG_VALUE)
|
|
2395
|
+
|
|
2396
|
+
# When Pandas DF's used and Pandas Index is saved, get list of levels to add as columns
|
|
2397
|
+
index_names_to_add = _get_index_labels(self.df, self.index_label)[0]
|
|
2398
|
+
for label in index_names_to_add:
|
|
2399
|
+
if label in df_columns:
|
|
2400
|
+
raise TeradataMlException(Messages.get_message(MessageCodes.INDEX_ALREADY_EXISTS, label),
|
|
2401
|
+
MessageCodes.INDEX_ALREADY_EXISTS)
|
|
2402
|
+
|
|
2403
|
+
df_columns = df_columns + index_names_to_add
|
|
2404
|
+
|
|
2405
|
+
if self.index_label is not None and self.index is False:
|
|
2406
|
+
raise TeradataMlException(Messages.get_message(MessageCodes.INVALID_INDEX_LABEL),
|
|
2407
|
+
MessageCodes.INVALID_INDEX_LABEL)
|
|
2408
|
+
|
|
2409
|
+
elif isinstance(self.df, tdmldf.DataFrame):
|
|
2410
|
+
# teradataml DataFrame's do not support saving pandas index/index_label
|
|
2411
|
+
if self.index_label is not None:
|
|
2412
|
+
raise TeradataMlException(Messages.get_message(MessageCodes.INVALID_ARG_VALUE,
|
|
2413
|
+
str(self.index_label), 'index_label', 'None'),
|
|
2414
|
+
MessageCodes.INVALID_ARG_VALUE)
|
|
2415
|
+
|
|
2416
|
+
if self.index is not False:
|
|
2417
|
+
raise TeradataMlException(Messages.get_message(MessageCodes.INVALID_ARG_VALUE,
|
|
2418
|
+
str(self.index), 'index', 'False'),
|
|
2419
|
+
MessageCodes.INVALID_ARG_VALUE)
|
|
2420
|
+
|
|
2421
|
+
# Check for number of columns
|
|
2422
|
+
if len(df_columns) > TeradataConstants.TABLE_COLUMN_LIMIT.value:
|
|
2423
|
+
raise TeradataMlException(Messages.get_message(MessageCodes.TD_MAX_COL_MESSAGE),
|
|
2424
|
+
MessageCodes.TD_MAX_COL_MESSAGE)
|
|
2425
|
+
|
|
2426
|
+
# Check for existence of Primary Index Columns
|
|
2427
|
+
pindex = self.primary_index
|
|
2428
|
+
if self.primary_index is not None:
|
|
2429
|
+
if isinstance(self.primary_index, str):
|
|
2430
|
+
pindex = [self.primary_index]
|
|
2431
|
+
|
|
2432
|
+
for column in pindex:
|
|
2433
|
+
if column not in df_columns:
|
|
2434
|
+
raise TeradataMlException(Messages.get_message(MessageCodes.INVALID_PRIMARY_INDEX),
|
|
2435
|
+
MessageCodes.INVALID_PRIMARY_INDEX)
|
|
2436
|
+
|
|
2437
|
+
# Verify types argument is a dictionary, non-empty, and contains appropriate columns
|
|
2438
|
+
if self.types is not None:
|
|
2439
|
+
# Verify types argument is non-empty when specified
|
|
2440
|
+
if not (self.types):
|
|
2441
|
+
raise TeradataMlException(Messages.get_message(MessageCodes.ARG_EMPTY, 'types'),
|
|
2442
|
+
MessageCodes.ARG_EMPTY)
|
|
2443
|
+
|
|
2444
|
+
# Check if all column names provided in types are valid DataFrame columns
|
|
2445
|
+
if any(key not in df_columns for key in self.types):
|
|
2446
|
+
# Only iterate entire types dictionary if an invalid column value passed
|
|
2447
|
+
for key in self.types:
|
|
2448
|
+
if key not in df_columns:
|
|
2449
|
+
raise TeradataMlException(Messages.get_message(MessageCodes.INVALID_ARG_VALUE,
|
|
2450
|
+
str(key), 'types', ', '.join(df_columns)),
|
|
2451
|
+
MessageCodes.INVALID_ARG_VALUE)
|
|
2452
|
+
|
|
2453
|
+
def _check_table_exists(self, is_table_exists):
|
|
2454
|
+
"""
|
|
2455
|
+
Internal function to raise an exception when table does not exists
|
|
2456
|
+
and if_exists is set to 'fail'.
|
|
2457
|
+
|
|
2458
|
+
PARAMETERS:
|
|
2459
|
+
is_table_exists:
|
|
2460
|
+
Required Argument.
|
|
2461
|
+
Specifies whether table exists or not.
|
|
2462
|
+
Types: bool
|
|
2463
|
+
|
|
2464
|
+
RETURNS:
|
|
2465
|
+
None
|
|
2466
|
+
|
|
2467
|
+
RAISES:
|
|
2468
|
+
TeradataMlException
|
|
2469
|
+
|
|
2470
|
+
EXAMPLES:
|
|
2471
|
+
_check_table_exists(True)
|
|
2472
|
+
"""
|
|
2473
|
+
# Raise an exception when the table exists and if_exists = 'fail'
|
|
2474
|
+
if is_table_exists and self.if_exists.lower() == 'fail':
|
|
2475
|
+
raise TeradataMlException(Messages.get_message(MessageCodes.TABLE_ALREADY_EXISTS, self.table_name),
|
|
2476
|
+
MessageCodes.TABLE_ALREADY_EXISTS)
|
|
2477
|
+
|
|
2478
|
+
def _create_or_replace_table(self, con, table_exists):
|
|
2479
|
+
"""
|
|
2480
|
+
Internal function to create the table if table does not exists on
|
|
2481
|
+
system. If table exists then drop the existing table and create
|
|
2482
|
+
new table.
|
|
2483
|
+
|
|
2484
|
+
PARAMETERS:
|
|
2485
|
+
con:
|
|
2486
|
+
Required Argument.
|
|
2487
|
+
A SQLAlchemy connectable (engine/connection) object
|
|
2488
|
+
Types: Teradata connection object
|
|
2489
|
+
|
|
2490
|
+
table_exists:
|
|
2491
|
+
Required Argument.
|
|
2492
|
+
Specifies whether table exists or not.
|
|
2493
|
+
Types: bool
|
|
2494
|
+
|
|
2495
|
+
Returns:
|
|
2496
|
+
None
|
|
2497
|
+
|
|
2498
|
+
RAISES:
|
|
2499
|
+
None
|
|
2500
|
+
|
|
2501
|
+
EXAMPLES:
|
|
2502
|
+
_create_or_replace_table(con, True)
|
|
2503
|
+
"""
|
|
2504
|
+
# If the table need to be replaced, let's drop the existing table first.
|
|
2505
|
+
if table_exists:
|
|
2506
|
+
UtilFuncs._drop_table(self._get_fully_qualified_table_name())
|
|
2507
|
+
|
|
2508
|
+
# Create target table for FastLoad
|
|
2509
|
+
self._create_table(con)
|
|
2510
|
+
|
|
2511
|
+
def _check_columns_compatibility(self, table_obj, cols):
|
|
2512
|
+
"""
|
|
2513
|
+
Internal function checks the columns' compatibility with already
|
|
2514
|
+
existing table.
|
|
2515
|
+
|
|
2516
|
+
PARAMETERS:
|
|
2517
|
+
table_obj:
|
|
2518
|
+
Required Argument.
|
|
2519
|
+
Specifies the table object.
|
|
2520
|
+
Types: sqlalchemy.sql.schema.Table
|
|
2521
|
+
|
|
2522
|
+
cols:
|
|
2523
|
+
Required Argument.
|
|
2524
|
+
Specifies list of column names.
|
|
2525
|
+
Types: list
|
|
2526
|
+
|
|
2527
|
+
RETURNS:
|
|
2528
|
+
None
|
|
2529
|
+
|
|
2530
|
+
RAISES:
|
|
2531
|
+
TeradataMlException
|
|
2532
|
+
|
|
2533
|
+
EXAMPLES:
|
|
2534
|
+
_check_columns_compatibility(table_obj, ['c1', 'c2'])
|
|
2535
|
+
"""
|
|
2536
|
+
cols_compatible = _check_columns_insertion_compatible(table_obj.c, cols,
|
|
2537
|
+
True)
|
|
2538
|
+
if not cols_compatible:
|
|
2539
|
+
raise TeradataMlException(Messages.get_message(MessageCodes.INSERTION_INCOMPATIBLE),
|
|
2540
|
+
MessageCodes.INSERTION_INCOMPATIBLE)
|
|
2541
|
+
|
|
2542
|
+
def _create_staging_table_and_load_csv_data(self, column_info=None, primary_index=None):
|
|
2543
|
+
"""
|
|
2544
|
+
Internal function to create the staging table and load the csv
|
|
2545
|
+
data into it.
|
|
2546
|
+
|
|
2547
|
+
PARAMETERS:
|
|
2548
|
+
column_info:
|
|
2549
|
+
Required Argument.
|
|
2550
|
+
Specifies the data types for columns to be saved in Vantage.
|
|
2551
|
+
Keys of this dictionary should be the name of the columns and values should be
|
|
2552
|
+
teradatasqlalchemy.types.
|
|
2553
|
+
Types: dict (key should be the name of column,
|
|
2554
|
+
value should be the data type of column)
|
|
2555
|
+
|
|
2556
|
+
primary_index:
|
|
2557
|
+
Optional Argument.
|
|
2558
|
+
Specifies primary index.
|
|
2559
|
+
Default Value: None
|
|
2560
|
+
Types: list of Strings (str)
|
|
2561
|
+
|
|
2562
|
+
RETURNS:
|
|
2563
|
+
dict
|
|
2564
|
+
|
|
2565
|
+
RAISES:
|
|
2566
|
+
None
|
|
2567
|
+
|
|
2568
|
+
EXAMPLES:
|
|
2569
|
+
_create_staging_table_and_load_csv_data(column_info={"id": INTEGER})
|
|
2570
|
+
_create_staging_table_and_load_csv_data(column_info={"id": INTEGER}, primary_index = ['id'])
|
|
2571
|
+
|
|
2572
|
+
"""
|
|
2573
|
+
stage_table_created = False
|
|
2574
|
+
try:
|
|
2575
|
+
# Generate the temporary table.
|
|
2576
|
+
stag_table_name = UtilFuncs._generate_temp_table_name(prefix="fl_staging",
|
|
2577
|
+
gc_on_quit=False,
|
|
2578
|
+
quote=False,
|
|
2579
|
+
table_type=TeradataConstants.TERADATA_TABLE)
|
|
2580
|
+
|
|
2581
|
+
# If configure.temp_object_type="VT", _generate_temp_table_name() retruns the
|
|
2582
|
+
# table name in fully qualified format. Because of this , test cases started
|
|
2583
|
+
# failing with Blank name in quotation mark. Hence, extracted only the table name.
|
|
2584
|
+
stag_table_name = UtilFuncs._extract_table_name(stag_table_name)
|
|
2585
|
+
|
|
2586
|
+
# Information about uniqueness of primary index and
|
|
2587
|
+
# SET/MULTISET property of existing table is not available,
|
|
2588
|
+
# so over-assuming to be False.
|
|
2589
|
+
unique = False
|
|
2590
|
+
set_table = False
|
|
2591
|
+
|
|
2592
|
+
if not primary_index:
|
|
2593
|
+
primary_index = None
|
|
2594
|
+
|
|
2595
|
+
# Create staging table where column info deduced from the existing table.
|
|
2596
|
+
_create_table(table_name=stag_table_name, columns=column_info,
|
|
2597
|
+
primary_index=primary_index, unique=unique,
|
|
2598
|
+
temporary=False, schema_name=self.schema_name,
|
|
2599
|
+
set_table=set_table)
|
|
2600
|
+
stage_table_created = True
|
|
2601
|
+
column_names = list(column_info.keys())
|
|
2602
|
+
|
|
2603
|
+
# Load the data from CSV to staging table.
|
|
2604
|
+
rc_dict = self._insert_from_csv_with_fastload(table_name=stag_table_name,
|
|
2605
|
+
column_names=column_names)
|
|
2606
|
+
|
|
2607
|
+
# Insert all rows from staging table to already existing table.
|
|
2608
|
+
df_utils._insert_all_from_table(self.table_name,
|
|
2609
|
+
stag_table_name,
|
|
2610
|
+
column_names,
|
|
2611
|
+
to_schema_name=self.schema_name,
|
|
2612
|
+
from_schema_name=self.schema_name)
|
|
2613
|
+
|
|
2614
|
+
return rc_dict
|
|
2615
|
+
finally:
|
|
2616
|
+
# Drop the staging table.
|
|
2617
|
+
if stage_table_created:
|
|
2618
|
+
UtilFuncs._drop_table(self._get_fully_qualified_table_name(stag_table_name))
|
|
2619
|
+
|
|
2620
|
+
#
|
|
2621
|
+
# These functions are specific to read_csv() without using FastloadCSV.
|
|
2622
|
+
#
|
|
2623
|
+
def _insert_from_csv_without_fastload(self, table_name=None, column_names=None):
|
|
2624
|
+
"""
|
|
2625
|
+
Internal function to read the csv file and load the data into
|
|
2626
|
+
a table in Vantage.
|
|
2627
|
+
|
|
2628
|
+
PARAMETERS:
|
|
2629
|
+
table_name:
|
|
2630
|
+
Optional Argument.
|
|
2631
|
+
Specifies the table name to load the data into.
|
|
2632
|
+
Types: String
|
|
2633
|
+
|
|
2634
|
+
column_names:
|
|
2635
|
+
Optional Argument.
|
|
2636
|
+
Specifies the list of column names.
|
|
2637
|
+
Types: list of Strings (str)
|
|
2638
|
+
|
|
2639
|
+
RETURNS:
|
|
2640
|
+
None
|
|
2641
|
+
|
|
2642
|
+
RAISES:
|
|
2643
|
+
None
|
|
2644
|
+
|
|
2645
|
+
EXAMPLES:
|
|
2646
|
+
_insert_from_csv_without_fastload(con, "my_table", [VARCHAR, DATE])
|
|
2647
|
+
|
|
2648
|
+
"""
|
|
2649
|
+
# Form the INSERT query.
|
|
2650
|
+
insert_stmt = self._form_insert_query(table=table_name,
|
|
2651
|
+
column_names=column_names)
|
|
2652
|
+
|
|
2653
|
+
# Empty queryband buffer before SQL call.
|
|
2654
|
+
UtilFuncs._set_queryband()
|
|
2655
|
+
try:
|
|
2656
|
+
conn = get_connection().connection
|
|
2657
|
+
cur = conn.cursor()
|
|
2658
|
+
|
|
2659
|
+
# Execute INSERT query.
|
|
2660
|
+
cur.execute(insert_stmt)
|
|
2661
|
+
except Exception:
|
|
2662
|
+
raise
|
|
2663
|
+
|
|
2664
|
+
def _check_index_compatibility(self, primary_index_1, primary_index_2):
|
|
2665
|
+
"""
|
|
2666
|
+
Internal function to check the compatibility of user provided primary index
|
|
2667
|
+
with primary index of already existing table.
|
|
2668
|
+
|
|
2669
|
+
PARAMETERS:
|
|
2670
|
+
primary_index_1:
|
|
2671
|
+
Required Argument.
|
|
2672
|
+
Specifies the primary index columns of existing table.
|
|
2673
|
+
Types: list of Strings (str)
|
|
2674
|
+
|
|
2675
|
+
primary_index_2:
|
|
2676
|
+
Required Argument.
|
|
2677
|
+
Specifies the primary index columns provided by user.
|
|
2678
|
+
Types: str OR list of Strings (str)
|
|
2679
|
+
|
|
2680
|
+
RETURNS:
|
|
2681
|
+
None
|
|
2682
|
+
|
|
2683
|
+
RAISES:
|
|
2684
|
+
TeradataMlException
|
|
2685
|
+
|
|
2686
|
+
EXAMPLES:
|
|
2687
|
+
_check_index_compatibility(['c1'], ['c1', 'c2'])
|
|
2688
|
+
_check_index_compatibility(['c1'], 'c1')
|
|
2689
|
+
"""
|
|
2690
|
+
if isinstance(primary_index_2, str):
|
|
2691
|
+
primary_index_2 = [primary_index_2]
|
|
2692
|
+
|
|
2693
|
+
if not (sorted(primary_index_1) == (sorted(primary_index_2))):
|
|
2694
|
+
raise TeradataMlException("Unable to perform insertion to existing table; Indexes do not match.",
|
|
2695
|
+
MessageCodes.INSERTION_INCOMPATIBLE)
|
|
2696
|
+
|
|
2697
|
+
def _process_pti_load_csv_data(self, con):
|
|
2698
|
+
"""
|
|
2699
|
+
Function processes the CSV data so that it can be loaded in a PTI table.
|
|
2700
|
+
To do so, it first creates a staging table and loads the data into same
|
|
2701
|
+
and the creates/loads into the PTI table using copy_to_sql().
|
|
2702
|
+
|
|
2703
|
+
PARAMETERS:
|
|
2704
|
+
con:
|
|
2705
|
+
Required Argument.
|
|
2706
|
+
Specifies a SQLAlchemy connectable (engine/connection) object.
|
|
2707
|
+
Types: sqlalchemy.engine.base.Engine
|
|
2708
|
+
|
|
2709
|
+
RETURNS:
|
|
2710
|
+
If catch_errors_warnings is set to True returns dict, otherwise None.
|
|
2711
|
+
|
|
2712
|
+
RAISES:
|
|
2713
|
+
ValueError
|
|
2714
|
+
|
|
2715
|
+
EXAMPLES:
|
|
2716
|
+
_process_pti_load_csv_data(con)
|
|
2717
|
+
|
|
2718
|
+
"""
|
|
2719
|
+
stag_table_name = ""
|
|
2720
|
+
try:
|
|
2721
|
+
# Types argument is required while appending the PTI table.
|
|
2722
|
+
if self._table_exists(con) and self.if_exists == "append" and self.types is None:
|
|
2723
|
+
_Validators._validate_argument_is_not_None(self.types, "types")
|
|
2724
|
+
|
|
2725
|
+
# Generate the staging table name.
|
|
2726
|
+
stag_table_name = UtilFuncs._generate_temp_table_name(prefix="rc_pti_staging",
|
|
2727
|
+
gc_on_quit=False,
|
|
2728
|
+
quote=False,
|
|
2729
|
+
table_type=TeradataConstants.TERADATA_TABLE)
|
|
2730
|
+
|
|
2731
|
+
# If configure.temp_object_type="VT", _generate_temp_table_name() retruns the
|
|
2732
|
+
# table name in fully qualified format. Because of this , test cases started
|
|
2733
|
+
# failing with Blank name in quotation mark. Hence, extracted only the table name.
|
|
2734
|
+
stag_table_name = UtilFuncs._extract_table_name(stag_table_name)
|
|
2735
|
+
|
|
2736
|
+
# Get the teradataml dataframe from staging table using read_csv()
|
|
2737
|
+
read_csv_output = read_csv(filepath=self.df, table_name=stag_table_name,
|
|
2738
|
+
types=self.types, sep=self.sep,
|
|
2739
|
+
quotechar=self.quotechar,
|
|
2740
|
+
schema_name=self.schema_name,
|
|
2741
|
+
save_errors=self.save_errors,
|
|
2742
|
+
catch_errors_warnings=self.catch_errors_warnings,
|
|
2743
|
+
use_fastload=self.use_fastload)
|
|
2744
|
+
|
|
2745
|
+
rc_dict = None
|
|
2746
|
+
if self.catch_errors_warnings:
|
|
2747
|
+
stage_table_tdml_df, rc_dict = read_csv_output
|
|
2748
|
+
else:
|
|
2749
|
+
stage_table_tdml_df = read_csv_output
|
|
2750
|
+
|
|
2751
|
+
# To create the PTI table use copy_to_sql().
|
|
2752
|
+
copy_to_sql(df=stage_table_tdml_df, table_name=self.table_name,
|
|
2753
|
+
schema_name=self.schema_name, if_exists=self.if_exists,
|
|
2754
|
+
primary_index=self.primary_index,
|
|
2755
|
+
temporary=self.temporary,
|
|
2756
|
+
primary_time_index_name=self.primary_time_index_name,
|
|
2757
|
+
timecode_column=self.timecode_column,
|
|
2758
|
+
timebucket_duration=self.timebucket_duration,
|
|
2759
|
+
timezero_date=self.timezero_date,
|
|
2760
|
+
columns_list=self.columns_list,
|
|
2761
|
+
sequence_column=self.sequence_column,
|
|
2762
|
+
seq_max=self.seq_max, set_table=self.set_table)
|
|
2763
|
+
|
|
2764
|
+
return rc_dict
|
|
2765
|
+
finally:
|
|
2766
|
+
if stag_table_name:
|
|
2767
|
+
UtilFuncs._drop_table(self._get_fully_qualified_table_name(stag_table_name))
|
|
2768
|
+
|
|
2769
|
+
# End of functions specific to read_csv().
|
|
2770
|
+
|
|
2771
|
+
# Functions specific to exporting Vantage table data into CSV
|
|
2772
|
+
# which can be done with teradatasql driver escape functions with or without
|
|
2773
|
+
# fastexport protocol.
|
|
2774
|
+
def _get_csv(self, require_fastexport=True, csv_file_name=None, **kwargs):
|
|
2775
|
+
"""
|
|
2776
|
+
DESCRIPTION:
|
|
2777
|
+
Internal function to export the data present in teradataml DataFrame into a CSV file
|
|
2778
|
+
using teradatasql driver's escape functions meant for writing to CSV. This can be done with or
|
|
2779
|
+
without Fastexport protocol.
|
|
2780
|
+
|
|
2781
|
+
PARAMETERS:
|
|
2782
|
+
require_fastexport:
|
|
2783
|
+
Optional Argument.
|
|
2784
|
+
Specifies whether fastexport protocol is required for data
|
|
2785
|
+
transfer.
|
|
2786
|
+
Default Value: True
|
|
2787
|
+
Types: boolean
|
|
2788
|
+
|
|
2789
|
+
csv_file_name:
|
|
2790
|
+
Required Argument.
|
|
2791
|
+
Specifies the name of CSV file to which data is to be exported.
|
|
2792
|
+
Default Value: None
|
|
2793
|
+
Types: Str
|
|
2794
|
+
|
|
2795
|
+
kwargs:
|
|
2796
|
+
Specifies keyword arguments. Argument "open_sessions"
|
|
2797
|
+
can be passed as keyword arguments.
|
|
2798
|
+
* "open_sessions" specifies the number of Teradata sessions to
|
|
2799
|
+
be opened for fastexport.
|
|
2800
|
+
|
|
2801
|
+
RETURNS:
|
|
2802
|
+
When "catch_errors_warnings" is set to True, the function returns
|
|
2803
|
+
a tuple containing:
|
|
2804
|
+
* Errors, if any, thrown by fastexport in a list of strings.
|
|
2805
|
+
* Warnings, if any, thrown by fastexport in a list of strings.
|
|
2806
|
+
|
|
2807
|
+
RAISES:
|
|
2808
|
+
TeradataMlException.
|
|
2809
|
+
|
|
2810
|
+
EXAMPLES:
|
|
2811
|
+
dt_obj = _DataTransferUtils(df)
|
|
2812
|
+
dt_obj._get_csv(require_fastexport = True, csv_file_name = "Test.csv")
|
|
2813
|
+
dt_obj._get_csv(csv_file_name = "Test.csv")
|
|
2814
|
+
dt_obj._get_csv(csv_file_name = "Test.csv", open_sessions = 2)
|
|
2815
|
+
|
|
2816
|
+
"""
|
|
2817
|
+
try:
|
|
2818
|
+
base_query = self._generate_to_pandas_base_query()
|
|
2819
|
+
base_query = base_query.lstrip()
|
|
2820
|
+
# Get open_sessions argument.
|
|
2821
|
+
open_sessions = kwargs.pop("open_sessions", None)
|
|
2822
|
+
if not require_fastexport and open_sessions is not None:
|
|
2823
|
+
raise TeradataMlException("'{0}' can only be used when '{1}' is set to True." \
|
|
2824
|
+
.format("open_sessions", "fastexport or require"),
|
|
2825
|
+
MessageCodes.DEPENDENT_ARGUMENT)
|
|
2826
|
+
|
|
2827
|
+
csv_export_query = self._generate_fastexport_query(base_query,
|
|
2828
|
+
require=require_fastexport,
|
|
2829
|
+
open_sessions=open_sessions,
|
|
2830
|
+
csv_file_name=csv_file_name)
|
|
2831
|
+
|
|
2832
|
+
UtilFuncs._execute_query(csv_export_query)
|
|
2833
|
+
err, warn = \
|
|
2834
|
+
self._process_fastexport_errors_warnings(csv_export_query)
|
|
2835
|
+
|
|
2836
|
+
print('\nData is successfully exported into {}'.format(csv_file_name))
|
|
2837
|
+
if self.catch_errors_warnings:
|
|
2838
|
+
return err, warn
|
|
2839
|
+
except TeradataMlException:
|
|
2840
|
+
raise
|