teradataml 17.20.0.6__py3-none-any.whl → 20.0.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of teradataml might be problematic. Click here for more details.
- teradataml/LICENSE-3RD-PARTY.pdf +0 -0
- teradataml/LICENSE.pdf +0 -0
- teradataml/README.md +238 -1
- teradataml/__init__.py +13 -3
- teradataml/_version.py +1 -1
- teradataml/analytics/Transformations.py +4 -4
- teradataml/analytics/__init__.py +0 -2
- teradataml/analytics/analytic_function_executor.py +3 -0
- teradataml/analytics/json_parser/utils.py +13 -12
- teradataml/analytics/sqle/DecisionTreePredict.py +15 -30
- teradataml/analytics/sqle/NaiveBayesPredict.py +11 -20
- teradataml/analytics/sqle/__init__.py +0 -13
- teradataml/analytics/utils.py +1 -0
- teradataml/analytics/valib.py +3 -0
- teradataml/automl/__init__.py +1628 -0
- teradataml/automl/custom_json_utils.py +1270 -0
- teradataml/automl/data_preparation.py +993 -0
- teradataml/automl/data_transformation.py +727 -0
- teradataml/automl/feature_engineering.py +1648 -0
- teradataml/automl/feature_exploration.py +547 -0
- teradataml/automl/model_evaluation.py +163 -0
- teradataml/automl/model_training.py +887 -0
- teradataml/catalog/__init__.py +0 -2
- teradataml/catalog/byom.py +49 -6
- teradataml/catalog/function_argument_mapper.py +0 -2
- teradataml/catalog/model_cataloging_utils.py +2 -1021
- teradataml/common/aed_utils.py +6 -2
- teradataml/common/constants.py +50 -58
- teradataml/common/deprecations.py +160 -0
- teradataml/common/garbagecollector.py +61 -104
- teradataml/common/messagecodes.py +27 -36
- teradataml/common/messages.py +11 -15
- teradataml/common/utils.py +205 -287
- teradataml/common/wrapper_utils.py +1 -110
- teradataml/context/context.py +150 -78
- teradataml/data/bank_churn.csv +10001 -0
- teradataml/data/bmi.csv +501 -0
- teradataml/data/docs/sqle/docs_17_10/BincodeFit.py +3 -3
- teradataml/data/docs/sqle/docs_17_10/BincodeTransform.py +6 -5
- teradataml/data/docs/sqle/docs_17_10/Fit.py +1 -1
- teradataml/data/docs/sqle/docs_17_10/OneHotEncodingTransform.py +1 -1
- teradataml/data/docs/sqle/docs_17_10/OutlierFilterTransform.py +1 -1
- teradataml/data/docs/sqle/docs_17_10/PolynomialFeaturesTransform.py +2 -2
- teradataml/data/docs/sqle/docs_17_10/RowNormalizeTransform.py +2 -1
- teradataml/data/docs/sqle/docs_17_10/ScaleTransform.py +1 -0
- teradataml/data/docs/sqle/docs_17_10/SimpleImputeTransform.py +1 -1
- teradataml/data/docs/sqle/docs_17_10/Transform.py +2 -1
- teradataml/data/docs/sqle/docs_17_20/BincodeFit.py +3 -3
- teradataml/data/docs/sqle/docs_17_20/BincodeTransform.py +6 -5
- teradataml/data/docs/sqle/docs_17_20/Fit.py +1 -1
- teradataml/data/docs/sqle/docs_17_20/GLM.py +1 -1
- teradataml/data/docs/sqle/docs_17_20/GLMPredictPerSegment.py +9 -10
- teradataml/data/docs/sqle/docs_17_20/KMeansPredict.py +3 -2
- teradataml/data/docs/sqle/docs_17_20/NaiveBayesTextClassifierPredict.py +16 -15
- teradataml/data/docs/sqle/docs_17_20/NaiveBayesTextClassifierTrainer.py +2 -2
- teradataml/data/docs/sqle/docs_17_20/NonLinearCombineFit.py +2 -2
- teradataml/data/docs/sqle/docs_17_20/NonLinearCombineTransform.py +8 -8
- teradataml/data/docs/sqle/docs_17_20/OneClassSVMPredict.py +21 -20
- teradataml/data/docs/sqle/docs_17_20/OneHotEncodingTransform.py +1 -1
- teradataml/data/docs/sqle/docs_17_20/OutlierFilterTransform.py +8 -3
- teradataml/data/docs/sqle/docs_17_20/PolynomialFeaturesTransform.py +6 -5
- teradataml/data/docs/sqle/docs_17_20/RandomProjectionTransform.py +6 -6
- teradataml/data/docs/sqle/docs_17_20/RowNormalizeTransform.py +2 -1
- teradataml/data/docs/sqle/docs_17_20/SVM.py +1 -1
- teradataml/data/docs/sqle/docs_17_20/SVMPredict.py +16 -16
- teradataml/data/docs/sqle/docs_17_20/ScaleTransform.py +1 -0
- teradataml/data/docs/sqle/docs_17_20/SimpleImputeTransform.py +3 -2
- teradataml/data/docs/sqle/docs_17_20/TDDecisionForestPredict.py +4 -4
- teradataml/data/docs/sqle/docs_17_20/TDGLMPredict.py +19 -19
- teradataml/data/docs/sqle/docs_17_20/TargetEncodingTransform.py +5 -4
- teradataml/data/docs/sqle/docs_17_20/Transform.py +2 -2
- teradataml/data/docs/sqle/docs_17_20/XGBoostPredict.py +9 -9
- teradataml/data/fish.csv +160 -0
- teradataml/data/glass_types.csv +215 -0
- teradataml/data/insurance.csv +1 -1
- teradataml/data/iris_data.csv +151 -0
- teradataml/data/jsons/sqle/17.10/TD_FunctionTransform.json +1 -0
- teradataml/data/jsons/sqle/17.10/TD_OneHotEncodingTransform.json +1 -0
- teradataml/data/jsons/sqle/17.10/TD_OutlierFilterTransform.json +1 -0
- teradataml/data/jsons/sqle/17.10/TD_PolynomialFeaturesTransform.json +1 -0
- teradataml/data/jsons/sqle/17.10/TD_RowNormalizeTransform.json +1 -0
- teradataml/data/jsons/sqle/17.10/TD_ScaleTransform.json +1 -0
- teradataml/data/jsons/sqle/17.10/TD_SimpleImputeTransform.json +1 -0
- teradataml/data/load_example_data.py +3 -0
- teradataml/data/multi_model_classification.csv +401 -0
- teradataml/data/multi_model_regression.csv +401 -0
- teradataml/data/openml_example.json +63 -0
- teradataml/data/scripts/deploy_script.py +65 -0
- teradataml/data/scripts/mapper.R +20 -0
- teradataml/data/scripts/sklearn/__init__.py +0 -0
- teradataml/data/scripts/sklearn/sklearn_fit.py +175 -0
- teradataml/data/scripts/sklearn/sklearn_fit_predict.py +135 -0
- teradataml/data/scripts/sklearn/sklearn_function.template +113 -0
- teradataml/data/scripts/sklearn/sklearn_model_selection_split.py +158 -0
- teradataml/data/scripts/sklearn/sklearn_neighbors.py +152 -0
- teradataml/data/scripts/sklearn/sklearn_score.py +128 -0
- teradataml/data/scripts/sklearn/sklearn_transform.py +179 -0
- teradataml/data/templates/open_source_ml.json +9 -0
- teradataml/data/teradataml_example.json +73 -1
- teradataml/data/test_classification.csv +101 -0
- teradataml/data/test_prediction.csv +101 -0
- teradataml/data/test_regression.csv +101 -0
- teradataml/data/train_multiclass.csv +101 -0
- teradataml/data/train_regression.csv +101 -0
- teradataml/data/train_regression_multiple_labels.csv +101 -0
- teradataml/data/wine_data.csv +1600 -0
- teradataml/dataframe/copy_to.py +79 -13
- teradataml/dataframe/data_transfer.py +8 -0
- teradataml/dataframe/dataframe.py +910 -311
- teradataml/dataframe/dataframe_utils.py +102 -5
- teradataml/dataframe/fastload.py +11 -3
- teradataml/dataframe/setop.py +15 -2
- teradataml/dataframe/sql.py +3735 -77
- teradataml/dataframe/sql_function_parameters.py +56 -5
- teradataml/dataframe/vantage_function_types.py +45 -1
- teradataml/dataframe/window.py +30 -29
- teradataml/dbutils/dbutils.py +18 -1
- teradataml/geospatial/geodataframe.py +18 -7
- teradataml/geospatial/geodataframecolumn.py +5 -0
- teradataml/hyperparameter_tuner/optimizer.py +910 -120
- teradataml/hyperparameter_tuner/utils.py +131 -37
- teradataml/lib/aed_0_1.dll +0 -0
- teradataml/lib/libaed_0_1.dylib +0 -0
- teradataml/lib/libaed_0_1.so +0 -0
- teradataml/libaed_0_1.dylib +0 -0
- teradataml/libaed_0_1.so +0 -0
- teradataml/opensource/__init__.py +1 -0
- teradataml/opensource/sklearn/__init__.py +1 -0
- teradataml/opensource/sklearn/_class.py +255 -0
- teradataml/opensource/sklearn/_sklearn_wrapper.py +1668 -0
- teradataml/opensource/sklearn/_wrapper_utils.py +268 -0
- teradataml/opensource/sklearn/constants.py +54 -0
- teradataml/options/__init__.py +3 -6
- teradataml/options/configure.py +21 -20
- teradataml/scriptmgmt/UserEnv.py +61 -5
- teradataml/scriptmgmt/lls_utils.py +135 -53
- teradataml/table_operators/Apply.py +38 -6
- teradataml/table_operators/Script.py +45 -308
- teradataml/table_operators/TableOperator.py +182 -591
- teradataml/table_operators/__init__.py +0 -1
- teradataml/table_operators/table_operator_util.py +32 -40
- teradataml/utils/validators.py +127 -3
- {teradataml-17.20.0.6.dist-info → teradataml-20.0.0.0.dist-info}/METADATA +243 -3
- {teradataml-17.20.0.6.dist-info → teradataml-20.0.0.0.dist-info}/RECORD +147 -391
- teradataml/analytics/mle/AdaBoost.py +0 -651
- teradataml/analytics/mle/AdaBoostPredict.py +0 -564
- teradataml/analytics/mle/Antiselect.py +0 -342
- teradataml/analytics/mle/Arima.py +0 -641
- teradataml/analytics/mle/ArimaPredict.py +0 -477
- teradataml/analytics/mle/Attribution.py +0 -1070
- teradataml/analytics/mle/Betweenness.py +0 -658
- teradataml/analytics/mle/Burst.py +0 -711
- teradataml/analytics/mle/CCM.py +0 -600
- teradataml/analytics/mle/CCMPrepare.py +0 -324
- teradataml/analytics/mle/CFilter.py +0 -460
- teradataml/analytics/mle/ChangePointDetection.py +0 -572
- teradataml/analytics/mle/ChangePointDetectionRT.py +0 -477
- teradataml/analytics/mle/Closeness.py +0 -737
- teradataml/analytics/mle/ConfusionMatrix.py +0 -420
- teradataml/analytics/mle/Correlation.py +0 -477
- teradataml/analytics/mle/Correlation2.py +0 -573
- teradataml/analytics/mle/CoxHazardRatio.py +0 -679
- teradataml/analytics/mle/CoxPH.py +0 -556
- teradataml/analytics/mle/CoxSurvival.py +0 -478
- teradataml/analytics/mle/CumulativeMovAvg.py +0 -363
- teradataml/analytics/mle/DTW.py +0 -623
- teradataml/analytics/mle/DWT.py +0 -564
- teradataml/analytics/mle/DWT2D.py +0 -599
- teradataml/analytics/mle/DecisionForest.py +0 -716
- teradataml/analytics/mle/DecisionForestEvaluator.py +0 -363
- teradataml/analytics/mle/DecisionForestPredict.py +0 -561
- teradataml/analytics/mle/DecisionTree.py +0 -830
- teradataml/analytics/mle/DecisionTreePredict.py +0 -528
- teradataml/analytics/mle/ExponentialMovAvg.py +0 -418
- teradataml/analytics/mle/FMeasure.py +0 -402
- teradataml/analytics/mle/FPGrowth.py +0 -734
- teradataml/analytics/mle/FrequentPaths.py +0 -695
- teradataml/analytics/mle/GLM.py +0 -558
- teradataml/analytics/mle/GLML1L2.py +0 -547
- teradataml/analytics/mle/GLML1L2Predict.py +0 -519
- teradataml/analytics/mle/GLMPredict.py +0 -529
- teradataml/analytics/mle/HMMDecoder.py +0 -945
- teradataml/analytics/mle/HMMEvaluator.py +0 -901
- teradataml/analytics/mle/HMMSupervised.py +0 -521
- teradataml/analytics/mle/HMMUnsupervised.py +0 -572
- teradataml/analytics/mle/Histogram.py +0 -561
- teradataml/analytics/mle/IDWT.py +0 -476
- teradataml/analytics/mle/IDWT2D.py +0 -493
- teradataml/analytics/mle/IdentityMatch.py +0 -763
- teradataml/analytics/mle/Interpolator.py +0 -918
- teradataml/analytics/mle/KMeans.py +0 -485
- teradataml/analytics/mle/KNN.py +0 -627
- teradataml/analytics/mle/KNNRecommender.py +0 -488
- teradataml/analytics/mle/KNNRecommenderPredict.py +0 -581
- teradataml/analytics/mle/LAR.py +0 -439
- teradataml/analytics/mle/LARPredict.py +0 -478
- teradataml/analytics/mle/LDA.py +0 -548
- teradataml/analytics/mle/LDAInference.py +0 -492
- teradataml/analytics/mle/LDATopicSummary.py +0 -464
- teradataml/analytics/mle/LevenshteinDistance.py +0 -450
- teradataml/analytics/mle/LinReg.py +0 -433
- teradataml/analytics/mle/LinRegPredict.py +0 -438
- teradataml/analytics/mle/MinHash.py +0 -544
- teradataml/analytics/mle/Modularity.py +0 -587
- teradataml/analytics/mle/NEREvaluator.py +0 -410
- teradataml/analytics/mle/NERExtractor.py +0 -595
- teradataml/analytics/mle/NERTrainer.py +0 -458
- teradataml/analytics/mle/NGrams.py +0 -570
- teradataml/analytics/mle/NPath.py +0 -634
- teradataml/analytics/mle/NTree.py +0 -549
- teradataml/analytics/mle/NaiveBayes.py +0 -462
- teradataml/analytics/mle/NaiveBayesPredict.py +0 -513
- teradataml/analytics/mle/NaiveBayesTextClassifier.py +0 -607
- teradataml/analytics/mle/NaiveBayesTextClassifier2.py +0 -531
- teradataml/analytics/mle/NaiveBayesTextClassifierPredict.py +0 -799
- teradataml/analytics/mle/NamedEntityFinder.py +0 -529
- teradataml/analytics/mle/NamedEntityFinderEvaluator.py +0 -414
- teradataml/analytics/mle/NamedEntityFinderTrainer.py +0 -396
- teradataml/analytics/mle/POSTagger.py +0 -417
- teradataml/analytics/mle/Pack.py +0 -411
- teradataml/analytics/mle/PageRank.py +0 -535
- teradataml/analytics/mle/PathAnalyzer.py +0 -426
- teradataml/analytics/mle/PathGenerator.py +0 -367
- teradataml/analytics/mle/PathStart.py +0 -464
- teradataml/analytics/mle/PathSummarizer.py +0 -470
- teradataml/analytics/mle/Pivot.py +0 -471
- teradataml/analytics/mle/ROC.py +0 -425
- teradataml/analytics/mle/RandomSample.py +0 -637
- teradataml/analytics/mle/RandomWalkSample.py +0 -490
- teradataml/analytics/mle/SAX.py +0 -779
- teradataml/analytics/mle/SVMDense.py +0 -677
- teradataml/analytics/mle/SVMDensePredict.py +0 -536
- teradataml/analytics/mle/SVMDenseSummary.py +0 -437
- teradataml/analytics/mle/SVMSparse.py +0 -557
- teradataml/analytics/mle/SVMSparsePredict.py +0 -553
- teradataml/analytics/mle/SVMSparseSummary.py +0 -435
- teradataml/analytics/mle/Sampling.py +0 -549
- teradataml/analytics/mle/Scale.py +0 -565
- teradataml/analytics/mle/ScaleByPartition.py +0 -496
- teradataml/analytics/mle/ScaleMap.py +0 -378
- teradataml/analytics/mle/ScaleSummary.py +0 -320
- teradataml/analytics/mle/SentenceExtractor.py +0 -363
- teradataml/analytics/mle/SentimentEvaluator.py +0 -432
- teradataml/analytics/mle/SentimentExtractor.py +0 -578
- teradataml/analytics/mle/SentimentTrainer.py +0 -405
- teradataml/analytics/mle/SeriesSplitter.py +0 -641
- teradataml/analytics/mle/Sessionize.py +0 -475
- teradataml/analytics/mle/SimpleMovAvg.py +0 -397
- teradataml/analytics/mle/StringSimilarity.py +0 -425
- teradataml/analytics/mle/TF.py +0 -389
- teradataml/analytics/mle/TFIDF.py +0 -504
- teradataml/analytics/mle/TextChunker.py +0 -414
- teradataml/analytics/mle/TextClassifier.py +0 -399
- teradataml/analytics/mle/TextClassifierEvaluator.py +0 -413
- teradataml/analytics/mle/TextClassifierTrainer.py +0 -565
- teradataml/analytics/mle/TextMorph.py +0 -494
- teradataml/analytics/mle/TextParser.py +0 -623
- teradataml/analytics/mle/TextTagger.py +0 -530
- teradataml/analytics/mle/TextTokenizer.py +0 -502
- teradataml/analytics/mle/UnivariateStatistics.py +0 -488
- teradataml/analytics/mle/Unpack.py +0 -526
- teradataml/analytics/mle/Unpivot.py +0 -438
- teradataml/analytics/mle/VarMax.py +0 -776
- teradataml/analytics/mle/VectorDistance.py +0 -762
- teradataml/analytics/mle/WeightedMovAvg.py +0 -400
- teradataml/analytics/mle/XGBoost.py +0 -842
- teradataml/analytics/mle/XGBoostPredict.py +0 -627
- teradataml/analytics/mle/__init__.py +0 -123
- teradataml/analytics/mle/json/adaboost_mle.json +0 -135
- teradataml/analytics/mle/json/adaboostpredict_mle.json +0 -85
- teradataml/analytics/mle/json/antiselect_mle.json +0 -34
- teradataml/analytics/mle/json/antiselect_mle_mle.json +0 -34
- teradataml/analytics/mle/json/arima_mle.json +0 -172
- teradataml/analytics/mle/json/arimapredict_mle.json +0 -52
- teradataml/analytics/mle/json/attribution_mle_mle.json +0 -143
- teradataml/analytics/mle/json/betweenness_mle.json +0 -97
- teradataml/analytics/mle/json/burst_mle.json +0 -140
- teradataml/analytics/mle/json/ccm_mle.json +0 -124
- teradataml/analytics/mle/json/ccmprepare_mle.json +0 -14
- teradataml/analytics/mle/json/cfilter_mle.json +0 -93
- teradataml/analytics/mle/json/changepointdetection_mle.json +0 -92
- teradataml/analytics/mle/json/changepointdetectionrt_mle.json +0 -78
- teradataml/analytics/mle/json/closeness_mle.json +0 -104
- teradataml/analytics/mle/json/confusionmatrix_mle.json +0 -79
- teradataml/analytics/mle/json/correlation_mle.json +0 -86
- teradataml/analytics/mle/json/correlationreduce_mle.json +0 -49
- teradataml/analytics/mle/json/coxhazardratio_mle.json +0 -89
- teradataml/analytics/mle/json/coxph_mle.json +0 -98
- teradataml/analytics/mle/json/coxsurvival_mle.json +0 -79
- teradataml/analytics/mle/json/cumulativemovavg_mle.json +0 -34
- teradataml/analytics/mle/json/decisionforest_mle.json +0 -167
- teradataml/analytics/mle/json/decisionforestevaluator_mle.json +0 -33
- teradataml/analytics/mle/json/decisionforestpredict_mle_mle.json +0 -74
- teradataml/analytics/mle/json/decisiontree_mle.json +0 -194
- teradataml/analytics/mle/json/decisiontreepredict_mle_mle.json +0 -86
- teradataml/analytics/mle/json/dtw_mle.json +0 -97
- teradataml/analytics/mle/json/dwt2d_mle.json +0 -116
- teradataml/analytics/mle/json/dwt_mle.json +0 -101
- teradataml/analytics/mle/json/exponentialmovavg_mle.json +0 -55
- teradataml/analytics/mle/json/fmeasure_mle.json +0 -58
- teradataml/analytics/mle/json/fpgrowth_mle.json +0 -159
- teradataml/analytics/mle/json/frequentpaths_mle.json +0 -129
- teradataml/analytics/mle/json/glm_mle.json +0 -111
- teradataml/analytics/mle/json/glml1l2_mle.json +0 -106
- teradataml/analytics/mle/json/glml1l2predict_mle.json +0 -57
- teradataml/analytics/mle/json/glmpredict_mle_mle.json +0 -74
- teradataml/analytics/mle/json/histogram_mle.json +0 -100
- teradataml/analytics/mle/json/hmmdecoder_mle.json +0 -192
- teradataml/analytics/mle/json/hmmevaluator_mle.json +0 -206
- teradataml/analytics/mle/json/hmmsupervised_mle.json +0 -91
- teradataml/analytics/mle/json/hmmunsupervised_mle.json +0 -114
- teradataml/analytics/mle/json/identitymatch_mle.json +0 -88
- teradataml/analytics/mle/json/idwt2d_mle.json +0 -73
- teradataml/analytics/mle/json/idwt_mle.json +0 -66
- teradataml/analytics/mle/json/interpolator_mle.json +0 -151
- teradataml/analytics/mle/json/kmeans_mle.json +0 -97
- teradataml/analytics/mle/json/knn_mle.json +0 -141
- teradataml/analytics/mle/json/knnrecommender_mle.json +0 -111
- teradataml/analytics/mle/json/knnrecommenderpredict_mle.json +0 -75
- teradataml/analytics/mle/json/lar_mle.json +0 -78
- teradataml/analytics/mle/json/larpredict_mle.json +0 -69
- teradataml/analytics/mle/json/lda_mle.json +0 -130
- teradataml/analytics/mle/json/ldainference_mle.json +0 -78
- teradataml/analytics/mle/json/ldatopicsummary_mle.json +0 -64
- teradataml/analytics/mle/json/levenshteindistance_mle.json +0 -92
- teradataml/analytics/mle/json/linreg_mle.json +0 -42
- teradataml/analytics/mle/json/linregpredict_mle.json +0 -56
- teradataml/analytics/mle/json/minhash_mle.json +0 -113
- teradataml/analytics/mle/json/modularity_mle.json +0 -91
- teradataml/analytics/mle/json/naivebayespredict_mle_mle.json +0 -85
- teradataml/analytics/mle/json/naivebayesreduce_mle.json +0 -52
- teradataml/analytics/mle/json/naivebayestextclassifierpredict_mle_mle.json +0 -147
- teradataml/analytics/mle/json/naivebayestextclassifiertrainer2_mle.json +0 -108
- teradataml/analytics/mle/json/naivebayestextclassifiertrainer_mle.json +0 -102
- teradataml/analytics/mle/json/namedentityfinder_mle.json +0 -84
- teradataml/analytics/mle/json/namedentityfinderevaluatorreduce_mle.json +0 -43
- teradataml/analytics/mle/json/namedentityfindertrainer_mle.json +0 -64
- teradataml/analytics/mle/json/nerevaluator_mle.json +0 -54
- teradataml/analytics/mle/json/nerextractor_mle.json +0 -87
- teradataml/analytics/mle/json/nertrainer_mle.json +0 -89
- teradataml/analytics/mle/json/ngrams_mle.json +0 -137
- teradataml/analytics/mle/json/ngramsplitter_mle_mle.json +0 -137
- teradataml/analytics/mle/json/npath@coprocessor_mle.json +0 -73
- teradataml/analytics/mle/json/ntree@coprocessor_mle.json +0 -123
- teradataml/analytics/mle/json/pack_mle.json +0 -58
- teradataml/analytics/mle/json/pack_mle_mle.json +0 -58
- teradataml/analytics/mle/json/pagerank_mle.json +0 -81
- teradataml/analytics/mle/json/pathanalyzer_mle.json +0 -63
- teradataml/analytics/mle/json/pathgenerator_mle.json +0 -40
- teradataml/analytics/mle/json/pathstart_mle.json +0 -62
- teradataml/analytics/mle/json/pathsummarizer_mle.json +0 -72
- teradataml/analytics/mle/json/pivoting_mle.json +0 -71
- teradataml/analytics/mle/json/postagger_mle.json +0 -51
- teradataml/analytics/mle/json/randomsample_mle.json +0 -131
- teradataml/analytics/mle/json/randomwalksample_mle.json +0 -85
- teradataml/analytics/mle/json/roc_mle.json +0 -73
- teradataml/analytics/mle/json/sampling_mle.json +0 -75
- teradataml/analytics/mle/json/sax_mle.json +0 -154
- teradataml/analytics/mle/json/scale_mle.json +0 -93
- teradataml/analytics/mle/json/scalebypartition_mle.json +0 -89
- teradataml/analytics/mle/json/scalemap_mle.json +0 -44
- teradataml/analytics/mle/json/scalesummary_mle.json +0 -14
- teradataml/analytics/mle/json/sentenceextractor_mle.json +0 -41
- teradataml/analytics/mle/json/sentimentevaluator_mle.json +0 -43
- teradataml/analytics/mle/json/sentimentextractor_mle.json +0 -100
- teradataml/analytics/mle/json/sentimenttrainer_mle.json +0 -68
- teradataml/analytics/mle/json/seriessplitter_mle.json +0 -133
- teradataml/analytics/mle/json/sessionize_mle_mle.json +0 -62
- teradataml/analytics/mle/json/simplemovavg_mle.json +0 -48
- teradataml/analytics/mle/json/stringsimilarity_mle.json +0 -50
- teradataml/analytics/mle/json/stringsimilarity_mle_mle.json +0 -50
- teradataml/analytics/mle/json/svmdense_mle.json +0 -165
- teradataml/analytics/mle/json/svmdensepredict_mle.json +0 -95
- teradataml/analytics/mle/json/svmdensesummary_mle.json +0 -58
- teradataml/analytics/mle/json/svmsparse_mle.json +0 -148
- teradataml/analytics/mle/json/svmsparsepredict_mle_mle.json +0 -103
- teradataml/analytics/mle/json/svmsparsesummary_mle.json +0 -57
- teradataml/analytics/mle/json/textchunker_mle.json +0 -40
- teradataml/analytics/mle/json/textclassifier_mle.json +0 -51
- teradataml/analytics/mle/json/textclassifierevaluator_mle.json +0 -43
- teradataml/analytics/mle/json/textclassifiertrainer_mle.json +0 -103
- teradataml/analytics/mle/json/textmorph_mle.json +0 -63
- teradataml/analytics/mle/json/textparser_mle.json +0 -166
- teradataml/analytics/mle/json/texttagger_mle.json +0 -81
- teradataml/analytics/mle/json/texttokenizer_mle.json +0 -91
- teradataml/analytics/mle/json/tf_mle.json +0 -33
- teradataml/analytics/mle/json/tfidf_mle.json +0 -34
- teradataml/analytics/mle/json/univariatestatistics_mle.json +0 -81
- teradataml/analytics/mle/json/unpack_mle.json +0 -91
- teradataml/analytics/mle/json/unpack_mle_mle.json +0 -91
- teradataml/analytics/mle/json/unpivoting_mle.json +0 -63
- teradataml/analytics/mle/json/varmax_mle.json +0 -176
- teradataml/analytics/mle/json/vectordistance_mle.json +0 -179
- teradataml/analytics/mle/json/weightedmovavg_mle.json +0 -48
- teradataml/analytics/mle/json/xgboost_mle.json +0 -178
- teradataml/analytics/mle/json/xgboostpredict_mle.json +0 -104
- teradataml/analytics/sqle/Antiselect.py +0 -321
- teradataml/analytics/sqle/Attribution.py +0 -603
- teradataml/analytics/sqle/DecisionForestPredict.py +0 -408
- teradataml/analytics/sqle/GLMPredict.py +0 -430
- teradataml/analytics/sqle/MovingAverage.py +0 -543
- teradataml/analytics/sqle/NGramSplitter.py +0 -548
- teradataml/analytics/sqle/NPath.py +0 -632
- teradataml/analytics/sqle/NaiveBayesTextClassifierPredict.py +0 -515
- teradataml/analytics/sqle/Pack.py +0 -388
- teradataml/analytics/sqle/SVMSparsePredict.py +0 -464
- teradataml/analytics/sqle/Sessionize.py +0 -390
- teradataml/analytics/sqle/StringSimilarity.py +0 -400
- teradataml/analytics/sqle/Unpack.py +0 -503
- teradataml/analytics/sqle/json/antiselect_sqle.json +0 -21
- teradataml/analytics/sqle/json/attribution_sqle.json +0 -92
- teradataml/analytics/sqle/json/decisionforestpredict_sqle.json +0 -48
- teradataml/analytics/sqle/json/glmpredict_sqle.json +0 -48
- teradataml/analytics/sqle/json/h2opredict_sqle.json +0 -63
- teradataml/analytics/sqle/json/movingaverage_sqle.json +0 -58
- teradataml/analytics/sqle/json/naivebayestextclassifierpredict_sqle.json +0 -76
- teradataml/analytics/sqle/json/ngramsplitter_sqle.json +0 -126
- teradataml/analytics/sqle/json/npath_sqle.json +0 -67
- teradataml/analytics/sqle/json/pack_sqle.json +0 -47
- teradataml/analytics/sqle/json/pmmlpredict_sqle.json +0 -55
- teradataml/analytics/sqle/json/sessionize_sqle.json +0 -43
- teradataml/analytics/sqle/json/stringsimilarity_sqle.json +0 -39
- teradataml/analytics/sqle/json/svmsparsepredict_sqle.json +0 -74
- teradataml/analytics/sqle/json/unpack_sqle.json +0 -80
- teradataml/catalog/model_cataloging.py +0 -980
- teradataml/config/mlengine_alias_definitions_v1.0 +0 -118
- teradataml/config/mlengine_alias_definitions_v1.1 +0 -127
- teradataml/config/mlengine_alias_definitions_v1.3 +0 -129
- teradataml/table_operators/sandbox_container_util.py +0 -643
- {teradataml-17.20.0.6.dist-info → teradataml-20.0.0.0.dist-info}/WHEEL +0 -0
- {teradataml-17.20.0.6.dist-info → teradataml-20.0.0.0.dist-info}/top_level.txt +0 -0
- {teradataml-17.20.0.6.dist-info → teradataml-20.0.0.0.dist-info}/zip-safe +0 -0
|
@@ -1,623 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/python
|
|
2
|
-
# ##################################################################
|
|
3
|
-
#
|
|
4
|
-
# Copyright 2018 Teradata. All rights reserved.
|
|
5
|
-
# TERADATA CONFIDENTIAL AND TRADE SECRET
|
|
6
|
-
#
|
|
7
|
-
# Primary Owner: Bhavana N (bhavana.n@teradata.com)
|
|
8
|
-
# Secondary Owner: Pankaj Purandare (pankajvinod.purandare@teradata.com)
|
|
9
|
-
#
|
|
10
|
-
# Version: 1.2
|
|
11
|
-
# Function Version: 1.14
|
|
12
|
-
#
|
|
13
|
-
# ##################################################################
|
|
14
|
-
|
|
15
|
-
import inspect
|
|
16
|
-
import time
|
|
17
|
-
from teradataml.common.wrapper_utils import AnalyticsWrapperUtils
|
|
18
|
-
from teradataml.common.utils import UtilFuncs
|
|
19
|
-
from teradataml.context.context import *
|
|
20
|
-
from teradataml.dataframe.dataframe import DataFrame
|
|
21
|
-
from teradataml.common.aed_utils import AedUtils
|
|
22
|
-
from teradataml.analytics.analytic_query_generator import AnalyticQueryGenerator
|
|
23
|
-
from teradataml.common.exceptions import TeradataMlException
|
|
24
|
-
from teradataml.common.messages import Messages
|
|
25
|
-
from teradataml.common.messagecodes import MessageCodes
|
|
26
|
-
from teradataml.common.constants import TeradataConstants
|
|
27
|
-
from teradataml.dataframe.dataframe_utils import DataFrameUtils as df_utils
|
|
28
|
-
from teradataml.options.display import display
|
|
29
|
-
|
|
30
|
-
class TextParser:
|
|
31
|
-
|
|
32
|
-
def __init__(self,
|
|
33
|
-
data = None,
|
|
34
|
-
text_column = None,
|
|
35
|
-
to_lower_case = True,
|
|
36
|
-
stemming = False,
|
|
37
|
-
delimiter = "[ \\t\\f\\r\\n]+",
|
|
38
|
-
total_words_num = False,
|
|
39
|
-
punctuation = "[.,!?]",
|
|
40
|
-
accumulate = None,
|
|
41
|
-
token_column = "token",
|
|
42
|
-
frequency_column = "frequency",
|
|
43
|
-
total_column = "total_count",
|
|
44
|
-
remove_stop_words = False,
|
|
45
|
-
position_column = "location",
|
|
46
|
-
list_positions = False,
|
|
47
|
-
output_by_word = True,
|
|
48
|
-
stemming_exceptions = None,
|
|
49
|
-
stop_words = None,
|
|
50
|
-
data_sequence_column = None,
|
|
51
|
-
data_order_column = None):
|
|
52
|
-
"""
|
|
53
|
-
DESCRIPTION:
|
|
54
|
-
The TextParser function tokenizes an input stream of words, optionally
|
|
55
|
-
stems them (reduces them to their root forms), and then outputs them.
|
|
56
|
-
The function can either output all words in one row or output each
|
|
57
|
-
word in its own row with (optionally) the number of times that the word appears.
|
|
58
|
-
The TextParser function uses Porter2 as the stemming algorithm.
|
|
59
|
-
The TextParser function reads a document into a database memory buffer and
|
|
60
|
-
creates a hash table. The dictionary for the document must not exceed available
|
|
61
|
-
memory; however, a million-word dictionary with an average word length of
|
|
62
|
-
ten bytes requires only 10 MB of memory.
|
|
63
|
-
This function can be used with real-time applications.
|
|
64
|
-
Note: TextParser uses files that are preinstalled on the ML Engine.
|
|
65
|
-
For details, see Preinstalled Files that functions Use.
|
|
66
|
-
|
|
67
|
-
PARAMETERS:
|
|
68
|
-
data:
|
|
69
|
-
Required Argument.
|
|
70
|
-
Specifies the teradataml DataFrame that contains the text to be tokenized.
|
|
71
|
-
|
|
72
|
-
data_order_column:
|
|
73
|
-
Optional Argument.
|
|
74
|
-
Specifies Order By columns for data.
|
|
75
|
-
Values to this argument can be provided as list, if multiple columns
|
|
76
|
-
are used for ordering.
|
|
77
|
-
Types: str OR list of Strings (str)
|
|
78
|
-
|
|
79
|
-
text_column:
|
|
80
|
-
Required Argument.
|
|
81
|
-
Specifies the name of the input column whose contents are to be
|
|
82
|
-
tokenized.
|
|
83
|
-
Types: str
|
|
84
|
-
|
|
85
|
-
to_lower_case:
|
|
86
|
-
Optional Argument.
|
|
87
|
-
Specifies whether to convert input text to lowercase.
|
|
88
|
-
Note: The function ignores this argument, if the "stemming" argument has the value
|
|
89
|
-
True.
|
|
90
|
-
Default Value: True
|
|
91
|
-
Types: bool
|
|
92
|
-
|
|
93
|
-
stemming:
|
|
94
|
-
Optional Argument.
|
|
95
|
-
Specifies whether to stem the tokens that is, whether to apply the
|
|
96
|
-
Porter2 stemming algorithm to each token to reduce it to its root
|
|
97
|
-
form. Before stemming, the function converts the input text to
|
|
98
|
-
lowercase and applies the remove_stop_words argument.
|
|
99
|
-
Default Value: False
|
|
100
|
-
Types: bool
|
|
101
|
-
|
|
102
|
-
delimiter:
|
|
103
|
-
Optional Argument.
|
|
104
|
-
Specifies a regular expression that represents the word delimiter.
|
|
105
|
-
Default Value: [ \\t\\f\\r\\n]+
|
|
106
|
-
Types: str
|
|
107
|
-
|
|
108
|
-
total_words_num:
|
|
109
|
-
Optional Argument.
|
|
110
|
-
Specifies whether to output a column that contains the total number
|
|
111
|
-
of words in the input document.
|
|
112
|
-
Default Value: False
|
|
113
|
-
Types: bool
|
|
114
|
-
|
|
115
|
-
punctuation:
|
|
116
|
-
Optional Argument.
|
|
117
|
-
Specifies a regular expression that represents the punctuation
|
|
118
|
-
characters to remove from the input text. With stemming (True), the
|
|
119
|
-
recommended value is "[\\\[.,?!:;~()\\\]]+".
|
|
120
|
-
Default Value: [.,!?]
|
|
121
|
-
Types: str
|
|
122
|
-
|
|
123
|
-
accumulate:
|
|
124
|
-
Optional Argument.
|
|
125
|
-
Specifies the names of the input columns to copy to the output teradataml DataFrame.
|
|
126
|
-
By default, the function copies all input columns to the output
|
|
127
|
-
teradtaml DataFrame.
|
|
128
|
-
Note: No accumulate column can be the same as token_column or
|
|
129
|
-
total_column.
|
|
130
|
-
Types: str OR list of Strings (str)
|
|
131
|
-
|
|
132
|
-
token_column:
|
|
133
|
-
Optional Argument.
|
|
134
|
-
Specifies the name of the output column that contains the tokens.
|
|
135
|
-
Default Value: token
|
|
136
|
-
Types: str
|
|
137
|
-
|
|
138
|
-
frequency_column:
|
|
139
|
-
Optional Argument.
|
|
140
|
-
Specifies the name of the output column that contains the frequency
|
|
141
|
-
of each token.
|
|
142
|
-
Default Value: frequency
|
|
143
|
-
Types: str
|
|
144
|
-
|
|
145
|
-
total_column:
|
|
146
|
-
Optional Argument.
|
|
147
|
-
Specifies the name of the output column that contains the total
|
|
148
|
-
number of words in the input document.
|
|
149
|
-
Default Value: total_count
|
|
150
|
-
Types: str
|
|
151
|
-
|
|
152
|
-
remove_stop_words:
|
|
153
|
-
Optional Argument.
|
|
154
|
-
Specifies whether to remove stop words from the input text before
|
|
155
|
-
parsing.
|
|
156
|
-
Default Value: False
|
|
157
|
-
Types: bool
|
|
158
|
-
|
|
159
|
-
position_column:
|
|
160
|
-
Optional Argument.
|
|
161
|
-
Specifies the name of the output column that contains the position of
|
|
162
|
-
a word within a document.
|
|
163
|
-
Default Value: location
|
|
164
|
-
Types: str
|
|
165
|
-
|
|
166
|
-
list_positions:
|
|
167
|
-
Optional Argument.
|
|
168
|
-
Specifies whether to output the position of a word in list form.
|
|
169
|
-
If the value is True, the function to output a row for each occurrence of the
|
|
170
|
-
word.
|
|
171
|
-
Note: The function ignores this argument if the output_by_word
|
|
172
|
-
argument has the value False.
|
|
173
|
-
Default Value: False
|
|
174
|
-
Types: bool
|
|
175
|
-
|
|
176
|
-
output_by_word:
|
|
177
|
-
Optional Argument.
|
|
178
|
-
Specifies whether to output each token of each input document in its
|
|
179
|
-
own row in the output teradataml DataFrame. If you specify False, then the
|
|
180
|
-
function outputs each tokenized input document in one row of the
|
|
181
|
-
output teradataml DataFrame.
|
|
182
|
-
Default Value: True
|
|
183
|
-
Types: bool
|
|
184
|
-
|
|
185
|
-
stemming_exceptions:
|
|
186
|
-
Optional Argument.
|
|
187
|
-
Specifies the location of the file that contains the stemming
|
|
188
|
-
exceptions. A stemming exception is a word followed by its stemmed
|
|
189
|
-
form. The word and its stemmed form are separated by white space.
|
|
190
|
-
Each stemming exception is on its own line in the file.
|
|
191
|
-
For example: bias bias news news goods goods lying lie ugly ugli sky sky early
|
|
192
|
-
earli
|
|
193
|
-
The words "lying", "ugly", and "early" are to become "lie",
|
|
194
|
-
"ugli", and "earli", respectively. The other words are not to change.
|
|
195
|
-
Types: str
|
|
196
|
-
|
|
197
|
-
stop_words:
|
|
198
|
-
Optional Argument.
|
|
199
|
-
Specifies the location of the file that contains the stop words
|
|
200
|
-
(words to ignore when parsing text). Each stop word is on its own
|
|
201
|
-
line in the file.
|
|
202
|
-
For example: a an the and this with but will
|
|
203
|
-
Types: str
|
|
204
|
-
|
|
205
|
-
data_sequence_column:
|
|
206
|
-
Optional Argument.
|
|
207
|
-
Specifies the list of column(s) that uniquely identifies each row of
|
|
208
|
-
the input argument "data". The argument is used to ensure
|
|
209
|
-
deterministic results for functions which produce results that vary
|
|
210
|
-
from run to run.
|
|
211
|
-
Types: str OR list of Strings (str)
|
|
212
|
-
|
|
213
|
-
RETURNS:
|
|
214
|
-
Instance of TextParser.
|
|
215
|
-
Output teradataml DataFrames can be accessed using attribute
|
|
216
|
-
references, such as TextParserObj.<attribute_name>.
|
|
217
|
-
Output teradataml DataFrame attribute name is:
|
|
218
|
-
result
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
RAISES:
|
|
222
|
-
TeradataMlException
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
EXAMPLES:
|
|
226
|
-
# Load example data.
|
|
227
|
-
load_example_data("textparser", ["complaints","complaints_mini"])
|
|
228
|
-
|
|
229
|
-
# Create teradataml DataFrame objects.
|
|
230
|
-
complaints = DataFrame.from_table("complaints")
|
|
231
|
-
complaints_mini = DataFrame.from_table("complaints_mini")
|
|
232
|
-
|
|
233
|
-
# Example 1 - StopWords without StemmingExceptions
|
|
234
|
-
text_parser_out1 = TextParser(data = complaints,
|
|
235
|
-
text_column = "text_data",
|
|
236
|
-
to_lower_case = True,
|
|
237
|
-
stemming = False,
|
|
238
|
-
punctuation = "\\\\[.,?\\\\!\\\\]",
|
|
239
|
-
accumulate = ["doc_id","category"],
|
|
240
|
-
remove_stop_words = True,
|
|
241
|
-
list_positions = True,
|
|
242
|
-
output_by_word = True,
|
|
243
|
-
stop_words = "stopwords.txt"
|
|
244
|
-
)
|
|
245
|
-
# Print the result DataFrame.
|
|
246
|
-
print(text_parser_out1.result)
|
|
247
|
-
|
|
248
|
-
# Example 2 - StemmingExceptions without StopWords
|
|
249
|
-
text_parser_out2 = TextParser(data = complaints_mini,
|
|
250
|
-
text_column = "text_data",
|
|
251
|
-
to_lower_case = True,
|
|
252
|
-
stemming = True,
|
|
253
|
-
punctuation = "\\\\[.,?\\\\!\\\\]",
|
|
254
|
-
accumulate = ["doc_id","category"],
|
|
255
|
-
output_by_word = False,
|
|
256
|
-
stemming_exceptions = "stemmingexception.txt"
|
|
257
|
-
)
|
|
258
|
-
|
|
259
|
-
# Print the result DataFrame.
|
|
260
|
-
print(text_parser_out2.result)
|
|
261
|
-
|
|
262
|
-
"""
|
|
263
|
-
|
|
264
|
-
# Start the timer to get the build time
|
|
265
|
-
_start_time = time.time()
|
|
266
|
-
|
|
267
|
-
self.data = data
|
|
268
|
-
self.text_column = text_column
|
|
269
|
-
self.to_lower_case = to_lower_case
|
|
270
|
-
self.stemming = stemming
|
|
271
|
-
self.delimiter = delimiter
|
|
272
|
-
self.total_words_num = total_words_num
|
|
273
|
-
self.punctuation = punctuation
|
|
274
|
-
self.accumulate = accumulate
|
|
275
|
-
self.token_column = token_column
|
|
276
|
-
self.frequency_column = frequency_column
|
|
277
|
-
self.total_column = total_column
|
|
278
|
-
self.remove_stop_words = remove_stop_words
|
|
279
|
-
self.position_column = position_column
|
|
280
|
-
self.list_positions = list_positions
|
|
281
|
-
self.output_by_word = output_by_word
|
|
282
|
-
self.stemming_exceptions = stemming_exceptions
|
|
283
|
-
self.stop_words = stop_words
|
|
284
|
-
self.data_sequence_column = data_sequence_column
|
|
285
|
-
self.data_order_column = data_order_column
|
|
286
|
-
|
|
287
|
-
# Create TeradataPyWrapperUtils instance which contains validation functions.
|
|
288
|
-
self.__awu = AnalyticsWrapperUtils()
|
|
289
|
-
self.__aed_utils = AedUtils()
|
|
290
|
-
|
|
291
|
-
# Create argument information matrix to do parameter checking
|
|
292
|
-
self.__arg_info_matrix = []
|
|
293
|
-
self.__arg_info_matrix.append(["data", self.data, False, (DataFrame)])
|
|
294
|
-
self.__arg_info_matrix.append(["data_order_column", self.data_order_column, True, (str,list)])
|
|
295
|
-
self.__arg_info_matrix.append(["text_column", self.text_column, False, (str)])
|
|
296
|
-
self.__arg_info_matrix.append(["to_lower_case", self.to_lower_case, True, (bool)])
|
|
297
|
-
self.__arg_info_matrix.append(["stemming", self.stemming, True, (bool)])
|
|
298
|
-
self.__arg_info_matrix.append(["delimiter", self.delimiter, True, (str)])
|
|
299
|
-
self.__arg_info_matrix.append(["total_words_num", self.total_words_num, True, (bool)])
|
|
300
|
-
self.__arg_info_matrix.append(["punctuation", self.punctuation, True, (str)])
|
|
301
|
-
self.__arg_info_matrix.append(["accumulate", self.accumulate, True, (str,list)])
|
|
302
|
-
self.__arg_info_matrix.append(["token_column", self.token_column, True, (str)])
|
|
303
|
-
self.__arg_info_matrix.append(["frequency_column", self.frequency_column, True, (str)])
|
|
304
|
-
self.__arg_info_matrix.append(["total_column", self.total_column, True, (str)])
|
|
305
|
-
self.__arg_info_matrix.append(["remove_stop_words", self.remove_stop_words, True, (bool)])
|
|
306
|
-
self.__arg_info_matrix.append(["position_column", self.position_column, True, (str)])
|
|
307
|
-
self.__arg_info_matrix.append(["list_positions", self.list_positions, True, (bool)])
|
|
308
|
-
self.__arg_info_matrix.append(["output_by_word", self.output_by_word, True, (bool)])
|
|
309
|
-
self.__arg_info_matrix.append(["stemming_exceptions", self.stemming_exceptions, True, (str)])
|
|
310
|
-
self.__arg_info_matrix.append(["stop_words", self.stop_words, True, (str)])
|
|
311
|
-
self.__arg_info_matrix.append(["data_sequence_column", self.data_sequence_column, True, (str,list)])
|
|
312
|
-
|
|
313
|
-
if inspect.stack()[1][3] != '_from_model_catalog':
|
|
314
|
-
# Perform the function validations
|
|
315
|
-
self.__validate()
|
|
316
|
-
# Generate the ML query
|
|
317
|
-
self.__form_tdml_query()
|
|
318
|
-
# Execute ML query
|
|
319
|
-
self.__execute()
|
|
320
|
-
# Get the prediction type
|
|
321
|
-
self._prediction_type = self.__awu._get_function_prediction_type(self)
|
|
322
|
-
|
|
323
|
-
# End the timer to get the build time
|
|
324
|
-
_end_time = time.time()
|
|
325
|
-
|
|
326
|
-
# Calculate the build time
|
|
327
|
-
self._build_time = (int)(_end_time - _start_time)
|
|
328
|
-
|
|
329
|
-
def __validate(self):
|
|
330
|
-
"""
|
|
331
|
-
Function to validate sqlmr function arguments, which verifies missing
|
|
332
|
-
arguments, input argument and table types. Also processes the
|
|
333
|
-
argument values.
|
|
334
|
-
"""
|
|
335
|
-
|
|
336
|
-
# Make sure that a non-NULL value has been supplied for all mandatory arguments
|
|
337
|
-
self.__awu._validate_missing_required_arguments(self.__arg_info_matrix)
|
|
338
|
-
|
|
339
|
-
# Make sure that a non-NULL value has been supplied correct type of argument
|
|
340
|
-
self.__awu._validate_argument_types(self.__arg_info_matrix)
|
|
341
|
-
|
|
342
|
-
# Check to make sure input table types are strings or data frame objects or of valid type.
|
|
343
|
-
self.__awu._validate_input_table_datatype(self.data, "data", None)
|
|
344
|
-
|
|
345
|
-
# Check whether the input columns passed to the argument are not empty.
|
|
346
|
-
# Also check whether the input columns passed to the argument valid or not.
|
|
347
|
-
self.__awu._validate_input_columns_not_empty(self.text_column, "text_column")
|
|
348
|
-
self.__awu._validate_dataframe_has_argument_columns(self.text_column, "text_column", self.data, "data", False)
|
|
349
|
-
|
|
350
|
-
self.__awu._validate_input_columns_not_empty(self.accumulate, "accumulate")
|
|
351
|
-
self.__awu._validate_dataframe_has_argument_columns(self.accumulate, "accumulate", self.data, "data", False)
|
|
352
|
-
|
|
353
|
-
self.__awu._validate_input_columns_not_empty(self.data_sequence_column, "data_sequence_column")
|
|
354
|
-
self.__awu._validate_dataframe_has_argument_columns(self.data_sequence_column, "data_sequence_column", self.data, "data", False)
|
|
355
|
-
|
|
356
|
-
self.__awu._validate_input_columns_not_empty(self.data_order_column, "data_order_column")
|
|
357
|
-
self.__awu._validate_dataframe_has_argument_columns(self.data_order_column, "data_order_column", self.data, "data", False)
|
|
358
|
-
|
|
359
|
-
# Validate that value passed to the output column argument is not empty.
|
|
360
|
-
self.__awu._validate_input_columns_not_empty(self.token_column, "token_column")
|
|
361
|
-
self.__awu._validate_input_columns_not_empty(self.frequency_column, "frequency_column")
|
|
362
|
-
self.__awu._validate_input_columns_not_empty(self.total_column, "total_column")
|
|
363
|
-
self.__awu._validate_input_columns_not_empty(self.position_column, "position_column")
|
|
364
|
-
|
|
365
|
-
def __form_tdml_query(self):
|
|
366
|
-
"""
|
|
367
|
-
Function to generate the analytical function queries. The function defines
|
|
368
|
-
variables and list of arguments required to form the query.
|
|
369
|
-
"""
|
|
370
|
-
|
|
371
|
-
# Output table arguments list
|
|
372
|
-
self.__func_output_args_sql_names = []
|
|
373
|
-
self.__func_output_args = []
|
|
374
|
-
|
|
375
|
-
# Model Cataloging related attributes.
|
|
376
|
-
self._sql_specific_attributes = {}
|
|
377
|
-
self._sql_formula_attribute_mapper = {}
|
|
378
|
-
self._target_column = None
|
|
379
|
-
self._algorithm_name = None
|
|
380
|
-
|
|
381
|
-
# Generate lists for rest of the function arguments
|
|
382
|
-
self.__func_other_arg_sql_names = []
|
|
383
|
-
self.__func_other_args = []
|
|
384
|
-
self.__func_other_arg_json_datatypes = []
|
|
385
|
-
|
|
386
|
-
self.__func_other_arg_sql_names.append("TextColumn")
|
|
387
|
-
self.__func_other_args.append(UtilFuncs._teradata_collapse_arglist(UtilFuncs._teradata_quote_arg(self.text_column, "\""), "'"))
|
|
388
|
-
self.__func_other_arg_json_datatypes.append("COLUMNS")
|
|
389
|
-
|
|
390
|
-
if self.accumulate is not None:
|
|
391
|
-
self.__func_other_arg_sql_names.append("Accumulate")
|
|
392
|
-
self.__func_other_args.append(UtilFuncs._teradata_collapse_arglist(UtilFuncs._teradata_quote_arg(self.accumulate, "\""), "'"))
|
|
393
|
-
self.__func_other_arg_json_datatypes.append("COLUMNS")
|
|
394
|
-
|
|
395
|
-
if self.to_lower_case is not None and self.to_lower_case != True:
|
|
396
|
-
self.__func_other_arg_sql_names.append("ToLowerCase")
|
|
397
|
-
self.__func_other_args.append(UtilFuncs._teradata_collapse_arglist(self.to_lower_case, "'"))
|
|
398
|
-
self.__func_other_arg_json_datatypes.append("BOOLEAN")
|
|
399
|
-
|
|
400
|
-
if self.stemming is not None and self.stemming != False:
|
|
401
|
-
self.__func_other_arg_sql_names.append("Stemming")
|
|
402
|
-
self.__func_other_args.append(UtilFuncs._teradata_collapse_arglist(self.stemming, "'"))
|
|
403
|
-
self.__func_other_arg_json_datatypes.append("BOOLEAN")
|
|
404
|
-
|
|
405
|
-
if self.output_by_word is not None and self.output_by_word != True:
|
|
406
|
-
self.__func_other_arg_sql_names.append("OutputByWord")
|
|
407
|
-
self.__func_other_args.append(UtilFuncs._teradata_collapse_arglist(self.output_by_word, "'"))
|
|
408
|
-
self.__func_other_arg_json_datatypes.append("BOOLEAN")
|
|
409
|
-
|
|
410
|
-
if self.stemming_exceptions is not None:
|
|
411
|
-
self.__func_other_arg_sql_names.append("StemmingExceptions")
|
|
412
|
-
self.__func_other_args.append(UtilFuncs._teradata_collapse_arglist(self.stemming_exceptions, "'"))
|
|
413
|
-
self.__func_other_arg_json_datatypes.append("STRING")
|
|
414
|
-
|
|
415
|
-
if self.remove_stop_words is not None and self.remove_stop_words != False:
|
|
416
|
-
self.__func_other_arg_sql_names.append("RemoveStopWords")
|
|
417
|
-
self.__func_other_args.append(UtilFuncs._teradata_collapse_arglist(self.remove_stop_words, "'"))
|
|
418
|
-
self.__func_other_arg_json_datatypes.append("BOOLEAN")
|
|
419
|
-
|
|
420
|
-
if self.stop_words is not None:
|
|
421
|
-
self.__func_other_arg_sql_names.append("StopWords")
|
|
422
|
-
self.__func_other_args.append(UtilFuncs._teradata_collapse_arglist(self.stop_words, "'"))
|
|
423
|
-
self.__func_other_arg_json_datatypes.append("STRING")
|
|
424
|
-
|
|
425
|
-
if self.delimiter is not None and self.delimiter != "[ \\t\\f\\r\\n]+":
|
|
426
|
-
self.__func_other_arg_sql_names.append("Delimiter")
|
|
427
|
-
self.__func_other_args.append(UtilFuncs._teradata_collapse_arglist(self.delimiter, "'"))
|
|
428
|
-
self.__func_other_arg_json_datatypes.append("STRING")
|
|
429
|
-
|
|
430
|
-
if self.total_words_num is not None and self.total_words_num != False:
|
|
431
|
-
self.__func_other_arg_sql_names.append("TotalWordsNum")
|
|
432
|
-
self.__func_other_args.append(UtilFuncs._teradata_collapse_arglist(self.total_words_num, "'"))
|
|
433
|
-
self.__func_other_arg_json_datatypes.append("BOOLEAN")
|
|
434
|
-
|
|
435
|
-
if self.punctuation is not None and self.punctuation != "[.,!?]":
|
|
436
|
-
self.__func_other_arg_sql_names.append("Punctuation")
|
|
437
|
-
self.__func_other_args.append(UtilFuncs._teradata_collapse_arglist(self.punctuation, "'"))
|
|
438
|
-
self.__func_other_arg_json_datatypes.append("STRING")
|
|
439
|
-
|
|
440
|
-
if self.list_positions is not None and self.list_positions != False:
|
|
441
|
-
self.__func_other_arg_sql_names.append("ListPositions")
|
|
442
|
-
self.__func_other_args.append(UtilFuncs._teradata_collapse_arglist(self.list_positions, "'"))
|
|
443
|
-
self.__func_other_arg_json_datatypes.append("BOOLEAN")
|
|
444
|
-
|
|
445
|
-
if self.token_column is not None and self.token_column != "token":
|
|
446
|
-
self.__func_other_arg_sql_names.append("TokenColumn")
|
|
447
|
-
self.__func_other_args.append(UtilFuncs._teradata_collapse_arglist(self.token_column, "'"))
|
|
448
|
-
self.__func_other_arg_json_datatypes.append("STRING")
|
|
449
|
-
|
|
450
|
-
if self.frequency_column is not None and self.frequency_column != "frequency":
|
|
451
|
-
self.__func_other_arg_sql_names.append("FrequencyColumn")
|
|
452
|
-
self.__func_other_args.append(UtilFuncs._teradata_collapse_arglist(self.frequency_column, "'"))
|
|
453
|
-
self.__func_other_arg_json_datatypes.append("STRING")
|
|
454
|
-
|
|
455
|
-
if self.total_column is not None and self.total_column != "total_count":
|
|
456
|
-
self.__func_other_arg_sql_names.append("TotalColumn")
|
|
457
|
-
self.__func_other_args.append(UtilFuncs._teradata_collapse_arglist(self.total_column, "'"))
|
|
458
|
-
self.__func_other_arg_json_datatypes.append("STRING")
|
|
459
|
-
|
|
460
|
-
if self.position_column is not None and self.position_column != "location":
|
|
461
|
-
self.__func_other_arg_sql_names.append("PositionColumn")
|
|
462
|
-
self.__func_other_args.append(UtilFuncs._teradata_collapse_arglist(self.position_column, "'"))
|
|
463
|
-
self.__func_other_arg_json_datatypes.append("STRING")
|
|
464
|
-
|
|
465
|
-
# Generate lists for rest of the function arguments
|
|
466
|
-
sequence_input_by_list = []
|
|
467
|
-
if self.data_sequence_column is not None:
|
|
468
|
-
sequence_input_by_list.append("input:" + UtilFuncs._teradata_collapse_arglist(self.data_sequence_column, ""))
|
|
469
|
-
|
|
470
|
-
if len(sequence_input_by_list) > 0:
|
|
471
|
-
self.__func_other_arg_sql_names.append("SequenceInputBy")
|
|
472
|
-
sequence_input_by_arg_value = UtilFuncs._teradata_collapse_arglist(sequence_input_by_list, "'")
|
|
473
|
-
self.__func_other_args.append(sequence_input_by_arg_value)
|
|
474
|
-
self.__func_other_arg_json_datatypes.append("STRING")
|
|
475
|
-
self._sql_specific_attributes["SequenceInputBy"] = sequence_input_by_arg_value
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
# Declare empty lists to hold input table information.
|
|
479
|
-
self.__func_input_arg_sql_names = []
|
|
480
|
-
self.__func_input_table_view_query = []
|
|
481
|
-
self.__func_input_dataframe_type = []
|
|
482
|
-
self.__func_input_distribution = []
|
|
483
|
-
self.__func_input_partition_by_cols = []
|
|
484
|
-
self.__func_input_order_by_cols = []
|
|
485
|
-
|
|
486
|
-
# Process data
|
|
487
|
-
self.__table_ref = self.__awu._teradata_on_clause_from_dataframe(self.data, False)
|
|
488
|
-
self.__func_input_distribution.append("FACT")
|
|
489
|
-
self.__func_input_arg_sql_names.append("input")
|
|
490
|
-
self.__func_input_table_view_query.append(self.__table_ref["ref"])
|
|
491
|
-
self.__func_input_dataframe_type.append(self.__table_ref["ref_type"])
|
|
492
|
-
self.__func_input_partition_by_cols.append("ANY")
|
|
493
|
-
self.__func_input_order_by_cols.append(UtilFuncs._teradata_collapse_arglist(self.data_order_column, "\""))
|
|
494
|
-
|
|
495
|
-
function_name = "TextParser"
|
|
496
|
-
# Create instance to generate SQLMR.
|
|
497
|
-
self.__aqg_obj = AnalyticQueryGenerator(function_name,
|
|
498
|
-
self.__func_input_arg_sql_names,
|
|
499
|
-
self.__func_input_table_view_query,
|
|
500
|
-
self.__func_input_dataframe_type,
|
|
501
|
-
self.__func_input_distribution,
|
|
502
|
-
self.__func_input_partition_by_cols,
|
|
503
|
-
self.__func_input_order_by_cols,
|
|
504
|
-
self.__func_other_arg_sql_names,
|
|
505
|
-
self.__func_other_args,
|
|
506
|
-
self.__func_other_arg_json_datatypes,
|
|
507
|
-
self.__func_output_args_sql_names,
|
|
508
|
-
self.__func_output_args,
|
|
509
|
-
engine="ENGINE_ML")
|
|
510
|
-
# Invoke call to SQL-MR generation.
|
|
511
|
-
self.sqlmr_query = self.__aqg_obj._gen_sqlmr_select_stmt_sql()
|
|
512
|
-
|
|
513
|
-
# Print SQL-MR query if requested to do so.
|
|
514
|
-
if display.print_sqlmr_query:
|
|
515
|
-
print(self.sqlmr_query)
|
|
516
|
-
|
|
517
|
-
# Set the algorithm name for Model Cataloging.
|
|
518
|
-
self._algorithm_name = self.__aqg_obj._get_alias_name_for_function(function_name)
|
|
519
|
-
|
|
520
|
-
def __execute(self):
|
|
521
|
-
"""
|
|
522
|
-
Function to execute SQL-MR queries.
|
|
523
|
-
Create DataFrames for the required SQL-MR outputs.
|
|
524
|
-
"""
|
|
525
|
-
# Generate STDOUT table name and add it to the output table list.
|
|
526
|
-
sqlmr_stdout_temp_tablename = UtilFuncs._generate_temp_table_name(prefix="td_sqlmr_out_", use_default_database=True, gc_on_quit=True, quote=False)
|
|
527
|
-
try:
|
|
528
|
-
# Generate the output.
|
|
529
|
-
UtilFuncs._create_view(sqlmr_stdout_temp_tablename, self.sqlmr_query)
|
|
530
|
-
except Exception as emsg:
|
|
531
|
-
raise TeradataMlException(Messages.get_message(MessageCodes.TDMLDF_EXEC_SQL_FAILED, str(emsg)), MessageCodes.TDMLDF_EXEC_SQL_FAILED)
|
|
532
|
-
|
|
533
|
-
# Update output table data frames.
|
|
534
|
-
self._mlresults = []
|
|
535
|
-
self.result = self.__awu._create_data_set_object(df_input=UtilFuncs._extract_table_name(sqlmr_stdout_temp_tablename), source_type="table", database_name=UtilFuncs._extract_db_name(sqlmr_stdout_temp_tablename))
|
|
536
|
-
self._mlresults.append(self.result)
|
|
537
|
-
|
|
538
|
-
def show_query(self):
|
|
539
|
-
"""
|
|
540
|
-
Function to return the underlying SQL query.
|
|
541
|
-
When model object is created using retrieve_model(), then None is returned.
|
|
542
|
-
"""
|
|
543
|
-
return self.sqlmr_query
|
|
544
|
-
|
|
545
|
-
def get_prediction_type(self):
|
|
546
|
-
"""
|
|
547
|
-
Function to return the Prediction type of the algorithm.
|
|
548
|
-
When model object is created using retrieve_model(), then the value returned is
|
|
549
|
-
as saved in the Model Catalog.
|
|
550
|
-
"""
|
|
551
|
-
return self._prediction_type
|
|
552
|
-
|
|
553
|
-
def get_target_column(self):
|
|
554
|
-
"""
|
|
555
|
-
Function to return the Target Column of the algorithm.
|
|
556
|
-
When model object is created using retrieve_model(), then the value returned is
|
|
557
|
-
as saved in the Model Catalog.
|
|
558
|
-
"""
|
|
559
|
-
return self._target_column
|
|
560
|
-
|
|
561
|
-
def get_build_time(self):
|
|
562
|
-
"""
|
|
563
|
-
Function to return the build time of the algorithm in seconds.
|
|
564
|
-
When model object is created using retrieve_model(), then the value returned is
|
|
565
|
-
as saved in the Model Catalog.
|
|
566
|
-
"""
|
|
567
|
-
return self._build_time
|
|
568
|
-
|
|
569
|
-
def _get_algorithm_name(self):
|
|
570
|
-
"""
|
|
571
|
-
Function to return the name of the algorithm.
|
|
572
|
-
"""
|
|
573
|
-
return self._algorithm_name
|
|
574
|
-
|
|
575
|
-
def _get_sql_specific_attributes(self):
|
|
576
|
-
"""
|
|
577
|
-
Function to return the dictionary containing the SQL specific attributes of the algorithm.
|
|
578
|
-
"""
|
|
579
|
-
return self._sql_specific_attributes
|
|
580
|
-
|
|
581
|
-
@classmethod
|
|
582
|
-
def _from_model_catalog(cls,
|
|
583
|
-
result = None,
|
|
584
|
-
**kwargs):
|
|
585
|
-
"""
|
|
586
|
-
Classmethod is used by Model Cataloging, to instantiate this wrapper class.
|
|
587
|
-
"""
|
|
588
|
-
kwargs.pop("result", None)
|
|
589
|
-
|
|
590
|
-
# Model Cataloging related attributes.
|
|
591
|
-
target_column = kwargs.pop("__target_column", None)
|
|
592
|
-
prediction_type = kwargs.pop("__prediction_type", None)
|
|
593
|
-
algorithm_name = kwargs.pop("__algorithm_name", None)
|
|
594
|
-
build_time = kwargs.pop("__build_time", None)
|
|
595
|
-
|
|
596
|
-
# Let's create an object of this class.
|
|
597
|
-
obj = cls(**kwargs)
|
|
598
|
-
obj.result = result
|
|
599
|
-
|
|
600
|
-
# Initialize the sqlmr_query class attribute.
|
|
601
|
-
obj.sqlmr_query = None
|
|
602
|
-
|
|
603
|
-
# Initialize the SQL specific Model Cataloging attributes.
|
|
604
|
-
obj._sql_specific_attributes = None
|
|
605
|
-
obj._target_column = target_column
|
|
606
|
-
obj._prediction_type = prediction_type
|
|
607
|
-
obj._algorithm_name = algorithm_name
|
|
608
|
-
obj._build_time = build_time
|
|
609
|
-
|
|
610
|
-
# Update output table data frames.
|
|
611
|
-
obj._mlresults = []
|
|
612
|
-
obj.result = obj.__awu._create_data_set_object(df_input=UtilFuncs._extract_table_name(obj.result), source_type="table", database_name=UtilFuncs._extract_db_name(obj.result))
|
|
613
|
-
obj._mlresults.append(obj.result)
|
|
614
|
-
return obj
|
|
615
|
-
|
|
616
|
-
def __repr__(self):
|
|
617
|
-
"""
|
|
618
|
-
Returns the string representation for a TextParser class instance.
|
|
619
|
-
"""
|
|
620
|
-
repr_string="############ STDOUT Output ############"
|
|
621
|
-
repr_string = "{}\n\n{}".format(repr_string,self.result)
|
|
622
|
-
return repr_string
|
|
623
|
-
|