bigquery-data-science 0.7.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- bigquery_data_science/__init__.py +58 -0
- bigquery_data_science/_skills/PLANS.md +76 -0
- bigquery_data_science/_skills/bigquery-ai/SKILL.md +66 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ai_agg.md +528 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ai_classify.md +450 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ai_count_tokens.md +180 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ai_detect_anomalies.md +322 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ai_embed.md +425 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ai_evaluate.md +368 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ai_forecast.md +437 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ai_generate.md +693 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ai_generate_bool.md +248 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ai_generate_double.md +254 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ai_generate_embedding.md +402 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ai_generate_int.md +252 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ai_generate_table.md +447 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ai_generate_text.md +356 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ai_if.md +373 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ai_key_drivers.md +331 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ai_parse_document.md +416 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ai_predict.md +559 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ai_score.md +310 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ai_search.md +465 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ai_similarity.md +371 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/catalog_search.md +951 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/content_analysis.md +284 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/content_moderation.md +369 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/data_enrichment.md +426 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/document_intelligence.md +470 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/document_rag.md +453 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/image_deduplication.md +770 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/log_analysis.md +668 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/metric_diagnostics.md +568 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ml_annotate_image.md +680 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ml_generate_embedding.md +217 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ml_generate_text.md +223 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ml_process_document.md +392 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ml_transcribe.md +815 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ml_translate.md +600 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ml_understand_text.md +673 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/multimodal_analysis.md +266 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/rag_pipeline.md +593 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/semantic_search.md +510 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/tabular_prediction.md +485 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/time_series_intelligence.md +489 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/vector_search.md +672 -0
- bigquery_data_science/_skills/bigquery-ai/reference/augmented-analytics.md +77 -0
- bigquery_data_science/_skills/bigquery-ai/reference/cloud-ai-service-models.md +88 -0
- bigquery_data_science/_skills/bigquery-ai/reference/document-processing.md +95 -0
- bigquery_data_science/_skills/bigquery-ai/reference/embeddings-and-search.md +166 -0
- bigquery_data_science/_skills/bigquery-ai/reference/general-purpose-generation.md +87 -0
- bigquery_data_science/_skills/bigquery-ai/reference/managed-functions.md +72 -0
- bigquery_data_science/_skills/bigquery-ai/reference/predictive-ai.md +113 -0
- bigquery_data_science/_skills/bigquery-ai/reference/workflows.md +56 -0
- bigquery_data_science/_skills/bigquery-ai/skill.manifest.json +109 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/INDEX.md +71 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/aiagg.md +67 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/aicausal_effect-covered-in-bigquery-ml.md +17 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/aiclassify.md +72 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/aicount_tokens.md +53 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/aidetect_anomalies.md +63 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/aiembed.md +85 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/aievaluate.md +114 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/aiforecast.md +123 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/aigenerate.md +69 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/aigenerate_bool.md +53 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/aigenerate_double.md +45 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/aigenerate_embedding.md +108 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/aigenerate_int.md +45 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/aigenerate_table.md +64 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/aigenerate_text.md +167 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/aiif.md +56 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/aikey_drivers.md +76 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/aiparse_document.md +91 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/aipredict.md +74 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/aiscore.md +52 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/aisearch.md +66 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/aisimilarity.md +79 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/choosing-between-these-and-the-ai-functions.md +11 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/hybrid-search-capability.md +165 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/mlannotate_image.md +24 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/mlgenerate_embedding.md +34 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/mlgenerate_text.md +65 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/mlprocess_document.md +125 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/mltranscribe.md +26 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/mltranslate.md +27 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/mlunderstand_text.md +32 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/multimodal-input-patterns-by-function.md +128 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/object-tables.md +109 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/objectref-and-objectrefruntime-schema-reference.md +50 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/objfetch_metadata.md +49 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/objget_access_url.md +66 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/objget_read_url.md +40 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/objlist.md +34 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/objmake_ref.md +58 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/per-row-failures-and-the-retry-pattern.md +19 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/setup-apis-connection-roles.md +39 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/two-input-shapes.md +22 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/vector_search.md +78 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/what-the-five-have-in-common.md +27 -0
- bigquery_data_science/_skills/bigquery-data-science/SKILL.md +69 -0
- bigquery_data_science/_skills/bigquery-data-science/reference/namespace-matrix.md +112 -0
- bigquery_data_science/_skills/bigquery-data-science/skill.manifest.json +13 -0
- bigquery_data_science/_skills/bigquery-ml/SKILL.md +70 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/airflow_with_kfp.md +379 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/anomaly_fraud_detection.md +195 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/arima_plus.md +635 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/arima_plus_xreg.md +453 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/autoencoder.md +470 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/automl_classifier.md +291 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/automl_regressor.md +274 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/boosted_tree_classifier.md +449 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/boosted_tree_regressor.md +400 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/bucketizing.md +201 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/causal_effect.md +773 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/churn_retention.md +271 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/cloud_workflows.md +370 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/composer_airflow.md +475 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/contribution_analysis.md +346 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/cross_validation.md +185 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/customer_segmentation.md +233 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/dataform.md +372 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/dbt.md +283 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/difference_in_differences.md +280 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/distance.md +195 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/dnn_classifier.md +326 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/dnn_regressor.md +308 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/embeddings_classification.md +426 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/encoding.md +238 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/ensembling.md +278 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/evaluation.md +445 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/exploration.md +779 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/export.md +321 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/feature_engineering.md +280 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/feature_store.md +652 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/feature_store_workflow.md +426 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/ga4_churn_prediction.md +285 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/hierarchical_forecasting.md +441 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/image.md +791 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/imported.md +426 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/kmeans.md +348 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/linear_regression.md +372 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/logistic_regression.md +390 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/matrix_factorization.md +404 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/model_monitoring.md +609 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/pca.md +357 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/price_elasticity_dml.md +286 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/propensity_score_matching.md +350 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/random_forest_classifier.md +445 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/random_forest_regressor.md +409 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/recommendation.md +281 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/regression_based_forecasting.md +491 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/remote.md +428 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/scalers.md +325 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/scheduled_queries.md +284 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/sql_scripting.md +243 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/survival_analysis.md +363 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/synthetic_control.md +221 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/text.md +215 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/time_series.md +669 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/transform_only.md +273 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/uplift_cate.md +241 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/vertex_kfp.md +360 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/wide_and_deep_classifier.md +316 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/wide_and_deep_regressor.md +298 -0
- bigquery_data_science/_skills/bigquery-ml/reference/classification.md +80 -0
- bigquery_data_science/_skills/bigquery-ml/reference/model-management.md +90 -0
- bigquery_data_science/_skills/bigquery-ml/reference/preprocessing-functions.md +234 -0
- bigquery_data_science/_skills/bigquery-ml/reference/regression.md +79 -0
- bigquery_data_science/_skills/bigquery-ml/reference/unsupervised-and-specialized.md +101 -0
- bigquery_data_science/_skills/bigquery-ml/reference/workflows-and-pipelines.md +111 -0
- bigquery_data_science/_skills/bigquery-ml/skill.manifest.json +177 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/1-model_type-catalog.md +28 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/2-task-mlevaluate-metric-columns.md +26 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/3-capability-matrix-per-model-type.md +21 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/4-explainability-weights-functions.md +14 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/5-connection-matrix.md +11 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/6-which-model-free-function-no-create-model-no-connection.md +15 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/7-point-in-time-feature-retrieval-which-of-the-two.md +11 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/INDEX.md +111 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/aicausal_effect.md +85 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/arima_plus-forecasting-lifecycle-functions.md +14 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/arima_plus-univariate-time-series-forecasting.md +132 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/arima_plus_xreg-multivariate-time-series-with-external-regressors.md +103 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/autoencoder.md +102 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/automl_classifier-automl_regressor.md +88 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/boosted_tree_classifier-boosted_tree_regressor.md +117 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/contribution_analysis.md +113 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/dnn_classifier-dnn_regressor.md +107 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/dnn_linear_combined_classifier-dnn_linear_combined_regressor-wide-and-deep.md +98 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/explainability-functions-mlexplain_predict-mlglobal_explain-mlfeature_importance.md +7 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/exploratory-data-analysis-mldescribe_data-mlcorrelation.md +11 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/export-model.md +92 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/feature-engineering-standalone-preprocessing-functions.md +13 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/image-preprocessing-functions-mldecode_image-mlresize_image-mlconvert_image_type-mlconvert_color_space.md +63 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/imported-models-tensorflow-tensorflow_lite-onnx-xgboost.md +36 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/kmeans.md +87 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/linear_reg-logistic_reg-generalized-linear-models.md +101 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/matrix_factorization.md +162 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mladvanced_weights.md +76 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlarima_coefficients.md +30 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlarima_evaluate.md +49 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlbag_of_words.md +47 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlbucketize.md +47 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlcentroids.md +46 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlconfusion_matrix.md +56 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlconvert_color_space.md +53 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlconvert_image_type.md +39 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlcorrelation.md +78 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mldecode_image.md +44 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mldescribe_data.md +74 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mldetect_anomalies.md +86 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mldistance.md +48 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlentity_features_at_time.md +56 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlevaluate.md +72 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlexplain_forecast.md +50 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlexplain_predict.md +66 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlfeature_cross.md +46 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlfeature_importance.md +47 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlfeature_info.md +50 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlfeatures_at_time.md +51 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlforecast.md +76 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlgenerate_embedding-from-a-bqml-pca-autoencoder-matrix_factorization-model.md +64 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlglobal_explain.md +63 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlhash_bucketize.md +47 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlholiday_info.md +46 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlimputer.md +46 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mllp_norm.md +47 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlmetrics.md +80 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlngrams.md +37 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlone_hot_encoder-mlmulti_hot_encoder-mllabel_encoder.md +67 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlpolynomial_expand.md +47 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlpredict.md +67 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlprincipal_component_info.md +42 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlprincipal_components.md +40 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlquantile_bucketize.md +43 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlrecommend.md +60 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlreconstruction_loss.md +48 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlresize_image.md +45 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlroc_curve.md +63 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mltf_idf.md +37 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mltfdv_describe.md +34 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mltfdv_validate.md +48 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mltraining_info.md +78 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mltransform-applying-a-saved-transform.md +52 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mltranspose-not-a-function-pointer.md +32 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mltrial_info.md +60 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlvalidate_data_drift.md +99 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlvalidate_data_skew.md +50 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlweights.md +84 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/model-free-time-series-decomposition-mltrend-mlseasonality-mldetect_change_points.md +100 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/model-introspection-mlfeature_info-mltraining_info-mltrial_info.md +7 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/model-monitoring-data-validation.md +23 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/numerical-scalers-mlstandard_scaler-mlmin_max_scaler-mlmax_abs_scaler-mlrobust_scaler-mlnormalizer.md +77 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/onnx.md +55 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/pca.md +91 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/point-in-time-feature-retrieval-mlfeatures_at_time-mlentity_features_at_time.md +25 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/quick-comparison.md +14 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/random_forest_classifier-random_forest_regressor.md +123 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/remote-remote-model-over-a-vertex-ai-endpoint-the-mechanism.md +87 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/tensorflow.md +48 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/tensorflow_lite.md +29 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/text-preprocessing-functions-mlngrams-mltf_idf-mlbag_of_words.md +18 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/timesfm-built-in-foundation-forecaster-see-bigquery-ai.md +28 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/transform_only.md +91 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/xgboost.md +72 -0
- bigquery_data_science/_skills/pinned-models.json +28 -0
- bigquery_data_science/catalog.py +1107 -0
- bigquery_data_science/cli.py +786 -0
- bigquery_data_science/installer.py +282 -0
- bigquery_data_science/mcp_server.py +317 -0
- bigquery_data_science/tooling/__init__.py +15 -0
- bigquery_data_science/tooling/convert_notebook.py +176 -0
- bigquery_data_science/tooling/link_policy.py +247 -0
- bigquery_data_science/tooling/manifest.py +152 -0
- bigquery_data_science/tooling/markdown_anchors.py +84 -0
- bigquery_data_science/tooling/narrative_drift.py +163 -0
- bigquery_data_science/tooling/pinned_models.py +216 -0
- bigquery_data_science/tooling/reference_structure.py +85 -0
- bigquery_data_science/tooling/sync_syntax.py +407 -0
- bigquery_data_science/tooling/validate.py +208 -0
- bigquery_data_science-0.7.0.dist-info/METADATA +306 -0
- bigquery_data_science-0.7.0.dist-info/RECORD +286 -0
- bigquery_data_science-0.7.0.dist-info/WHEEL +4 -0
- bigquery_data_science-0.7.0.dist-info/entry_points.txt +5 -0
- bigquery_data_science-0.7.0.dist-info/licenses/LICENSE +201 -0
|
@@ -0,0 +1,673 @@
|
|
|
1
|
+
> **Source (`statmike/bigquery-data-science`):** **Notebook:** [`bigquery-ai/functions/ml_understand_text/ml_understand_text.ipynb`](https://github.com/statmike/bigquery-data-science/blob/main/bigquery-ai/functions/ml_understand_text/ml_understand_text.ipynb) · **SQL:** [`bigquery-ai/functions/ml_understand_text/ml_understand_text.sql`](https://github.com/statmike/bigquery-data-science/blob/main/bigquery-ai/functions/ml_understand_text/ml_understand_text.sql)
|
|
2
|
+
|
|
3
|
+
# ML.UNDERSTAND_TEXT — BigQuery AI Functions
|
|
4
|
+
|
|
5
|
+
`ML.UNDERSTAND_TEXT` sends text from a BigQuery table to the [Cloud Natural Language API](https://cloud.google.com/natural-language/docs) and returns its analysis as JSON. One function covers five different analyses — sentiment, entities, entity sentiment, syntax, and content classification — selected by a string option. It is a table-valued function that runs through a **remote model**, so there is a `CREATE MODEL` step, but nothing trains: the model object is a handle on a pre-trained Google API.
|
|
6
|
+
|
|
7
|
+
**One of five Cloud AI service models.** `ML.UNDERSTAND_TEXT` belongs to a small family of BigQuery ML functions that call a pre-trained Cloud AI service through `CREATE MODEL … REMOTE WITH CONNECTION … OPTIONS(REMOTE_SERVICE_TYPE = …)`. The five members and their service types:
|
|
8
|
+
|
|
9
|
+
| Function | `REMOTE_SERVICE_TYPE` | Service | Input |
|
|
10
|
+
|---|---|---|---|
|
|
11
|
+
| `ML.UNDERSTAND_TEXT` *(this notebook)* | `CLOUD_AI_NATURAL_LANGUAGE_V1` | Cloud Natural Language | A table or query with a `text_content` column |
|
|
12
|
+
| `functions/ml_translate` (`ML.TRANSLATE`) | `CLOUD_AI_TRANSLATE_V3` | Cloud Translation | A table or query with a `text_content` column |
|
|
13
|
+
| `functions/ml_annotate_image` (`ML.ANNOTATE_IMAGE`) | `CLOUD_AI_VISION_V1` | Cloud Vision | An object table of images |
|
|
14
|
+
| `functions/ml_transcribe` (`ML.TRANSCRIBE`) | `CLOUD_AI_SPEECH_TO_TEXT_V2` | Speech-to-Text | An object table of audio files |
|
|
15
|
+
| `functions/ml_process_document` (`ML.PROCESS_DOCUMENT`) | `CLOUD_AI_DOCUMENT_V1` | Document AI | An object table of documents |
|
|
16
|
+
|
|
17
|
+
The shared setup — connection, service account roles, `CREATE MODEL` — is written once in `reference/cloud-ai-service-models.md` (Cloud AI Service Models). This notebook repeats the parts you need to run it standalone.
|
|
18
|
+
|
|
19
|
+
**The five analyses, all through one function:**
|
|
20
|
+
|
|
21
|
+
| `nlu_option` | What comes back |
|
|
22
|
+
|---|---|
|
|
23
|
+
| `ANALYZE_SENTIMENT` | Document and per-sentence sentiment, as a score and a magnitude |
|
|
24
|
+
| `ANALYZE_ENTITIES` | Named things in the text, with a type and a salience |
|
|
25
|
+
| `ANALYZE_ENTITY_SENTIMENT` | The same entities, each carrying its own sentiment |
|
|
26
|
+
| `ANALYZE_SYNTAX` | Tokens with part of speech, lemma and dependency edges |
|
|
27
|
+
| `CLASSIFY_TEXT` | Content categories from a fixed taxonomy, with confidences |
|
|
28
|
+
|
|
29
|
+
**When to use it:** you want a fixed, priced, non-generative read on text — sentiment scores that mean the same thing every quarter, entity types from a closed vocabulary, categories from a published taxonomy.
|
|
30
|
+
|
|
31
|
+
**Alternative:** `functions/ai_generate_double` (`AI.GENERATE_DOUBLE`), `functions/ai_generate_table` (`AI.GENERATE_TABLE`) or `functions/ai_score` (`AI.SCORE`) with a prompt. Example 9 runs sentiment both ways over the same rows.
|
|
32
|
+
|
|
33
|
+
**Two rules to know before you start:**
|
|
34
|
+
- The function reads a column named exactly `text_content`. A table without one errors. Example 8 shows the error and the one-line fix.
|
|
35
|
+
- Language support differs **per analysis**, and an unsupported language fails the row, not the job. Example 4 catches that live.
|
|
36
|
+
|
|
37
|
+
---
|
|
38
|
+
## Setup
|
|
39
|
+
|
|
40
|
+
This function needs two resources: a **Cloud Resource connection** whose service account may call the Cloud Natural Language API, and a **remote model** that names the service. Both are created below and both are idempotent.
|
|
41
|
+
|
|
42
|
+
> See the `setup` (Setup Reference) for connections and remote models in general, and `reference/cloud-ai-service-models.md` (Cloud AI Service Models) for this family in particular.
|
|
43
|
+
|
|
44
|
+
```python
|
|
45
|
+
PROJECT_ID = 'statmike-mlops-349915' # <-- Replace with your project ID
|
|
46
|
+
LOCATION = 'US' # BigQuery dataset location — this family runs in US or EU
|
|
47
|
+
DATASET_ID = 'bq_ai_functions' # Shared dataset across all notebooks
|
|
48
|
+
CONNECTION_ID = 'bq_ai_functions' # Shared connection
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
### Environment
|
|
52
|
+
|
|
53
|
+
> **Already set up the project environment?** The cell below is a no-op — packages are already in your kernel. See the `setup` (Setup Reference) for details.
|
|
54
|
+
>
|
|
55
|
+
> **Running standalone** (Colab, Colab Enterprise, Vertex AI Workbench)? The cell below installs required packages into your current kernel.
|
|
56
|
+
|
|
57
|
+
```python
|
|
58
|
+
from google.cloud import bigquery
|
|
59
|
+
import pandas as pd
|
|
60
|
+
|
|
61
|
+
client = bigquery.Client(project=PROJECT_ID)
|
|
62
|
+
pd.set_option('display.max_colwidth', None)
|
|
63
|
+
|
|
64
|
+
# Create the shared dataset (idempotent)
|
|
65
|
+
dataset_ref = bigquery.DatasetReference(PROJECT_ID, DATASET_ID)
|
|
66
|
+
dataset = bigquery.Dataset(dataset_ref)
|
|
67
|
+
dataset.location = LOCATION
|
|
68
|
+
client.create_dataset(dataset, exists_ok=True)
|
|
69
|
+
print(f'Dataset {PROJECT_ID}.{DATASET_ID} ready')
|
|
70
|
+
|
|
71
|
+
# Register %%bigquery cell magic (auto-loaded in Colab, needed elsewhere)
|
|
72
|
+
%load_ext bigquery_magics
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
### The APIs
|
|
76
|
+
|
|
77
|
+
Three services are involved: BigQuery itself, the BigQuery Connection service that owns the connection, and the Cloud Natural Language API that does the work. The cell enables any that are off.
|
|
78
|
+
|
|
79
|
+
```python
|
|
80
|
+
import subprocess as _sp
|
|
81
|
+
|
|
82
|
+
REQUIRED_APIS = [
|
|
83
|
+
'bigquery.googleapis.com',
|
|
84
|
+
'bigqueryconnection.googleapis.com',
|
|
85
|
+
'language.googleapis.com',
|
|
86
|
+
]
|
|
87
|
+
|
|
88
|
+
enabled = set(_sp.run(['gcloud', 'services', 'list', '--enabled',
|
|
89
|
+
'--format=value(config.name)', f'--project={PROJECT_ID}'],
|
|
90
|
+
capture_output=True, text=True, check=True).stdout.split())
|
|
91
|
+
missing = [api for api in REQUIRED_APIS if api not in enabled]
|
|
92
|
+
for api in missing:
|
|
93
|
+
_sp.run(['gcloud', 'services', 'enable', api, f'--project={PROJECT_ID}'],
|
|
94
|
+
capture_output=True, text=True, check=True)
|
|
95
|
+
|
|
96
|
+
print(f'Already enabled: {", ".join(sorted(set(REQUIRED_APIS) & enabled)) or "none"}')
|
|
97
|
+
print(f'Enabled now: {", ".join(missing) or "none"}')
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
### The connection and its service account
|
|
101
|
+
|
|
102
|
+
The connection's service account — not your user account — is what calls the Cloud Natural Language API. `ML.UNDERSTAND_TEXT` itself needs two grants at the project level, both named on [this function's setup page](https://cloud.google.com/bigquery/docs/analyze-text). A third belongs to the `AI.GENERATE_DOUBLE` comparison later on, not to this function:
|
|
103
|
+
|
|
104
|
+
| Role | Why |
|
|
105
|
+
|---|---|
|
|
106
|
+
| `roles/serviceusage.serviceUsageConsumer` | Lets the service account consume an API in this project. **Without it every call fails**, even though `CREATE MODEL` succeeds. |
|
|
107
|
+
| `roles/bigquery.connectionUser` | Lets the service account use the connection it belongs to. |
|
|
108
|
+
| `roles/aiplatform.user` | Only for Example 9, which calls `AI.GENERATE_DOUBLE` through the same connection to compare against Cloud Natural Language. Not part of `ML.UNDERSTAND_TEXT`. |
|
|
109
|
+
|
|
110
|
+
> **Two roles is the whole list for this function, and that is not true across the family.** The Cloud Natural Language API has no predefined IAM role of its own — `serviceusage.services.use` is the permission that gates it — so the documented pair is sufficient. `functions/ml_translate` (`ML.TRANSLATE`) needs a third role (`roles/cloudtranslate.user`) that its setup page does not mention, and `functions/ml_transcribe` (`ML.TRANSCRIBE`) needs `roles/speech.client`. The pattern to carry: the BigQuery-side roles get the request to the service, and then the service applies whatever IAM it has of its own.
|
|
111
|
+
|
|
112
|
+
> **`CREATE MODEL` succeeding tells you nothing about permissions.** Model creation for this family does not contact the service, so it completes against a connection that cannot call anything. The first failure arrives at call time, not at setup time.
|
|
113
|
+
|
|
114
|
+
The cell below prints which roles the service account already had and which it granted, so the run is auditable. It needs project-level IAM admin rights; if you do not have them, hand the three role names to someone who does.
|
|
115
|
+
|
|
116
|
+
```python
|
|
117
|
+
import subprocess as _sp, json as _json
|
|
118
|
+
|
|
119
|
+
# Create the Cloud Resource connection (idempotent — an existing one is left alone)
|
|
120
|
+
_sp.run(['bq', 'mk', '--connection', '--location', LOCATION,
|
|
121
|
+
'--connection_type', 'CLOUD_RESOURCE',
|
|
122
|
+
'--project_id', PROJECT_ID, CONNECTION_ID],
|
|
123
|
+
capture_output=True, text=True)
|
|
124
|
+
|
|
125
|
+
# The connection's auto-provisioned service account is the identity that calls the Cloud AI service
|
|
126
|
+
r = _sp.run(['bq', 'show', '--connection', '--format=json',
|
|
127
|
+
'--project_id', PROJECT_ID, '--location', LOCATION, CONNECTION_ID],
|
|
128
|
+
capture_output=True, text=True, check=True)
|
|
129
|
+
sa = _json.loads(r.stdout)['cloudResource']['serviceAccountId']
|
|
130
|
+
print(f'Connection service account: {sa}')
|
|
131
|
+
|
|
132
|
+
REQUIRED_ROLES = [
|
|
133
|
+
'roles/serviceusage.serviceUsageConsumer',
|
|
134
|
+
'roles/bigquery.connectionUser',
|
|
135
|
+
'roles/aiplatform.user', # only for the AI.GENERATE_DOUBLE comparison in Example 9
|
|
136
|
+
]
|
|
137
|
+
|
|
138
|
+
def roles_held():
|
|
139
|
+
out = _sp.run(['gcloud', 'projects', 'get-iam-policy', PROJECT_ID,
|
|
140
|
+
'--flatten=bindings[].members',
|
|
141
|
+
f'--filter=bindings.members:{sa}',
|
|
142
|
+
'--format=value(bindings.role)'],
|
|
143
|
+
capture_output=True, text=True, check=True)
|
|
144
|
+
return set(out.stdout.split())
|
|
145
|
+
|
|
146
|
+
held = roles_held()
|
|
147
|
+
missing = [role for role in REQUIRED_ROLES if role not in held]
|
|
148
|
+
for role in missing:
|
|
149
|
+
_sp.run(['gcloud', 'projects', 'add-iam-policy-binding', PROJECT_ID,
|
|
150
|
+
f'--member=serviceAccount:{sa}', f'--role={role}', '--condition=None', '--quiet'],
|
|
151
|
+
capture_output=True, text=True, check=True)
|
|
152
|
+
|
|
153
|
+
print(f'Already held: {", ".join(sorted(set(REQUIRED_ROLES) & held)) or "none"}')
|
|
154
|
+
print(f'Granted now: {", ".join(missing) or "none"}')
|
|
155
|
+
```
|
|
156
|
+
|
|
157
|
+
### The remote model
|
|
158
|
+
|
|
159
|
+
Nothing trains here. `CREATE MODEL` writes a model object that records the connection and the service type; the Cloud Natural Language API is called later, per row, by `ML.UNDERSTAND_TEXT`. There is no `ML.EVALUATE`, no weights, and no training data — the model's own metadata says so, and Example 10 reads it.
|
|
160
|
+
|
|
161
|
+
One model serves all five analyses. The `nlu_option` is an argument to the function, not an option on the model.
|
|
162
|
+
|
|
163
|
+
```python
|
|
164
|
+
client.query(f'''
|
|
165
|
+
CREATE OR REPLACE MODEL `{PROJECT_ID}.{DATASET_ID}.ml_understand_text_model`
|
|
166
|
+
REMOTE WITH CONNECTION `{PROJECT_ID}.{LOCATION}.{CONNECTION_ID}`
|
|
167
|
+
OPTIONS (REMOTE_SERVICE_TYPE = 'CLOUD_AI_NATURAL_LANGUAGE_V1')
|
|
168
|
+
''').result()
|
|
169
|
+
print('Model ml_understand_text_model ready')
|
|
170
|
+
```
|
|
171
|
+
|
|
172
|
+
### The data — eight pieces of customer feedback in five languages
|
|
173
|
+
|
|
174
|
+
The languages are deliberate: they are how Example 4 and Example 5 show that support is per analysis, not per API. The text column is named `body`, not `text_content`, because that is what a real table looks like.
|
|
175
|
+
|
|
176
|
+
```python
|
|
177
|
+
client.query(f'''
|
|
178
|
+
CREATE OR REPLACE TABLE `{PROJECT_ID}.{DATASET_ID}.ml_understand_text_feedback` AS
|
|
179
|
+
SELECT * FROM UNNEST([
|
|
180
|
+
STRUCT(1 AS feedback_id, 'en' AS expected_language, 'The battery lasts all day but the charger gets far too hot to touch.' AS body),
|
|
181
|
+
STRUCT(2, 'en', 'Shipping from Acme Logistics in Denver was fast and the packaging was perfect.'),
|
|
182
|
+
STRUCT(3, 'es', 'La batería dura todo el día, pero el cargador se calienta demasiado.'),
|
|
183
|
+
STRUCT(4, 'ja', '画面はとても綺麗ですが、アプリの動作が少し遅いです。'),
|
|
184
|
+
STRUCT(5, 'en', 'That actor on TV makes movies in Hollywood and also stars in a variety of popular new TV shows about cooking competitions and travel documentaries filmed across Europe.'),
|
|
185
|
+
STRUCT(6, 'ko', '가격 대비 성능은 훌륭하지만 배송이 너무 오래 걸렸습니다.'),
|
|
186
|
+
STRUCT(7, 'vi', 'Pin dùng được cả ngày nhưng bộ sạc quá nóng.'),
|
|
187
|
+
STRUCT(8, 'en', 'The Northwind Audio team in Seattle replaced my headphones within two days of my email.')
|
|
188
|
+
])
|
|
189
|
+
''').result()
|
|
190
|
+
|
|
191
|
+
client.query(f'''
|
|
192
|
+
SELECT feedback_id, expected_language, body
|
|
193
|
+
FROM `{PROJECT_ID}.{DATASET_ID}.ml_understand_text_feedback`
|
|
194
|
+
ORDER BY feedback_id
|
|
195
|
+
''').to_dataframe()
|
|
196
|
+
```
|
|
197
|
+
|
|
198
|
+
### Wait for the grants to take effect
|
|
199
|
+
|
|
200
|
+
A role binding is not usable the instant `gcloud` returns. On a fresh grant the first call still fails with the permission error, so this cell probes with the smallest possible analysis and waits on evidence rather than on a clock. It prints how long it took — on a project where the roles were already in place, that is zero seconds.
|
|
201
|
+
|
|
202
|
+
```python
|
|
203
|
+
import time
|
|
204
|
+
from google.api_core.exceptions import BadRequest, Forbidden
|
|
205
|
+
|
|
206
|
+
probe_sql = f'''
|
|
207
|
+
SELECT FLOAT64(ml_understand_text_result.document_sentiment.score) AS score
|
|
208
|
+
FROM ML.UNDERSTAND_TEXT(
|
|
209
|
+
MODEL `{PROJECT_ID}.{DATASET_ID}.ml_understand_text_model`,
|
|
210
|
+
(SELECT 'This is wonderful.' AS text_content),
|
|
211
|
+
STRUCT('ANALYZE_SENTIMENT' AS nlu_option)
|
|
212
|
+
)
|
|
213
|
+
'''
|
|
214
|
+
|
|
215
|
+
waited = 0
|
|
216
|
+
while True:
|
|
217
|
+
try:
|
|
218
|
+
print(f"Probe succeeded after {waited}s: sentiment score {list(client.query(probe_sql).result())[0]['score']}")
|
|
219
|
+
break
|
|
220
|
+
except (BadRequest, Forbidden) as e:
|
|
221
|
+
if 'denied' not in str(e) and 'permission' not in str(e):
|
|
222
|
+
raise
|
|
223
|
+
if waited >= 300:
|
|
224
|
+
raise RuntimeError('IAM grants did not become usable within 5 minutes')
|
|
225
|
+
print(f' not usable yet ({waited}s elapsed)')
|
|
226
|
+
time.sleep(15)
|
|
227
|
+
waited += 15
|
|
228
|
+
```
|
|
229
|
+
|
|
230
|
+
---
|
|
231
|
+
## Examples — SQL
|
|
232
|
+
|
|
233
|
+
### 1. `ANALYZE_SENTIMENT` — the raw shape, then the columns
|
|
234
|
+
|
|
235
|
+
Every analysis returns one JSON column, `ml_understand_text_result`, plus a per-row `ml_understand_text_status`. Here is the JSON for one row before any unpacking.
|
|
236
|
+
|
|
237
|
+
```python
|
|
238
|
+
query = f'''
|
|
239
|
+
SELECT feedback_id, ml_understand_text_result, ml_understand_text_status
|
|
240
|
+
FROM ML.UNDERSTAND_TEXT(
|
|
241
|
+
MODEL `{PROJECT_ID}.{DATASET_ID}.ml_understand_text_model`,
|
|
242
|
+
(SELECT feedback_id, body AS text_content
|
|
243
|
+
FROM `{PROJECT_ID}.{DATASET_ID}.ml_understand_text_feedback`
|
|
244
|
+
WHERE feedback_id = 1),
|
|
245
|
+
STRUCT('ANALYZE_SENTIMENT' AS nlu_option)
|
|
246
|
+
)
|
|
247
|
+
'''
|
|
248
|
+
print(client.query(query).to_dataframe()['ml_understand_text_result'][0])
|
|
249
|
+
```
|
|
250
|
+
|
|
251
|
+
Two numbers describe the sentiment and they are not the same measurement:
|
|
252
|
+
|
|
253
|
+
- **`score`** runs from −1 to 1 and says which way the text leans.
|
|
254
|
+
- **`magnitude`** is unbounded and says how much emotion is present regardless of direction. A long, evenly mixed review has a high magnitude and a score near zero; a short flat statement has both near zero.
|
|
255
|
+
|
|
256
|
+
Every row in this table is a single sentence, so magnitude comes back as the absolute value of the score below; the two separate as documents get longer and mix directions.
|
|
257
|
+
|
|
258
|
+
The API also reports the language it detected, which is worth selecting — it is the input to knowing whether the *next* analysis you want supports this row.
|
|
259
|
+
|
|
260
|
+
```python
|
|
261
|
+
query = f'''
|
|
262
|
+
SELECT
|
|
263
|
+
feedback_id,
|
|
264
|
+
text_content,
|
|
265
|
+
STRING(ml_understand_text_result.language) AS detected_language,
|
|
266
|
+
FLOAT64(ml_understand_text_result.document_sentiment.score) AS score,
|
|
267
|
+
FLOAT64(ml_understand_text_result.document_sentiment.magnitude) AS magnitude,
|
|
268
|
+
ml_understand_text_status
|
|
269
|
+
FROM ML.UNDERSTAND_TEXT(
|
|
270
|
+
MODEL `{PROJECT_ID}.{DATASET_ID}.ml_understand_text_model`,
|
|
271
|
+
(SELECT feedback_id, body AS text_content
|
|
272
|
+
FROM `{PROJECT_ID}.{DATASET_ID}.ml_understand_text_feedback`),
|
|
273
|
+
STRUCT('ANALYZE_SENTIMENT' AS nlu_option)
|
|
274
|
+
)
|
|
275
|
+
ORDER BY feedback_id
|
|
276
|
+
'''
|
|
277
|
+
sentiment = client.query(query).to_dataframe()
|
|
278
|
+
sentiment
|
|
279
|
+
```
|
|
280
|
+
|
|
281
|
+
### 2. `flatten_json_output` — let the function do the unpacking
|
|
282
|
+
|
|
283
|
+
Set `flatten_json_output` to `TRUE` and the JSON column is replaced by separate columns. Which columns you get depends on the analysis, so the flattened shape is not stable across `nlu_option` values — the cell prints the column list for two of them.
|
|
284
|
+
|
|
285
|
+
```python
|
|
286
|
+
for option in ['ANALYZE_SENTIMENT', 'CLASSIFY_TEXT']:
|
|
287
|
+
df = client.query(f'''
|
|
288
|
+
SELECT *
|
|
289
|
+
FROM ML.UNDERSTAND_TEXT(
|
|
290
|
+
MODEL `{PROJECT_ID}.{DATASET_ID}.ml_understand_text_model`,
|
|
291
|
+
(SELECT feedback_id, body AS text_content
|
|
292
|
+
FROM `{PROJECT_ID}.{DATASET_ID}.ml_understand_text_feedback`
|
|
293
|
+
WHERE feedback_id = 5),
|
|
294
|
+
STRUCT('{option}' AS nlu_option, TRUE AS flatten_json_output)
|
|
295
|
+
)
|
|
296
|
+
''').to_dataframe()
|
|
297
|
+
print(f'{option}:')
|
|
298
|
+
print(f' columns: {list(df.columns)}')
|
|
299
|
+
print()
|
|
300
|
+
```
|
|
301
|
+
|
|
302
|
+
The flattened columns are still JSON *values* where the content is nested — `sentences` and `categories` come back as JSON arrays, not as rows. Flattening saves you the top-level accessor, not the `UNNEST`.
|
|
303
|
+
|
|
304
|
+
```python
|
|
305
|
+
query = f'''
|
|
306
|
+
SELECT feedback_id, language, sentiment, sentences
|
|
307
|
+
FROM ML.UNDERSTAND_TEXT(
|
|
308
|
+
MODEL `{PROJECT_ID}.{DATASET_ID}.ml_understand_text_model`,
|
|
309
|
+
(SELECT feedback_id, body AS text_content
|
|
310
|
+
FROM `{PROJECT_ID}.{DATASET_ID}.ml_understand_text_feedback`
|
|
311
|
+
WHERE feedback_id = 1),
|
|
312
|
+
STRUCT('ANALYZE_SENTIMENT' AS nlu_option, TRUE AS flatten_json_output)
|
|
313
|
+
)
|
|
314
|
+
'''
|
|
315
|
+
client.query(query).to_dataframe()
|
|
316
|
+
```
|
|
317
|
+
|
|
318
|
+
### 3. `ANALYZE_ENTITIES` — one row per thing mentioned
|
|
319
|
+
|
|
320
|
+
`entities` is a JSON array, so `JSON_QUERY_ARRAY` plus `UNNEST` turns one row of feedback into one row per entity. Each carries a `type` from a closed vocabulary and a `salience` — how central the entity is to the text, summing to roughly 1 across the entities of a document.
|
|
321
|
+
|
|
322
|
+
```python
|
|
323
|
+
query = f'''
|
|
324
|
+
SELECT
|
|
325
|
+
feedback_id,
|
|
326
|
+
STRING(entity.name) AS entity,
|
|
327
|
+
STRING(entity.type) AS entity_type,
|
|
328
|
+
ROUND(FLOAT64(entity.salience), 3) AS salience
|
|
329
|
+
FROM ML.UNDERSTAND_TEXT(
|
|
330
|
+
MODEL `{PROJECT_ID}.{DATASET_ID}.ml_understand_text_model`,
|
|
331
|
+
(SELECT feedback_id, body AS text_content
|
|
332
|
+
FROM `{PROJECT_ID}.{DATASET_ID}.ml_understand_text_feedback`
|
|
333
|
+
WHERE feedback_id IN (2, 8)),
|
|
334
|
+
STRUCT('ANALYZE_ENTITIES' AS nlu_option)
|
|
335
|
+
),
|
|
336
|
+
UNNEST(JSON_QUERY_ARRAY(ml_understand_text_result.entities)) AS entity
|
|
337
|
+
ORDER BY feedback_id, salience DESC
|
|
338
|
+
'''
|
|
339
|
+
client.query(query).to_dataframe()
|
|
340
|
+
```
|
|
341
|
+
|
|
342
|
+
Read the `entity_type` column before you build on it. The type is the model's guess from context, not a lookup against a registry of companies and places: one of the two company names above comes back as `ORGANIZATION` and the other does not. Salience can also be absent for an entity that is not really *about* anything, which arrives as NULL rather than zero. Treat the types as a signal to filter on, not as ground truth to key a join on.
|
|
343
|
+
|
|
344
|
+
### 4. `ANALYZE_ENTITY_SENTIMENT`, and the per-row failure that does not fail the job
|
|
345
|
+
|
|
346
|
+
Same entities, each with its own sentiment — useful when one review praises the battery and condemns the charger.
|
|
347
|
+
|
|
348
|
+
This is also the example where `ml_understand_text_status` earns its place. Entity sentiment supports fewer languages than sentiment analysis does. The rows in an unsupported language come back **empty with an error string**, while the job itself succeeds. If you select only the result column you will never see it.
|
|
349
|
+
|
|
350
|
+
```python
|
|
351
|
+
query = f'''
|
|
352
|
+
SELECT
|
|
353
|
+
feedback_id,
|
|
354
|
+
expected_language,
|
|
355
|
+
ml_understand_text_status,
|
|
356
|
+
ARRAY_LENGTH(JSON_QUERY_ARRAY(ml_understand_text_result.entities)) AS entity_count
|
|
357
|
+
FROM ML.UNDERSTAND_TEXT(
|
|
358
|
+
MODEL `{PROJECT_ID}.{DATASET_ID}.ml_understand_text_model`,
|
|
359
|
+
(SELECT feedback_id, expected_language, body AS text_content
|
|
360
|
+
FROM `{PROJECT_ID}.{DATASET_ID}.ml_understand_text_feedback`),
|
|
361
|
+
STRUCT('ANALYZE_ENTITY_SENTIMENT' AS nlu_option)
|
|
362
|
+
)
|
|
363
|
+
ORDER BY feedback_id
|
|
364
|
+
'''
|
|
365
|
+
entity_sentiment = client.query(query).to_dataframe()
|
|
366
|
+
entity_sentiment
|
|
367
|
+
```
|
|
368
|
+
|
|
369
|
+
The rows that did work carry a sentiment per entity — and an entity can still come back with an empty sentiment object, which surfaces as NULL rather than as zero:
|
|
370
|
+
|
|
371
|
+
```python
|
|
372
|
+
query = f'''
|
|
373
|
+
SELECT
|
|
374
|
+
feedback_id,
|
|
375
|
+
STRING(entity.name) AS entity,
|
|
376
|
+
ROUND(FLOAT64(entity.sentiment.score), 2) AS entity_score,
|
|
377
|
+
ROUND(FLOAT64(entity.sentiment.magnitude), 2) AS entity_magnitude
|
|
378
|
+
FROM ML.UNDERSTAND_TEXT(
|
|
379
|
+
MODEL `{PROJECT_ID}.{DATASET_ID}.ml_understand_text_model`,
|
|
380
|
+
(SELECT feedback_id, body AS text_content
|
|
381
|
+
FROM `{PROJECT_ID}.{DATASET_ID}.ml_understand_text_feedback`
|
|
382
|
+
WHERE feedback_id = 1),
|
|
383
|
+
STRUCT('ANALYZE_ENTITY_SENTIMENT' AS nlu_option)
|
|
384
|
+
),
|
|
385
|
+
UNNEST(JSON_QUERY_ARRAY(ml_understand_text_result.entities)) AS entity
|
|
386
|
+
ORDER BY entity_score
|
|
387
|
+
'''
|
|
388
|
+
client.query(query).to_dataframe()
|
|
389
|
+
```
|
|
390
|
+
|
|
391
|
+
**The retry pattern.** Because failures are per row, the fix is to select the failed rows back out and call again — the same shape the documentation recommends for the `RESOURCE EXHAUSTED` case, where a row failed for load rather than for language. For an unsupported language the retry will fail identically, which is itself the answer: check the status column, then decide whether to retry the row or route it somewhere else.
|
|
392
|
+
|
|
393
|
+
```python
|
|
394
|
+
failed = entity_sentiment[entity_sentiment['ml_understand_text_status'] != '']
|
|
395
|
+
print(f'{len(failed)} of {len(entity_sentiment)} rows carry an error while the job succeeded:')
|
|
396
|
+
for _, row in failed.iterrows():
|
|
397
|
+
print(f" feedback_id {row['feedback_id']} ({row['expected_language']}): {row['ml_understand_text_status']}")
|
|
398
|
+
```
|
|
399
|
+
|
|
400
|
+
### 5. `ANALYZE_SYNTAX` — tokens, parts of speech, dependencies
|
|
401
|
+
|
|
402
|
+
The most granular analysis: every token with its lemma, part of speech and dependency edge. The dependency edge is what makes it more than a tokenizer — `head_token_index` points at the token this one attaches to, so the sentence's grammatical structure is recoverable.
|
|
403
|
+
|
|
404
|
+
Language support does not nest neatly between the analyses: the Korean row that entity sentiment refused is analyzed here without complaint, and the Vietnamese row is refused by both. Per analysis, per row — read the status column.
|
|
405
|
+
|
|
406
|
+
```python
|
|
407
|
+
query = f'''
|
|
408
|
+
SELECT
|
|
409
|
+
feedback_id,
|
|
410
|
+
expected_language,
|
|
411
|
+
STRING(ml_understand_text_result.language) AS detected_language,
|
|
412
|
+
ARRAY_LENGTH(JSON_QUERY_ARRAY(ml_understand_text_result.tokens)) AS token_count,
|
|
413
|
+
ml_understand_text_status
|
|
414
|
+
FROM ML.UNDERSTAND_TEXT(
|
|
415
|
+
MODEL `{PROJECT_ID}.{DATASET_ID}.ml_understand_text_model`,
|
|
416
|
+
(SELECT feedback_id, expected_language, body AS text_content
|
|
417
|
+
FROM `{PROJECT_ID}.{DATASET_ID}.ml_understand_text_feedback`),
|
|
418
|
+
STRUCT('ANALYZE_SYNTAX' AS nlu_option)
|
|
419
|
+
)
|
|
420
|
+
ORDER BY feedback_id
|
|
421
|
+
'''
|
|
422
|
+
client.query(query).to_dataframe()
|
|
423
|
+
```
|
|
424
|
+
|
|
425
|
+
```python
|
|
426
|
+
query = f'''
|
|
427
|
+
SELECT
|
|
428
|
+
STRING(token.text.content) AS token,
|
|
429
|
+
STRING(token.part_of_speech.tag) AS part_of_speech,
|
|
430
|
+
STRING(token.lemma) AS lemma,
|
|
431
|
+
STRING(token.dependency_edge.label) AS dependency,
|
|
432
|
+
INT64(token.dependency_edge.head_token_index) AS head_index
|
|
433
|
+
FROM ML.UNDERSTAND_TEXT(
|
|
434
|
+
MODEL `{PROJECT_ID}.{DATASET_ID}.ml_understand_text_model`,
|
|
435
|
+
(SELECT body AS text_content
|
|
436
|
+
FROM `{PROJECT_ID}.{DATASET_ID}.ml_understand_text_feedback`
|
|
437
|
+
WHERE feedback_id = 1),
|
|
438
|
+
STRUCT('ANALYZE_SYNTAX' AS nlu_option)
|
|
439
|
+
),
|
|
440
|
+
UNNEST(JSON_QUERY_ARRAY(ml_understand_text_result.tokens)) AS token
|
|
441
|
+
'''
|
|
442
|
+
client.query(query).to_dataframe()
|
|
443
|
+
```
|
|
444
|
+
|
|
445
|
+
### 6. `CLASSIFY_TEXT` — categories from a fixed taxonomy
|
|
446
|
+
|
|
447
|
+
Content categories from [a published taxonomy](https://cloud.google.com/natural-language/docs/categories), returned as a path with a confidence. Several can come back for one document, ordered by confidence, and short text can produce a single low-confidence guess where a longer passage produces a ranked list.
|
|
448
|
+
|
|
449
|
+
```python
|
|
450
|
+
query = f'''
|
|
451
|
+
SELECT
|
|
452
|
+
feedback_id,
|
|
453
|
+
STRING(category.name) AS category,
|
|
454
|
+
ROUND(FLOAT64(category.confidence), 3) AS confidence
|
|
455
|
+
FROM ML.UNDERSTAND_TEXT(
|
|
456
|
+
MODEL `{PROJECT_ID}.{DATASET_ID}.ml_understand_text_model`,
|
|
457
|
+
(SELECT feedback_id, body AS text_content
|
|
458
|
+
FROM `{PROJECT_ID}.{DATASET_ID}.ml_understand_text_feedback`),
|
|
459
|
+
STRUCT('CLASSIFY_TEXT' AS nlu_option)
|
|
460
|
+
),
|
|
461
|
+
UNNEST(JSON_QUERY_ARRAY(ml_understand_text_result.categories)) AS category
|
|
462
|
+
ORDER BY feedback_id, confidence DESC
|
|
463
|
+
'''
|
|
464
|
+
client.query(query).to_dataframe()
|
|
465
|
+
```
|
|
466
|
+
|
|
467
|
+
`CLASSIFY_TEXT` is the one analysis that rejects `encoding_type`, which is a compile-time error rather than a row-level one.
|
|
468
|
+
|
|
469
|
+
```python
|
|
470
|
+
try:
|
|
471
|
+
client.query(f'''
|
|
472
|
+
SELECT *
|
|
473
|
+
FROM ML.UNDERSTAND_TEXT(
|
|
474
|
+
MODEL `{PROJECT_ID}.{DATASET_ID}.ml_understand_text_model`,
|
|
475
|
+
(SELECT body AS text_content
|
|
476
|
+
FROM `{PROJECT_ID}.{DATASET_ID}.ml_understand_text_feedback`
|
|
477
|
+
WHERE feedback_id = 5),
|
|
478
|
+
STRUCT('CLASSIFY_TEXT' AS nlu_option, 'UTF8' AS encoding_type)
|
|
479
|
+
)
|
|
480
|
+
''').result()
|
|
481
|
+
print('No error — encoding_type was accepted.')
|
|
482
|
+
except BadRequest as e:
|
|
483
|
+
print('Failed, as expected:\n')
|
|
484
|
+
print(str(e).split(';')[0])
|
|
485
|
+
```
|
|
486
|
+
|
|
487
|
+
### 7. `encoding_type` — the option that turns offsets on
|
|
488
|
+
|
|
489
|
+
Every analysis that returns text spans reports a `begin_offset`, and the default `encoding_type` of `NONE` means those offsets are not computed. Ask for an encoding and the offsets appear — measured **in that encoding's units**, which is the part worth seeing rather than reading about. The text below is Spanish, so accented characters make UTF-8 bytes and UTF-16 code units diverge.
|
|
490
|
+
|
|
491
|
+
```python
|
|
492
|
+
rows = []
|
|
493
|
+
for encoding in ['NONE', 'UTF8', 'UTF16', 'UTF32']:
|
|
494
|
+
df = client.query(f'''
|
|
495
|
+
SELECT
|
|
496
|
+
STRING(entity.name) AS entity,
|
|
497
|
+
INT64(entity.mentions[0].text.begin_offset) AS begin_offset
|
|
498
|
+
FROM ML.UNDERSTAND_TEXT(
|
|
499
|
+
MODEL `{PROJECT_ID}.{DATASET_ID}.ml_understand_text_model`,
|
|
500
|
+
(SELECT body AS text_content
|
|
501
|
+
FROM `{PROJECT_ID}.{DATASET_ID}.ml_understand_text_feedback`
|
|
502
|
+
WHERE feedback_id = 3),
|
|
503
|
+
STRUCT('ANALYZE_ENTITIES' AS nlu_option, '{encoding}' AS encoding_type)
|
|
504
|
+
),
|
|
505
|
+
UNNEST(JSON_QUERY_ARRAY(ml_understand_text_result.entities)) AS entity
|
|
506
|
+
''').to_dataframe()
|
|
507
|
+
rows.append(df.drop_duplicates('entity').set_index('entity')['begin_offset'].rename(encoding))
|
|
508
|
+
|
|
509
|
+
text = client.query(f'''
|
|
510
|
+
SELECT body FROM `{PROJECT_ID}.{DATASET_ID}.ml_understand_text_feedback` WHERE feedback_id = 3
|
|
511
|
+
''').to_dataframe()['body'][0]
|
|
512
|
+
print(text)
|
|
513
|
+
print()
|
|
514
|
+
pd.concat(rows, axis=1)
|
|
515
|
+
```
|
|
516
|
+
|
|
517
|
+
`NONE` returns `-1` for every offset: not "position zero" but "not computed". Where the encodings differ, the difference is the accented characters ahead of the entity — each one costs an extra byte in UTF-8 and nothing in UTF-16 or UTF-32. If you plan to slice the original string using these offsets, the encoding you ask for has to match the units your slicing code counts in.
|
|
518
|
+
|
|
519
|
+
### 8. The `text_content` rule, and what breaks without it
|
|
520
|
+
|
|
521
|
+
Pass the table directly and the call fails, because the table's text column is named `body`. The error is caught and printed rather than allowed to stop the notebook.
|
|
522
|
+
|
|
523
|
+
```python
|
|
524
|
+
try:
|
|
525
|
+
client.query(f'''
|
|
526
|
+
SELECT *
|
|
527
|
+
FROM ML.UNDERSTAND_TEXT(
|
|
528
|
+
MODEL `{PROJECT_ID}.{DATASET_ID}.ml_understand_text_model`,
|
|
529
|
+
TABLE `{PROJECT_ID}.{DATASET_ID}.ml_understand_text_feedback`,
|
|
530
|
+
STRUCT('ANALYZE_SENTIMENT' AS nlu_option)
|
|
531
|
+
)
|
|
532
|
+
''').result()
|
|
533
|
+
print('No error — this table has a text_content column after all.')
|
|
534
|
+
except BadRequest as e:
|
|
535
|
+
print('Failed, as expected:\n')
|
|
536
|
+
print(str(e).split(';')[0])
|
|
537
|
+
```
|
|
538
|
+
|
|
539
|
+
Two fixes, and they are not equivalent:
|
|
540
|
+
|
|
541
|
+
1. **Alias in a subquery** — `(SELECT body AS text_content FROM …)`. What every other example here does. Anything else the subquery selects passes through to the output, which is how `feedback_id` and `expected_language` survive the call.
|
|
542
|
+
2. **Name the column `text_content` in the table itself.** Then `TABLE …` works directly. Worth doing only if the table exists to be analyzed.
|
|
543
|
+
|
|
544
|
+
### 9. Is it deterministic, and how does an LLM compare?
|
|
545
|
+
|
|
546
|
+
Worth knowing before you build a pipeline on it, and not documented either way. The same sentiment call runs twice with the query cache **off** — a cached result would answer the wrong question — and the scores are compared. The same comparison runs against `AI.GENERATE_DOUBLE` asked for a sentiment score on the same scale.
|
|
547
|
+
|
|
548
|
+
```python
|
|
549
|
+
no_cache = bigquery.QueryJobConfig(use_query_cache=False)
|
|
550
|
+
|
|
551
|
+
nlu_sql = f'''
|
|
552
|
+
SELECT feedback_id, FLOAT64(ml_understand_text_result.document_sentiment.score) AS score
|
|
553
|
+
FROM ML.UNDERSTAND_TEXT(
|
|
554
|
+
MODEL `{PROJECT_ID}.{DATASET_ID}.ml_understand_text_model`,
|
|
555
|
+
(SELECT feedback_id, body AS text_content
|
|
556
|
+
FROM `{PROJECT_ID}.{DATASET_ID}.ml_understand_text_feedback`),
|
|
557
|
+
STRUCT('ANALYZE_SENTIMENT' AS nlu_option)
|
|
558
|
+
)
|
|
559
|
+
ORDER BY feedback_id
|
|
560
|
+
'''
|
|
561
|
+
|
|
562
|
+
llm_sql = f'''
|
|
563
|
+
SELECT
|
|
564
|
+
feedback_id,
|
|
565
|
+
(AI.GENERATE_DOUBLE(
|
|
566
|
+
CONCAT('Rate the sentiment of this customer feedback from -1.0 (negative) to 1.0 (positive): ', body)
|
|
567
|
+
)).result AS score
|
|
568
|
+
FROM `{PROJECT_ID}.{DATASET_ID}.ml_understand_text_feedback`
|
|
569
|
+
ORDER BY feedback_id
|
|
570
|
+
'''
|
|
571
|
+
|
|
572
|
+
frames = {}
|
|
573
|
+
for label, sql in [('ML.UNDERSTAND_TEXT', nlu_sql), ('AI.GENERATE_DOUBLE', llm_sql)]:
|
|
574
|
+
run1 = client.query(sql, job_config=no_cache).to_dataframe()
|
|
575
|
+
run2 = client.query(sql, job_config=no_cache).to_dataframe()
|
|
576
|
+
same = (run1['score'] == run2['score']).sum()
|
|
577
|
+
print(f'{label:<19}: {same} of {len(run1)} scores identical across two cache-disabled runs')
|
|
578
|
+
frames[f'{label} run 1'] = run1.set_index('feedback_id')['score']
|
|
579
|
+
frames[f'{label} run 2'] = run2.set_index('feedback_id')['score']
|
|
580
|
+
|
|
581
|
+
print()
|
|
582
|
+
pd.concat(frames, axis=1)
|
|
583
|
+
```
|
|
584
|
+
|
|
585
|
+
The two do not answer quite the same question even when they agree on direction. `ML.UNDERSTAND_TEXT` returns a calibrated score from a fixed model with a companion magnitude; `AI.GENERATE_DOUBLE` returns whatever number the prompt talks the model into, on whatever scale you asked for, with no magnitude and no per-sentence breakdown.
|
|
586
|
+
|
|
587
|
+
### 10. What the model object actually is
|
|
588
|
+
|
|
589
|
+
Models are not part of `INFORMATION_SCHEMA` — the `TABLES` view below lists the dataset's tables and the model is not among them — so the model's metadata is read through the client API. What comes back is the clearest statement that nothing trained: `model_type` is unspecified, the single training run carries an empty `trainingOptions` and an empty `evaluationMetrics`, and the only substance in the record is `remoteModelInfo`, holding the service type and the connection. `ML.EVALUATE`, the function every trained BigQuery ML model answers, refuses outright.
|
|
590
|
+
|
|
591
|
+
```python
|
|
592
|
+
import json
|
|
593
|
+
|
|
594
|
+
# INFORMATION_SCHEMA covers tables — models are not listed here
|
|
595
|
+
display(client.query(f'''
|
|
596
|
+
SELECT table_name, table_type
|
|
597
|
+
FROM `{PROJECT_ID}.{DATASET_ID}`.INFORMATION_SCHEMA.TABLES
|
|
598
|
+
ORDER BY table_name
|
|
599
|
+
''').to_dataframe())
|
|
600
|
+
|
|
601
|
+
# Model metadata comes from the client API instead
|
|
602
|
+
model = client.get_model(f'{PROJECT_ID}.{DATASET_ID}.ml_understand_text_model')
|
|
603
|
+
print(json.dumps(model.to_api_repr(), indent=2))
|
|
604
|
+
print()
|
|
605
|
+
|
|
606
|
+
try:
|
|
607
|
+
client.query(f'''
|
|
608
|
+
SELECT * FROM ML.EVALUATE(MODEL `{PROJECT_ID}.{DATASET_ID}.ml_understand_text_model`)
|
|
609
|
+
''').result()
|
|
610
|
+
print('ML.EVALUATE returned a result.')
|
|
611
|
+
except Exception as e:
|
|
612
|
+
print('ML.EVALUATE on a Cloud AI service model:\n')
|
|
613
|
+
print(str(e).split(';')[0])
|
|
614
|
+
```
|
|
615
|
+
|
|
616
|
+
---
|
|
617
|
+
## Examples — `%%bigquery` Magics
|
|
618
|
+
|
|
619
|
+
`%%bigquery` cannot interpolate Python variables into the SQL body, so model and table references are written out in full.
|
|
620
|
+
|
|
621
|
+
```sql
|
|
622
|
+
%%bigquery --project {PROJECT_ID}
|
|
623
|
+
|
|
624
|
+
SELECT
|
|
625
|
+
feedback_id,
|
|
626
|
+
STRING(ml_understand_text_result.language) AS detected_language,
|
|
627
|
+
ROUND(FLOAT64(ml_understand_text_result.document_sentiment.score), 2) AS score,
|
|
628
|
+
ROUND(FLOAT64(ml_understand_text_result.document_sentiment.magnitude), 2) AS magnitude
|
|
629
|
+
FROM ML.UNDERSTAND_TEXT(
|
|
630
|
+
MODEL `statmike-mlops-349915.bq_ai_functions.ml_understand_text_model`,
|
|
631
|
+
(SELECT feedback_id, body AS text_content
|
|
632
|
+
FROM `statmike-mlops-349915.bq_ai_functions.ml_understand_text_feedback`),
|
|
633
|
+
STRUCT('ANALYZE_SENTIMENT' AS nlu_option)
|
|
634
|
+
)
|
|
635
|
+
ORDER BY feedback_id
|
|
636
|
+
```
|
|
637
|
+
|
|
638
|
+
---
|
|
639
|
+
## Examples — BigFrames
|
|
640
|
+
|
|
641
|
+
BigFrames has no `ML.UNDERSTAND_TEXT` wrapper. Run the SQL through `read_gbq_query()` and get a BigFrames DataFrame back.
|
|
642
|
+
|
|
643
|
+
```python
|
|
644
|
+
import bigframes.pandas as bpd
|
|
645
|
+
|
|
646
|
+
bpd.options.bigquery.project = PROJECT_ID
|
|
647
|
+
bpd.options.bigquery.location = LOCATION
|
|
648
|
+
|
|
649
|
+
session = bpd.get_global_session()
|
|
650
|
+
df_bf = session.read_gbq_query(f'''
|
|
651
|
+
SELECT
|
|
652
|
+
feedback_id,
|
|
653
|
+
STRING(category.name) AS category,
|
|
654
|
+
ROUND(FLOAT64(category.confidence), 3) AS confidence
|
|
655
|
+
FROM ML.UNDERSTAND_TEXT(
|
|
656
|
+
MODEL `{PROJECT_ID}.{DATASET_ID}.ml_understand_text_model`,
|
|
657
|
+
(SELECT feedback_id, body AS text_content
|
|
658
|
+
FROM `{PROJECT_ID}.{DATASET_ID}.ml_understand_text_feedback`),
|
|
659
|
+
STRUCT('CLASSIFY_TEXT' AS nlu_option)
|
|
660
|
+
),
|
|
661
|
+
UNNEST(JSON_QUERY_ARRAY(ml_understand_text_result.categories)) AS category
|
|
662
|
+
''')
|
|
663
|
+
df_bf.to_pandas().sort_values(['feedback_id', 'confidence'], ascending=[True, False]).reset_index(drop=True)
|
|
664
|
+
```
|
|
665
|
+
|
|
666
|
+
---
|
|
667
|
+
## Which one should you reach for?
|
|
668
|
+
|
|
669
|
+
**Reach for `ML.UNDERSTAND_TEXT`** when you want the same answer next quarter: a fixed model behind a versioned API, sentiment on a stated scale, entity types and category paths from closed vocabularies, and a per-character price you can forecast. Nothing about the output moves when a generative model default changes.
|
|
670
|
+
|
|
671
|
+
**Reach for the `AI.*` functions** when the analysis you want is not one of the five — aspect-level summaries, your own category list, extraction into a schema you define — or when the taxonomy here does not match your business. `AI.GENERATE_TABLE` will produce columns you specify; `ML.UNDERSTAND_TEXT` produces the columns it has.
|
|
672
|
+
|
|
673
|
+
**What the notebook measured, not argued:** Example 9 runs both under a disabled query cache and prints run-to-run stability for each. Examples 4 and 5 show the coverage limit you inherit with the fixed model — per-analysis language support, reported per row.
|