bigquery-data-science 0.7.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- bigquery_data_science/__init__.py +58 -0
- bigquery_data_science/_skills/PLANS.md +76 -0
- bigquery_data_science/_skills/bigquery-ai/SKILL.md +66 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ai_agg.md +528 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ai_classify.md +450 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ai_count_tokens.md +180 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ai_detect_anomalies.md +322 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ai_embed.md +425 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ai_evaluate.md +368 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ai_forecast.md +437 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ai_generate.md +693 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ai_generate_bool.md +248 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ai_generate_double.md +254 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ai_generate_embedding.md +402 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ai_generate_int.md +252 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ai_generate_table.md +447 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ai_generate_text.md +356 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ai_if.md +373 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ai_key_drivers.md +331 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ai_parse_document.md +416 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ai_predict.md +559 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ai_score.md +310 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ai_search.md +465 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ai_similarity.md +371 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/catalog_search.md +951 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/content_analysis.md +284 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/content_moderation.md +369 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/data_enrichment.md +426 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/document_intelligence.md +470 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/document_rag.md +453 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/image_deduplication.md +770 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/log_analysis.md +668 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/metric_diagnostics.md +568 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ml_annotate_image.md +680 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ml_generate_embedding.md +217 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ml_generate_text.md +223 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ml_process_document.md +392 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ml_transcribe.md +815 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ml_translate.md +600 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ml_understand_text.md +673 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/multimodal_analysis.md +266 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/rag_pipeline.md +593 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/semantic_search.md +510 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/tabular_prediction.md +485 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/time_series_intelligence.md +489 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/vector_search.md +672 -0
- bigquery_data_science/_skills/bigquery-ai/reference/augmented-analytics.md +77 -0
- bigquery_data_science/_skills/bigquery-ai/reference/cloud-ai-service-models.md +88 -0
- bigquery_data_science/_skills/bigquery-ai/reference/document-processing.md +95 -0
- bigquery_data_science/_skills/bigquery-ai/reference/embeddings-and-search.md +166 -0
- bigquery_data_science/_skills/bigquery-ai/reference/general-purpose-generation.md +87 -0
- bigquery_data_science/_skills/bigquery-ai/reference/managed-functions.md +72 -0
- bigquery_data_science/_skills/bigquery-ai/reference/predictive-ai.md +113 -0
- bigquery_data_science/_skills/bigquery-ai/reference/workflows.md +56 -0
- bigquery_data_science/_skills/bigquery-ai/skill.manifest.json +109 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/INDEX.md +71 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/aiagg.md +67 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/aicausal_effect-covered-in-bigquery-ml.md +17 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/aiclassify.md +72 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/aicount_tokens.md +53 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/aidetect_anomalies.md +63 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/aiembed.md +85 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/aievaluate.md +114 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/aiforecast.md +123 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/aigenerate.md +69 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/aigenerate_bool.md +53 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/aigenerate_double.md +45 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/aigenerate_embedding.md +108 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/aigenerate_int.md +45 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/aigenerate_table.md +64 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/aigenerate_text.md +167 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/aiif.md +56 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/aikey_drivers.md +76 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/aiparse_document.md +91 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/aipredict.md +74 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/aiscore.md +52 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/aisearch.md +66 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/aisimilarity.md +79 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/choosing-between-these-and-the-ai-functions.md +11 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/hybrid-search-capability.md +165 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/mlannotate_image.md +24 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/mlgenerate_embedding.md +34 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/mlgenerate_text.md +65 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/mlprocess_document.md +125 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/mltranscribe.md +26 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/mltranslate.md +27 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/mlunderstand_text.md +32 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/multimodal-input-patterns-by-function.md +128 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/object-tables.md +109 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/objectref-and-objectrefruntime-schema-reference.md +50 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/objfetch_metadata.md +49 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/objget_access_url.md +66 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/objget_read_url.md +40 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/objlist.md +34 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/objmake_ref.md +58 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/per-row-failures-and-the-retry-pattern.md +19 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/setup-apis-connection-roles.md +39 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/two-input-shapes.md +22 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/vector_search.md +78 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/what-the-five-have-in-common.md +27 -0
- bigquery_data_science/_skills/bigquery-data-science/SKILL.md +69 -0
- bigquery_data_science/_skills/bigquery-data-science/reference/namespace-matrix.md +112 -0
- bigquery_data_science/_skills/bigquery-data-science/skill.manifest.json +13 -0
- bigquery_data_science/_skills/bigquery-ml/SKILL.md +70 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/airflow_with_kfp.md +379 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/anomaly_fraud_detection.md +195 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/arima_plus.md +635 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/arima_plus_xreg.md +453 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/autoencoder.md +470 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/automl_classifier.md +291 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/automl_regressor.md +274 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/boosted_tree_classifier.md +449 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/boosted_tree_regressor.md +400 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/bucketizing.md +201 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/causal_effect.md +773 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/churn_retention.md +271 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/cloud_workflows.md +370 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/composer_airflow.md +475 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/contribution_analysis.md +346 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/cross_validation.md +185 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/customer_segmentation.md +233 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/dataform.md +372 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/dbt.md +283 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/difference_in_differences.md +280 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/distance.md +195 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/dnn_classifier.md +326 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/dnn_regressor.md +308 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/embeddings_classification.md +426 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/encoding.md +238 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/ensembling.md +278 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/evaluation.md +445 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/exploration.md +779 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/export.md +321 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/feature_engineering.md +280 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/feature_store.md +652 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/feature_store_workflow.md +426 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/ga4_churn_prediction.md +285 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/hierarchical_forecasting.md +441 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/image.md +791 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/imported.md +426 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/kmeans.md +348 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/linear_regression.md +372 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/logistic_regression.md +390 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/matrix_factorization.md +404 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/model_monitoring.md +609 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/pca.md +357 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/price_elasticity_dml.md +286 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/propensity_score_matching.md +350 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/random_forest_classifier.md +445 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/random_forest_regressor.md +409 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/recommendation.md +281 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/regression_based_forecasting.md +491 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/remote.md +428 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/scalers.md +325 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/scheduled_queries.md +284 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/sql_scripting.md +243 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/survival_analysis.md +363 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/synthetic_control.md +221 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/text.md +215 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/time_series.md +669 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/transform_only.md +273 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/uplift_cate.md +241 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/vertex_kfp.md +360 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/wide_and_deep_classifier.md +316 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/wide_and_deep_regressor.md +298 -0
- bigquery_data_science/_skills/bigquery-ml/reference/classification.md +80 -0
- bigquery_data_science/_skills/bigquery-ml/reference/model-management.md +90 -0
- bigquery_data_science/_skills/bigquery-ml/reference/preprocessing-functions.md +234 -0
- bigquery_data_science/_skills/bigquery-ml/reference/regression.md +79 -0
- bigquery_data_science/_skills/bigquery-ml/reference/unsupervised-and-specialized.md +101 -0
- bigquery_data_science/_skills/bigquery-ml/reference/workflows-and-pipelines.md +111 -0
- bigquery_data_science/_skills/bigquery-ml/skill.manifest.json +177 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/1-model_type-catalog.md +28 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/2-task-mlevaluate-metric-columns.md +26 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/3-capability-matrix-per-model-type.md +21 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/4-explainability-weights-functions.md +14 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/5-connection-matrix.md +11 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/6-which-model-free-function-no-create-model-no-connection.md +15 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/7-point-in-time-feature-retrieval-which-of-the-two.md +11 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/INDEX.md +111 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/aicausal_effect.md +85 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/arima_plus-forecasting-lifecycle-functions.md +14 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/arima_plus-univariate-time-series-forecasting.md +132 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/arima_plus_xreg-multivariate-time-series-with-external-regressors.md +103 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/autoencoder.md +102 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/automl_classifier-automl_regressor.md +88 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/boosted_tree_classifier-boosted_tree_regressor.md +117 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/contribution_analysis.md +113 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/dnn_classifier-dnn_regressor.md +107 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/dnn_linear_combined_classifier-dnn_linear_combined_regressor-wide-and-deep.md +98 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/explainability-functions-mlexplain_predict-mlglobal_explain-mlfeature_importance.md +7 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/exploratory-data-analysis-mldescribe_data-mlcorrelation.md +11 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/export-model.md +92 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/feature-engineering-standalone-preprocessing-functions.md +13 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/image-preprocessing-functions-mldecode_image-mlresize_image-mlconvert_image_type-mlconvert_color_space.md +63 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/imported-models-tensorflow-tensorflow_lite-onnx-xgboost.md +36 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/kmeans.md +87 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/linear_reg-logistic_reg-generalized-linear-models.md +101 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/matrix_factorization.md +162 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mladvanced_weights.md +76 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlarima_coefficients.md +30 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlarima_evaluate.md +49 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlbag_of_words.md +47 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlbucketize.md +47 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlcentroids.md +46 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlconfusion_matrix.md +56 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlconvert_color_space.md +53 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlconvert_image_type.md +39 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlcorrelation.md +78 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mldecode_image.md +44 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mldescribe_data.md +74 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mldetect_anomalies.md +86 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mldistance.md +48 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlentity_features_at_time.md +56 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlevaluate.md +72 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlexplain_forecast.md +50 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlexplain_predict.md +66 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlfeature_cross.md +46 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlfeature_importance.md +47 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlfeature_info.md +50 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlfeatures_at_time.md +51 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlforecast.md +76 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlgenerate_embedding-from-a-bqml-pca-autoencoder-matrix_factorization-model.md +64 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlglobal_explain.md +63 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlhash_bucketize.md +47 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlholiday_info.md +46 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlimputer.md +46 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mllp_norm.md +47 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlmetrics.md +80 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlngrams.md +37 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlone_hot_encoder-mlmulti_hot_encoder-mllabel_encoder.md +67 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlpolynomial_expand.md +47 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlpredict.md +67 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlprincipal_component_info.md +42 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlprincipal_components.md +40 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlquantile_bucketize.md +43 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlrecommend.md +60 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlreconstruction_loss.md +48 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlresize_image.md +45 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlroc_curve.md +63 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mltf_idf.md +37 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mltfdv_describe.md +34 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mltfdv_validate.md +48 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mltraining_info.md +78 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mltransform-applying-a-saved-transform.md +52 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mltranspose-not-a-function-pointer.md +32 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mltrial_info.md +60 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlvalidate_data_drift.md +99 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlvalidate_data_skew.md +50 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlweights.md +84 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/model-free-time-series-decomposition-mltrend-mlseasonality-mldetect_change_points.md +100 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/model-introspection-mlfeature_info-mltraining_info-mltrial_info.md +7 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/model-monitoring-data-validation.md +23 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/numerical-scalers-mlstandard_scaler-mlmin_max_scaler-mlmax_abs_scaler-mlrobust_scaler-mlnormalizer.md +77 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/onnx.md +55 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/pca.md +91 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/point-in-time-feature-retrieval-mlfeatures_at_time-mlentity_features_at_time.md +25 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/quick-comparison.md +14 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/random_forest_classifier-random_forest_regressor.md +123 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/remote-remote-model-over-a-vertex-ai-endpoint-the-mechanism.md +87 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/tensorflow.md +48 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/tensorflow_lite.md +29 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/text-preprocessing-functions-mlngrams-mltf_idf-mlbag_of_words.md +18 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/timesfm-built-in-foundation-forecaster-see-bigquery-ai.md +28 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/transform_only.md +91 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/xgboost.md +72 -0
- bigquery_data_science/_skills/pinned-models.json +28 -0
- bigquery_data_science/catalog.py +1107 -0
- bigquery_data_science/cli.py +786 -0
- bigquery_data_science/installer.py +282 -0
- bigquery_data_science/mcp_server.py +317 -0
- bigquery_data_science/tooling/__init__.py +15 -0
- bigquery_data_science/tooling/convert_notebook.py +176 -0
- bigquery_data_science/tooling/link_policy.py +247 -0
- bigquery_data_science/tooling/manifest.py +152 -0
- bigquery_data_science/tooling/markdown_anchors.py +84 -0
- bigquery_data_science/tooling/narrative_drift.py +163 -0
- bigquery_data_science/tooling/pinned_models.py +216 -0
- bigquery_data_science/tooling/reference_structure.py +85 -0
- bigquery_data_science/tooling/sync_syntax.py +407 -0
- bigquery_data_science/tooling/validate.py +208 -0
- bigquery_data_science-0.7.0.dist-info/METADATA +306 -0
- bigquery_data_science-0.7.0.dist-info/RECORD +286 -0
- bigquery_data_science-0.7.0.dist-info/WHEEL +4 -0
- bigquery_data_science-0.7.0.dist-info/entry_points.txt +5 -0
- bigquery_data_science-0.7.0.dist-info/licenses/LICENSE +201 -0
|
@@ -0,0 +1,693 @@
|
|
|
1
|
+
> **Source (`statmike/bigquery-data-science`):** **Notebook:** [`bigquery-ai/functions/ai_generate/ai_generate.ipynb`](https://github.com/statmike/bigquery-data-science/blob/main/bigquery-ai/functions/ai_generate/ai_generate.ipynb) · **SQL:** [`bigquery-ai/functions/ai_generate/ai_generate.sql`](https://github.com/statmike/bigquery-data-science/blob/main/bigquery-ai/functions/ai_generate/ai_generate.sql)
|
|
2
|
+
|
|
3
|
+
# AI.GENERATE — BigQuery AI Functions
|
|
4
|
+
|
|
5
|
+
`AI.GENERATE` is the simplest way to call a Gemini model from SQL. It's a **scalar function** — use it in SELECT, WHERE, or anywhere you'd use a regular function. No model creation required.
|
|
6
|
+
|
|
7
|
+
**What it does:** Sends a prompt to a Gemini model and returns generated text or structured output.
|
|
8
|
+
|
|
9
|
+
**When to use it:**
|
|
10
|
+
- You want to call Gemini from SQL with minimal setup
|
|
11
|
+
- You need free-form text generation or structured output with typed columns
|
|
12
|
+
- You're working with Gemini models only (for Claude/Llama/Mistral, use `functions/ai_generate_text` (`AI.GENERATE_TEXT`))
|
|
13
|
+
|
|
14
|
+
**Alternatives:**
|
|
15
|
+
- `functions/ai_generate_text` (`AI.GENERATE_TEXT`) — table-valued function, supports non-Gemini models, requires `CREATE MODEL`
|
|
16
|
+
- `functions/ai_if` (`AI.IF`), `functions/ai_score` (`AI.SCORE`), `functions/ai_classify` (`AI.CLASSIFY`) — simplified interfaces for common tasks with auto-optimized prompts
|
|
17
|
+
- `functions/ai_generate_bool` (`AI.GENERATE_BOOL`), `functions/ai_generate_double` (`AI.GENERATE_DOUBLE`), `functions/ai_generate_int` (`AI.GENERATE_INT`) — typed scalar variants returning `BOOL`, `FLOAT64`, `INT64`
|
|
18
|
+
|
|
19
|
+
**Multimodal:** Supports document, image, and video input via `reference/unstructured-data-infrastructure.md#objectref-and-objectrefruntime-schema-reference` (ObjectRef). Pass a STRUCT prompt with `ObjectRef` fields to analyze unstructured data from Cloud Storage.
|
|
20
|
+
|
|
21
|
+
**Thinking:** Supports extended reasoning via `model_params` — use `thinking_budget` (Gemini 2.5) or `thinking_level` (Gemini 3.0+) to control reasoning depth.
|
|
22
|
+
|
|
23
|
+
**Featured in:** `workflows/content_analysis` (Content Analysis Pipeline) | `workflows/data_enrichment` (Data Enrichment) | `workflows/rag_pipeline` (RAG Pipeline) | `workflows/document_intelligence` (Document Intelligence) | `workflows/content_moderation` (Content Moderation) | `workflows/multimodal_analysis` (Multimodal Analysis) | `workflows/document_rag` (Document RAG Pipeline) | `workflows/metric_diagnostics` (Metric Diagnostics) | `workflows/tabular_prediction` (Zero-Shot Tabular Prediction)
|
|
24
|
+
|
|
25
|
+
**References:** `RESOURCES.md` (Full syntax reference) | [Official documentation](https://cloud.google.com/bigquery/docs/reference/standard-sql/bigqueryml-syntax-ai-generate) | `setup` (Setup guide)
|
|
26
|
+
|
|
27
|
+
---
|
|
28
|
+
## Setup
|
|
29
|
+
|
|
30
|
+
Set your project and location, authenticate, and create a temporary dataset for this notebook.
|
|
31
|
+
|
|
32
|
+
> `AI.GENERATE` doesn't require a connection or model — it uses end-user credentials and defaults to `gemini-2.5-flash`. The [multimodal examples](#examples--multimodal-with-objectref) later in this notebook add a connection for GCS access. See the `setup` (Setup Reference) for details.
|
|
33
|
+
|
|
34
|
+
```python
|
|
35
|
+
PROJECT_ID = 'statmike-mlops-349915' # <-- Replace with your project ID
|
|
36
|
+
LOCATION = 'US' # BigQuery dataset location
|
|
37
|
+
DATASET_ID = 'bq_ai_functions' # Shared dataset across all notebooks
|
|
38
|
+
CONNECTION_ID = 'bq_ai_functions' # Cloud resource connection (used by multimodal examples)
|
|
39
|
+
BUCKET = PROJECT_ID # GCS bucket (same name as project)
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
### Environment
|
|
43
|
+
|
|
44
|
+
> **Already set up the project environment?** The cell below is a no-op — packages are already in your kernel. See the `setup` (Setup Reference) for details.
|
|
45
|
+
>
|
|
46
|
+
> **Running standalone** (Colab, Colab Enterprise, Vertex AI Workbench)? The cell below installs required packages into your current kernel.
|
|
47
|
+
|
|
48
|
+
```python
|
|
49
|
+
from google.cloud import bigquery
|
|
50
|
+
import pandas as pd
|
|
51
|
+
|
|
52
|
+
client = bigquery.Client(project=PROJECT_ID)
|
|
53
|
+
pd.set_option('display.max_colwidth', None)
|
|
54
|
+
|
|
55
|
+
# Create the shared dataset (idempotent)
|
|
56
|
+
dataset_ref = bigquery.DatasetReference(PROJECT_ID, DATASET_ID)
|
|
57
|
+
dataset = bigquery.Dataset(dataset_ref)
|
|
58
|
+
dataset.location = LOCATION
|
|
59
|
+
client.create_dataset(dataset, exists_ok=True)
|
|
60
|
+
print(f'Dataset {PROJECT_ID}.{DATASET_ID} ready')
|
|
61
|
+
|
|
62
|
+
# Register %%bigquery cell magic (auto-loaded in Colab, needed elsewhere)
|
|
63
|
+
%load_ext bigquery_magics
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
### Schema helper — Pydantic to BigQuery
|
|
67
|
+
|
|
68
|
+
For simple schemas, you can write the `output_schema` string directly. For complex schemas with descriptions, array types, and many fields, defining a Pydantic model is cleaner and less error-prone.
|
|
69
|
+
|
|
70
|
+
This helper converts a Pydantic `BaseModel` to the formats that `AI.GENERATE` (SQL string) and BigFrames (dict) expect.
|
|
71
|
+
|
|
72
|
+
```python
|
|
73
|
+
from pydantic import BaseModel, Field
|
|
74
|
+
from typing import get_origin, get_args
|
|
75
|
+
|
|
76
|
+
# Python type → BigQuery type
|
|
77
|
+
_BQ_TYPES = {str: 'STRING', int: 'INT64', float: 'FLOAT64', bool: 'BOOL'}
|
|
78
|
+
|
|
79
|
+
def _bq_type(annotation) -> str:
|
|
80
|
+
"""Map a Python type annotation to a BigQuery type string."""
|
|
81
|
+
if get_origin(annotation) is list:
|
|
82
|
+
inner = _BQ_TYPES[get_args(annotation)[0]]
|
|
83
|
+
return f'ARRAY<{inner}>'
|
|
84
|
+
return _BQ_TYPES[annotation]
|
|
85
|
+
|
|
86
|
+
def bq_schema(model: type[BaseModel]) -> str:
|
|
87
|
+
"""Convert a Pydantic model to a BigQuery output_schema string.
|
|
88
|
+
|
|
89
|
+
Supports str, int, float, bool, and list[T].
|
|
90
|
+
Field descriptions become OPTIONS(description = '...').
|
|
91
|
+
"""
|
|
92
|
+
fields = []
|
|
93
|
+
for name, info in model.model_fields.items():
|
|
94
|
+
field_str = f'{name} {_bq_type(info.annotation)}'
|
|
95
|
+
if info.description:
|
|
96
|
+
field_str += f" OPTIONS(description = '{info.description}')"
|
|
97
|
+
fields.append(field_str)
|
|
98
|
+
return ', '.join(fields)
|
|
99
|
+
|
|
100
|
+
def bf_schema(model: type[BaseModel]) -> dict[str, str]:
|
|
101
|
+
"""Convert a Pydantic model to a BigFrames output_schema dict."""
|
|
102
|
+
return {name: _bq_type(info.annotation) for name, info in model.model_fields.items()}
|
|
103
|
+
```
|
|
104
|
+
|
|
105
|
+
---
|
|
106
|
+
## Examples — SQL
|
|
107
|
+
|
|
108
|
+
Progressive examples from simplest to most advanced. Each cell adds one new concept.
|
|
109
|
+
|
|
110
|
+
### 1. Simplest possible call
|
|
111
|
+
|
|
112
|
+
`AI.GENERATE` takes a prompt string and returns a STRUCT. Access the generated text with `.result`.
|
|
113
|
+
|
|
114
|
+
```python
|
|
115
|
+
query = """
|
|
116
|
+
SELECT
|
|
117
|
+
(AI.GENERATE('What is BigQuery?')).result AS answer
|
|
118
|
+
"""
|
|
119
|
+
df = client.query(query).to_dataframe()
|
|
120
|
+
print(df.iloc[0]['answer'])
|
|
121
|
+
```
|
|
122
|
+
|
|
123
|
+
### 2. Using column values in prompts
|
|
124
|
+
|
|
125
|
+
Use `CONCAT` to build prompts from column values. `AI.GENERATE` runs once per row.
|
|
126
|
+
|
|
127
|
+
```python
|
|
128
|
+
query = """
|
|
129
|
+
SELECT
|
|
130
|
+
city,
|
|
131
|
+
(AI.GENERATE(
|
|
132
|
+
CONCAT('What country is ', city, ' in? Answer in one word.')
|
|
133
|
+
)).result AS country
|
|
134
|
+
FROM UNNEST(['Tokyo', 'Paris', 'Nairobi', 'Lima', 'Sydney']) AS city
|
|
135
|
+
"""
|
|
136
|
+
client.query(query).to_dataframe()
|
|
137
|
+
```
|
|
138
|
+
|
|
139
|
+
### 3. Accessing the full response
|
|
140
|
+
|
|
141
|
+
`AI.GENERATE` returns a STRUCT with three fields:
|
|
142
|
+
- `.result` — the generated text (STRING)
|
|
143
|
+
- `.full_response` — the complete API response (JSON)
|
|
144
|
+
- `.status` — error message if failed, empty if successful (STRING)
|
|
145
|
+
|
|
146
|
+
```python
|
|
147
|
+
import json
|
|
148
|
+
|
|
149
|
+
query = """
|
|
150
|
+
SELECT AI.GENERATE('Explain cloud computing in one sentence.') AS response
|
|
151
|
+
"""
|
|
152
|
+
df = client.query(query).to_dataframe()
|
|
153
|
+
|
|
154
|
+
print('result:', df.iloc[0]['response']['result'])
|
|
155
|
+
print('status:', repr(df.iloc[0]['response']['status']))
|
|
156
|
+
|
|
157
|
+
# Explore the full_response JSON structure
|
|
158
|
+
full = json.loads(df.iloc[0]['response']['full_response'])
|
|
159
|
+
print('\nfull_response keys:', list(full.keys()))
|
|
160
|
+
if 'candidates' in full:
|
|
161
|
+
print('candidate keys:', list(full['candidates'][0].keys()))
|
|
162
|
+
if 'usageMetadata' in full:
|
|
163
|
+
print('usage:', full['usageMetadata'])
|
|
164
|
+
```
|
|
165
|
+
|
|
166
|
+
### 4. Specifying an endpoint
|
|
167
|
+
|
|
168
|
+
Override the default model (`gemini-2.5-flash`) with any Gemini model.
|
|
169
|
+
|
|
170
|
+
```python
|
|
171
|
+
query = """
|
|
172
|
+
SELECT
|
|
173
|
+
(AI.GENERATE(
|
|
174
|
+
'Write a haiku about data warehouses.',
|
|
175
|
+
endpoint => 'gemini-2.5-pro'
|
|
176
|
+
)).result AS haiku
|
|
177
|
+
"""
|
|
178
|
+
df = client.query(query).to_dataframe()
|
|
179
|
+
print(df.iloc[0]['haiku'])
|
|
180
|
+
```
|
|
181
|
+
|
|
182
|
+
### 5. Structured output with `output_schema`
|
|
183
|
+
|
|
184
|
+
Provide an `output_schema` to get typed columns instead of free-form text. The `result` field is replaced by your custom schema fields.
|
|
185
|
+
|
|
186
|
+
```python
|
|
187
|
+
query = """
|
|
188
|
+
SELECT
|
|
189
|
+
animal,
|
|
190
|
+
result.habitat,
|
|
191
|
+
result.average_lifespan_years,
|
|
192
|
+
result.is_endangered
|
|
193
|
+
FROM
|
|
194
|
+
UNNEST(['Eagle', 'Salmon', 'Cobra']) AS animal,
|
|
195
|
+
UNNEST([
|
|
196
|
+
AI.GENERATE(
|
|
197
|
+
CONCAT('Give me facts about: ', animal),
|
|
198
|
+
output_schema => 'habitat STRING, average_lifespan_years INT64, is_endangered BOOL'
|
|
199
|
+
)
|
|
200
|
+
]) AS result
|
|
201
|
+
"""
|
|
202
|
+
client.query(query).to_dataframe()
|
|
203
|
+
```
|
|
204
|
+
|
|
205
|
+
### 6. Structured output with Pydantic schemas
|
|
206
|
+
|
|
207
|
+
For complex schemas with descriptions and array types, define a Pydantic model and convert it with `bq_schema()`. `Field(description=...)` maps to BigQuery's `OPTIONS(description = '...')`.
|
|
208
|
+
|
|
209
|
+
```python
|
|
210
|
+
class ProductAnalysis(BaseModel):
|
|
211
|
+
category: str = Field(description='Product category like Electronics, Apparel, Kitchen')
|
|
212
|
+
price_range_usd: str = Field(description='Estimated price range like $50-$100')
|
|
213
|
+
key_features: list[str] = Field(description='Top 3 selling points')
|
|
214
|
+
needs_batteries: bool = Field(description='Whether the product typically requires batteries')
|
|
215
|
+
|
|
216
|
+
# See what the converter produces
|
|
217
|
+
print(bq_schema(ProductAnalysis))
|
|
218
|
+
```
|
|
219
|
+
|
|
220
|
+
```python
|
|
221
|
+
output_schema = bq_schema(ProductAnalysis)
|
|
222
|
+
|
|
223
|
+
query = f'''
|
|
224
|
+
SELECT
|
|
225
|
+
product,
|
|
226
|
+
result.category,
|
|
227
|
+
result.price_range_usd,
|
|
228
|
+
result.key_features,
|
|
229
|
+
result.needs_batteries
|
|
230
|
+
FROM
|
|
231
|
+
UNNEST(['laptop', 'running shoes', 'espresso machine']) AS product,
|
|
232
|
+
UNNEST([
|
|
233
|
+
AI.GENERATE(
|
|
234
|
+
CONCAT('Analyze this product for an e-commerce listing: ', product),
|
|
235
|
+
output_schema => """{output_schema}"""
|
|
236
|
+
)
|
|
237
|
+
]) AS result
|
|
238
|
+
'''
|
|
239
|
+
client.query(query).to_dataframe()
|
|
240
|
+
```
|
|
241
|
+
|
|
242
|
+
### 7. Controlling generation with `model_params`
|
|
243
|
+
|
|
244
|
+
`model_params` accepts any field from the [Gemini `generateContent` request body](https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/gemini#request_body) (except `contents`). Common uses: temperature, top_p, max_output_tokens.
|
|
245
|
+
|
|
246
|
+
```python
|
|
247
|
+
query = """
|
|
248
|
+
SELECT
|
|
249
|
+
(AI.GENERATE(
|
|
250
|
+
'Invent a creative name for a new coffee shop. Just the name, nothing else.',
|
|
251
|
+
model_params => JSON '{"generation_config": {"temperature": 1.5, "max_output_tokens": 50}}'
|
|
252
|
+
)).result AS creative_name
|
|
253
|
+
"""
|
|
254
|
+
df = client.query(query).to_dataframe()
|
|
255
|
+
print(df.iloc[0]['creative_name'])
|
|
256
|
+
```
|
|
257
|
+
|
|
258
|
+
### 8. Thinking budget
|
|
259
|
+
|
|
260
|
+
Gemini 2.5 models support "thinking" — internal reasoning before responding. Control the thinking budget (in tokens) to balance quality vs cost.
|
|
261
|
+
|
|
262
|
+
> **Gemini 3.0+:** Use `thinking_level` (`LOW`, `MEDIUM`, `HIGH`) in `thinking_config` instead of `thinking_budget`:
|
|
263
|
+
> ```sql
|
|
264
|
+
> model_params => JSON '{"generation_config": {"thinking_config": {"thinking_level": "LOW"}}}'
|
|
265
|
+
> ```
|
|
266
|
+
|
|
267
|
+
```python
|
|
268
|
+
query = """
|
|
269
|
+
SELECT
|
|
270
|
+
(AI.GENERATE(
|
|
271
|
+
'What is the sum of all prime numbers less than 50?',
|
|
272
|
+
model_params => JSON '{"generation_config": {"thinking_config": {"thinking_budget": 2048}}}'
|
|
273
|
+
)).result AS answer
|
|
274
|
+
"""
|
|
275
|
+
df = client.query(query).to_dataframe()
|
|
276
|
+
print(df.iloc[0]['answer'])
|
|
277
|
+
```
|
|
278
|
+
|
|
279
|
+
### 9. Grounding with Google Search
|
|
280
|
+
|
|
281
|
+
Enable Google Search grounding via the `tools` field in `model_params`. The model searches the web for current information before responding.
|
|
282
|
+
|
|
283
|
+
```python
|
|
284
|
+
query = """
|
|
285
|
+
SELECT
|
|
286
|
+
(AI.GENERATE(
|
|
287
|
+
'What were the top 3 news stories today?',
|
|
288
|
+
model_params => JSON '{"tools": [{"googleSearch": {}}]}'
|
|
289
|
+
)).result AS grounded_answer
|
|
290
|
+
"""
|
|
291
|
+
df = client.query(query).to_dataframe()
|
|
292
|
+
print(df.iloc[0]['grounded_answer'])
|
|
293
|
+
```
|
|
294
|
+
|
|
295
|
+
### 10. Processing table data at scale
|
|
296
|
+
|
|
297
|
+
**Best practice:** When using `LIMIT`, materialize selected rows first to avoid re-evaluating the subquery and incurring extra Vertex AI charges.
|
|
298
|
+
|
|
299
|
+
```python
|
|
300
|
+
class SentimentResult(BaseModel):
|
|
301
|
+
sentiment: str
|
|
302
|
+
confidence: float
|
|
303
|
+
|
|
304
|
+
output_schema = bq_schema(SentimentResult)
|
|
305
|
+
|
|
306
|
+
query = f"""
|
|
307
|
+
WITH selected_reviews AS (
|
|
308
|
+
SELECT *
|
|
309
|
+
FROM UNNEST([
|
|
310
|
+
STRUCT('r1' AS id, 'The product is amazing, fast delivery!' AS review),
|
|
311
|
+
STRUCT('r2', 'Terrible quality. Broke after one day.'),
|
|
312
|
+
STRUCT('r3', 'Average product, nothing special.'),
|
|
313
|
+
STRUCT('r4', 'Great value for the price. Highly recommend!'),
|
|
314
|
+
STRUCT('r5', 'Worst purchase ever. Do not buy.')
|
|
315
|
+
])
|
|
316
|
+
)
|
|
317
|
+
SELECT id, review, response.sentiment, response.confidence
|
|
318
|
+
FROM (
|
|
319
|
+
SELECT
|
|
320
|
+
id,
|
|
321
|
+
review,
|
|
322
|
+
AI.GENERATE(
|
|
323
|
+
CONCAT('Classify this review sentiment as positive, negative, or neutral: ', review),
|
|
324
|
+
output_schema => '{output_schema}'
|
|
325
|
+
) AS response
|
|
326
|
+
FROM selected_reviews
|
|
327
|
+
)
|
|
328
|
+
"""
|
|
329
|
+
client.query(query).to_dataframe()
|
|
330
|
+
```
|
|
331
|
+
|
|
332
|
+
---
|
|
333
|
+
## Examples — Multimodal with ObjectRef
|
|
334
|
+
|
|
335
|
+
`AI.GENERATE` can process documents, images, and video stored in Cloud Storage. `OBJ.MAKE_REF` turns a GCS URI into an **ObjectRef** — a pointer the function resolves for you:
|
|
336
|
+
|
|
337
|
+
```
|
|
338
|
+
OBJ.MAKE_REF(uri, connection) → ObjectRef — pass this straight to the function
|
|
339
|
+
```
|
|
340
|
+
|
|
341
|
+
Pass `ObjectRef` values in a STRUCT prompt with `prompt` and `object_refs` fields. `OBJ.MAKE_REF` is for files you already know by URI; an object table's `ref` column covers a bucket you keep tracking; and `OBJ.LIST` (GA) — example 14 below — covers the middle case, every file under a wildcard path, with no table at all. `OBJ.FETCH_METADATA` (content type and size) and `OBJ.GET_ACCESS_URL` (a signed `ObjectRefRuntime` URL) still have their uses — displaying an object, delegated access, an explicit TTL — but the AI functions take the `ObjectRef` itself. See the `reference/unstructured-data-infrastructure.md#objectref-and-objectrefruntime-schema-reference` (ObjectRef reference) for details.
|
|
342
|
+
|
|
343
|
+
### Multimodal setup — connection and sample documents
|
|
344
|
+
|
|
345
|
+
ObjectRef requires a `setup` (Cloud resource connection) to access GCS. The cells below create a connection (if needed) and upload two sample documents.
|
|
346
|
+
|
|
347
|
+
```python
|
|
348
|
+
import subprocess as _sp
|
|
349
|
+
|
|
350
|
+
# Create the connection (idempotent — succeeds even if it already exists)
|
|
351
|
+
_sp.run(
|
|
352
|
+
['bq', 'mk', '--connection', '--location', LOCATION,
|
|
353
|
+
'--connection_type', 'CLOUD_RESOURCE',
|
|
354
|
+
'--project_id', PROJECT_ID, CONNECTION_ID],
|
|
355
|
+
capture_output=True, text=True
|
|
356
|
+
)
|
|
357
|
+
|
|
358
|
+
# Get the connection's service account
|
|
359
|
+
import json as _json
|
|
360
|
+
_conn = _sp.run(
|
|
361
|
+
['bq', 'show', '--connection', '--format=json',
|
|
362
|
+
f'{PROJECT_ID}.{LOCATION}.{CONNECTION_ID}'],
|
|
363
|
+
capture_output=True, text=True
|
|
364
|
+
)
|
|
365
|
+
sa = _json.loads(_conn.stdout)['cloudResource']['serviceAccountId']
|
|
366
|
+
print(f'Connection SA: {sa}')
|
|
367
|
+
|
|
368
|
+
# Grant GCS read access (needed for ObjectRef to read documents)
|
|
369
|
+
_sp.run(
|
|
370
|
+
['gcloud', 'projects', 'add-iam-policy-binding', PROJECT_ID,
|
|
371
|
+
f'--member=serviceAccount:{sa}', '--role=roles/storage.objectViewer', '--quiet'],
|
|
372
|
+
capture_output=True, text=True
|
|
373
|
+
)
|
|
374
|
+
print('Granted roles/storage.objectViewer')
|
|
375
|
+
```
|
|
376
|
+
|
|
377
|
+
```python
|
|
378
|
+
from google.cloud import storage as _storage
|
|
379
|
+
from pathlib import Path
|
|
380
|
+
|
|
381
|
+
_gcs = _storage.Client(project=PROJECT_ID)
|
|
382
|
+
_bucket = _gcs.bucket(BUCKET)
|
|
383
|
+
_prefix = 'bq_ai_functions/ai_generate'
|
|
384
|
+
|
|
385
|
+
# Find the data directory (works from repo checkout or notebook directory)
|
|
386
|
+
_data = Path('../../data/documents')
|
|
387
|
+
if not _data.exists():
|
|
388
|
+
_data = Path('data/documents')
|
|
389
|
+
|
|
390
|
+
# Upload one invoice and one receipt
|
|
391
|
+
for subdir, filename in [('invoices', 'invoice_001.pdf'), ('receipts', 'receipt_001.pdf')]:
|
|
392
|
+
blob = _bucket.blob(f'{_prefix}/{filename}')
|
|
393
|
+
if not blob.exists():
|
|
394
|
+
blob.upload_from_filename(str(_data / subdir / filename))
|
|
395
|
+
print(f'Uploaded {filename} → gs://{BUCKET}/{_prefix}/{filename}')
|
|
396
|
+
else:
|
|
397
|
+
print(f'Already exists: gs://{BUCKET}/{_prefix}/{filename}')
|
|
398
|
+
```
|
|
399
|
+
|
|
400
|
+
### 11. Describe a document with ObjectRef
|
|
401
|
+
|
|
402
|
+
`OBJ.MAKE_REF` turns a GCS URI into an `ObjectRef` that `AI.GENERATE` can read. Pass it in a STRUCT with `prompt` (text) and `object_refs` (array of references) fields.
|
|
403
|
+
|
|
404
|
+
```python
|
|
405
|
+
query = f"""
|
|
406
|
+
SELECT
|
|
407
|
+
(AI.GENERATE(
|
|
408
|
+
STRUCT(
|
|
409
|
+
'Describe what this document is and summarize its key details.' AS prompt,
|
|
410
|
+
[OBJ.MAKE_REF(
|
|
411
|
+
'gs://{BUCKET}/bq_ai_functions/ai_generate/invoice_001.pdf',
|
|
412
|
+
'{PROJECT_ID}.{LOCATION}.{CONNECTION_ID}'
|
|
413
|
+
)] AS object_refs
|
|
414
|
+
)
|
|
415
|
+
)).result AS description
|
|
416
|
+
"""
|
|
417
|
+
df = client.query(query).to_dataframe()
|
|
418
|
+
print(df.iloc[0]['description'])
|
|
419
|
+
```
|
|
420
|
+
|
|
421
|
+
### 12. Extract structured data from a document
|
|
422
|
+
|
|
423
|
+
Combine ObjectRef with `output_schema` to extract typed fields from documents — no Document AI processor needed.
|
|
424
|
+
|
|
425
|
+
```python
|
|
426
|
+
query = f"""
|
|
427
|
+
SELECT
|
|
428
|
+
result.vendor_name,
|
|
429
|
+
result.invoice_number,
|
|
430
|
+
result.total_amount,
|
|
431
|
+
result.currency,
|
|
432
|
+
result.invoice_date,
|
|
433
|
+
result.line_item_count
|
|
434
|
+
FROM UNNEST([
|
|
435
|
+
AI.GENERATE(
|
|
436
|
+
STRUCT(
|
|
437
|
+
'Extract the key fields from this invoice.' AS prompt,
|
|
438
|
+
[OBJ.MAKE_REF(
|
|
439
|
+
'gs://{BUCKET}/bq_ai_functions/ai_generate/invoice_001.pdf',
|
|
440
|
+
'{PROJECT_ID}.{LOCATION}.{CONNECTION_ID}'
|
|
441
|
+
)] AS object_refs
|
|
442
|
+
),
|
|
443
|
+
output_schema => 'vendor_name STRING, invoice_number STRING, total_amount FLOAT64, currency STRING, invoice_date STRING, line_item_count INT64'
|
|
444
|
+
)
|
|
445
|
+
]) AS result
|
|
446
|
+
"""
|
|
447
|
+
client.query(query).to_dataframe()
|
|
448
|
+
```
|
|
449
|
+
|
|
450
|
+
### 13. Process multiple documents
|
|
451
|
+
|
|
452
|
+
Process different document types in a single query by building ObjectRef from a list of URIs.
|
|
453
|
+
|
|
454
|
+
```python
|
|
455
|
+
query = f"""
|
|
456
|
+
SELECT
|
|
457
|
+
uri,
|
|
458
|
+
result.document_type,
|
|
459
|
+
result.total_amount,
|
|
460
|
+
result.date,
|
|
461
|
+
result.summary
|
|
462
|
+
FROM
|
|
463
|
+
UNNEST([
|
|
464
|
+
'gs://{BUCKET}/bq_ai_functions/ai_generate/invoice_001.pdf',
|
|
465
|
+
'gs://{BUCKET}/bq_ai_functions/ai_generate/receipt_001.pdf'
|
|
466
|
+
]) AS uri,
|
|
467
|
+
UNNEST([
|
|
468
|
+
AI.GENERATE(
|
|
469
|
+
STRUCT(
|
|
470
|
+
'Identify the document type and extract key details.' AS prompt,
|
|
471
|
+
[OBJ.MAKE_REF(uri, '{PROJECT_ID}.{LOCATION}.{CONNECTION_ID}')] AS object_refs
|
|
472
|
+
),
|
|
473
|
+
output_schema => 'document_type STRING, total_amount FLOAT64, date STRING, summary STRING'
|
|
474
|
+
)
|
|
475
|
+
]) AS result
|
|
476
|
+
"""
|
|
477
|
+
client.query(query).to_dataframe()
|
|
478
|
+
```
|
|
479
|
+
|
|
480
|
+
### 14. Discover documents with `OBJ.LIST`
|
|
481
|
+
|
|
482
|
+
Example 13 hand-built the URI list. `OBJ.LIST` builds it for you: one wildcard path (at most one `*` per segment, no trailing `/`) returns every matching object's metadata — `uri`, `content_type`, `size`, `md5_hash`, `updated`, `generation` — plus a `ref` column that already *is* an `ObjectRef`, so the AI call takes it directly. No object table, no DDL, nothing to clean up.
|
|
483
|
+
|
|
484
|
+
The second argument is the connection. Omit it and the returned references use **direct access** — the objects are read as whoever runs the query, which works on a public bucket and on any bucket where you hold `storage.objectViewer`; pass it, as here, for delegated access through the connection's service account. Use an object table instead when you need a persistent listing that keeps tracking new arrivals.
|
|
485
|
+
|
|
486
|
+
```python
|
|
487
|
+
query = f"""
|
|
488
|
+
SELECT
|
|
489
|
+
uri,
|
|
490
|
+
content_type,
|
|
491
|
+
size,
|
|
492
|
+
result.document_type,
|
|
493
|
+
result.total_amount,
|
|
494
|
+
result.summary
|
|
495
|
+
FROM
|
|
496
|
+
OBJ.LIST('gs://{BUCKET}/bq_ai_functions/ai_generate/*.pdf', '{PROJECT_ID}.{LOCATION}.{CONNECTION_ID}'),
|
|
497
|
+
UNNEST([
|
|
498
|
+
AI.GENERATE(
|
|
499
|
+
STRUCT(
|
|
500
|
+
'Identify the document type and extract key details.' AS prompt,
|
|
501
|
+
[ref] AS object_refs
|
|
502
|
+
),
|
|
503
|
+
output_schema => 'document_type STRING, total_amount FLOAT64, summary STRING'
|
|
504
|
+
)
|
|
505
|
+
]) AS result
|
|
506
|
+
ORDER BY uri
|
|
507
|
+
"""
|
|
508
|
+
client.query(query).to_dataframe()
|
|
509
|
+
```
|
|
510
|
+
|
|
511
|
+
---
|
|
512
|
+
## Examples — `%%bigquery` Magics
|
|
513
|
+
|
|
514
|
+
The same examples using IPython magic commands. Magics let you write SQL directly in notebook cells without Python string wrapping.
|
|
515
|
+
|
|
516
|
+
Key patterns:
|
|
517
|
+
- `%%bigquery` — run SQL, display results inline
|
|
518
|
+
- `%%bigquery df` — run SQL, capture results into a pandas DataFrame
|
|
519
|
+
|
|
520
|
+
### Basic call with `%%bigquery`
|
|
521
|
+
|
|
522
|
+
```sql
|
|
523
|
+
%%bigquery --project {PROJECT_ID}
|
|
524
|
+
|
|
525
|
+
SELECT
|
|
526
|
+
city,
|
|
527
|
+
(AI.GENERATE(
|
|
528
|
+
CONCAT('What country is ', city, ' in? Answer in one word.')
|
|
529
|
+
)).result AS country
|
|
530
|
+
FROM UNNEST(['Tokyo', 'Paris', 'Nairobi', 'Lima', 'Sydney']) AS city
|
|
531
|
+
```
|
|
532
|
+
|
|
533
|
+
### Capture results to a DataFrame with `%%bigquery df`
|
|
534
|
+
|
|
535
|
+
```sql
|
|
536
|
+
%%bigquery df_animals --project {PROJECT_ID}
|
|
537
|
+
|
|
538
|
+
SELECT
|
|
539
|
+
animal,
|
|
540
|
+
result.habitat,
|
|
541
|
+
result.average_lifespan_years,
|
|
542
|
+
result.is_endangered
|
|
543
|
+
FROM
|
|
544
|
+
UNNEST(['Eagle', 'Salmon', 'Cobra']) AS animal,
|
|
545
|
+
UNNEST([
|
|
546
|
+
AI.GENERATE(
|
|
547
|
+
CONCAT('Give me facts about: ', animal),
|
|
548
|
+
output_schema => 'habitat STRING, average_lifespan_years INT64, is_endangered BOOL'
|
|
549
|
+
)
|
|
550
|
+
]) AS result
|
|
551
|
+
```
|
|
552
|
+
|
|
553
|
+
```python
|
|
554
|
+
# Now df_animals is a regular pandas DataFrame
|
|
555
|
+
print(f'Shape: {df_animals.shape}')
|
|
556
|
+
print(f'Columns: {list(df_animals.columns)}')
|
|
557
|
+
df_animals
|
|
558
|
+
```
|
|
559
|
+
|
|
560
|
+
### Structured output with field descriptions
|
|
561
|
+
|
|
562
|
+
```sql
|
|
563
|
+
%%bigquery df_products --project {PROJECT_ID}
|
|
564
|
+
|
|
565
|
+
SELECT
|
|
566
|
+
product,
|
|
567
|
+
result.category,
|
|
568
|
+
result.price_range_usd,
|
|
569
|
+
result.key_features,
|
|
570
|
+
result.needs_batteries
|
|
571
|
+
FROM
|
|
572
|
+
UNNEST(['laptop', 'running shoes', 'espresso machine']) AS product,
|
|
573
|
+
UNNEST([
|
|
574
|
+
AI.GENERATE(
|
|
575
|
+
CONCAT('Analyze this product for an e-commerce listing: ', product),
|
|
576
|
+
output_schema => """
|
|
577
|
+
category STRING OPTIONS(description = 'Product category like Electronics, Apparel, Kitchen'),
|
|
578
|
+
price_range_usd STRING OPTIONS(description = 'Estimated price range like $50-$100'),
|
|
579
|
+
key_features ARRAY<STRING> OPTIONS(description = 'Top 3 selling points'),
|
|
580
|
+
needs_batteries BOOL OPTIONS(description = 'Whether the product typically requires batteries')
|
|
581
|
+
"""
|
|
582
|
+
)
|
|
583
|
+
]) AS result
|
|
584
|
+
```
|
|
585
|
+
|
|
586
|
+
```python
|
|
587
|
+
df_products
|
|
588
|
+
```
|
|
589
|
+
|
|
590
|
+
### Grounding with Google Search
|
|
591
|
+
|
|
592
|
+
```sql
|
|
593
|
+
%%bigquery --project {PROJECT_ID}
|
|
594
|
+
|
|
595
|
+
SELECT
|
|
596
|
+
(AI.GENERATE(
|
|
597
|
+
'What were the top 3 news stories today?',
|
|
598
|
+
model_params => JSON '{"tools": [{"googleSearch": {}}]}'
|
|
599
|
+
)).result AS grounded_answer
|
|
600
|
+
```
|
|
601
|
+
|
|
602
|
+
---
|
|
603
|
+
## Examples — BigFrames
|
|
604
|
+
|
|
605
|
+
BigFrames provides a pandas-like interface to BigQuery. The `bigframes.bigquery.ai.generate()` function wraps `AI.GENERATE` for use in Python.
|
|
606
|
+
|
|
607
|
+
**Key difference from SQL:** Prompts use a **tuple pattern** — a tuple of string literals and Series that get concatenated:
|
|
608
|
+
```python
|
|
609
|
+
bbq.ai.generate(("Summarize: ", df["text"]))
|
|
610
|
+
```
|
|
611
|
+
|
|
612
|
+
```python
|
|
613
|
+
import bigframes.pandas as bpd
|
|
614
|
+
import bigframes.bigquery as bbq
|
|
615
|
+
|
|
616
|
+
bpd.options.bigquery.project = PROJECT_ID
|
|
617
|
+
bpd.options.bigquery.location = LOCATION
|
|
618
|
+
```
|
|
619
|
+
|
|
620
|
+
### Basic generation
|
|
621
|
+
|
|
622
|
+
```python
|
|
623
|
+
# Create a BigFrames DataFrame
|
|
624
|
+
df = bpd.DataFrame({'city': ['Tokyo', 'Paris', 'Nairobi', 'Lima', 'Sydney']})
|
|
625
|
+
|
|
626
|
+
# AI.GENERATE with tuple prompt: string literal + Series
|
|
627
|
+
df['response'] = bbq.ai.generate(
|
|
628
|
+
('What country is ', df['city'], ' in? Answer in one word.')
|
|
629
|
+
)
|
|
630
|
+
|
|
631
|
+
# The result is a struct — access .result with struct accessor
|
|
632
|
+
df['country'] = df['response'].struct.field('result')
|
|
633
|
+
df[['city', 'country']].to_pandas()
|
|
634
|
+
```
|
|
635
|
+
|
|
636
|
+
### Structured output with Pydantic + `bf_schema()`
|
|
637
|
+
|
|
638
|
+
BigFrames takes `output_schema` as a `dict[str, str]`. Use `bf_schema()` to convert the same Pydantic model.
|
|
639
|
+
|
|
640
|
+
```python
|
|
641
|
+
class AnimalFacts(BaseModel):
|
|
642
|
+
habitat: str
|
|
643
|
+
average_lifespan_years: int
|
|
644
|
+
is_endangered: bool
|
|
645
|
+
|
|
646
|
+
df_animals = bpd.DataFrame({'animal': ['Eagle', 'Salmon', 'Cobra']})
|
|
647
|
+
|
|
648
|
+
df_animals['facts'] = bbq.ai.generate(
|
|
649
|
+
('Give me facts about: ', df_animals['animal']),
|
|
650
|
+
output_schema=bf_schema(AnimalFacts)
|
|
651
|
+
)
|
|
652
|
+
|
|
653
|
+
# Extract structured fields
|
|
654
|
+
df_animals['habitat'] = df_animals['facts'].struct.field('habitat')
|
|
655
|
+
df_animals['lifespan'] = df_animals['facts'].struct.field('average_lifespan_years')
|
|
656
|
+
df_animals['endangered'] = df_animals['facts'].struct.field('is_endangered')
|
|
657
|
+
|
|
658
|
+
df_animals[['animal', 'habitat', 'lifespan', 'endangered']].to_pandas()
|
|
659
|
+
```
|
|
660
|
+
|
|
661
|
+
### Complex schema with descriptions
|
|
662
|
+
|
|
663
|
+
```python
|
|
664
|
+
# Reuse the ProductAnalysis model defined earlier
|
|
665
|
+
# bf_schema() extracts the types (descriptions are SQL-only via bq_schema())
|
|
666
|
+
print(bf_schema(ProductAnalysis))
|
|
667
|
+
|
|
668
|
+
df_products = bpd.DataFrame({'product': ['laptop', 'running shoes', 'espresso machine']})
|
|
669
|
+
|
|
670
|
+
df_products['analysis'] = bbq.ai.generate(
|
|
671
|
+
('Analyze this product for an e-commerce listing: ', df_products['product']),
|
|
672
|
+
output_schema=bf_schema(ProductAnalysis)
|
|
673
|
+
)
|
|
674
|
+
|
|
675
|
+
df_products['category'] = df_products['analysis'].struct.field('category')
|
|
676
|
+
df_products['price'] = df_products['analysis'].struct.field('price_range_usd')
|
|
677
|
+
df_products['features'] = df_products['analysis'].struct.field('key_features')
|
|
678
|
+
|
|
679
|
+
df_products[['product', 'category', 'price', 'features']].to_pandas()
|
|
680
|
+
```
|
|
681
|
+
|
|
682
|
+
### Specifying an endpoint
|
|
683
|
+
|
|
684
|
+
```python
|
|
685
|
+
df = bpd.DataFrame({'prompt': ['Write a haiku about data warehouses.']})
|
|
686
|
+
|
|
687
|
+
result = bbq.ai.generate(
|
|
688
|
+
df['prompt'],
|
|
689
|
+
endpoint='gemini-2.5-pro'
|
|
690
|
+
)
|
|
691
|
+
|
|
692
|
+
result.struct.field('result').to_pandas()
|
|
693
|
+
```
|