bigquery-data-science 0.7.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- bigquery_data_science/__init__.py +58 -0
- bigquery_data_science/_skills/PLANS.md +76 -0
- bigquery_data_science/_skills/bigquery-ai/SKILL.md +66 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ai_agg.md +528 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ai_classify.md +450 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ai_count_tokens.md +180 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ai_detect_anomalies.md +322 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ai_embed.md +425 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ai_evaluate.md +368 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ai_forecast.md +437 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ai_generate.md +693 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ai_generate_bool.md +248 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ai_generate_double.md +254 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ai_generate_embedding.md +402 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ai_generate_int.md +252 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ai_generate_table.md +447 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ai_generate_text.md +356 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ai_if.md +373 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ai_key_drivers.md +331 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ai_parse_document.md +416 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ai_predict.md +559 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ai_score.md +310 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ai_search.md +465 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ai_similarity.md +371 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/catalog_search.md +951 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/content_analysis.md +284 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/content_moderation.md +369 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/data_enrichment.md +426 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/document_intelligence.md +470 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/document_rag.md +453 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/image_deduplication.md +770 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/log_analysis.md +668 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/metric_diagnostics.md +568 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ml_annotate_image.md +680 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ml_generate_embedding.md +217 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ml_generate_text.md +223 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ml_process_document.md +392 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ml_transcribe.md +815 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ml_translate.md +600 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ml_understand_text.md +673 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/multimodal_analysis.md +266 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/rag_pipeline.md +593 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/semantic_search.md +510 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/tabular_prediction.md +485 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/time_series_intelligence.md +489 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/vector_search.md +672 -0
- bigquery_data_science/_skills/bigquery-ai/reference/augmented-analytics.md +77 -0
- bigquery_data_science/_skills/bigquery-ai/reference/cloud-ai-service-models.md +88 -0
- bigquery_data_science/_skills/bigquery-ai/reference/document-processing.md +95 -0
- bigquery_data_science/_skills/bigquery-ai/reference/embeddings-and-search.md +166 -0
- bigquery_data_science/_skills/bigquery-ai/reference/general-purpose-generation.md +87 -0
- bigquery_data_science/_skills/bigquery-ai/reference/managed-functions.md +72 -0
- bigquery_data_science/_skills/bigquery-ai/reference/predictive-ai.md +113 -0
- bigquery_data_science/_skills/bigquery-ai/reference/workflows.md +56 -0
- bigquery_data_science/_skills/bigquery-ai/skill.manifest.json +109 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/INDEX.md +71 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/aiagg.md +67 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/aicausal_effect-covered-in-bigquery-ml.md +17 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/aiclassify.md +72 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/aicount_tokens.md +53 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/aidetect_anomalies.md +63 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/aiembed.md +85 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/aievaluate.md +114 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/aiforecast.md +123 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/aigenerate.md +69 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/aigenerate_bool.md +53 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/aigenerate_double.md +45 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/aigenerate_embedding.md +108 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/aigenerate_int.md +45 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/aigenerate_table.md +64 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/aigenerate_text.md +167 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/aiif.md +56 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/aikey_drivers.md +76 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/aiparse_document.md +91 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/aipredict.md +74 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/aiscore.md +52 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/aisearch.md +66 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/aisimilarity.md +79 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/choosing-between-these-and-the-ai-functions.md +11 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/hybrid-search-capability.md +165 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/mlannotate_image.md +24 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/mlgenerate_embedding.md +34 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/mlgenerate_text.md +65 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/mlprocess_document.md +125 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/mltranscribe.md +26 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/mltranslate.md +27 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/mlunderstand_text.md +32 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/multimodal-input-patterns-by-function.md +128 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/object-tables.md +109 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/objectref-and-objectrefruntime-schema-reference.md +50 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/objfetch_metadata.md +49 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/objget_access_url.md +66 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/objget_read_url.md +40 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/objlist.md +34 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/objmake_ref.md +58 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/per-row-failures-and-the-retry-pattern.md +19 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/setup-apis-connection-roles.md +39 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/two-input-shapes.md +22 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/vector_search.md +78 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/what-the-five-have-in-common.md +27 -0
- bigquery_data_science/_skills/bigquery-data-science/SKILL.md +69 -0
- bigquery_data_science/_skills/bigquery-data-science/reference/namespace-matrix.md +112 -0
- bigquery_data_science/_skills/bigquery-data-science/skill.manifest.json +13 -0
- bigquery_data_science/_skills/bigquery-ml/SKILL.md +70 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/airflow_with_kfp.md +379 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/anomaly_fraud_detection.md +195 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/arima_plus.md +635 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/arima_plus_xreg.md +453 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/autoencoder.md +470 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/automl_classifier.md +291 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/automl_regressor.md +274 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/boosted_tree_classifier.md +449 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/boosted_tree_regressor.md +400 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/bucketizing.md +201 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/causal_effect.md +773 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/churn_retention.md +271 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/cloud_workflows.md +370 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/composer_airflow.md +475 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/contribution_analysis.md +346 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/cross_validation.md +185 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/customer_segmentation.md +233 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/dataform.md +372 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/dbt.md +283 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/difference_in_differences.md +280 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/distance.md +195 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/dnn_classifier.md +326 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/dnn_regressor.md +308 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/embeddings_classification.md +426 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/encoding.md +238 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/ensembling.md +278 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/evaluation.md +445 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/exploration.md +779 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/export.md +321 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/feature_engineering.md +280 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/feature_store.md +652 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/feature_store_workflow.md +426 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/ga4_churn_prediction.md +285 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/hierarchical_forecasting.md +441 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/image.md +791 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/imported.md +426 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/kmeans.md +348 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/linear_regression.md +372 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/logistic_regression.md +390 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/matrix_factorization.md +404 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/model_monitoring.md +609 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/pca.md +357 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/price_elasticity_dml.md +286 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/propensity_score_matching.md +350 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/random_forest_classifier.md +445 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/random_forest_regressor.md +409 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/recommendation.md +281 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/regression_based_forecasting.md +491 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/remote.md +428 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/scalers.md +325 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/scheduled_queries.md +284 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/sql_scripting.md +243 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/survival_analysis.md +363 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/synthetic_control.md +221 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/text.md +215 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/time_series.md +669 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/transform_only.md +273 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/uplift_cate.md +241 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/vertex_kfp.md +360 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/wide_and_deep_classifier.md +316 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/wide_and_deep_regressor.md +298 -0
- bigquery_data_science/_skills/bigquery-ml/reference/classification.md +80 -0
- bigquery_data_science/_skills/bigquery-ml/reference/model-management.md +90 -0
- bigquery_data_science/_skills/bigquery-ml/reference/preprocessing-functions.md +234 -0
- bigquery_data_science/_skills/bigquery-ml/reference/regression.md +79 -0
- bigquery_data_science/_skills/bigquery-ml/reference/unsupervised-and-specialized.md +101 -0
- bigquery_data_science/_skills/bigquery-ml/reference/workflows-and-pipelines.md +111 -0
- bigquery_data_science/_skills/bigquery-ml/skill.manifest.json +177 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/1-model_type-catalog.md +28 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/2-task-mlevaluate-metric-columns.md +26 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/3-capability-matrix-per-model-type.md +21 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/4-explainability-weights-functions.md +14 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/5-connection-matrix.md +11 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/6-which-model-free-function-no-create-model-no-connection.md +15 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/7-point-in-time-feature-retrieval-which-of-the-two.md +11 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/INDEX.md +111 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/aicausal_effect.md +85 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/arima_plus-forecasting-lifecycle-functions.md +14 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/arima_plus-univariate-time-series-forecasting.md +132 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/arima_plus_xreg-multivariate-time-series-with-external-regressors.md +103 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/autoencoder.md +102 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/automl_classifier-automl_regressor.md +88 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/boosted_tree_classifier-boosted_tree_regressor.md +117 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/contribution_analysis.md +113 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/dnn_classifier-dnn_regressor.md +107 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/dnn_linear_combined_classifier-dnn_linear_combined_regressor-wide-and-deep.md +98 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/explainability-functions-mlexplain_predict-mlglobal_explain-mlfeature_importance.md +7 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/exploratory-data-analysis-mldescribe_data-mlcorrelation.md +11 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/export-model.md +92 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/feature-engineering-standalone-preprocessing-functions.md +13 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/image-preprocessing-functions-mldecode_image-mlresize_image-mlconvert_image_type-mlconvert_color_space.md +63 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/imported-models-tensorflow-tensorflow_lite-onnx-xgboost.md +36 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/kmeans.md +87 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/linear_reg-logistic_reg-generalized-linear-models.md +101 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/matrix_factorization.md +162 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mladvanced_weights.md +76 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlarima_coefficients.md +30 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlarima_evaluate.md +49 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlbag_of_words.md +47 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlbucketize.md +47 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlcentroids.md +46 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlconfusion_matrix.md +56 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlconvert_color_space.md +53 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlconvert_image_type.md +39 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlcorrelation.md +78 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mldecode_image.md +44 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mldescribe_data.md +74 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mldetect_anomalies.md +86 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mldistance.md +48 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlentity_features_at_time.md +56 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlevaluate.md +72 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlexplain_forecast.md +50 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlexplain_predict.md +66 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlfeature_cross.md +46 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlfeature_importance.md +47 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlfeature_info.md +50 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlfeatures_at_time.md +51 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlforecast.md +76 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlgenerate_embedding-from-a-bqml-pca-autoencoder-matrix_factorization-model.md +64 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlglobal_explain.md +63 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlhash_bucketize.md +47 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlholiday_info.md +46 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlimputer.md +46 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mllp_norm.md +47 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlmetrics.md +80 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlngrams.md +37 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlone_hot_encoder-mlmulti_hot_encoder-mllabel_encoder.md +67 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlpolynomial_expand.md +47 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlpredict.md +67 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlprincipal_component_info.md +42 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlprincipal_components.md +40 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlquantile_bucketize.md +43 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlrecommend.md +60 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlreconstruction_loss.md +48 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlresize_image.md +45 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlroc_curve.md +63 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mltf_idf.md +37 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mltfdv_describe.md +34 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mltfdv_validate.md +48 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mltraining_info.md +78 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mltransform-applying-a-saved-transform.md +52 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mltranspose-not-a-function-pointer.md +32 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mltrial_info.md +60 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlvalidate_data_drift.md +99 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlvalidate_data_skew.md +50 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlweights.md +84 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/model-free-time-series-decomposition-mltrend-mlseasonality-mldetect_change_points.md +100 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/model-introspection-mlfeature_info-mltraining_info-mltrial_info.md +7 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/model-monitoring-data-validation.md +23 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/numerical-scalers-mlstandard_scaler-mlmin_max_scaler-mlmax_abs_scaler-mlrobust_scaler-mlnormalizer.md +77 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/onnx.md +55 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/pca.md +91 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/point-in-time-feature-retrieval-mlfeatures_at_time-mlentity_features_at_time.md +25 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/quick-comparison.md +14 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/random_forest_classifier-random_forest_regressor.md +123 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/remote-remote-model-over-a-vertex-ai-endpoint-the-mechanism.md +87 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/tensorflow.md +48 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/tensorflow_lite.md +29 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/text-preprocessing-functions-mlngrams-mltf_idf-mlbag_of_words.md +18 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/timesfm-built-in-foundation-forecaster-see-bigquery-ai.md +28 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/transform_only.md +91 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/xgboost.md +72 -0
- bigquery_data_science/_skills/pinned-models.json +28 -0
- bigquery_data_science/catalog.py +1107 -0
- bigquery_data_science/cli.py +786 -0
- bigquery_data_science/installer.py +282 -0
- bigquery_data_science/mcp_server.py +317 -0
- bigquery_data_science/tooling/__init__.py +15 -0
- bigquery_data_science/tooling/convert_notebook.py +176 -0
- bigquery_data_science/tooling/link_policy.py +247 -0
- bigquery_data_science/tooling/manifest.py +152 -0
- bigquery_data_science/tooling/markdown_anchors.py +84 -0
- bigquery_data_science/tooling/narrative_drift.py +163 -0
- bigquery_data_science/tooling/pinned_models.py +216 -0
- bigquery_data_science/tooling/reference_structure.py +85 -0
- bigquery_data_science/tooling/sync_syntax.py +407 -0
- bigquery_data_science/tooling/validate.py +208 -0
- bigquery_data_science-0.7.0.dist-info/METADATA +306 -0
- bigquery_data_science-0.7.0.dist-info/RECORD +286 -0
- bigquery_data_science-0.7.0.dist-info/WHEEL +4 -0
- bigquery_data_science-0.7.0.dist-info/entry_points.txt +5 -0
- bigquery_data_science-0.7.0.dist-info/licenses/LICENSE +201 -0
|
@@ -0,0 +1,425 @@
|
|
|
1
|
+
> **Source (`statmike/bigquery-data-science`):** **Notebook:** [`bigquery-ai/functions/ai_embed/ai_embed.ipynb`](https://github.com/statmike/bigquery-data-science/blob/main/bigquery-ai/functions/ai_embed/ai_embed.ipynb) · **SQL:** [`bigquery-ai/functions/ai_embed/ai_embed.sql`](https://github.com/statmike/bigquery-data-science/blob/main/bigquery-ai/functions/ai_embed/ai_embed.sql)
|
|
2
|
+
|
|
3
|
+
# AI.EMBED — BigQuery AI Functions
|
|
4
|
+
|
|
5
|
+
`AI.EMBED` is a scalar function that creates text or image embeddings. Specify an endpoint directly, or use the built-in `embeddinggemma-300m` model with no Vertex AI charges — no model creation required either way.
|
|
6
|
+
|
|
7
|
+
**When to use it:**
|
|
8
|
+
- You need embeddings for individual values in SELECT, WHERE, or JOIN
|
|
9
|
+
- You want to create embeddings without CREATE MODEL
|
|
10
|
+
- You need to embed a small number of values inline
|
|
11
|
+
- You want zero-cost text embedding using the built-in model (`embeddinggemma-300m`)
|
|
12
|
+
|
|
13
|
+
**Alternatives:**
|
|
14
|
+
- `functions/ai_generate_embedding` (`AI.GENERATE_EMBEDDING`) — TVF for batch embedding, requires CREATE MODEL
|
|
15
|
+
- `functions/ai_similarity` (`AI.SIMILARITY`) — Cosine similarity between two inputs (uses embeddings internally)
|
|
16
|
+
- `functions/vector_search` (`VECTOR_SEARCH`) — Top-K nearest neighbor search on pre-computed embeddings
|
|
17
|
+
|
|
18
|
+
**Featured in:** `workflows/semantic_search` (Semantic Search System) | `workflows/rag_pipeline` (RAG Pipeline) | `workflows/multimodal_analysis` (Multimodal Analysis) | `workflows/document_rag` (Document RAG Pipeline) | `workflows/log_analysis` (Log Analysis) | `workflows/catalog_search` (Catalog Search) | `workflows/image_deduplication` (Image Deduplication)
|
|
19
|
+
|
|
20
|
+
**Multimodal:** Supports image input via `reference/unstructured-data-infrastructure.md#objectref-and-objectrefruntime-schema-reference` (ObjectRef) with `multimodalembedding@001`. For PDFs, use `gemini-embedding-2` (GA), which embeds PDFs directly without rendering to images.
|
|
21
|
+
|
|
22
|
+
**References:** `RESOURCES.md` (Full syntax reference) | [Official documentation](https://cloud.google.com/bigquery/docs/reference/standard-sql/bigqueryml-syntax-ai-embed) | `setup` (Setup guide)
|
|
23
|
+
|
|
24
|
+
---
|
|
25
|
+
## Setup
|
|
26
|
+
|
|
27
|
+
Set your project and location, authenticate, and create a temporary dataset for this notebook.
|
|
28
|
+
|
|
29
|
+
> Text embedding uses end-user credentials automatically — no connection needed. **Multimodal** embedding (images) requires a connection. See the `setup` (Setup Reference) for details.
|
|
30
|
+
|
|
31
|
+
```python
|
|
32
|
+
PROJECT_ID = 'statmike-mlops-349915' # <-- Replace with your project ID
|
|
33
|
+
LOCATION = 'US' # BigQuery dataset location
|
|
34
|
+
DATASET_ID = 'bq_ai_functions' # Shared dataset across all notebooks
|
|
35
|
+
CONNECTION_ID = 'bq_ai_functions' # Shared connection
|
|
36
|
+
BUCKET = PROJECT_ID # GCS bucket (same name as project)
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
### Environment
|
|
40
|
+
|
|
41
|
+
> **Already set up the project environment?** The cell below is a no-op — packages are already in your kernel. See the `setup` (Setup Reference) for details.
|
|
42
|
+
>
|
|
43
|
+
> **Running standalone** (Colab, Colab Enterprise, Vertex AI Workbench)? The cell below installs required packages into your current kernel.
|
|
44
|
+
|
|
45
|
+
```python
|
|
46
|
+
from google.cloud import bigquery
|
|
47
|
+
import pandas as pd
|
|
48
|
+
|
|
49
|
+
client = bigquery.Client(project=PROJECT_ID)
|
|
50
|
+
pd.set_option('display.max_colwidth', None)
|
|
51
|
+
|
|
52
|
+
# Create the shared dataset (idempotent)
|
|
53
|
+
dataset_ref = bigquery.DatasetReference(PROJECT_ID, DATASET_ID)
|
|
54
|
+
dataset = bigquery.Dataset(dataset_ref)
|
|
55
|
+
dataset.location = LOCATION
|
|
56
|
+
client.create_dataset(dataset, exists_ok=True)
|
|
57
|
+
print(f'Dataset {PROJECT_ID}.{DATASET_ID} ready')
|
|
58
|
+
|
|
59
|
+
# Register %%bigquery cell magic (auto-loaded in Colab, needed elsewhere)
|
|
60
|
+
%load_ext bigquery_magics
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
### Connection Setup
|
|
64
|
+
|
|
65
|
+
Multimodal embedding (images) requires a connection with `aiplatform.user` and `storage.objectViewer` roles. The connection is shared across notebooks.
|
|
66
|
+
|
|
67
|
+
```python
|
|
68
|
+
import subprocess as _sp, json as _json
|
|
69
|
+
|
|
70
|
+
# Create connection (idempotent)
|
|
71
|
+
_sp.run(['bq', 'mk', '--connection', '--location', LOCATION,
|
|
72
|
+
'--connection_type', 'CLOUD_RESOURCE',
|
|
73
|
+
'--project_id', PROJECT_ID, CONNECTION_ID],
|
|
74
|
+
capture_output=True, text=True)
|
|
75
|
+
|
|
76
|
+
# Get service account and grant required roles
|
|
77
|
+
r = _sp.run(['bq', 'show', '--connection', '--format=json',
|
|
78
|
+
'--project_id', PROJECT_ID, '--location', LOCATION, CONNECTION_ID],
|
|
79
|
+
capture_output=True, text=True, check=True)
|
|
80
|
+
sa = _json.loads(r.stdout)['cloudResource']['serviceAccountId']
|
|
81
|
+
for role in ['roles/aiplatform.user', 'roles/storage.objectViewer']:
|
|
82
|
+
_sp.run(['gcloud', 'projects', 'add-iam-policy-binding', PROJECT_ID,
|
|
83
|
+
f'--member=serviceAccount:{sa}', f'--role={role}', '--quiet'],
|
|
84
|
+
capture_output=True, text=True)
|
|
85
|
+
print(f'Connection {CONNECTION_ID} ready (SA: {sa})')
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
---
|
|
89
|
+
## Examples — SQL
|
|
90
|
+
|
|
91
|
+
Progressive examples from simplest to most advanced. Each cell adds one new concept.
|
|
92
|
+
|
|
93
|
+
### 1. Basic text embedding
|
|
94
|
+
|
|
95
|
+
`AI.EMBED` takes content and an endpoint, returns a STRUCT with `result` (the embedding vector) and `status`.
|
|
96
|
+
|
|
97
|
+
**Note:** Either `endpoint` or `model` is required. When using Vertex AI models, `endpoint` must include the model version. When using the built-in model, use `model` instead (see Example 6).
|
|
98
|
+
|
|
99
|
+
```python
|
|
100
|
+
query = """
|
|
101
|
+
SELECT
|
|
102
|
+
text,
|
|
103
|
+
ARRAY_LENGTH((AI.EMBED(
|
|
104
|
+
content => text,
|
|
105
|
+
endpoint => 'text-embedding-005'
|
|
106
|
+
)).result) AS embedding_dimensions
|
|
107
|
+
FROM UNNEST(['BigQuery is a data warehouse.', 'Cloud computing scales on demand.']) AS text
|
|
108
|
+
"""
|
|
109
|
+
client.query(query).to_dataframe()
|
|
110
|
+
```
|
|
111
|
+
|
|
112
|
+
### 2. Viewing the embedding vector
|
|
113
|
+
|
|
114
|
+
The embedding is an `ARRAY<FLOAT64>` — a dense vector of floating-point numbers.
|
|
115
|
+
|
|
116
|
+
```python
|
|
117
|
+
query = """
|
|
118
|
+
SELECT
|
|
119
|
+
(AI.EMBED(
|
|
120
|
+
content => 'What is machine learning?',
|
|
121
|
+
endpoint => 'text-embedding-005'
|
|
122
|
+
)).result AS embedding
|
|
123
|
+
"""
|
|
124
|
+
df = client.query(query).to_dataframe()
|
|
125
|
+
embedding = df.iloc[0]['embedding']
|
|
126
|
+
print(f'Dimensions: {len(embedding)}')
|
|
127
|
+
print(f'First 5 values: {embedding[:5]}')
|
|
128
|
+
```
|
|
129
|
+
|
|
130
|
+
### 3. Using task_type for retrieval
|
|
131
|
+
|
|
132
|
+
`task_type` tells the embedding model the intended use of the text, which changes how the embedding is generated. This is a critical configuration choice.
|
|
133
|
+
|
|
134
|
+
**Asymmetric task types** — documents and queries use *different* task types. Use this pattern for retrieval, where what you index differs in intent from what you search with:
|
|
135
|
+
|
|
136
|
+
| Use Case | Document Task Type | Query Task Type |
|
|
137
|
+
|----------|-------------------|-----------------|
|
|
138
|
+
| Search | `RETRIEVAL_DOCUMENT` | `RETRIEVAL_QUERY` |
|
|
139
|
+
| Question Answering | `RETRIEVAL_DOCUMENT` | `QUESTION_ANSWERING` |
|
|
140
|
+
| Fact Checking | `RETRIEVAL_DOCUMENT` | `FACT_VERIFICATION` |
|
|
141
|
+
| Code Retrieval | `RETRIEVAL_DOCUMENT` | `CODE_RETRIEVAL_QUERY` |
|
|
142
|
+
|
|
143
|
+
**Symmetric task types** — both sides use the *same* task type:
|
|
144
|
+
- `SEMANTIC_SIMILARITY` — comparing how similar two texts are
|
|
145
|
+
- `CLASSIFICATION` — grouping texts by category
|
|
146
|
+
- `CLUSTERING` — organizing texts into clusters
|
|
147
|
+
|
|
148
|
+
> **Key point:** If your use case doesn't align with a specific task type, use `RETRIEVAL_DOCUMENT` when indexing and `RETRIEVAL_QUERY` when searching.
|
|
149
|
+
|
|
150
|
+
```python
|
|
151
|
+
query = """
|
|
152
|
+
SELECT
|
|
153
|
+
text,
|
|
154
|
+
(AI.EMBED(
|
|
155
|
+
content => text,
|
|
156
|
+
endpoint => 'text-embedding-005',
|
|
157
|
+
task_type => 'RETRIEVAL_DOCUMENT'
|
|
158
|
+
)).result AS embedding
|
|
159
|
+
FROM UNNEST([
|
|
160
|
+
'BigQuery is a serverless data warehouse.',
|
|
161
|
+
'Cloud Functions runs event-driven code.',
|
|
162
|
+
'Cloud Storage stores objects in buckets.'
|
|
163
|
+
]) AS text
|
|
164
|
+
"""
|
|
165
|
+
df = client.query(query).to_dataframe()
|
|
166
|
+
for _, row in df.iterrows():
|
|
167
|
+
print(f'{row["text"][:40]}... -> {len(row["embedding"])} dims')
|
|
168
|
+
```
|
|
169
|
+
|
|
170
|
+
### 4. Computing cosine similarity manually
|
|
171
|
+
|
|
172
|
+
Embed two texts and compute cosine similarity using the ML.DISTANCE function.
|
|
173
|
+
|
|
174
|
+
```python
|
|
175
|
+
query = """
|
|
176
|
+
WITH embeddings AS (
|
|
177
|
+
SELECT
|
|
178
|
+
text,
|
|
179
|
+
(AI.EMBED(content => text, endpoint => 'text-embedding-005')).result AS vec
|
|
180
|
+
FROM UNNEST([
|
|
181
|
+
'BigQuery is a data warehouse',
|
|
182
|
+
'BigQuery stores and analyzes large datasets',
|
|
183
|
+
'I enjoy hiking in the mountains'
|
|
184
|
+
]) AS text
|
|
185
|
+
)
|
|
186
|
+
SELECT
|
|
187
|
+
a.text AS text_a,
|
|
188
|
+
b.text AS text_b,
|
|
189
|
+
1 - ML.DISTANCE(a.vec, b.vec, 'COSINE') AS cosine_similarity
|
|
190
|
+
FROM embeddings a
|
|
191
|
+
CROSS JOIN embeddings b
|
|
192
|
+
WHERE a.text < b.text
|
|
193
|
+
ORDER BY cosine_similarity DESC
|
|
194
|
+
"""
|
|
195
|
+
client.query(query).to_dataframe()
|
|
196
|
+
```
|
|
197
|
+
|
|
198
|
+
### 5. Saving embeddings to a table
|
|
199
|
+
|
|
200
|
+
For reuse across queries, materialize embeddings into a table.
|
|
201
|
+
|
|
202
|
+
```python
|
|
203
|
+
query = f'''
|
|
204
|
+
CREATE OR REPLACE TABLE `{PROJECT_ID}.{DATASET_ID}.ai_embed_documents` AS
|
|
205
|
+
SELECT
|
|
206
|
+
id,
|
|
207
|
+
text,
|
|
208
|
+
(AI.EMBED(content => text, endpoint => 'text-embedding-005', task_type => 'RETRIEVAL_DOCUMENT')).result AS embedding
|
|
209
|
+
FROM UNNEST([
|
|
210
|
+
STRUCT(1 AS id, 'BigQuery is a serverless data warehouse for analytics.' AS text),
|
|
211
|
+
STRUCT(2, 'Cloud Functions lets you run code in response to events.'),
|
|
212
|
+
STRUCT(3, 'Cloud Storage provides object storage for any amount of data.'),
|
|
213
|
+
STRUCT(4, 'Kubernetes Engine runs containerized applications at scale.'),
|
|
214
|
+
STRUCT(5, 'Pub/Sub is a messaging service for event-driven systems.')
|
|
215
|
+
])
|
|
216
|
+
'''
|
|
217
|
+
client.query(query).result()
|
|
218
|
+
|
|
219
|
+
# Verify
|
|
220
|
+
client.query(f'SELECT id, text, ARRAY_LENGTH(embedding) AS dims FROM `{PROJECT_ID}.{DATASET_ID}.ai_embed_documents` ORDER BY id').to_dataframe()
|
|
221
|
+
```
|
|
222
|
+
|
|
223
|
+
### 6. Built-in model (no Vertex AI charges)
|
|
224
|
+
|
|
225
|
+
Use `model => 'embeddinggemma-300m'` for zero-cost text embedding. Data stays in BigQuery and uses BQ compute slots — no Vertex AI endpoint call, no connection needed, no per-token billing. Returns 768-dimension vectors.
|
|
226
|
+
|
|
227
|
+
This is ideal for cost-sensitive workloads or when you want to avoid Vertex AI dependencies entirely.
|
|
228
|
+
|
|
229
|
+
> **Note:** `embeddinggemma-300m` is in Preview. When using `model`, you cannot specify `endpoint`, `title`, `model_params`, or `connection_id`.
|
|
230
|
+
|
|
231
|
+
```python
|
|
232
|
+
query = """
|
|
233
|
+
SELECT
|
|
234
|
+
text,
|
|
235
|
+
ARRAY_LENGTH((AI.EMBED(
|
|
236
|
+
content => text,
|
|
237
|
+
model => 'embeddinggemma-300m'
|
|
238
|
+
)).result) AS embedding_dimensions
|
|
239
|
+
FROM UNNEST(['BigQuery is a data warehouse.', 'Cloud computing scales on demand.']) AS text
|
|
240
|
+
"""
|
|
241
|
+
client.query(query).to_dataframe()
|
|
242
|
+
```
|
|
243
|
+
|
|
244
|
+
### 7. Comparing embedding models
|
|
245
|
+
|
|
246
|
+
`AI.EMBED` supports multiple embedding models with different dimensions, capabilities, and pricing. Here's a side-by-side comparison:
|
|
247
|
+
|
|
248
|
+
| Model | Param | Type | Max Dims | Cost |
|
|
249
|
+
|-------|-------|------|----------|------|
|
|
250
|
+
| `embeddinggemma-300m` | `model` | Built-in text | 768 | BQ slots only |
|
|
251
|
+
| `text-embedding-005` | `endpoint` | Vertex AI text | 768 | Vertex AI per-token |
|
|
252
|
+
| `gemini-embedding-001` | `endpoint` | Vertex AI text | 3072 | Vertex AI per-token |
|
|
253
|
+
|
|
254
|
+
```python
|
|
255
|
+
query = """
|
|
256
|
+
SELECT 'embeddinggemma-300m (built-in)' AS model,
|
|
257
|
+
ARRAY_LENGTH((AI.EMBED(content => 'BigQuery is a data warehouse', model => 'embeddinggemma-300m')).result) AS dims
|
|
258
|
+
UNION ALL
|
|
259
|
+
SELECT 'text-embedding-005',
|
|
260
|
+
ARRAY_LENGTH((AI.EMBED(content => 'BigQuery is a data warehouse', endpoint => 'text-embedding-005')).result)
|
|
261
|
+
UNION ALL
|
|
262
|
+
SELECT 'gemini-embedding-001',
|
|
263
|
+
ARRAY_LENGTH((AI.EMBED(content => 'BigQuery is a data warehouse', endpoint => 'gemini-embedding-001')).result)
|
|
264
|
+
"""
|
|
265
|
+
client.query(query).to_dataframe()
|
|
266
|
+
```
|
|
267
|
+
|
|
268
|
+
---
|
|
269
|
+
## Multimodal — Document Embedding
|
|
270
|
+
|
|
271
|
+
`AI.EMBED` supports image input via ObjectRef with `multimodalembedding@001`. A `connection_id` is required for multimodal. Returns 1408-dimension vectors by default (configurable: 128, 256, 512, 1408).
|
|
272
|
+
|
|
273
|
+
> **Also GA:** `gemini-embedding-2` embeds PDFs directly — no rendering to PNG needed — plus audio, video, and a STRUCT that mixes text with files, at up to 3072 dimensions. It rejects `task_type` and `title`, returns no `statistics` column, and is served from multi-regional endpoints only. See `reference/embedding-generation-and-semantic-search.md` (Embedding Generation and Semantic Search) for details. The examples below use `multimodalembedding@001` with rendered PNGs.
|
|
274
|
+
|
|
275
|
+
Below we render invoice and receipt PDFs from this project's `data/documents` (document set) to PNG images, upload them to GCS, then embed and compare them. The embeddings should show that documents of the same type cluster together.
|
|
276
|
+
|
|
277
|
+
The ObjectRef pattern: `OBJ.MAKE_REF(uri, connection)` builds the reference inline — no object table needed, and nothing to sign.
|
|
278
|
+
|
|
279
|
+
```python
|
|
280
|
+
import shutil, subprocess
|
|
281
|
+
|
|
282
|
+
if not shutil.which('pdftoppm'):
|
|
283
|
+
subprocess.check_call(['sudo', 'apt-get', 'install', '-y', '-qq', 'poppler-utils'])
|
|
284
|
+
print('Installed poppler-utils (provides pdftoppm)')
|
|
285
|
+
else:
|
|
286
|
+
print('pdftoppm already available')
|
|
287
|
+
```
|
|
288
|
+
|
|
289
|
+
```python
|
|
290
|
+
import subprocess, io
|
|
291
|
+
from pathlib import Path
|
|
292
|
+
from google.cloud import storage
|
|
293
|
+
|
|
294
|
+
# Locate the document directory
|
|
295
|
+
data_dir = Path('../../data/documents')
|
|
296
|
+
if not data_dir.exists():
|
|
297
|
+
data_dir = Path('data/documents')
|
|
298
|
+
|
|
299
|
+
# Pick 3 invoices and 3 receipts
|
|
300
|
+
docs = [
|
|
301
|
+
('invoice_1.png', data_dir / 'invoices' / 'invoice_001.pdf'),
|
|
302
|
+
('invoice_2.png', data_dir / 'invoices' / 'invoice_002.pdf'),
|
|
303
|
+
('invoice_3.png', data_dir / 'invoices' / 'invoice_003.pdf'),
|
|
304
|
+
('receipt_1.png', data_dir / 'receipts' / 'receipt_001.pdf'),
|
|
305
|
+
('receipt_2.png', data_dir / 'receipts' / 'receipt_002.pdf'),
|
|
306
|
+
('receipt_3.png', data_dir / 'receipts' / 'receipt_003.pdf'),
|
|
307
|
+
]
|
|
308
|
+
|
|
309
|
+
gcs = storage.Client(project=PROJECT_ID)
|
|
310
|
+
bucket = gcs.bucket(BUCKET)
|
|
311
|
+
prefix = 'bq_ai_functions/ai_embed'
|
|
312
|
+
|
|
313
|
+
for name, pdf_path in docs:
|
|
314
|
+
# Render first page of PDF to PNG using pdftoppm (poppler)
|
|
315
|
+
result = subprocess.run(
|
|
316
|
+
['pdftoppm', '-png', '-f', '1', '-l', '1', '-r', '150', str(pdf_path)],
|
|
317
|
+
capture_output=True
|
|
318
|
+
)
|
|
319
|
+
blob = bucket.blob(f'{prefix}/{name}')
|
|
320
|
+
blob.upload_from_string(result.stdout, content_type='image/png')
|
|
321
|
+
|
|
322
|
+
print(f'Rendered and uploaded {len(docs)} document images to gs://{BUCKET}/{prefix}/')
|
|
323
|
+
```
|
|
324
|
+
|
|
325
|
+
### 8. Embed a document image
|
|
326
|
+
|
|
327
|
+
Use an inline `OBJ.MAKE_REF` to embed an invoice image with `multimodalembedding@001`. The `connection_id` parameter authorizes GCS access.
|
|
328
|
+
|
|
329
|
+
```python
|
|
330
|
+
query = f"""
|
|
331
|
+
SELECT
|
|
332
|
+
'invoice_1.png' AS document,
|
|
333
|
+
ARRAY_LENGTH((AI.EMBED(
|
|
334
|
+
content => OBJ.MAKE_REF(
|
|
335
|
+
'gs://{BUCKET}/bq_ai_functions/ai_embed/invoice_1.png',
|
|
336
|
+
'{PROJECT_ID}.{LOCATION}.{CONNECTION_ID}'
|
|
337
|
+
),
|
|
338
|
+
endpoint => 'multimodalembedding@001',
|
|
339
|
+
connection_id => '{PROJECT_ID}.{LOCATION}.{CONNECTION_ID}'
|
|
340
|
+
)).result) AS embedding_dimensions
|
|
341
|
+
"""
|
|
342
|
+
client.query(query).to_dataframe()
|
|
343
|
+
```
|
|
344
|
+
|
|
345
|
+
### 9. Compare document embeddings
|
|
346
|
+
|
|
347
|
+
Embed all 6 documents and compute pairwise cosine similarity using `ML.DISTANCE`. Documents of the same type (invoices vs receipts) should cluster together with higher similarity scores.
|
|
348
|
+
|
|
349
|
+
```python
|
|
350
|
+
query = f"""
|
|
351
|
+
WITH doc_embeddings AS (
|
|
352
|
+
SELECT
|
|
353
|
+
doc_name,
|
|
354
|
+
(AI.EMBED(
|
|
355
|
+
content => OBJ.MAKE_REF(
|
|
356
|
+
CONCAT('gs://{BUCKET}/bq_ai_functions/ai_embed/', doc_name),
|
|
357
|
+
'{PROJECT_ID}.{LOCATION}.{CONNECTION_ID}'
|
|
358
|
+
),
|
|
359
|
+
endpoint => 'multimodalembedding@001',
|
|
360
|
+
connection_id => '{PROJECT_ID}.{LOCATION}.{CONNECTION_ID}'
|
|
361
|
+
)).result AS vec
|
|
362
|
+
FROM UNNEST([
|
|
363
|
+
'invoice_1.png', 'invoice_2.png', 'invoice_3.png',
|
|
364
|
+
'receipt_1.png', 'receipt_2.png', 'receipt_3.png'
|
|
365
|
+
]) AS doc_name
|
|
366
|
+
)
|
|
367
|
+
SELECT
|
|
368
|
+
a.doc_name AS doc_a,
|
|
369
|
+
b.doc_name AS doc_b,
|
|
370
|
+
ROUND(1 - ML.DISTANCE(a.vec, b.vec, 'COSINE'), 4) AS cosine_similarity
|
|
371
|
+
FROM doc_embeddings a
|
|
372
|
+
CROSS JOIN doc_embeddings b
|
|
373
|
+
WHERE a.doc_name < b.doc_name
|
|
374
|
+
ORDER BY cosine_similarity DESC
|
|
375
|
+
"""
|
|
376
|
+
client.query(query).to_dataframe()
|
|
377
|
+
```
|
|
378
|
+
|
|
379
|
+
---
|
|
380
|
+
## Examples — `%%bigquery` Magics
|
|
381
|
+
|
|
382
|
+
The same examples using IPython magic commands. Magics let you write SQL directly in notebook cells without Python string wrapping.
|
|
383
|
+
|
|
384
|
+
Key patterns:
|
|
385
|
+
- `%%bigquery` — run SQL, display results inline
|
|
386
|
+
- `%%bigquery df` — run SQL, capture results into a pandas DataFrame
|
|
387
|
+
|
|
388
|
+
### Basic embedding with `%%bigquery`
|
|
389
|
+
|
|
390
|
+
```sql
|
|
391
|
+
%%bigquery --project {PROJECT_ID}
|
|
392
|
+
|
|
393
|
+
SELECT
|
|
394
|
+
text,
|
|
395
|
+
ARRAY_LENGTH((AI.EMBED(
|
|
396
|
+
content => text,
|
|
397
|
+
endpoint => 'text-embedding-005'
|
|
398
|
+
)).result) AS embedding_dimensions
|
|
399
|
+
FROM UNNEST(['Machine learning', 'Deep learning', 'Natural language processing']) AS text
|
|
400
|
+
```
|
|
401
|
+
|
|
402
|
+
---
|
|
403
|
+
## Examples — BigFrames
|
|
404
|
+
|
|
405
|
+
`AI.EMBED` has no direct BigFrames equivalent. Use `session.read_gbq_query()` to execute AI.EMBED SQL from BigFrames.
|
|
406
|
+
|
|
407
|
+
```python
|
|
408
|
+
import bigframes.pandas as bpd
|
|
409
|
+
|
|
410
|
+
bpd.options.bigquery.project = PROJECT_ID
|
|
411
|
+
bpd.options.bigquery.location = LOCATION
|
|
412
|
+
```
|
|
413
|
+
|
|
414
|
+
### Running AI.EMBED via read_gbq_query
|
|
415
|
+
|
|
416
|
+
```python
|
|
417
|
+
query = """
|
|
418
|
+
SELECT
|
|
419
|
+
text,
|
|
420
|
+
ARRAY_LENGTH((AI.EMBED(content => text, endpoint => 'text-embedding-005')).result) AS embedding_dims
|
|
421
|
+
FROM UNNEST(['BigQuery', 'Cloud Functions', 'Cloud Storage']) AS text
|
|
422
|
+
"""
|
|
423
|
+
df = bpd.read_gbq_query(query)
|
|
424
|
+
df.to_pandas()
|
|
425
|
+
```
|