bigquery-data-science 0.7.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- bigquery_data_science/__init__.py +58 -0
- bigquery_data_science/_skills/PLANS.md +76 -0
- bigquery_data_science/_skills/bigquery-ai/SKILL.md +66 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ai_agg.md +528 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ai_classify.md +450 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ai_count_tokens.md +180 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ai_detect_anomalies.md +322 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ai_embed.md +425 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ai_evaluate.md +368 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ai_forecast.md +437 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ai_generate.md +693 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ai_generate_bool.md +248 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ai_generate_double.md +254 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ai_generate_embedding.md +402 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ai_generate_int.md +252 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ai_generate_table.md +447 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ai_generate_text.md +356 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ai_if.md +373 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ai_key_drivers.md +331 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ai_parse_document.md +416 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ai_predict.md +559 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ai_score.md +310 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ai_search.md +465 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ai_similarity.md +371 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/catalog_search.md +951 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/content_analysis.md +284 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/content_moderation.md +369 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/data_enrichment.md +426 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/document_intelligence.md +470 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/document_rag.md +453 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/image_deduplication.md +770 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/log_analysis.md +668 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/metric_diagnostics.md +568 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ml_annotate_image.md +680 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ml_generate_embedding.md +217 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ml_generate_text.md +223 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ml_process_document.md +392 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ml_transcribe.md +815 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ml_translate.md +600 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ml_understand_text.md +673 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/multimodal_analysis.md +266 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/rag_pipeline.md +593 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/semantic_search.md +510 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/tabular_prediction.md +485 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/time_series_intelligence.md +489 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/vector_search.md +672 -0
- bigquery_data_science/_skills/bigquery-ai/reference/augmented-analytics.md +77 -0
- bigquery_data_science/_skills/bigquery-ai/reference/cloud-ai-service-models.md +88 -0
- bigquery_data_science/_skills/bigquery-ai/reference/document-processing.md +95 -0
- bigquery_data_science/_skills/bigquery-ai/reference/embeddings-and-search.md +166 -0
- bigquery_data_science/_skills/bigquery-ai/reference/general-purpose-generation.md +87 -0
- bigquery_data_science/_skills/bigquery-ai/reference/managed-functions.md +72 -0
- bigquery_data_science/_skills/bigquery-ai/reference/predictive-ai.md +113 -0
- bigquery_data_science/_skills/bigquery-ai/reference/workflows.md +56 -0
- bigquery_data_science/_skills/bigquery-ai/skill.manifest.json +109 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/INDEX.md +71 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/aiagg.md +67 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/aicausal_effect-covered-in-bigquery-ml.md +17 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/aiclassify.md +72 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/aicount_tokens.md +53 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/aidetect_anomalies.md +63 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/aiembed.md +85 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/aievaluate.md +114 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/aiforecast.md +123 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/aigenerate.md +69 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/aigenerate_bool.md +53 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/aigenerate_double.md +45 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/aigenerate_embedding.md +108 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/aigenerate_int.md +45 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/aigenerate_table.md +64 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/aigenerate_text.md +167 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/aiif.md +56 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/aikey_drivers.md +76 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/aiparse_document.md +91 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/aipredict.md +74 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/aiscore.md +52 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/aisearch.md +66 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/aisimilarity.md +79 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/choosing-between-these-and-the-ai-functions.md +11 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/hybrid-search-capability.md +165 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/mlannotate_image.md +24 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/mlgenerate_embedding.md +34 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/mlgenerate_text.md +65 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/mlprocess_document.md +125 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/mltranscribe.md +26 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/mltranslate.md +27 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/mlunderstand_text.md +32 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/multimodal-input-patterns-by-function.md +128 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/object-tables.md +109 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/objectref-and-objectrefruntime-schema-reference.md +50 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/objfetch_metadata.md +49 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/objget_access_url.md +66 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/objget_read_url.md +40 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/objlist.md +34 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/objmake_ref.md +58 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/per-row-failures-and-the-retry-pattern.md +19 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/setup-apis-connection-roles.md +39 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/two-input-shapes.md +22 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/vector_search.md +78 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/what-the-five-have-in-common.md +27 -0
- bigquery_data_science/_skills/bigquery-data-science/SKILL.md +69 -0
- bigquery_data_science/_skills/bigquery-data-science/reference/namespace-matrix.md +112 -0
- bigquery_data_science/_skills/bigquery-data-science/skill.manifest.json +13 -0
- bigquery_data_science/_skills/bigquery-ml/SKILL.md +70 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/airflow_with_kfp.md +379 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/anomaly_fraud_detection.md +195 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/arima_plus.md +635 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/arima_plus_xreg.md +453 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/autoencoder.md +470 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/automl_classifier.md +291 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/automl_regressor.md +274 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/boosted_tree_classifier.md +449 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/boosted_tree_regressor.md +400 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/bucketizing.md +201 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/causal_effect.md +773 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/churn_retention.md +271 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/cloud_workflows.md +370 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/composer_airflow.md +475 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/contribution_analysis.md +346 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/cross_validation.md +185 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/customer_segmentation.md +233 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/dataform.md +372 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/dbt.md +283 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/difference_in_differences.md +280 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/distance.md +195 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/dnn_classifier.md +326 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/dnn_regressor.md +308 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/embeddings_classification.md +426 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/encoding.md +238 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/ensembling.md +278 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/evaluation.md +445 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/exploration.md +779 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/export.md +321 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/feature_engineering.md +280 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/feature_store.md +652 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/feature_store_workflow.md +426 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/ga4_churn_prediction.md +285 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/hierarchical_forecasting.md +441 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/image.md +791 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/imported.md +426 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/kmeans.md +348 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/linear_regression.md +372 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/logistic_regression.md +390 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/matrix_factorization.md +404 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/model_monitoring.md +609 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/pca.md +357 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/price_elasticity_dml.md +286 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/propensity_score_matching.md +350 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/random_forest_classifier.md +445 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/random_forest_regressor.md +409 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/recommendation.md +281 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/regression_based_forecasting.md +491 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/remote.md +428 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/scalers.md +325 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/scheduled_queries.md +284 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/sql_scripting.md +243 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/survival_analysis.md +363 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/synthetic_control.md +221 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/text.md +215 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/time_series.md +669 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/transform_only.md +273 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/uplift_cate.md +241 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/vertex_kfp.md +360 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/wide_and_deep_classifier.md +316 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/wide_and_deep_regressor.md +298 -0
- bigquery_data_science/_skills/bigquery-ml/reference/classification.md +80 -0
- bigquery_data_science/_skills/bigquery-ml/reference/model-management.md +90 -0
- bigquery_data_science/_skills/bigquery-ml/reference/preprocessing-functions.md +234 -0
- bigquery_data_science/_skills/bigquery-ml/reference/regression.md +79 -0
- bigquery_data_science/_skills/bigquery-ml/reference/unsupervised-and-specialized.md +101 -0
- bigquery_data_science/_skills/bigquery-ml/reference/workflows-and-pipelines.md +111 -0
- bigquery_data_science/_skills/bigquery-ml/skill.manifest.json +177 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/1-model_type-catalog.md +28 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/2-task-mlevaluate-metric-columns.md +26 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/3-capability-matrix-per-model-type.md +21 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/4-explainability-weights-functions.md +14 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/5-connection-matrix.md +11 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/6-which-model-free-function-no-create-model-no-connection.md +15 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/7-point-in-time-feature-retrieval-which-of-the-two.md +11 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/INDEX.md +111 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/aicausal_effect.md +85 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/arima_plus-forecasting-lifecycle-functions.md +14 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/arima_plus-univariate-time-series-forecasting.md +132 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/arima_plus_xreg-multivariate-time-series-with-external-regressors.md +103 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/autoencoder.md +102 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/automl_classifier-automl_regressor.md +88 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/boosted_tree_classifier-boosted_tree_regressor.md +117 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/contribution_analysis.md +113 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/dnn_classifier-dnn_regressor.md +107 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/dnn_linear_combined_classifier-dnn_linear_combined_regressor-wide-and-deep.md +98 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/explainability-functions-mlexplain_predict-mlglobal_explain-mlfeature_importance.md +7 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/exploratory-data-analysis-mldescribe_data-mlcorrelation.md +11 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/export-model.md +92 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/feature-engineering-standalone-preprocessing-functions.md +13 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/image-preprocessing-functions-mldecode_image-mlresize_image-mlconvert_image_type-mlconvert_color_space.md +63 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/imported-models-tensorflow-tensorflow_lite-onnx-xgboost.md +36 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/kmeans.md +87 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/linear_reg-logistic_reg-generalized-linear-models.md +101 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/matrix_factorization.md +162 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mladvanced_weights.md +76 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlarima_coefficients.md +30 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlarima_evaluate.md +49 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlbag_of_words.md +47 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlbucketize.md +47 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlcentroids.md +46 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlconfusion_matrix.md +56 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlconvert_color_space.md +53 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlconvert_image_type.md +39 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlcorrelation.md +78 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mldecode_image.md +44 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mldescribe_data.md +74 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mldetect_anomalies.md +86 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mldistance.md +48 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlentity_features_at_time.md +56 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlevaluate.md +72 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlexplain_forecast.md +50 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlexplain_predict.md +66 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlfeature_cross.md +46 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlfeature_importance.md +47 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlfeature_info.md +50 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlfeatures_at_time.md +51 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlforecast.md +76 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlgenerate_embedding-from-a-bqml-pca-autoencoder-matrix_factorization-model.md +64 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlglobal_explain.md +63 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlhash_bucketize.md +47 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlholiday_info.md +46 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlimputer.md +46 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mllp_norm.md +47 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlmetrics.md +80 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlngrams.md +37 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlone_hot_encoder-mlmulti_hot_encoder-mllabel_encoder.md +67 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlpolynomial_expand.md +47 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlpredict.md +67 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlprincipal_component_info.md +42 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlprincipal_components.md +40 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlquantile_bucketize.md +43 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlrecommend.md +60 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlreconstruction_loss.md +48 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlresize_image.md +45 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlroc_curve.md +63 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mltf_idf.md +37 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mltfdv_describe.md +34 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mltfdv_validate.md +48 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mltraining_info.md +78 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mltransform-applying-a-saved-transform.md +52 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mltranspose-not-a-function-pointer.md +32 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mltrial_info.md +60 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlvalidate_data_drift.md +99 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlvalidate_data_skew.md +50 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlweights.md +84 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/model-free-time-series-decomposition-mltrend-mlseasonality-mldetect_change_points.md +100 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/model-introspection-mlfeature_info-mltraining_info-mltrial_info.md +7 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/model-monitoring-data-validation.md +23 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/numerical-scalers-mlstandard_scaler-mlmin_max_scaler-mlmax_abs_scaler-mlrobust_scaler-mlnormalizer.md +77 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/onnx.md +55 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/pca.md +91 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/point-in-time-feature-retrieval-mlfeatures_at_time-mlentity_features_at_time.md +25 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/quick-comparison.md +14 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/random_forest_classifier-random_forest_regressor.md +123 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/remote-remote-model-over-a-vertex-ai-endpoint-the-mechanism.md +87 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/tensorflow.md +48 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/tensorflow_lite.md +29 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/text-preprocessing-functions-mlngrams-mltf_idf-mlbag_of_words.md +18 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/timesfm-built-in-foundation-forecaster-see-bigquery-ai.md +28 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/transform_only.md +91 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/xgboost.md +72 -0
- bigquery_data_science/_skills/pinned-models.json +28 -0
- bigquery_data_science/catalog.py +1107 -0
- bigquery_data_science/cli.py +786 -0
- bigquery_data_science/installer.py +282 -0
- bigquery_data_science/mcp_server.py +317 -0
- bigquery_data_science/tooling/__init__.py +15 -0
- bigquery_data_science/tooling/convert_notebook.py +176 -0
- bigquery_data_science/tooling/link_policy.py +247 -0
- bigquery_data_science/tooling/manifest.py +152 -0
- bigquery_data_science/tooling/markdown_anchors.py +84 -0
- bigquery_data_science/tooling/narrative_drift.py +163 -0
- bigquery_data_science/tooling/pinned_models.py +216 -0
- bigquery_data_science/tooling/reference_structure.py +85 -0
- bigquery_data_science/tooling/sync_syntax.py +407 -0
- bigquery_data_science/tooling/validate.py +208 -0
- bigquery_data_science-0.7.0.dist-info/METADATA +306 -0
- bigquery_data_science-0.7.0.dist-info/RECORD +286 -0
- bigquery_data_science-0.7.0.dist-info/WHEEL +4 -0
- bigquery_data_science-0.7.0.dist-info/entry_points.txt +5 -0
- bigquery_data_science-0.7.0.dist-info/licenses/LICENSE +201 -0
|
@@ -0,0 +1,404 @@
|
|
|
1
|
+
> **Source (`statmike/bigquery-data-science`):** **Notebook:** [`bigquery-ml/models/matrix_factorization/matrix_factorization.ipynb`](https://github.com/statmike/bigquery-data-science/blob/main/bigquery-ml/models/matrix_factorization/matrix_factorization.ipynb) · **SQL:** [`bigquery-ml/models/matrix_factorization/matrix_factorization.sql`](https://github.com/statmike/bigquery-data-science/blob/main/bigquery-ml/models/matrix_factorization/matrix_factorization.sql)
|
|
2
|
+
|
|
3
|
+
# Matrix Factorization — BigQuery ML
|
|
4
|
+
|
|
5
|
+
Train a **collaborative-filtering recommendation** model with `CREATE MODEL` (model_type = `MATRIX_FACTORIZATION`) — factorizes a sparse (user, item, rating) matrix into low-dimensional user and item latent factor vectors — then walk the full model lifecycle: evaluate, recommend, inspect factors, generate embeddings, item-item similarity search, introspect features/training, and tune hyperparameters.
|
|
6
|
+
|
|
7
|
+
**Lifecycle:** `CREATE MODEL` → `ML.EVALUATE` → `ML.RECOMMEND` (user→items, item→users) → `ML.WEIGHTS` → `ML.GENERATE_EMBEDDING` → `VECTOR_SEARCH` → `ML.FEATURE_INFO` → `ML.TRAINING_INFO` → hyperparameter tuning (`ML.TRIAL_INFO`)
|
|
8
|
+
|
|
9
|
+
**The one model type in this entire project that needs real infrastructure setup first:**
|
|
10
|
+
- **`MATRIX_FACTORIZATION` cannot train under on-demand (per-byte) pricing** — verified directly: attempting `CREATE MODEL` without a reservation fails immediately with `"Training Matrix Factorization models is not available for on-demand usage."` Every other model type in this project trains fine on-demand; this one requires a BigQuery slot reservation.
|
|
11
|
+
- **The reservation's edition matters too, not just its existence** — verified: a `STANDARD` edition reservation still fails, with a *different* error (`"Using BQML related functionalities is disallowed in STANDARD edition"`). `ENTERPRISE` (or higher) is required for any BQML training, matrix factorization included.
|
|
12
|
+
- This notebook creates a small **autoscaling** BigQuery Editions reservation (`ENTERPRISE`, 0 baseline slots, scales up to 100 only while a job actually runs) in Setup, and deletes it in Cleanup. This is **not** a capacity commitment — no upfront purchase, no minimum term; it bills per-second only for slot-time actually consumed while queries run, and idles at zero cost otherwise.
|
|
13
|
+
- **A newly created reservation assignment needs time to propagate** — verified: `CREATE MODEL` failed on the very first attempt right after creating and assigning a fresh reservation, and succeeded only after waiting.
|
|
14
|
+
- No `TRANSFORM` clause exists for this model type — the training query must produce exactly a (user, item, rating) triple.
|
|
15
|
+
|
|
16
|
+
**Data:** [`bigquery-public-data.google_analytics_sample`](https://console.cloud.google.com/marketplace/product/bigquery-public-datasets) (Google Merchandise Store demo data, July 2017) — an **IMPLICIT** feedback signal built from product-detail-page view counts per (visitor, product) pair: 42,178 users × 320 items across ~1.3M interactions. `feedback_type='IMPLICIT'` fits this data (a behavioral proxy, not an explicit star rating) — see `RESOURCES.md` (RESOURCES.md) for how `EXPLICIT` feedback differs (different options, different `ML.EVALUATE` metrics).
|
|
17
|
+
|
|
18
|
+
**References:** `RESOURCES.md` (Full reference) | [CREATE MODEL (matrix factorization) docs](https://cloud.google.com/bigquery/docs/reference/standard-sql/bigqueryml-syntax-create-matrix-factorization) | [ML.RECOMMEND docs](https://cloud.google.com/bigquery/docs/reference/standard-sql/bigqueryml-syntax-recommend) | `setup` (Setup guide)
|
|
19
|
+
|
|
20
|
+
> **Cost note (measured, not estimated):** this notebook creates a temporary BigQuery Editions reservation to enable training — there's no capacity commitment and no charge while it sits idle, but training under `ENTERPRISE` autoscale bills per slot-hour of actual usage. Building and validating this notebook (base model + tuning job + the BigFrames example below, each retrained at least once) consumed **~6.4 cumulative slot-hours** of real, measured usage — check the [current BigQuery Editions pricing](https://cloud.google.com/bigquery/pricing) to estimate the actual cost for your project/region before running. The reservation is deleted in Cleanup; if a run is interrupted before Cleanup, delete it manually (see Cleanup for the exact commands) to avoid ongoing charges.
|
|
21
|
+
|
|
22
|
+
---
|
|
23
|
+
## Setup
|
|
24
|
+
|
|
25
|
+
Set your project and location, authenticate, and create a shared dataset for the model.
|
|
26
|
+
|
|
27
|
+
```python
|
|
28
|
+
PROJECT_ID = 'statmike-mlops-349915' # <-- Replace with your project ID
|
|
29
|
+
LOCATION = 'US' # BigQuery dataset location
|
|
30
|
+
DATASET_ID = 'bq_ml' # Shared dataset across all bigquery-ml notebooks
|
|
31
|
+
RESERVATION_ID = 'bqml-mf-temp' # Temporary reservation name (Setup creates it, Cleanup removes it)
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
### Environment
|
|
35
|
+
|
|
36
|
+
> **Already set up the project environment?** The cell below is a no-op — packages are already in your kernel. See the `setup` (Setup Reference) for details.
|
|
37
|
+
>
|
|
38
|
+
> **Running standalone** (Colab, Colab Enterprise, Vertex AI Workbench)? The cell below installs required packages into your current kernel.
|
|
39
|
+
|
|
40
|
+
```python
|
|
41
|
+
from google.cloud import bigquery
|
|
42
|
+
import pandas as pd
|
|
43
|
+
|
|
44
|
+
client = bigquery.Client(project=PROJECT_ID)
|
|
45
|
+
pd.set_option('display.max_colwidth', None)
|
|
46
|
+
|
|
47
|
+
# Create the shared dataset (idempotent)
|
|
48
|
+
dataset_ref = bigquery.DatasetReference(PROJECT_ID, DATASET_ID)
|
|
49
|
+
dataset = bigquery.Dataset(dataset_ref)
|
|
50
|
+
dataset.location = LOCATION
|
|
51
|
+
client.create_dataset(dataset, exists_ok=True)
|
|
52
|
+
print(f'Dataset {PROJECT_ID}.{DATASET_ID} ready')
|
|
53
|
+
|
|
54
|
+
# Register %%bigquery cell magic (auto-loaded in Colab, needed elsewhere)
|
|
55
|
+
%load_ext bigquery_magics
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
### Create a temporary slot reservation (required for this model type only)
|
|
59
|
+
|
|
60
|
+
`ENTERPRISE` edition, 0 baseline slots, autoscaling up to 100 slots — no capacity commitment, billed per-second only while queries actually run. Assigned to this project for `QUERY`-type jobs (BQML training runs as a `QUERY` job). Waits ~90 seconds afterward for the assignment to actually take effect — a real, verified propagation delay, not an arbitrary pause.
|
|
61
|
+
|
|
62
|
+
**Idempotent:** if this cell is re-run (e.g. after an interrupted prior run whose Cleanup never executed), it detects and reuses the existing reservation/assignment instead of erroring or creating a duplicate — and skips the 90-second wait when reusing an already-propagated assignment.
|
|
63
|
+
|
|
64
|
+
```python
|
|
65
|
+
import time
|
|
66
|
+
from google.cloud import bigquery_reservation_v1
|
|
67
|
+
from google.api_core.exceptions import AlreadyExists
|
|
68
|
+
|
|
69
|
+
reservation_client = bigquery_reservation_v1.ReservationServiceClient()
|
|
70
|
+
reservation_parent = f'projects/{PROJECT_ID}/locations/{LOCATION}'
|
|
71
|
+
reservation_name = f'{reservation_parent}/reservations/{RESERVATION_ID}'
|
|
72
|
+
|
|
73
|
+
# Idempotent: reuse an existing reservation/assignment (e.g. from an interrupted
|
|
74
|
+
# prior run) instead of erroring or creating a duplicate.
|
|
75
|
+
try:
|
|
76
|
+
reservation = reservation_client.create_reservation(
|
|
77
|
+
parent=reservation_parent,
|
|
78
|
+
reservation_id=RESERVATION_ID,
|
|
79
|
+
reservation=bigquery_reservation_v1.Reservation(
|
|
80
|
+
slot_capacity=0,
|
|
81
|
+
edition=bigquery_reservation_v1.Edition.ENTERPRISE,
|
|
82
|
+
autoscale=bigquery_reservation_v1.Reservation.Autoscale(max_slots=100),
|
|
83
|
+
ignore_idle_slots=True,
|
|
84
|
+
),
|
|
85
|
+
)
|
|
86
|
+
print(f'Created {reservation.name}')
|
|
87
|
+
except AlreadyExists:
|
|
88
|
+
reservation = reservation_client.get_reservation(name=reservation_name)
|
|
89
|
+
print(f'Reservation already exists — reusing {reservation.name}')
|
|
90
|
+
|
|
91
|
+
existing_assignments = list(reservation_client.list_assignments(parent=reservation.name))
|
|
92
|
+
if existing_assignments:
|
|
93
|
+
assignment = existing_assignments[0]
|
|
94
|
+
print(f'Assignment already exists — reusing {assignment.name}')
|
|
95
|
+
else:
|
|
96
|
+
assignment = reservation_client.create_assignment(
|
|
97
|
+
parent=reservation.name,
|
|
98
|
+
assignment=bigquery_reservation_v1.Assignment(
|
|
99
|
+
job_type=bigquery_reservation_v1.Assignment.JobType.QUERY,
|
|
100
|
+
assignee=f'projects/{PROJECT_ID}',
|
|
101
|
+
),
|
|
102
|
+
)
|
|
103
|
+
print(f'Created {assignment.name}, waiting ~90s for it to propagate...')
|
|
104
|
+
time.sleep(90)
|
|
105
|
+
|
|
106
|
+
print('Ready to train.')
|
|
107
|
+
```
|
|
108
|
+
|
|
109
|
+
---
|
|
110
|
+
## Step 1 — Create the model with `CREATE MODEL`
|
|
111
|
+
|
|
112
|
+
`feedback_type = 'IMPLICIT'` fits this data — `view_count` is a behavioral proxy signal, not an explicit star rating. `num_factors = 16` is a deliberate, modest choice for a 320-item catalog. There is no `TRANSFORM` clause for this model type — the training `SELECT` must produce exactly a (user, item, rating) triple.
|
|
113
|
+
|
|
114
|
+
```python
|
|
115
|
+
query = f"""
|
|
116
|
+
CREATE OR REPLACE MODEL `{PROJECT_ID}.{DATASET_ID}.matrix_factorization_ga`
|
|
117
|
+
OPTIONS(
|
|
118
|
+
model_type = 'MATRIX_FACTORIZATION',
|
|
119
|
+
feedback_type = 'IMPLICIT',
|
|
120
|
+
user_col = 'visitor_id',
|
|
121
|
+
item_col = 'product_name',
|
|
122
|
+
rating_col = 'view_count',
|
|
123
|
+
num_factors = 16
|
|
124
|
+
) AS
|
|
125
|
+
SELECT
|
|
126
|
+
fullVisitorId AS visitor_id,
|
|
127
|
+
product.v2ProductName AS product_name,
|
|
128
|
+
COUNT(*) AS view_count
|
|
129
|
+
FROM `bigquery-public-data.google_analytics_sample.ga_sessions_*`,
|
|
130
|
+
UNNEST(hits) AS hits,
|
|
131
|
+
UNNEST(hits.product) AS product
|
|
132
|
+
WHERE _TABLE_SUFFIX BETWEEN '20170701' AND '20170801'
|
|
133
|
+
AND product.v2ProductName IS NOT NULL AND product.v2ProductName != '(not set)'
|
|
134
|
+
GROUP BY visitor_id, product_name
|
|
135
|
+
"""
|
|
136
|
+
client.query(query).result()
|
|
137
|
+
print('Model matrix_factorization_ga created')
|
|
138
|
+
```
|
|
139
|
+
|
|
140
|
+
---
|
|
141
|
+
## Step 2 — Evaluate with `ML.EVALUATE`
|
|
142
|
+
|
|
143
|
+
IMPLICIT feedback gets ranking-style metrics — `mean_average_precision` and `normalized_discounted_cumulative_gain` (both higher-is-better, bounded [0,1]), plus `mean_squared_error` and `average_rank`. `EXPLICIT` models get a different, regression-style metric set instead (see `RESOURCES.md` (RESOURCES.md)) — the two feedback types aren't evaluated the same way.
|
|
144
|
+
|
|
145
|
+
> **Verified:** retraining this exact model (same name, same SQL) does not reproduce `mean_average_precision` exactly — one training run reached `0.873`, another reached `0.860`. Matrix factorization's WALS training shows measurable retraining variance, similar to `models/kmeans` (K-Means)/`RANDOM_FOREST_*` elsewhere in this project, unlike PCA's full determinism or the DNN family's bit-for-bit reproducibility under a fixed name.
|
|
146
|
+
|
|
147
|
+
```python
|
|
148
|
+
query = f"""
|
|
149
|
+
SELECT *
|
|
150
|
+
FROM ML.EVALUATE(MODEL `{PROJECT_ID}.{DATASET_ID}.matrix_factorization_ga`)
|
|
151
|
+
"""
|
|
152
|
+
client.query(query).to_dataframe()
|
|
153
|
+
```
|
|
154
|
+
|
|
155
|
+
---
|
|
156
|
+
## Step 3 — Recommend items for a user with `ML.RECOMMEND`
|
|
157
|
+
|
|
158
|
+
Passing only the user column scores every item for that user.
|
|
159
|
+
|
|
160
|
+
> **GOTCHA (verified):** calling this for a `visitor_id` **not present** in the training data does **not** error — it silently returns a full ranked list. `RESOURCES.md` (RESOURCES.md)'s general cold-start note ("cannot recommend for users or items absent from training data") describes the intent, not the observed runtime behavior: comparing the output for a confirmed real trained user against an obviously fake user ID shows completely different top-5 lists, consistent with the fake user falling back to item-side bias/popularity alone rather than genuine personalization (no user factor was ever learned for it). Don't assume a non-empty `ML.RECOMMEND` result means the input user actually existed in training.
|
|
161
|
+
|
|
162
|
+
```python
|
|
163
|
+
query = f"""
|
|
164
|
+
SELECT visitor_id, product_name, predicted_view_count_confidence
|
|
165
|
+
FROM ML.RECOMMEND(
|
|
166
|
+
MODEL `{PROJECT_ID}.{DATASET_ID}.matrix_factorization_ga`,
|
|
167
|
+
(SELECT '9790296909741758431' AS visitor_id)
|
|
168
|
+
)
|
|
169
|
+
ORDER BY predicted_view_count_confidence DESC
|
|
170
|
+
LIMIT 5
|
|
171
|
+
"""
|
|
172
|
+
client.query(query).to_dataframe()
|
|
173
|
+
```
|
|
174
|
+
|
|
175
|
+
---
|
|
176
|
+
## Step 4 — Recommend users for an item with `ML.RECOMMEND`
|
|
177
|
+
|
|
178
|
+
Passing only the item column scores every user for that item — the symmetric counterpart to Step 3.
|
|
179
|
+
|
|
180
|
+
```python
|
|
181
|
+
query = f"""
|
|
182
|
+
SELECT visitor_id, product_name, predicted_view_count_confidence
|
|
183
|
+
FROM ML.RECOMMEND(
|
|
184
|
+
MODEL `{PROJECT_ID}.{DATASET_ID}.matrix_factorization_ga`,
|
|
185
|
+
(SELECT 'Google Youth Girl Hoodie' AS product_name)
|
|
186
|
+
)
|
|
187
|
+
ORDER BY predicted_view_count_confidence DESC
|
|
188
|
+
LIMIT 5
|
|
189
|
+
"""
|
|
190
|
+
client.query(query).to_dataframe()
|
|
191
|
+
```
|
|
192
|
+
|
|
193
|
+
---
|
|
194
|
+
## Step 5 — Inspect the learned factors with `ML.WEIGHTS`
|
|
195
|
+
|
|
196
|
+
One row per user AND per item (distinguished by `processed_input`), each with a `factor_weights` `ARRAY<STRUCT<factor, weight>>` (length = `num_factors`) and an `intercept` (per-user/per-item bias term).
|
|
197
|
+
|
|
198
|
+
> **GOTCHA (verified):** there is also exactly one extra row with `processed_input=NULL` and `feature='global__INTERCEPT__'` — the single global bias term shared across the whole model, separate from every user's/item's own intercept.
|
|
199
|
+
|
|
200
|
+
```python
|
|
201
|
+
query = f"""
|
|
202
|
+
SELECT processed_input, COUNT(*) AS n
|
|
203
|
+
FROM ML.WEIGHTS(MODEL `{PROJECT_ID}.{DATASET_ID}.matrix_factorization_ga`)
|
|
204
|
+
GROUP BY processed_input
|
|
205
|
+
"""
|
|
206
|
+
client.query(query).to_dataframe()
|
|
207
|
+
```
|
|
208
|
+
|
|
209
|
+
---
|
|
210
|
+
## Step 6 — Factor vectors as embeddings with `ML.GENERATE_EMBEDDING`
|
|
211
|
+
|
|
212
|
+
> **GOTCHA (verified):** for `MATRIX_FACTORIZATION`, `ML.GENERATE_EMBEDDING` takes **only the model** — no input table/query argument at all. Passing one errors immediately: `"Function ML.GENERATE_EMBEDDING for MATRIX_FACTORIZATION models only expects 1 argument but 2 were passed."` This is a genuinely different signature from `models/pca` (PCA)/`models/autoencoder` (Autoencoder) (both require a 2nd input-data argument) — it returns embeddings for **every** user and item in the model in one call, mirroring `ML.WEIGHTS`' `processed_input`/`feature` shape rather than `ML.PREDICT`'s.
|
|
213
|
+
>
|
|
214
|
+
> **Second GOTCHA (verified):** the embedding array has **`num_factors + 1` elements (17 here, not 16)** — one more than `ML.WEIGHTS`' `factor_weights` array for the same model (confirmed 16 elements there, matching `num_factors` exactly). The extra element is almost certainly the per-user/per-item `intercept` appended onto the raw factor vector. Don't assume the embedding array length always equals `num_factors` for this model type.
|
|
215
|
+
|
|
216
|
+
```python
|
|
217
|
+
query = f"""
|
|
218
|
+
SELECT processed_input, feature, ml_generate_embedding_result
|
|
219
|
+
FROM ML.GENERATE_EMBEDDING(MODEL `{PROJECT_ID}.{DATASET_ID}.matrix_factorization_ga`)
|
|
220
|
+
LIMIT 5
|
|
221
|
+
"""
|
|
222
|
+
client.query(query).to_dataframe()
|
|
223
|
+
```
|
|
224
|
+
|
|
225
|
+
---
|
|
226
|
+
## Step 7 — Item-item similarity with `VECTOR_SEARCH`
|
|
227
|
+
|
|
228
|
+
Materialize the item embeddings (Step 6's output filtered to `processed_input='product_name'`) into a real table — `VECTOR_SEARCH` doesn't accept `ML.GENERATE_EMBEDDING` output directly as its base-table argument, the same limitation already verified in `models/autoencoder` (Autoencoder).
|
|
229
|
+
|
|
230
|
+
```python
|
|
231
|
+
query = f"""
|
|
232
|
+
CREATE OR REPLACE TABLE `{PROJECT_ID}.{DATASET_ID}.matrix_factorization_item_embeddings` AS
|
|
233
|
+
SELECT feature AS product_name, ml_generate_embedding_result AS embedding
|
|
234
|
+
FROM ML.GENERATE_EMBEDDING(MODEL `{PROJECT_ID}.{DATASET_ID}.matrix_factorization_ga`)
|
|
235
|
+
WHERE processed_input = 'product_name'
|
|
236
|
+
"""
|
|
237
|
+
client.query(query).result()
|
|
238
|
+
print('Table matrix_factorization_item_embeddings created')
|
|
239
|
+
```
|
|
240
|
+
|
|
241
|
+
> **Verified:** querying "Google Youth Girl Hoodie" surfaces other youth apparel (youth tees, youth t-shirts) as its nearest neighbors — the learned item factors capture real product-category structure, not just noise.
|
|
242
|
+
|
|
243
|
+
```python
|
|
244
|
+
query = f"""
|
|
245
|
+
SELECT query.product_name AS query_product, base.product_name AS similar_product, distance
|
|
246
|
+
FROM VECTOR_SEARCH(
|
|
247
|
+
TABLE `{PROJECT_ID}.{DATASET_ID}.matrix_factorization_item_embeddings`, 'embedding',
|
|
248
|
+
(SELECT product_name, embedding
|
|
249
|
+
FROM `{PROJECT_ID}.{DATASET_ID}.matrix_factorization_item_embeddings`
|
|
250
|
+
WHERE product_name = 'Google Youth Girl Hoodie'),
|
|
251
|
+
top_k => 6,
|
|
252
|
+
distance_type => 'COSINE'
|
|
253
|
+
)
|
|
254
|
+
ORDER BY distance
|
|
255
|
+
"""
|
|
256
|
+
client.query(query).to_dataframe()
|
|
257
|
+
```
|
|
258
|
+
|
|
259
|
+
---
|
|
260
|
+
## Step 8 — Introspect the training columns with `ML.FEATURE_INFO`
|
|
261
|
+
|
|
262
|
+
```python
|
|
263
|
+
query = f"""
|
|
264
|
+
SELECT *
|
|
265
|
+
FROM ML.FEATURE_INFO(MODEL `{PROJECT_ID}.{DATASET_ID}.matrix_factorization_ga`)
|
|
266
|
+
"""
|
|
267
|
+
client.query(query).to_dataframe()
|
|
268
|
+
```
|
|
269
|
+
|
|
270
|
+
---
|
|
271
|
+
## Step 9 — Training loss curve with `ML.TRAINING_INFO`
|
|
272
|
+
|
|
273
|
+
> **Verified:** `eval_loss` is `NULL` throughout for this model — unlike the supervised model types in this project, `MATRIX_FACTORIZATION`'s `ML.TRAINING_INFO` doesn't populate a held-out `eval_loss` column here.
|
|
274
|
+
|
|
275
|
+
```python
|
|
276
|
+
query = f"""
|
|
277
|
+
SELECT *
|
|
278
|
+
FROM ML.TRAINING_INFO(MODEL `{PROJECT_ID}.{DATASET_ID}.matrix_factorization_ga`)
|
|
279
|
+
"""
|
|
280
|
+
client.query(query).to_dataframe()
|
|
281
|
+
```
|
|
282
|
+
|
|
283
|
+
---
|
|
284
|
+
## Step 10 — Hyperparameter tuning
|
|
285
|
+
|
|
286
|
+
Tunes `num_factors`, `l2_reg`, and `wals_alpha` (IMPLICIT-only) together, maximizing `mean_average_precision` (the IMPLICIT default objective).
|
|
287
|
+
|
|
288
|
+
> **Verified:** tuning improved on Step 2's baseline here — the best trial (`num_factors=26`, `l2_reg=2.88`, `wals_alpha=49.2`) reached `mean_average_precision=0.899`, versus this run's untuned baseline of `0.860`. A separate training of the identical baseline and tuning config reached different specific numbers (baseline `0.873`, best tuned trial `0.915` with `wals_alpha=30.8` instead) — consistent with Step 2's retraining-variance finding, the exact hyperparameter search results aren't bit-for-bit reproducible either. **The qualitative result held in both runs: tuning beat the untuned baseline.** Contrast with `models/autoencoder` (Autoencoder), where an equally-sized 4-trial search failed to beat its untuned baseline at all.
|
|
289
|
+
|
|
290
|
+
```python
|
|
291
|
+
query = f"""
|
|
292
|
+
CREATE OR REPLACE MODEL `{PROJECT_ID}.{DATASET_ID}.matrix_factorization_ga_tuned`
|
|
293
|
+
OPTIONS(
|
|
294
|
+
model_type = 'MATRIX_FACTORIZATION',
|
|
295
|
+
feedback_type = 'IMPLICIT',
|
|
296
|
+
user_col = 'visitor_id',
|
|
297
|
+
item_col = 'product_name',
|
|
298
|
+
rating_col = 'view_count',
|
|
299
|
+
num_factors = HPARAM_RANGE(8, 32),
|
|
300
|
+
l2_reg = HPARAM_RANGE(0.1, 10.0),
|
|
301
|
+
wals_alpha = HPARAM_RANGE(20.0, 60.0),
|
|
302
|
+
num_trials = 4,
|
|
303
|
+
max_parallel_trials = 2
|
|
304
|
+
) AS
|
|
305
|
+
SELECT
|
|
306
|
+
fullVisitorId AS visitor_id,
|
|
307
|
+
product.v2ProductName AS product_name,
|
|
308
|
+
COUNT(*) AS view_count
|
|
309
|
+
FROM `bigquery-public-data.google_analytics_sample.ga_sessions_*`,
|
|
310
|
+
UNNEST(hits) AS hits,
|
|
311
|
+
UNNEST(hits.product) AS product
|
|
312
|
+
WHERE _TABLE_SUFFIX BETWEEN '20170701' AND '20170801'
|
|
313
|
+
AND product.v2ProductName IS NOT NULL AND product.v2ProductName != '(not set)'
|
|
314
|
+
GROUP BY visitor_id, product_name
|
|
315
|
+
"""
|
|
316
|
+
client.query(query).result()
|
|
317
|
+
print('Tuned model created')
|
|
318
|
+
```
|
|
319
|
+
|
|
320
|
+
```python
|
|
321
|
+
query = f"""
|
|
322
|
+
SELECT
|
|
323
|
+
trial_id,
|
|
324
|
+
hyperparameters,
|
|
325
|
+
hparam_tuning_evaluation_metrics.mean_average_precision AS mean_average_precision,
|
|
326
|
+
is_optimal
|
|
327
|
+
FROM ML.TRIAL_INFO(MODEL `{PROJECT_ID}.{DATASET_ID}.matrix_factorization_ga_tuned`)
|
|
328
|
+
ORDER BY mean_average_precision DESC
|
|
329
|
+
"""
|
|
330
|
+
client.query(query).to_dataframe()
|
|
331
|
+
```
|
|
332
|
+
|
|
333
|
+
Calling `ML.EVALUATE` on a tuned model with no extra arguments returns every trial's metrics in one result (one row per `trial_id`) — there's no `STRUCT(trial_id)` 2-argument form for `ML.EVALUATE` the way `ML.RECOMMEND`/`ML.PREDICT` accept one; passing a `STRUCT` as a 2nd argument errors (`"argument 2 must be a relation"`).
|
|
334
|
+
|
|
335
|
+
```python
|
|
336
|
+
query = f"""
|
|
337
|
+
SELECT *
|
|
338
|
+
FROM ML.EVALUATE(MODEL `{PROJECT_ID}.{DATASET_ID}.matrix_factorization_ga_tuned`)
|
|
339
|
+
"""
|
|
340
|
+
client.query(query).to_dataframe()
|
|
341
|
+
```
|
|
342
|
+
|
|
343
|
+
---
|
|
344
|
+
## Examples — `%%bigquery` Magics
|
|
345
|
+
|
|
346
|
+
The same operations using IPython magic commands — write SQL directly in cells without Python string wrapping.
|
|
347
|
+
|
|
348
|
+
- `%%bigquery` — run SQL, display results inline
|
|
349
|
+
- `%%bigquery df` — run SQL, capture results into a pandas DataFrame
|
|
350
|
+
|
|
351
|
+
### Evaluate with `%%bigquery`
|
|
352
|
+
|
|
353
|
+
```sql
|
|
354
|
+
%%bigquery --project {PROJECT_ID}
|
|
355
|
+
|
|
356
|
+
SELECT *
|
|
357
|
+
FROM ML.EVALUATE(MODEL `statmike-mlops-349915.bq_ml.matrix_factorization_ga`)
|
|
358
|
+
```
|
|
359
|
+
|
|
360
|
+
---
|
|
361
|
+
## Examples — BigFrames
|
|
362
|
+
|
|
363
|
+
BigFrames provides a scikit-learn-style API (`bigframes.ml`) that trains BigQuery ML models under the hood. **Matrix factorization has a genuine first-class wrapper: `bigframes.ml.decomposition.MatrixFactorization`.**
|
|
364
|
+
|
|
365
|
+
> **GOTCHA (verified against the live installed constructor signature):** `MatrixFactorization(feedback_type=..., num_factors=..., user_col=..., item_col=..., rating_col=..., l2_reg=...)` — there is **no `wals_alpha` parameter**, unlike the SQL `CREATE MODEL` syntax. For `feedback_type='implicit'`, BigFrames trains with BigQuery ML's default `wals_alpha` (40) with no way to override it through this wrapper — use SQL directly if you need to set `wals_alpha`.
|
|
366
|
+
|
|
367
|
+
```python
|
|
368
|
+
import bigframes.pandas as bpd
|
|
369
|
+
|
|
370
|
+
bpd.close_session() # Reset session to apply project/location settings
|
|
371
|
+
bpd.options.bigquery.project = PROJECT_ID
|
|
372
|
+
bpd.options.bigquery.location = LOCATION
|
|
373
|
+
```
|
|
374
|
+
|
|
375
|
+
```python
|
|
376
|
+
from bigframes.ml.decomposition import MatrixFactorization
|
|
377
|
+
|
|
378
|
+
query = f"""
|
|
379
|
+
SELECT
|
|
380
|
+
fullVisitorId AS visitor_id,
|
|
381
|
+
product.v2ProductName AS product_name,
|
|
382
|
+
COUNT(*) AS view_count
|
|
383
|
+
FROM `bigquery-public-data.google_analytics_sample.ga_sessions_*`,
|
|
384
|
+
UNNEST(hits) AS hits,
|
|
385
|
+
UNNEST(hits.product) AS product
|
|
386
|
+
WHERE _TABLE_SUFFIX BETWEEN '20170701' AND '20170801'
|
|
387
|
+
AND product.v2ProductName IS NOT NULL AND product.v2ProductName != '(not set)'
|
|
388
|
+
GROUP BY visitor_id, product_name
|
|
389
|
+
"""
|
|
390
|
+
df = bpd.read_gbq(query)
|
|
391
|
+
|
|
392
|
+
# Train (creates a BigQuery ML model behind the scenes)
|
|
393
|
+
model = MatrixFactorization(
|
|
394
|
+
feedback_type='implicit',
|
|
395
|
+
num_factors=16,
|
|
396
|
+
user_col='visitor_id',
|
|
397
|
+
item_col='product_name',
|
|
398
|
+
rating_col='view_count',
|
|
399
|
+
)
|
|
400
|
+
model.fit(df)
|
|
401
|
+
|
|
402
|
+
# Evaluate
|
|
403
|
+
model.score(df).to_pandas()
|
|
404
|
+
```
|