bigquery-data-science 0.7.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- bigquery_data_science/__init__.py +58 -0
- bigquery_data_science/_skills/PLANS.md +76 -0
- bigquery_data_science/_skills/bigquery-ai/SKILL.md +66 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ai_agg.md +528 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ai_classify.md +450 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ai_count_tokens.md +180 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ai_detect_anomalies.md +322 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ai_embed.md +425 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ai_evaluate.md +368 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ai_forecast.md +437 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ai_generate.md +693 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ai_generate_bool.md +248 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ai_generate_double.md +254 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ai_generate_embedding.md +402 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ai_generate_int.md +252 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ai_generate_table.md +447 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ai_generate_text.md +356 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ai_if.md +373 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ai_key_drivers.md +331 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ai_parse_document.md +416 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ai_predict.md +559 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ai_score.md +310 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ai_search.md +465 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ai_similarity.md +371 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/catalog_search.md +951 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/content_analysis.md +284 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/content_moderation.md +369 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/data_enrichment.md +426 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/document_intelligence.md +470 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/document_rag.md +453 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/image_deduplication.md +770 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/log_analysis.md +668 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/metric_diagnostics.md +568 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ml_annotate_image.md +680 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ml_generate_embedding.md +217 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ml_generate_text.md +223 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ml_process_document.md +392 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ml_transcribe.md +815 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ml_translate.md +600 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/ml_understand_text.md +673 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/multimodal_analysis.md +266 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/rag_pipeline.md +593 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/semantic_search.md +510 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/tabular_prediction.md +485 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/time_series_intelligence.md +489 -0
- bigquery_data_science/_skills/bigquery-ai/narrative/vector_search.md +672 -0
- bigquery_data_science/_skills/bigquery-ai/reference/augmented-analytics.md +77 -0
- bigquery_data_science/_skills/bigquery-ai/reference/cloud-ai-service-models.md +88 -0
- bigquery_data_science/_skills/bigquery-ai/reference/document-processing.md +95 -0
- bigquery_data_science/_skills/bigquery-ai/reference/embeddings-and-search.md +166 -0
- bigquery_data_science/_skills/bigquery-ai/reference/general-purpose-generation.md +87 -0
- bigquery_data_science/_skills/bigquery-ai/reference/managed-functions.md +72 -0
- bigquery_data_science/_skills/bigquery-ai/reference/predictive-ai.md +113 -0
- bigquery_data_science/_skills/bigquery-ai/reference/workflows.md +56 -0
- bigquery_data_science/_skills/bigquery-ai/skill.manifest.json +109 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/INDEX.md +71 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/aiagg.md +67 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/aicausal_effect-covered-in-bigquery-ml.md +17 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/aiclassify.md +72 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/aicount_tokens.md +53 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/aidetect_anomalies.md +63 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/aiembed.md +85 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/aievaluate.md +114 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/aiforecast.md +123 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/aigenerate.md +69 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/aigenerate_bool.md +53 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/aigenerate_double.md +45 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/aigenerate_embedding.md +108 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/aigenerate_int.md +45 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/aigenerate_table.md +64 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/aigenerate_text.md +167 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/aiif.md +56 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/aikey_drivers.md +76 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/aiparse_document.md +91 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/aipredict.md +74 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/aiscore.md +52 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/aisearch.md +66 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/aisimilarity.md +79 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/choosing-between-these-and-the-ai-functions.md +11 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/hybrid-search-capability.md +165 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/mlannotate_image.md +24 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/mlgenerate_embedding.md +34 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/mlgenerate_text.md +65 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/mlprocess_document.md +125 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/mltranscribe.md +26 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/mltranslate.md +27 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/mlunderstand_text.md +32 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/multimodal-input-patterns-by-function.md +128 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/object-tables.md +109 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/objectref-and-objectrefruntime-schema-reference.md +50 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/objfetch_metadata.md +49 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/objget_access_url.md +66 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/objget_read_url.md +40 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/objlist.md +34 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/objmake_ref.md +58 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/per-row-failures-and-the-retry-pattern.md +19 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/setup-apis-connection-roles.md +39 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/two-input-shapes.md +22 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/vector_search.md +78 -0
- bigquery_data_science/_skills/bigquery-ai/syntax/what-the-five-have-in-common.md +27 -0
- bigquery_data_science/_skills/bigquery-data-science/SKILL.md +69 -0
- bigquery_data_science/_skills/bigquery-data-science/reference/namespace-matrix.md +112 -0
- bigquery_data_science/_skills/bigquery-data-science/skill.manifest.json +13 -0
- bigquery_data_science/_skills/bigquery-ml/SKILL.md +70 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/airflow_with_kfp.md +379 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/anomaly_fraud_detection.md +195 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/arima_plus.md +635 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/arima_plus_xreg.md +453 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/autoencoder.md +470 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/automl_classifier.md +291 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/automl_regressor.md +274 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/boosted_tree_classifier.md +449 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/boosted_tree_regressor.md +400 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/bucketizing.md +201 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/causal_effect.md +773 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/churn_retention.md +271 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/cloud_workflows.md +370 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/composer_airflow.md +475 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/contribution_analysis.md +346 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/cross_validation.md +185 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/customer_segmentation.md +233 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/dataform.md +372 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/dbt.md +283 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/difference_in_differences.md +280 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/distance.md +195 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/dnn_classifier.md +326 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/dnn_regressor.md +308 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/embeddings_classification.md +426 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/encoding.md +238 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/ensembling.md +278 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/evaluation.md +445 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/exploration.md +779 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/export.md +321 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/feature_engineering.md +280 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/feature_store.md +652 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/feature_store_workflow.md +426 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/ga4_churn_prediction.md +285 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/hierarchical_forecasting.md +441 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/image.md +791 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/imported.md +426 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/kmeans.md +348 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/linear_regression.md +372 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/logistic_regression.md +390 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/matrix_factorization.md +404 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/model_monitoring.md +609 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/pca.md +357 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/price_elasticity_dml.md +286 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/propensity_score_matching.md +350 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/random_forest_classifier.md +445 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/random_forest_regressor.md +409 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/recommendation.md +281 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/regression_based_forecasting.md +491 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/remote.md +428 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/scalers.md +325 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/scheduled_queries.md +284 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/sql_scripting.md +243 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/survival_analysis.md +363 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/synthetic_control.md +221 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/text.md +215 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/time_series.md +669 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/transform_only.md +273 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/uplift_cate.md +241 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/vertex_kfp.md +360 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/wide_and_deep_classifier.md +316 -0
- bigquery_data_science/_skills/bigquery-ml/narrative/wide_and_deep_regressor.md +298 -0
- bigquery_data_science/_skills/bigquery-ml/reference/classification.md +80 -0
- bigquery_data_science/_skills/bigquery-ml/reference/model-management.md +90 -0
- bigquery_data_science/_skills/bigquery-ml/reference/preprocessing-functions.md +234 -0
- bigquery_data_science/_skills/bigquery-ml/reference/regression.md +79 -0
- bigquery_data_science/_skills/bigquery-ml/reference/unsupervised-and-specialized.md +101 -0
- bigquery_data_science/_skills/bigquery-ml/reference/workflows-and-pipelines.md +111 -0
- bigquery_data_science/_skills/bigquery-ml/skill.manifest.json +177 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/1-model_type-catalog.md +28 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/2-task-mlevaluate-metric-columns.md +26 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/3-capability-matrix-per-model-type.md +21 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/4-explainability-weights-functions.md +14 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/5-connection-matrix.md +11 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/6-which-model-free-function-no-create-model-no-connection.md +15 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/7-point-in-time-feature-retrieval-which-of-the-two.md +11 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/INDEX.md +111 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/aicausal_effect.md +85 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/arima_plus-forecasting-lifecycle-functions.md +14 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/arima_plus-univariate-time-series-forecasting.md +132 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/arima_plus_xreg-multivariate-time-series-with-external-regressors.md +103 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/autoencoder.md +102 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/automl_classifier-automl_regressor.md +88 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/boosted_tree_classifier-boosted_tree_regressor.md +117 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/contribution_analysis.md +113 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/dnn_classifier-dnn_regressor.md +107 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/dnn_linear_combined_classifier-dnn_linear_combined_regressor-wide-and-deep.md +98 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/explainability-functions-mlexplain_predict-mlglobal_explain-mlfeature_importance.md +7 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/exploratory-data-analysis-mldescribe_data-mlcorrelation.md +11 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/export-model.md +92 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/feature-engineering-standalone-preprocessing-functions.md +13 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/image-preprocessing-functions-mldecode_image-mlresize_image-mlconvert_image_type-mlconvert_color_space.md +63 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/imported-models-tensorflow-tensorflow_lite-onnx-xgboost.md +36 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/kmeans.md +87 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/linear_reg-logistic_reg-generalized-linear-models.md +101 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/matrix_factorization.md +162 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mladvanced_weights.md +76 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlarima_coefficients.md +30 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlarima_evaluate.md +49 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlbag_of_words.md +47 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlbucketize.md +47 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlcentroids.md +46 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlconfusion_matrix.md +56 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlconvert_color_space.md +53 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlconvert_image_type.md +39 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlcorrelation.md +78 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mldecode_image.md +44 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mldescribe_data.md +74 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mldetect_anomalies.md +86 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mldistance.md +48 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlentity_features_at_time.md +56 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlevaluate.md +72 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlexplain_forecast.md +50 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlexplain_predict.md +66 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlfeature_cross.md +46 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlfeature_importance.md +47 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlfeature_info.md +50 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlfeatures_at_time.md +51 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlforecast.md +76 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlgenerate_embedding-from-a-bqml-pca-autoencoder-matrix_factorization-model.md +64 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlglobal_explain.md +63 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlhash_bucketize.md +47 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlholiday_info.md +46 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlimputer.md +46 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mllp_norm.md +47 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlmetrics.md +80 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlngrams.md +37 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlone_hot_encoder-mlmulti_hot_encoder-mllabel_encoder.md +67 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlpolynomial_expand.md +47 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlpredict.md +67 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlprincipal_component_info.md +42 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlprincipal_components.md +40 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlquantile_bucketize.md +43 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlrecommend.md +60 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlreconstruction_loss.md +48 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlresize_image.md +45 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlroc_curve.md +63 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mltf_idf.md +37 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mltfdv_describe.md +34 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mltfdv_validate.md +48 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mltraining_info.md +78 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mltransform-applying-a-saved-transform.md +52 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mltranspose-not-a-function-pointer.md +32 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mltrial_info.md +60 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlvalidate_data_drift.md +99 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlvalidate_data_skew.md +50 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/mlweights.md +84 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/model-free-time-series-decomposition-mltrend-mlseasonality-mldetect_change_points.md +100 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/model-introspection-mlfeature_info-mltraining_info-mltrial_info.md +7 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/model-monitoring-data-validation.md +23 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/numerical-scalers-mlstandard_scaler-mlmin_max_scaler-mlmax_abs_scaler-mlrobust_scaler-mlnormalizer.md +77 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/onnx.md +55 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/pca.md +91 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/point-in-time-feature-retrieval-mlfeatures_at_time-mlentity_features_at_time.md +25 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/quick-comparison.md +14 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/random_forest_classifier-random_forest_regressor.md +123 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/remote-remote-model-over-a-vertex-ai-endpoint-the-mechanism.md +87 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/tensorflow.md +48 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/tensorflow_lite.md +29 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/text-preprocessing-functions-mlngrams-mltf_idf-mlbag_of_words.md +18 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/timesfm-built-in-foundation-forecaster-see-bigquery-ai.md +28 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/transform_only.md +91 -0
- bigquery_data_science/_skills/bigquery-ml/syntax/xgboost.md +72 -0
- bigquery_data_science/_skills/pinned-models.json +28 -0
- bigquery_data_science/catalog.py +1107 -0
- bigquery_data_science/cli.py +786 -0
- bigquery_data_science/installer.py +282 -0
- bigquery_data_science/mcp_server.py +317 -0
- bigquery_data_science/tooling/__init__.py +15 -0
- bigquery_data_science/tooling/convert_notebook.py +176 -0
- bigquery_data_science/tooling/link_policy.py +247 -0
- bigquery_data_science/tooling/manifest.py +152 -0
- bigquery_data_science/tooling/markdown_anchors.py +84 -0
- bigquery_data_science/tooling/narrative_drift.py +163 -0
- bigquery_data_science/tooling/pinned_models.py +216 -0
- bigquery_data_science/tooling/reference_structure.py +85 -0
- bigquery_data_science/tooling/sync_syntax.py +407 -0
- bigquery_data_science/tooling/validate.py +208 -0
- bigquery_data_science-0.7.0.dist-info/METADATA +306 -0
- bigquery_data_science-0.7.0.dist-info/RECORD +286 -0
- bigquery_data_science-0.7.0.dist-info/WHEEL +4 -0
- bigquery_data_science-0.7.0.dist-info/entry_points.txt +5 -0
- bigquery_data_science-0.7.0.dist-info/licenses/LICENSE +201 -0
|
@@ -0,0 +1,348 @@
|
|
|
1
|
+
> **Source (`statmike/bigquery-data-science`):** **Notebook:** [`bigquery-ml/models/kmeans/kmeans.ipynb`](https://github.com/statmike/bigquery-data-science/blob/main/bigquery-ml/models/kmeans/kmeans.ipynb) · **SQL:** [`bigquery-ml/models/kmeans/kmeans.sql`](https://github.com/statmike/bigquery-data-science/blob/main/bigquery-ml/models/kmeans/kmeans.sql)
|
|
2
|
+
|
|
3
|
+
# K-Means — BigQuery ML
|
|
4
|
+
|
|
5
|
+
Train an **unsupervised clustering** model with `CREATE MODEL` (model_type = `KMEANS`) — no label column, just `k` groups discovered from the data itself — then walk the full model lifecycle: evaluate, predict, visualize, inspect centroids, detect anomalies, apply in-model preprocessing, and tune hyperparameters. No data leaves BigQuery, and no separate training infrastructure is needed.
|
|
6
|
+
|
|
7
|
+
**Lifecycle:** `CREATE MODEL` → `ML.EVALUATE` → `ML.PREDICT` → visualize → `ML.CENTROIDS` → `ML.FEATURE_INFO` → `ML.DETECT_ANOMALIES` → `TRANSFORM` clause → hyperparameter tuning (`ML.TRIAL_INFO`)
|
|
8
|
+
|
|
9
|
+
**This is the first unsupervised model in this project — a few things are genuinely different from every classifier/regressor notebook so far:**
|
|
10
|
+
- No `input_label_cols` — the model has no target to predict, only groups to discover.
|
|
11
|
+
- `ML.EVALUATE`'s metrics (`davies_bouldin_index`, `mean_squared_distance`) measure internal cluster quality, **not** agreement with any ground truth — there is no `roc_auc`/`r2_score` equivalent.
|
|
12
|
+
- To sanity-check whether the clusters mean anything, this notebook carries the real `species` column through `ML.PREDICT` purely as an **external check** — it's never used to train the model.
|
|
13
|
+
- **Verified: K-means retraining is genuinely non-deterministic here**, even with `kmeans_init_method = 'KMEANS++'` — `davies_bouldin_index` and the specific cluster-to-species alignment both vary across separate `CREATE OR REPLACE MODEL` calls on identical SQL (see Step 8 for the full story).
|
|
14
|
+
|
|
15
|
+
**When to use K-means:**
|
|
16
|
+
- Customer/market segmentation and grouping unlabeled records
|
|
17
|
+
- Exploratory analysis to discover natural structure in data
|
|
18
|
+
- Unsupervised anomaly/outlier detection (distance-to-centroid based, via `ML.DETECT_ANOMALIES`)
|
|
19
|
+
- Fast, in-SQL clustering at BigQuery scale with no labels required
|
|
20
|
+
|
|
21
|
+
**Data:** [`bigquery-public-data.ml_datasets.penguins`](https://console.cloud.google.com/marketplace/product/bigquery-public-datasets) — cluster on four physical measurements (`culmen_length_mm`, `culmen_depth_mm`, `flipper_length_mm`, `body_mass_g`). Same dataset as `models/linear_regression` (Linear Regression), `models/boosted_tree_regressor` (Boosted Tree Regressor), `models/random_forest_regressor` (Random Forest Regressor), `models/dnn_regressor` (DNN Regressor), and `models/wide_and_deep_regressor` (Wide & Deep Regressor), but used completely differently here — no label, and `species`/`island`/`sex` are not training features (only `species` is used afterward, as an external check). 342 of 344 rows remain after filtering `body_mass_g IS NOT NULL` — a different row count than the regression notebooks, which also filter on `sex`.
|
|
22
|
+
|
|
23
|
+
**References:** `RESOURCES.md` (Full reference) | [CREATE MODEL (K-means) docs](https://cloud.google.com/bigquery/docs/reference/standard-sql/bigqueryml-syntax-create-kmeans) | `setup` (Setup guide)
|
|
24
|
+
|
|
25
|
+
---
|
|
26
|
+
## Setup
|
|
27
|
+
|
|
28
|
+
Set your project and location, authenticate, and create a shared dataset for the model.
|
|
29
|
+
|
|
30
|
+
> K-means trains on data already in BigQuery — no connection or remote model required. The model is created with `CREATE MODEL` and stored in your dataset. See the `setup` (Setup Reference) for details.
|
|
31
|
+
|
|
32
|
+
```python
|
|
33
|
+
PROJECT_ID = 'statmike-mlops-349915' # <-- Replace with your project ID
|
|
34
|
+
LOCATION = 'US' # BigQuery dataset location
|
|
35
|
+
DATASET_ID = 'bq_ml' # Shared dataset across all bigquery-ml notebooks
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
### Environment
|
|
39
|
+
|
|
40
|
+
> **Already set up the project environment?** The cell below is a no-op — packages are already in your kernel. See the `setup` (Setup Reference) for details.
|
|
41
|
+
>
|
|
42
|
+
> **Running standalone** (Colab, Colab Enterprise, Vertex AI Workbench)? The cell below installs required packages into your current kernel.
|
|
43
|
+
|
|
44
|
+
```python
|
|
45
|
+
from google.cloud import bigquery
|
|
46
|
+
import pandas as pd
|
|
47
|
+
|
|
48
|
+
client = bigquery.Client(project=PROJECT_ID)
|
|
49
|
+
pd.set_option('display.max_colwidth', None)
|
|
50
|
+
|
|
51
|
+
# Create the shared dataset (idempotent)
|
|
52
|
+
dataset_ref = bigquery.DatasetReference(PROJECT_ID, DATASET_ID)
|
|
53
|
+
dataset = bigquery.Dataset(dataset_ref)
|
|
54
|
+
dataset.location = LOCATION
|
|
55
|
+
client.create_dataset(dataset, exists_ok=True)
|
|
56
|
+
print(f'Dataset {PROJECT_ID}.{DATASET_ID} ready')
|
|
57
|
+
|
|
58
|
+
# Register %%bigquery cell magic (auto-loaded in Colab, needed elsewhere)
|
|
59
|
+
%load_ext bigquery_magics
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
---
|
|
63
|
+
## Step 1 — Create the model with `CREATE MODEL`
|
|
64
|
+
|
|
65
|
+
`CREATE MODEL` trains and stores the model in your dataset. There's no `input_label_cols` — every selected column is a clustering feature. `num_clusters = 3` is a deliberate, informed choice (there are 3 penguin species in this data), not a default guess. `kmeans_init_method = 'KMEANS++'` gives more stable, generally better convergence than the `RANDOM` default — though see Step 9: even with `KMEANS++`, retraining is not fully deterministic. `standardize_features` defaults to `TRUE` — important here since `body_mass_g` (thousands of grams) and `culmen_depth_mm` (tens of mm) are on very different scales.
|
|
66
|
+
|
|
67
|
+
```python
|
|
68
|
+
query = f"""
|
|
69
|
+
CREATE OR REPLACE MODEL `{PROJECT_ID}.{DATASET_ID}.kmeans_penguins`
|
|
70
|
+
OPTIONS(
|
|
71
|
+
model_type = 'KMEANS',
|
|
72
|
+
num_clusters = 3,
|
|
73
|
+
kmeans_init_method = 'KMEANS++',
|
|
74
|
+
distance_type = 'EUCLIDEAN'
|
|
75
|
+
) AS
|
|
76
|
+
SELECT culmen_length_mm, culmen_depth_mm, flipper_length_mm, body_mass_g
|
|
77
|
+
FROM `bigquery-public-data.ml_datasets.penguins`
|
|
78
|
+
WHERE body_mass_g IS NOT NULL
|
|
79
|
+
"""
|
|
80
|
+
client.query(query).result()
|
|
81
|
+
print('Model kmeans_penguins created')
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
---
|
|
85
|
+
## Step 2 — Evaluate with `ML.EVALUATE`
|
|
86
|
+
|
|
87
|
+
`davies_bouldin_index` (lower is better — more separated clusters) and `mean_squared_distance` (lower is better — tighter clusters). Neither metric uses or needs a label — this is intrinsic cluster quality, not agreement with any ground truth.
|
|
88
|
+
|
|
89
|
+
```python
|
|
90
|
+
query = f"""
|
|
91
|
+
SELECT *
|
|
92
|
+
FROM ML.EVALUATE(MODEL `{PROJECT_ID}.{DATASET_ID}.kmeans_penguins`)
|
|
93
|
+
"""
|
|
94
|
+
client.query(query).to_dataframe()
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
---
|
|
98
|
+
## Step 3 — Predict with `ML.PREDICT`
|
|
99
|
+
|
|
100
|
+
`ML.PREDICT` returns `CENTROID_ID` (the assigned cluster) and `NEAREST_CENTROIDS_DISTANCE` (distance to every centroid). `species` is passed through untouched — it was never a training feature — purely so we can check, as an external validation, how well the unsupervised clusters line up with the real species labels.
|
|
101
|
+
|
|
102
|
+
> **Result below, honestly reported as-is:** one cluster recovers Gentoo penguins almost perfectly, while the other two mix Adelie and Chinstrap — those two species overlap more in body measurements than either does with Gentoo. Unsupervised clustering finds real structure here, but not a perfect species partition. **Caveat (see Step 8): K-means retraining is non-deterministic**, so re-running this exact notebook can shift which species end up cleanly separated and which overlap — don't treat this specific split as guaranteed to reproduce bit-for-bit.
|
|
103
|
+
|
|
104
|
+
```python
|
|
105
|
+
query = f"""
|
|
106
|
+
SELECT species, CENTROID_ID, COUNT(*) AS n
|
|
107
|
+
FROM ML.PREDICT(
|
|
108
|
+
MODEL `{PROJECT_ID}.{DATASET_ID}.kmeans_penguins`,
|
|
109
|
+
(SELECT species, culmen_length_mm, culmen_depth_mm, flipper_length_mm, body_mass_g
|
|
110
|
+
FROM `bigquery-public-data.ml_datasets.penguins`
|
|
111
|
+
WHERE body_mass_g IS NOT NULL)
|
|
112
|
+
)
|
|
113
|
+
GROUP BY species, CENTROID_ID
|
|
114
|
+
ORDER BY species, CENTROID_ID
|
|
115
|
+
"""
|
|
116
|
+
client.query(query).to_dataframe()
|
|
117
|
+
```
|
|
118
|
+
|
|
119
|
+
---
|
|
120
|
+
## Step 4 — Visualize the clusters
|
|
121
|
+
|
|
122
|
+
Plot two of the clustering features against each other, colored by the *predicted* cluster and shaped by the *actual* species — a quick visual read on where the clusters agree and disagree with species.
|
|
123
|
+
|
|
124
|
+
```python
|
|
125
|
+
import matplotlib.pyplot as plt
|
|
126
|
+
|
|
127
|
+
query = f"""
|
|
128
|
+
SELECT species, CENTROID_ID, flipper_length_mm, body_mass_g
|
|
129
|
+
FROM ML.PREDICT(
|
|
130
|
+
MODEL `{PROJECT_ID}.{DATASET_ID}.kmeans_penguins`,
|
|
131
|
+
(SELECT species, culmen_length_mm, culmen_depth_mm, flipper_length_mm, body_mass_g
|
|
132
|
+
FROM `bigquery-public-data.ml_datasets.penguins`
|
|
133
|
+
WHERE body_mass_g IS NOT NULL)
|
|
134
|
+
)
|
|
135
|
+
"""
|
|
136
|
+
df = client.query(query).to_dataframe()
|
|
137
|
+
|
|
138
|
+
cluster_colors = {1: '#4285F4', 2: '#EA4335', 3: '#34A853'}
|
|
139
|
+
species_markers = dict(zip(sorted(df['species'].unique()), ['o', '^', 's']))
|
|
140
|
+
|
|
141
|
+
fig, ax = plt.subplots(figsize=(7, 6))
|
|
142
|
+
for species, marker in species_markers.items():
|
|
143
|
+
subset = df[df['species'] == species]
|
|
144
|
+
ax.scatter(
|
|
145
|
+
subset['flipper_length_mm'], subset['body_mass_g'],
|
|
146
|
+
c=[cluster_colors[c] for c in subset['CENTROID_ID']],
|
|
147
|
+
marker=marker, label=species, edgecolors='k', linewidths=0.3, s=50
|
|
148
|
+
)
|
|
149
|
+
ax.set_xlabel('Flipper Length (mm)')
|
|
150
|
+
ax.set_ylabel('Body Mass (g)')
|
|
151
|
+
ax.set_title('K-Means Clusters (color) vs Actual Species (marker shape)')
|
|
152
|
+
ax.legend(loc='upper left', fontsize=8)
|
|
153
|
+
plt.tight_layout()
|
|
154
|
+
plt.show()
|
|
155
|
+
```
|
|
156
|
+
|
|
157
|
+
---
|
|
158
|
+
## Step 5 — Inspect cluster centers with `ML.CENTROIDS`
|
|
159
|
+
|
|
160
|
+
One row per (`centroid_id`, `feature`) with the centroid's coordinate on that feature — this is how you interpret what each cluster "means" (e.g. the cluster with the highest `body_mass_g` and `flipper_length_mm` is consistent with it being the Gentoo cluster, the largest of the three species).
|
|
161
|
+
|
|
162
|
+
```python
|
|
163
|
+
query = f"""
|
|
164
|
+
SELECT *
|
|
165
|
+
FROM ML.CENTROIDS(MODEL `{PROJECT_ID}.{DATASET_ID}.kmeans_penguins`)
|
|
166
|
+
ORDER BY centroid_id, feature
|
|
167
|
+
"""
|
|
168
|
+
client.query(query).to_dataframe()
|
|
169
|
+
```
|
|
170
|
+
|
|
171
|
+
---
|
|
172
|
+
## Step 6 — Introspect the training features with `ML.FEATURE_INFO`
|
|
173
|
+
|
|
174
|
+
```python
|
|
175
|
+
query = f"""
|
|
176
|
+
SELECT *
|
|
177
|
+
FROM ML.FEATURE_INFO(MODEL `{PROJECT_ID}.{DATASET_ID}.kmeans_penguins`)
|
|
178
|
+
"""
|
|
179
|
+
client.query(query).to_dataframe()
|
|
180
|
+
```
|
|
181
|
+
|
|
182
|
+
---
|
|
183
|
+
## Step 7 — Detect outliers with `ML.DETECT_ANOMALIES`
|
|
184
|
+
|
|
185
|
+
`contamination = 0.05` flags the 5% of rows with the largest `normalized_distance` from their nearest centroid as anomalies — no labels required.
|
|
186
|
+
|
|
187
|
+
> **GOTCHA (verified):** for `KMEANS`, the input-data argument is **required** — calling this with only `(MODEL, STRUCT(contamination))` errors immediately: `"DETECT_ANOMALIES expects 3 arguments for KMEANS models but 2 were passed."` Always pass the scoring data as the 3rd argument.
|
|
188
|
+
|
|
189
|
+
```python
|
|
190
|
+
query = f"""
|
|
191
|
+
SELECT *
|
|
192
|
+
FROM ML.DETECT_ANOMALIES(
|
|
193
|
+
MODEL `{PROJECT_ID}.{DATASET_ID}.kmeans_penguins`,
|
|
194
|
+
STRUCT(0.05 AS contamination),
|
|
195
|
+
(SELECT culmen_length_mm, culmen_depth_mm, flipper_length_mm, body_mass_g
|
|
196
|
+
FROM `bigquery-public-data.ml_datasets.penguins`
|
|
197
|
+
WHERE body_mass_g IS NOT NULL)
|
|
198
|
+
)
|
|
199
|
+
WHERE is_anomaly
|
|
200
|
+
ORDER BY normalized_distance DESC
|
|
201
|
+
"""
|
|
202
|
+
client.query(query).to_dataframe()
|
|
203
|
+
```
|
|
204
|
+
|
|
205
|
+
---
|
|
206
|
+
## Step 8 — In-model preprocessing with the `TRANSFORM` clause
|
|
207
|
+
|
|
208
|
+
Adds `island` as a categorical feature alongside the four (explicitly scaled) numeric measurements. `ML.STANDARD_SCALER` here is redundant with the model's own `standardize_features = TRUE` default, but shown for clarity.
|
|
209
|
+
|
|
210
|
+
> **GOTCHA (verified):** K-means retraining is genuinely non-deterministic here, even with `kmeans_init_method = 'KMEANS++'`. Retraining both the baseline (Step 1) and this model multiple times on identical SQL shows `davies_bouldin_index` and the specific cluster-to-species alignment both vary meaningfully across separate `CREATE OR REPLACE MODEL` calls on the exact same SQL — sometimes the baseline's `davies_bouldin_index` is lower, sometimes this transform's is; sometimes Gentoo ends up in one clean cluster, sometimes it's split, regardless of whether `island` was included (the BigFrames section below independently confirms this too — same config, a third different `davies_bouldin_index`). **This means a single before/after comparison is not reliable evidence that a specific feature change causes a specific effect** on either the intrinsic metric or external alignment — you'd need to retrain each config multiple times and look at the range, not one sample. Separately: `davies_bouldin_index` measures internal cluster separation in whatever feature space you give it — it says nothing about whether clusters line up with any domain-meaningful grouping, so don't treat a lower value alone as proof of a "better" or "more useful" clustering.
|
|
211
|
+
|
|
212
|
+
```python
|
|
213
|
+
query = f"""
|
|
214
|
+
CREATE OR REPLACE MODEL `{PROJECT_ID}.{DATASET_ID}.kmeans_penguins_transform`
|
|
215
|
+
TRANSFORM(
|
|
216
|
+
ML.STANDARD_SCALER(culmen_length_mm) OVER() AS culmen_length_mm,
|
|
217
|
+
ML.STANDARD_SCALER(culmen_depth_mm) OVER() AS culmen_depth_mm,
|
|
218
|
+
ML.STANDARD_SCALER(flipper_length_mm) OVER() AS flipper_length_mm,
|
|
219
|
+
ML.STANDARD_SCALER(body_mass_g) OVER() AS body_mass_g,
|
|
220
|
+
island
|
|
221
|
+
)
|
|
222
|
+
OPTIONS(
|
|
223
|
+
model_type = 'KMEANS',
|
|
224
|
+
num_clusters = 3,
|
|
225
|
+
kmeans_init_method = 'KMEANS++',
|
|
226
|
+
distance_type = 'EUCLIDEAN'
|
|
227
|
+
) AS
|
|
228
|
+
SELECT culmen_length_mm, culmen_depth_mm, flipper_length_mm, body_mass_g, island
|
|
229
|
+
FROM `bigquery-public-data.ml_datasets.penguins`
|
|
230
|
+
WHERE body_mass_g IS NOT NULL
|
|
231
|
+
"""
|
|
232
|
+
client.query(query).result()
|
|
233
|
+
print('Model kmeans_penguins_transform created')
|
|
234
|
+
```
|
|
235
|
+
|
|
236
|
+
```python
|
|
237
|
+
# Confirm with ML.EVALUATE -- see the markdown above for why comparing
|
|
238
|
+
# this number to Step 2's isn't as simple as "higher/lower is the effect
|
|
239
|
+
# of adding island."
|
|
240
|
+
query = f"""
|
|
241
|
+
SELECT *
|
|
242
|
+
FROM ML.EVALUATE(MODEL `{PROJECT_ID}.{DATASET_ID}.kmeans_penguins_transform`)
|
|
243
|
+
"""
|
|
244
|
+
client.query(query).to_dataframe()
|
|
245
|
+
```
|
|
246
|
+
|
|
247
|
+
```python
|
|
248
|
+
query = f"""
|
|
249
|
+
SELECT species, CENTROID_ID, COUNT(*) AS n
|
|
250
|
+
FROM ML.PREDICT(
|
|
251
|
+
MODEL `{PROJECT_ID}.{DATASET_ID}.kmeans_penguins_transform`,
|
|
252
|
+
(SELECT species, culmen_length_mm, culmen_depth_mm, flipper_length_mm, body_mass_g, island
|
|
253
|
+
FROM `bigquery-public-data.ml_datasets.penguins`
|
|
254
|
+
WHERE body_mass_g IS NOT NULL)
|
|
255
|
+
)
|
|
256
|
+
GROUP BY species, CENTROID_ID
|
|
257
|
+
ORDER BY species, CENTROID_ID
|
|
258
|
+
"""
|
|
259
|
+
client.query(query).to_dataframe()
|
|
260
|
+
```
|
|
261
|
+
|
|
262
|
+
---
|
|
263
|
+
## Step 9 — Hyperparameter tuning
|
|
264
|
+
|
|
265
|
+
`num_clusters` is the primary tunable option, searched via `HPARAM_RANGE` with the default `davies_bouldin_index` objective (minimize).
|
|
266
|
+
|
|
267
|
+
> **Result below, honestly reported as-is — and another instance of the non-determinism from Step 8:** this run's tuning selected `num_clusters=2` as optimal, not 3. A separate pre-validation run of this exact same search space selected `num_clusters=3` (matching the true species count) instead. Both are real, both are "correct" outputs of the identical tuning configuration — they just landed on different local optima. This reinforces the same lesson: don't treat a single HP-tuning run's chosen value as the definitive answer for `num_clusters` on this kind of data; if the choice matters, run the search more than once (or with more trials) and look for a value that keeps winning, not just whichever won once.
|
|
268
|
+
|
|
269
|
+
```python
|
|
270
|
+
query = f"""
|
|
271
|
+
CREATE OR REPLACE MODEL `{PROJECT_ID}.{DATASET_ID}.kmeans_penguins_tuned`
|
|
272
|
+
OPTIONS(
|
|
273
|
+
model_type = 'KMEANS',
|
|
274
|
+
num_clusters = HPARAM_RANGE(2, 10),
|
|
275
|
+
kmeans_init_method = 'KMEANS++',
|
|
276
|
+
distance_type = 'EUCLIDEAN',
|
|
277
|
+
num_trials = 6,
|
|
278
|
+
max_parallel_trials = 3,
|
|
279
|
+
hparam_tuning_objectives = ['davies_bouldin_index']
|
|
280
|
+
) AS
|
|
281
|
+
SELECT culmen_length_mm, culmen_depth_mm, flipper_length_mm, body_mass_g
|
|
282
|
+
FROM `bigquery-public-data.ml_datasets.penguins`
|
|
283
|
+
WHERE body_mass_g IS NOT NULL
|
|
284
|
+
"""
|
|
285
|
+
client.query(query).result()
|
|
286
|
+
print('Tuned model created')
|
|
287
|
+
```
|
|
288
|
+
|
|
289
|
+
```python
|
|
290
|
+
query = f"""
|
|
291
|
+
SELECT
|
|
292
|
+
trial_id,
|
|
293
|
+
hyperparameters,
|
|
294
|
+
hparam_tuning_evaluation_metrics.davies_bouldin_index AS davies_bouldin_index,
|
|
295
|
+
is_optimal
|
|
296
|
+
FROM ML.TRIAL_INFO(MODEL `{PROJECT_ID}.{DATASET_ID}.kmeans_penguins_tuned`)
|
|
297
|
+
ORDER BY davies_bouldin_index ASC
|
|
298
|
+
"""
|
|
299
|
+
client.query(query).to_dataframe()
|
|
300
|
+
```
|
|
301
|
+
|
|
302
|
+
---
|
|
303
|
+
## Examples — `%%bigquery` Magics
|
|
304
|
+
|
|
305
|
+
The same operations using IPython magic commands — write SQL directly in cells without Python string wrapping.
|
|
306
|
+
|
|
307
|
+
- `%%bigquery` — run SQL, display results inline
|
|
308
|
+
- `%%bigquery df` — run SQL, capture results into a pandas DataFrame
|
|
309
|
+
|
|
310
|
+
### Evaluate with `%%bigquery`
|
|
311
|
+
|
|
312
|
+
```sql
|
|
313
|
+
%%bigquery --project {PROJECT_ID}
|
|
314
|
+
|
|
315
|
+
SELECT *
|
|
316
|
+
FROM ML.EVALUATE(MODEL `statmike-mlops-349915.bq_ml.kmeans_penguins`)
|
|
317
|
+
```
|
|
318
|
+
|
|
319
|
+
---
|
|
320
|
+
## Examples — BigFrames
|
|
321
|
+
|
|
322
|
+
BigFrames provides a scikit-learn-style API (`bigframes.ml`) that trains BigQuery ML models under the hood. Unlike DNN/wide-and-deep, **K-means has a genuine first-class wrapper**: `bigframes.ml.cluster.KMeans`.
|
|
323
|
+
|
|
324
|
+
```python
|
|
325
|
+
import bigframes.pandas as bpd
|
|
326
|
+
|
|
327
|
+
bpd.close_session() # Reset session to apply project/location settings
|
|
328
|
+
bpd.options.bigquery.project = PROJECT_ID
|
|
329
|
+
bpd.options.bigquery.location = LOCATION
|
|
330
|
+
```
|
|
331
|
+
|
|
332
|
+
```python
|
|
333
|
+
from bigframes.ml.cluster import KMeans
|
|
334
|
+
|
|
335
|
+
# Load data as a BigFrames DataFrame
|
|
336
|
+
df = bpd.read_gbq('bigquery-public-data.ml_datasets.penguins')
|
|
337
|
+
df = df[df['body_mass_g'].notnull()]
|
|
338
|
+
|
|
339
|
+
feature_cols = ['culmen_length_mm', 'culmen_depth_mm', 'flipper_length_mm', 'body_mass_g']
|
|
340
|
+
X = df[feature_cols]
|
|
341
|
+
|
|
342
|
+
# Train (creates a BigQuery ML model behind the scenes)
|
|
343
|
+
model = KMeans(n_clusters=3)
|
|
344
|
+
model.fit(X)
|
|
345
|
+
|
|
346
|
+
# Evaluate
|
|
347
|
+
model.score(X).to_pandas()
|
|
348
|
+
```
|