bigquery-data-science 0.7.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (286) hide show
  1. bigquery_data_science/__init__.py +58 -0
  2. bigquery_data_science/_skills/PLANS.md +76 -0
  3. bigquery_data_science/_skills/bigquery-ai/SKILL.md +66 -0
  4. bigquery_data_science/_skills/bigquery-ai/narrative/ai_agg.md +528 -0
  5. bigquery_data_science/_skills/bigquery-ai/narrative/ai_classify.md +450 -0
  6. bigquery_data_science/_skills/bigquery-ai/narrative/ai_count_tokens.md +180 -0
  7. bigquery_data_science/_skills/bigquery-ai/narrative/ai_detect_anomalies.md +322 -0
  8. bigquery_data_science/_skills/bigquery-ai/narrative/ai_embed.md +425 -0
  9. bigquery_data_science/_skills/bigquery-ai/narrative/ai_evaluate.md +368 -0
  10. bigquery_data_science/_skills/bigquery-ai/narrative/ai_forecast.md +437 -0
  11. bigquery_data_science/_skills/bigquery-ai/narrative/ai_generate.md +693 -0
  12. bigquery_data_science/_skills/bigquery-ai/narrative/ai_generate_bool.md +248 -0
  13. bigquery_data_science/_skills/bigquery-ai/narrative/ai_generate_double.md +254 -0
  14. bigquery_data_science/_skills/bigquery-ai/narrative/ai_generate_embedding.md +402 -0
  15. bigquery_data_science/_skills/bigquery-ai/narrative/ai_generate_int.md +252 -0
  16. bigquery_data_science/_skills/bigquery-ai/narrative/ai_generate_table.md +447 -0
  17. bigquery_data_science/_skills/bigquery-ai/narrative/ai_generate_text.md +356 -0
  18. bigquery_data_science/_skills/bigquery-ai/narrative/ai_if.md +373 -0
  19. bigquery_data_science/_skills/bigquery-ai/narrative/ai_key_drivers.md +331 -0
  20. bigquery_data_science/_skills/bigquery-ai/narrative/ai_parse_document.md +416 -0
  21. bigquery_data_science/_skills/bigquery-ai/narrative/ai_predict.md +559 -0
  22. bigquery_data_science/_skills/bigquery-ai/narrative/ai_score.md +310 -0
  23. bigquery_data_science/_skills/bigquery-ai/narrative/ai_search.md +465 -0
  24. bigquery_data_science/_skills/bigquery-ai/narrative/ai_similarity.md +371 -0
  25. bigquery_data_science/_skills/bigquery-ai/narrative/catalog_search.md +951 -0
  26. bigquery_data_science/_skills/bigquery-ai/narrative/content_analysis.md +284 -0
  27. bigquery_data_science/_skills/bigquery-ai/narrative/content_moderation.md +369 -0
  28. bigquery_data_science/_skills/bigquery-ai/narrative/data_enrichment.md +426 -0
  29. bigquery_data_science/_skills/bigquery-ai/narrative/document_intelligence.md +470 -0
  30. bigquery_data_science/_skills/bigquery-ai/narrative/document_rag.md +453 -0
  31. bigquery_data_science/_skills/bigquery-ai/narrative/image_deduplication.md +770 -0
  32. bigquery_data_science/_skills/bigquery-ai/narrative/log_analysis.md +668 -0
  33. bigquery_data_science/_skills/bigquery-ai/narrative/metric_diagnostics.md +568 -0
  34. bigquery_data_science/_skills/bigquery-ai/narrative/ml_annotate_image.md +680 -0
  35. bigquery_data_science/_skills/bigquery-ai/narrative/ml_generate_embedding.md +217 -0
  36. bigquery_data_science/_skills/bigquery-ai/narrative/ml_generate_text.md +223 -0
  37. bigquery_data_science/_skills/bigquery-ai/narrative/ml_process_document.md +392 -0
  38. bigquery_data_science/_skills/bigquery-ai/narrative/ml_transcribe.md +815 -0
  39. bigquery_data_science/_skills/bigquery-ai/narrative/ml_translate.md +600 -0
  40. bigquery_data_science/_skills/bigquery-ai/narrative/ml_understand_text.md +673 -0
  41. bigquery_data_science/_skills/bigquery-ai/narrative/multimodal_analysis.md +266 -0
  42. bigquery_data_science/_skills/bigquery-ai/narrative/rag_pipeline.md +593 -0
  43. bigquery_data_science/_skills/bigquery-ai/narrative/semantic_search.md +510 -0
  44. bigquery_data_science/_skills/bigquery-ai/narrative/tabular_prediction.md +485 -0
  45. bigquery_data_science/_skills/bigquery-ai/narrative/time_series_intelligence.md +489 -0
  46. bigquery_data_science/_skills/bigquery-ai/narrative/vector_search.md +672 -0
  47. bigquery_data_science/_skills/bigquery-ai/reference/augmented-analytics.md +77 -0
  48. bigquery_data_science/_skills/bigquery-ai/reference/cloud-ai-service-models.md +88 -0
  49. bigquery_data_science/_skills/bigquery-ai/reference/document-processing.md +95 -0
  50. bigquery_data_science/_skills/bigquery-ai/reference/embeddings-and-search.md +166 -0
  51. bigquery_data_science/_skills/bigquery-ai/reference/general-purpose-generation.md +87 -0
  52. bigquery_data_science/_skills/bigquery-ai/reference/managed-functions.md +72 -0
  53. bigquery_data_science/_skills/bigquery-ai/reference/predictive-ai.md +113 -0
  54. bigquery_data_science/_skills/bigquery-ai/reference/workflows.md +56 -0
  55. bigquery_data_science/_skills/bigquery-ai/skill.manifest.json +109 -0
  56. bigquery_data_science/_skills/bigquery-ai/syntax/INDEX.md +71 -0
  57. bigquery_data_science/_skills/bigquery-ai/syntax/aiagg.md +67 -0
  58. bigquery_data_science/_skills/bigquery-ai/syntax/aicausal_effect-covered-in-bigquery-ml.md +17 -0
  59. bigquery_data_science/_skills/bigquery-ai/syntax/aiclassify.md +72 -0
  60. bigquery_data_science/_skills/bigquery-ai/syntax/aicount_tokens.md +53 -0
  61. bigquery_data_science/_skills/bigquery-ai/syntax/aidetect_anomalies.md +63 -0
  62. bigquery_data_science/_skills/bigquery-ai/syntax/aiembed.md +85 -0
  63. bigquery_data_science/_skills/bigquery-ai/syntax/aievaluate.md +114 -0
  64. bigquery_data_science/_skills/bigquery-ai/syntax/aiforecast.md +123 -0
  65. bigquery_data_science/_skills/bigquery-ai/syntax/aigenerate.md +69 -0
  66. bigquery_data_science/_skills/bigquery-ai/syntax/aigenerate_bool.md +53 -0
  67. bigquery_data_science/_skills/bigquery-ai/syntax/aigenerate_double.md +45 -0
  68. bigquery_data_science/_skills/bigquery-ai/syntax/aigenerate_embedding.md +108 -0
  69. bigquery_data_science/_skills/bigquery-ai/syntax/aigenerate_int.md +45 -0
  70. bigquery_data_science/_skills/bigquery-ai/syntax/aigenerate_table.md +64 -0
  71. bigquery_data_science/_skills/bigquery-ai/syntax/aigenerate_text.md +167 -0
  72. bigquery_data_science/_skills/bigquery-ai/syntax/aiif.md +56 -0
  73. bigquery_data_science/_skills/bigquery-ai/syntax/aikey_drivers.md +76 -0
  74. bigquery_data_science/_skills/bigquery-ai/syntax/aiparse_document.md +91 -0
  75. bigquery_data_science/_skills/bigquery-ai/syntax/aipredict.md +74 -0
  76. bigquery_data_science/_skills/bigquery-ai/syntax/aiscore.md +52 -0
  77. bigquery_data_science/_skills/bigquery-ai/syntax/aisearch.md +66 -0
  78. bigquery_data_science/_skills/bigquery-ai/syntax/aisimilarity.md +79 -0
  79. bigquery_data_science/_skills/bigquery-ai/syntax/choosing-between-these-and-the-ai-functions.md +11 -0
  80. bigquery_data_science/_skills/bigquery-ai/syntax/hybrid-search-capability.md +165 -0
  81. bigquery_data_science/_skills/bigquery-ai/syntax/mlannotate_image.md +24 -0
  82. bigquery_data_science/_skills/bigquery-ai/syntax/mlgenerate_embedding.md +34 -0
  83. bigquery_data_science/_skills/bigquery-ai/syntax/mlgenerate_text.md +65 -0
  84. bigquery_data_science/_skills/bigquery-ai/syntax/mlprocess_document.md +125 -0
  85. bigquery_data_science/_skills/bigquery-ai/syntax/mltranscribe.md +26 -0
  86. bigquery_data_science/_skills/bigquery-ai/syntax/mltranslate.md +27 -0
  87. bigquery_data_science/_skills/bigquery-ai/syntax/mlunderstand_text.md +32 -0
  88. bigquery_data_science/_skills/bigquery-ai/syntax/multimodal-input-patterns-by-function.md +128 -0
  89. bigquery_data_science/_skills/bigquery-ai/syntax/object-tables.md +109 -0
  90. bigquery_data_science/_skills/bigquery-ai/syntax/objectref-and-objectrefruntime-schema-reference.md +50 -0
  91. bigquery_data_science/_skills/bigquery-ai/syntax/objfetch_metadata.md +49 -0
  92. bigquery_data_science/_skills/bigquery-ai/syntax/objget_access_url.md +66 -0
  93. bigquery_data_science/_skills/bigquery-ai/syntax/objget_read_url.md +40 -0
  94. bigquery_data_science/_skills/bigquery-ai/syntax/objlist.md +34 -0
  95. bigquery_data_science/_skills/bigquery-ai/syntax/objmake_ref.md +58 -0
  96. bigquery_data_science/_skills/bigquery-ai/syntax/per-row-failures-and-the-retry-pattern.md +19 -0
  97. bigquery_data_science/_skills/bigquery-ai/syntax/setup-apis-connection-roles.md +39 -0
  98. bigquery_data_science/_skills/bigquery-ai/syntax/two-input-shapes.md +22 -0
  99. bigquery_data_science/_skills/bigquery-ai/syntax/vector_search.md +78 -0
  100. bigquery_data_science/_skills/bigquery-ai/syntax/what-the-five-have-in-common.md +27 -0
  101. bigquery_data_science/_skills/bigquery-data-science/SKILL.md +69 -0
  102. bigquery_data_science/_skills/bigquery-data-science/reference/namespace-matrix.md +112 -0
  103. bigquery_data_science/_skills/bigquery-data-science/skill.manifest.json +13 -0
  104. bigquery_data_science/_skills/bigquery-ml/SKILL.md +70 -0
  105. bigquery_data_science/_skills/bigquery-ml/narrative/airflow_with_kfp.md +379 -0
  106. bigquery_data_science/_skills/bigquery-ml/narrative/anomaly_fraud_detection.md +195 -0
  107. bigquery_data_science/_skills/bigquery-ml/narrative/arima_plus.md +635 -0
  108. bigquery_data_science/_skills/bigquery-ml/narrative/arima_plus_xreg.md +453 -0
  109. bigquery_data_science/_skills/bigquery-ml/narrative/autoencoder.md +470 -0
  110. bigquery_data_science/_skills/bigquery-ml/narrative/automl_classifier.md +291 -0
  111. bigquery_data_science/_skills/bigquery-ml/narrative/automl_regressor.md +274 -0
  112. bigquery_data_science/_skills/bigquery-ml/narrative/boosted_tree_classifier.md +449 -0
  113. bigquery_data_science/_skills/bigquery-ml/narrative/boosted_tree_regressor.md +400 -0
  114. bigquery_data_science/_skills/bigquery-ml/narrative/bucketizing.md +201 -0
  115. bigquery_data_science/_skills/bigquery-ml/narrative/causal_effect.md +773 -0
  116. bigquery_data_science/_skills/bigquery-ml/narrative/churn_retention.md +271 -0
  117. bigquery_data_science/_skills/bigquery-ml/narrative/cloud_workflows.md +370 -0
  118. bigquery_data_science/_skills/bigquery-ml/narrative/composer_airflow.md +475 -0
  119. bigquery_data_science/_skills/bigquery-ml/narrative/contribution_analysis.md +346 -0
  120. bigquery_data_science/_skills/bigquery-ml/narrative/cross_validation.md +185 -0
  121. bigquery_data_science/_skills/bigquery-ml/narrative/customer_segmentation.md +233 -0
  122. bigquery_data_science/_skills/bigquery-ml/narrative/dataform.md +372 -0
  123. bigquery_data_science/_skills/bigquery-ml/narrative/dbt.md +283 -0
  124. bigquery_data_science/_skills/bigquery-ml/narrative/difference_in_differences.md +280 -0
  125. bigquery_data_science/_skills/bigquery-ml/narrative/distance.md +195 -0
  126. bigquery_data_science/_skills/bigquery-ml/narrative/dnn_classifier.md +326 -0
  127. bigquery_data_science/_skills/bigquery-ml/narrative/dnn_regressor.md +308 -0
  128. bigquery_data_science/_skills/bigquery-ml/narrative/embeddings_classification.md +426 -0
  129. bigquery_data_science/_skills/bigquery-ml/narrative/encoding.md +238 -0
  130. bigquery_data_science/_skills/bigquery-ml/narrative/ensembling.md +278 -0
  131. bigquery_data_science/_skills/bigquery-ml/narrative/evaluation.md +445 -0
  132. bigquery_data_science/_skills/bigquery-ml/narrative/exploration.md +779 -0
  133. bigquery_data_science/_skills/bigquery-ml/narrative/export.md +321 -0
  134. bigquery_data_science/_skills/bigquery-ml/narrative/feature_engineering.md +280 -0
  135. bigquery_data_science/_skills/bigquery-ml/narrative/feature_store.md +652 -0
  136. bigquery_data_science/_skills/bigquery-ml/narrative/feature_store_workflow.md +426 -0
  137. bigquery_data_science/_skills/bigquery-ml/narrative/ga4_churn_prediction.md +285 -0
  138. bigquery_data_science/_skills/bigquery-ml/narrative/hierarchical_forecasting.md +441 -0
  139. bigquery_data_science/_skills/bigquery-ml/narrative/image.md +791 -0
  140. bigquery_data_science/_skills/bigquery-ml/narrative/imported.md +426 -0
  141. bigquery_data_science/_skills/bigquery-ml/narrative/kmeans.md +348 -0
  142. bigquery_data_science/_skills/bigquery-ml/narrative/linear_regression.md +372 -0
  143. bigquery_data_science/_skills/bigquery-ml/narrative/logistic_regression.md +390 -0
  144. bigquery_data_science/_skills/bigquery-ml/narrative/matrix_factorization.md +404 -0
  145. bigquery_data_science/_skills/bigquery-ml/narrative/model_monitoring.md +609 -0
  146. bigquery_data_science/_skills/bigquery-ml/narrative/pca.md +357 -0
  147. bigquery_data_science/_skills/bigquery-ml/narrative/price_elasticity_dml.md +286 -0
  148. bigquery_data_science/_skills/bigquery-ml/narrative/propensity_score_matching.md +350 -0
  149. bigquery_data_science/_skills/bigquery-ml/narrative/random_forest_classifier.md +445 -0
  150. bigquery_data_science/_skills/bigquery-ml/narrative/random_forest_regressor.md +409 -0
  151. bigquery_data_science/_skills/bigquery-ml/narrative/recommendation.md +281 -0
  152. bigquery_data_science/_skills/bigquery-ml/narrative/regression_based_forecasting.md +491 -0
  153. bigquery_data_science/_skills/bigquery-ml/narrative/remote.md +428 -0
  154. bigquery_data_science/_skills/bigquery-ml/narrative/scalers.md +325 -0
  155. bigquery_data_science/_skills/bigquery-ml/narrative/scheduled_queries.md +284 -0
  156. bigquery_data_science/_skills/bigquery-ml/narrative/sql_scripting.md +243 -0
  157. bigquery_data_science/_skills/bigquery-ml/narrative/survival_analysis.md +363 -0
  158. bigquery_data_science/_skills/bigquery-ml/narrative/synthetic_control.md +221 -0
  159. bigquery_data_science/_skills/bigquery-ml/narrative/text.md +215 -0
  160. bigquery_data_science/_skills/bigquery-ml/narrative/time_series.md +669 -0
  161. bigquery_data_science/_skills/bigquery-ml/narrative/transform_only.md +273 -0
  162. bigquery_data_science/_skills/bigquery-ml/narrative/uplift_cate.md +241 -0
  163. bigquery_data_science/_skills/bigquery-ml/narrative/vertex_kfp.md +360 -0
  164. bigquery_data_science/_skills/bigquery-ml/narrative/wide_and_deep_classifier.md +316 -0
  165. bigquery_data_science/_skills/bigquery-ml/narrative/wide_and_deep_regressor.md +298 -0
  166. bigquery_data_science/_skills/bigquery-ml/reference/classification.md +80 -0
  167. bigquery_data_science/_skills/bigquery-ml/reference/model-management.md +90 -0
  168. bigquery_data_science/_skills/bigquery-ml/reference/preprocessing-functions.md +234 -0
  169. bigquery_data_science/_skills/bigquery-ml/reference/regression.md +79 -0
  170. bigquery_data_science/_skills/bigquery-ml/reference/unsupervised-and-specialized.md +101 -0
  171. bigquery_data_science/_skills/bigquery-ml/reference/workflows-and-pipelines.md +111 -0
  172. bigquery_data_science/_skills/bigquery-ml/skill.manifest.json +177 -0
  173. bigquery_data_science/_skills/bigquery-ml/syntax/1-model_type-catalog.md +28 -0
  174. bigquery_data_science/_skills/bigquery-ml/syntax/2-task-mlevaluate-metric-columns.md +26 -0
  175. bigquery_data_science/_skills/bigquery-ml/syntax/3-capability-matrix-per-model-type.md +21 -0
  176. bigquery_data_science/_skills/bigquery-ml/syntax/4-explainability-weights-functions.md +14 -0
  177. bigquery_data_science/_skills/bigquery-ml/syntax/5-connection-matrix.md +11 -0
  178. bigquery_data_science/_skills/bigquery-ml/syntax/6-which-model-free-function-no-create-model-no-connection.md +15 -0
  179. bigquery_data_science/_skills/bigquery-ml/syntax/7-point-in-time-feature-retrieval-which-of-the-two.md +11 -0
  180. bigquery_data_science/_skills/bigquery-ml/syntax/INDEX.md +111 -0
  181. bigquery_data_science/_skills/bigquery-ml/syntax/aicausal_effect.md +85 -0
  182. bigquery_data_science/_skills/bigquery-ml/syntax/arima_plus-forecasting-lifecycle-functions.md +14 -0
  183. bigquery_data_science/_skills/bigquery-ml/syntax/arima_plus-univariate-time-series-forecasting.md +132 -0
  184. bigquery_data_science/_skills/bigquery-ml/syntax/arima_plus_xreg-multivariate-time-series-with-external-regressors.md +103 -0
  185. bigquery_data_science/_skills/bigquery-ml/syntax/autoencoder.md +102 -0
  186. bigquery_data_science/_skills/bigquery-ml/syntax/automl_classifier-automl_regressor.md +88 -0
  187. bigquery_data_science/_skills/bigquery-ml/syntax/boosted_tree_classifier-boosted_tree_regressor.md +117 -0
  188. bigquery_data_science/_skills/bigquery-ml/syntax/contribution_analysis.md +113 -0
  189. bigquery_data_science/_skills/bigquery-ml/syntax/dnn_classifier-dnn_regressor.md +107 -0
  190. bigquery_data_science/_skills/bigquery-ml/syntax/dnn_linear_combined_classifier-dnn_linear_combined_regressor-wide-and-deep.md +98 -0
  191. bigquery_data_science/_skills/bigquery-ml/syntax/explainability-functions-mlexplain_predict-mlglobal_explain-mlfeature_importance.md +7 -0
  192. bigquery_data_science/_skills/bigquery-ml/syntax/exploratory-data-analysis-mldescribe_data-mlcorrelation.md +11 -0
  193. bigquery_data_science/_skills/bigquery-ml/syntax/export-model.md +92 -0
  194. bigquery_data_science/_skills/bigquery-ml/syntax/feature-engineering-standalone-preprocessing-functions.md +13 -0
  195. bigquery_data_science/_skills/bigquery-ml/syntax/image-preprocessing-functions-mldecode_image-mlresize_image-mlconvert_image_type-mlconvert_color_space.md +63 -0
  196. bigquery_data_science/_skills/bigquery-ml/syntax/imported-models-tensorflow-tensorflow_lite-onnx-xgboost.md +36 -0
  197. bigquery_data_science/_skills/bigquery-ml/syntax/kmeans.md +87 -0
  198. bigquery_data_science/_skills/bigquery-ml/syntax/linear_reg-logistic_reg-generalized-linear-models.md +101 -0
  199. bigquery_data_science/_skills/bigquery-ml/syntax/matrix_factorization.md +162 -0
  200. bigquery_data_science/_skills/bigquery-ml/syntax/mladvanced_weights.md +76 -0
  201. bigquery_data_science/_skills/bigquery-ml/syntax/mlarima_coefficients.md +30 -0
  202. bigquery_data_science/_skills/bigquery-ml/syntax/mlarima_evaluate.md +49 -0
  203. bigquery_data_science/_skills/bigquery-ml/syntax/mlbag_of_words.md +47 -0
  204. bigquery_data_science/_skills/bigquery-ml/syntax/mlbucketize.md +47 -0
  205. bigquery_data_science/_skills/bigquery-ml/syntax/mlcentroids.md +46 -0
  206. bigquery_data_science/_skills/bigquery-ml/syntax/mlconfusion_matrix.md +56 -0
  207. bigquery_data_science/_skills/bigquery-ml/syntax/mlconvert_color_space.md +53 -0
  208. bigquery_data_science/_skills/bigquery-ml/syntax/mlconvert_image_type.md +39 -0
  209. bigquery_data_science/_skills/bigquery-ml/syntax/mlcorrelation.md +78 -0
  210. bigquery_data_science/_skills/bigquery-ml/syntax/mldecode_image.md +44 -0
  211. bigquery_data_science/_skills/bigquery-ml/syntax/mldescribe_data.md +74 -0
  212. bigquery_data_science/_skills/bigquery-ml/syntax/mldetect_anomalies.md +86 -0
  213. bigquery_data_science/_skills/bigquery-ml/syntax/mldistance.md +48 -0
  214. bigquery_data_science/_skills/bigquery-ml/syntax/mlentity_features_at_time.md +56 -0
  215. bigquery_data_science/_skills/bigquery-ml/syntax/mlevaluate.md +72 -0
  216. bigquery_data_science/_skills/bigquery-ml/syntax/mlexplain_forecast.md +50 -0
  217. bigquery_data_science/_skills/bigquery-ml/syntax/mlexplain_predict.md +66 -0
  218. bigquery_data_science/_skills/bigquery-ml/syntax/mlfeature_cross.md +46 -0
  219. bigquery_data_science/_skills/bigquery-ml/syntax/mlfeature_importance.md +47 -0
  220. bigquery_data_science/_skills/bigquery-ml/syntax/mlfeature_info.md +50 -0
  221. bigquery_data_science/_skills/bigquery-ml/syntax/mlfeatures_at_time.md +51 -0
  222. bigquery_data_science/_skills/bigquery-ml/syntax/mlforecast.md +76 -0
  223. bigquery_data_science/_skills/bigquery-ml/syntax/mlgenerate_embedding-from-a-bqml-pca-autoencoder-matrix_factorization-model.md +64 -0
  224. bigquery_data_science/_skills/bigquery-ml/syntax/mlglobal_explain.md +63 -0
  225. bigquery_data_science/_skills/bigquery-ml/syntax/mlhash_bucketize.md +47 -0
  226. bigquery_data_science/_skills/bigquery-ml/syntax/mlholiday_info.md +46 -0
  227. bigquery_data_science/_skills/bigquery-ml/syntax/mlimputer.md +46 -0
  228. bigquery_data_science/_skills/bigquery-ml/syntax/mllp_norm.md +47 -0
  229. bigquery_data_science/_skills/bigquery-ml/syntax/mlmetrics.md +80 -0
  230. bigquery_data_science/_skills/bigquery-ml/syntax/mlngrams.md +37 -0
  231. bigquery_data_science/_skills/bigquery-ml/syntax/mlone_hot_encoder-mlmulti_hot_encoder-mllabel_encoder.md +67 -0
  232. bigquery_data_science/_skills/bigquery-ml/syntax/mlpolynomial_expand.md +47 -0
  233. bigquery_data_science/_skills/bigquery-ml/syntax/mlpredict.md +67 -0
  234. bigquery_data_science/_skills/bigquery-ml/syntax/mlprincipal_component_info.md +42 -0
  235. bigquery_data_science/_skills/bigquery-ml/syntax/mlprincipal_components.md +40 -0
  236. bigquery_data_science/_skills/bigquery-ml/syntax/mlquantile_bucketize.md +43 -0
  237. bigquery_data_science/_skills/bigquery-ml/syntax/mlrecommend.md +60 -0
  238. bigquery_data_science/_skills/bigquery-ml/syntax/mlreconstruction_loss.md +48 -0
  239. bigquery_data_science/_skills/bigquery-ml/syntax/mlresize_image.md +45 -0
  240. bigquery_data_science/_skills/bigquery-ml/syntax/mlroc_curve.md +63 -0
  241. bigquery_data_science/_skills/bigquery-ml/syntax/mltf_idf.md +37 -0
  242. bigquery_data_science/_skills/bigquery-ml/syntax/mltfdv_describe.md +34 -0
  243. bigquery_data_science/_skills/bigquery-ml/syntax/mltfdv_validate.md +48 -0
  244. bigquery_data_science/_skills/bigquery-ml/syntax/mltraining_info.md +78 -0
  245. bigquery_data_science/_skills/bigquery-ml/syntax/mltransform-applying-a-saved-transform.md +52 -0
  246. bigquery_data_science/_skills/bigquery-ml/syntax/mltranspose-not-a-function-pointer.md +32 -0
  247. bigquery_data_science/_skills/bigquery-ml/syntax/mltrial_info.md +60 -0
  248. bigquery_data_science/_skills/bigquery-ml/syntax/mlvalidate_data_drift.md +99 -0
  249. bigquery_data_science/_skills/bigquery-ml/syntax/mlvalidate_data_skew.md +50 -0
  250. bigquery_data_science/_skills/bigquery-ml/syntax/mlweights.md +84 -0
  251. bigquery_data_science/_skills/bigquery-ml/syntax/model-free-time-series-decomposition-mltrend-mlseasonality-mldetect_change_points.md +100 -0
  252. bigquery_data_science/_skills/bigquery-ml/syntax/model-introspection-mlfeature_info-mltraining_info-mltrial_info.md +7 -0
  253. bigquery_data_science/_skills/bigquery-ml/syntax/model-monitoring-data-validation.md +23 -0
  254. bigquery_data_science/_skills/bigquery-ml/syntax/numerical-scalers-mlstandard_scaler-mlmin_max_scaler-mlmax_abs_scaler-mlrobust_scaler-mlnormalizer.md +77 -0
  255. bigquery_data_science/_skills/bigquery-ml/syntax/onnx.md +55 -0
  256. bigquery_data_science/_skills/bigquery-ml/syntax/pca.md +91 -0
  257. bigquery_data_science/_skills/bigquery-ml/syntax/point-in-time-feature-retrieval-mlfeatures_at_time-mlentity_features_at_time.md +25 -0
  258. bigquery_data_science/_skills/bigquery-ml/syntax/quick-comparison.md +14 -0
  259. bigquery_data_science/_skills/bigquery-ml/syntax/random_forest_classifier-random_forest_regressor.md +123 -0
  260. bigquery_data_science/_skills/bigquery-ml/syntax/remote-remote-model-over-a-vertex-ai-endpoint-the-mechanism.md +87 -0
  261. bigquery_data_science/_skills/bigquery-ml/syntax/tensorflow.md +48 -0
  262. bigquery_data_science/_skills/bigquery-ml/syntax/tensorflow_lite.md +29 -0
  263. bigquery_data_science/_skills/bigquery-ml/syntax/text-preprocessing-functions-mlngrams-mltf_idf-mlbag_of_words.md +18 -0
  264. bigquery_data_science/_skills/bigquery-ml/syntax/timesfm-built-in-foundation-forecaster-see-bigquery-ai.md +28 -0
  265. bigquery_data_science/_skills/bigquery-ml/syntax/transform_only.md +91 -0
  266. bigquery_data_science/_skills/bigquery-ml/syntax/xgboost.md +72 -0
  267. bigquery_data_science/_skills/pinned-models.json +28 -0
  268. bigquery_data_science/catalog.py +1107 -0
  269. bigquery_data_science/cli.py +786 -0
  270. bigquery_data_science/installer.py +282 -0
  271. bigquery_data_science/mcp_server.py +317 -0
  272. bigquery_data_science/tooling/__init__.py +15 -0
  273. bigquery_data_science/tooling/convert_notebook.py +176 -0
  274. bigquery_data_science/tooling/link_policy.py +247 -0
  275. bigquery_data_science/tooling/manifest.py +152 -0
  276. bigquery_data_science/tooling/markdown_anchors.py +84 -0
  277. bigquery_data_science/tooling/narrative_drift.py +163 -0
  278. bigquery_data_science/tooling/pinned_models.py +216 -0
  279. bigquery_data_science/tooling/reference_structure.py +85 -0
  280. bigquery_data_science/tooling/sync_syntax.py +407 -0
  281. bigquery_data_science/tooling/validate.py +208 -0
  282. bigquery_data_science-0.7.0.dist-info/METADATA +306 -0
  283. bigquery_data_science-0.7.0.dist-info/RECORD +286 -0
  284. bigquery_data_science-0.7.0.dist-info/WHEEL +4 -0
  285. bigquery_data_science-0.7.0.dist-info/entry_points.txt +5 -0
  286. bigquery_data_science-0.7.0.dist-info/licenses/LICENSE +201 -0
@@ -0,0 +1,316 @@
1
+ > **Source (`statmike/bigquery-data-science`):** **Notebook:** [`bigquery-ml/models/wide_and_deep_classifier/wide_and_deep_classifier.ipynb`](https://github.com/statmike/bigquery-data-science/blob/main/bigquery-ml/models/wide_and_deep_classifier/wide_and_deep_classifier.ipynb) · **SQL:** [`bigquery-ml/models/wide_and_deep_classifier/wide_and_deep_classifier.sql`](https://github.com/statmike/bigquery-data-science/blob/main/bigquery-ml/models/wide_and_deep_classifier/wide_and_deep_classifier.sql)
2
+
3
+ # Wide & Deep Classifier — BigQuery ML
4
+
5
+ Train a **binary classification** model with `CREATE MODEL` (model_type = `DNN_LINEAR_COMBINED_CLASSIFIER`) — a jointly-trained combination of a **wide** linear model (memorizes feature interactions) and a **deep** neural network (generalizes), trained with TensorFlow inside BigQuery — then walk the full model lifecycle: evaluate, predict, explain, inspect training, apply in-model preprocessing, and tune hyperparameters. No data leaves BigQuery, and no separate training infrastructure is needed.
6
+
7
+ **Lifecycle:** `CREATE MODEL` → `ML.EVALUATE` → `ML.CONFUSION_MATRIX` / `ML.ROC_CURVE` → `ML.PREDICT` → `ML.EXPLAIN_PREDICT` / `ML.GLOBAL_EXPLAIN` → `ML.FEATURE_INFO` / `ML.TRAINING_INFO` → `TRANSFORM` clause → hyperparameter tuning (`ML.TRIAL_INFO`)
8
+
9
+ **When to use wide & deep:**
10
+ - Large, sparse categorical features (high-cardinality IDs) common in ranking/recommendation problems
11
+ - You want both memorization of specific feature combinations (wide) and generalization to unseen ones (deep)
12
+ - A plain DNN underfits sparse signals but a linear-only model underfits interactions
13
+ - Structured/tabular data — compare `roc_auc` directly against `models/logistic_regression` (Logistic Regression), `models/boosted_tree_classifier` (Boosted Tree Classifier), `models/random_forest_classifier` (Random Forest Classifier), and `models/dnn_classifier` (DNN Classifier) on the same data
14
+
15
+ **Same training-cost profile as `models/dnn_classifier` (DNN Classifier)** — this is TensorFlow training inside BigQuery, so expect multi-minute `CREATE MODEL` calls even on modest data, well beyond the tree models.
16
+
17
+ **Data:** [`bigquery-public-data.ml_datasets.census_adult_income`](https://console.cloud.google.com/marketplace/product/bigquery-public-datasets) — predict whether income exceeds $50K/year from census attributes. **Same data + label as `models/logistic_regression` (Logistic Regression), `models/boosted_tree_classifier` (Boosted Tree Classifier), `models/random_forest_classifier` (Random Forest Classifier), and `models/dnn_classifier` (DNN Classifier)** — compare `roc_auc` across all five techniques.
18
+
19
+ **References:** `RESOURCES.md` (Full reference) | [CREATE MODEL (wide-and-deep) docs](https://cloud.google.com/bigquery/docs/reference/standard-sql/bigqueryml-syntax-create-wnd-models) | `setup` (Setup guide)
20
+
21
+ ---
22
+ ## Setup
23
+
24
+ Set your project and location, authenticate, and create a shared dataset for the model.
25
+
26
+ > Wide & deep models train on data already in BigQuery — no connection or remote model required. The model is created with `CREATE MODEL` and stored in your dataset. See the `setup` (Setup Reference) for details.
27
+
28
+ ```python
29
+ PROJECT_ID = 'statmike-mlops-349915' # <-- Replace with your project ID
30
+ LOCATION = 'US' # BigQuery dataset location
31
+ DATASET_ID = 'bq_ml' # Shared dataset across all bigquery-ml notebooks
32
+ ```
33
+
34
+ ### Environment
35
+
36
+ > **Already set up the project environment?** The cell below is a no-op — packages are already in your kernel. See the `setup` (Setup Reference) for details.
37
+ >
38
+ > **Running standalone** (Colab, Colab Enterprise, Vertex AI Workbench)? The cell below installs required packages into your current kernel.
39
+
40
+ ```python
41
+ from google.cloud import bigquery
42
+ import pandas as pd
43
+
44
+ client = bigquery.Client(project=PROJECT_ID)
45
+ pd.set_option('display.max_colwidth', None)
46
+
47
+ # Create the shared dataset (idempotent)
48
+ dataset_ref = bigquery.DatasetReference(PROJECT_ID, DATASET_ID)
49
+ dataset = bigquery.Dataset(dataset_ref)
50
+ dataset.location = LOCATION
51
+ client.create_dataset(dataset, exists_ok=True)
52
+ print(f'Dataset {PROJECT_ID}.{DATASET_ID} ready')
53
+
54
+ # Register %%bigquery cell magic (auto-loaded in Colab, needed elsewhere)
55
+ %load_ext bigquery_magics
56
+ ```
57
+
58
+ ---
59
+ ## Step 1 — Create the model with `CREATE MODEL`
60
+
61
+ `CREATE MODEL` trains and stores the model in your dataset. `hidden_units` define the **deep** side's fully-connected layers; the **wide** (linear) side is implicit and uses the same input features. `auto_class_weights` balances the classes; `enable_global_explain` is **required** to use `ML.GLOBAL_EXPLAIN` later.
62
+
63
+ > **Verified:** like `DNN_CLASSIFIER`, this tolerates unscaled numeric inputs reasonably well on this dataset — `early_stop` still cuts training to 2 iterations, but `roc_auc` lands competitively (~0.89). Contrast with `models/wide_and_deep_regressor` (Wide & Deep Regressor), where the same default settings on the small penguins dataset produce a badly broken model (same finding as `models/dnn_regressor` (DNN Regressor)).
64
+
65
+ ```python
66
+ query = f"""
67
+ CREATE OR REPLACE MODEL `{PROJECT_ID}.{DATASET_ID}.wide_deep_classifier_income`
68
+ OPTIONS(
69
+ model_type = 'DNN_LINEAR_COMBINED_CLASSIFIER',
70
+ input_label_cols = ['income_bracket'],
71
+ hidden_units = [64, 32],
72
+ activation_fn = 'RELU',
73
+ dropout = 0.15,
74
+ max_iterations = 20,
75
+ early_stop = TRUE,
76
+ auto_class_weights = TRUE,
77
+ data_split_method = 'AUTO_SPLIT',
78
+ enable_global_explain = TRUE
79
+ ) AS
80
+ SELECT
81
+ age, workclass, education, education_num, marital_status, occupation,
82
+ relationship, race, sex, hours_per_week, native_country, income_bracket
83
+ FROM `bigquery-public-data.ml_datasets.census_adult_income`
84
+ """
85
+ client.query(query).result()
86
+ print('Model wide_deep_classifier_income created')
87
+ ```
88
+
89
+ ---
90
+ ## Step 2 — Evaluate with `ML.EVALUATE`
91
+
92
+ `ML.EVALUATE` returns standard classification metrics on the automatically held-out evaluation split. Compare `roc_auc` against `models/logistic_regression` (Logistic Regression), `models/boosted_tree_classifier` (Boosted Tree Classifier), `models/random_forest_classifier` (Random Forest Classifier), and `models/dnn_classifier` (DNN Classifier), which all train on the exact same data.
93
+
94
+ ```python
95
+ query = f"""
96
+ SELECT *
97
+ FROM ML.EVALUATE(MODEL `{PROJECT_ID}.{DATASET_ID}.wide_deep_classifier_income`)
98
+ """
99
+ client.query(query).to_dataframe()
100
+ ```
101
+
102
+ ---
103
+ ## Step 3 — Confusion matrix and ROC curve
104
+
105
+ `ML.CONFUSION_MATRIX` shows counts of predicted vs. actual classes. `ML.ROC_CURVE` returns the recall / false-positive-rate tradeoff across thresholds.
106
+
107
+ ```python
108
+ query = f"""
109
+ SELECT *
110
+ FROM ML.CONFUSION_MATRIX(MODEL `{PROJECT_ID}.{DATASET_ID}.wide_deep_classifier_income`)
111
+ """
112
+ client.query(query).to_dataframe()
113
+ ```
114
+
115
+ ```python
116
+ import matplotlib.pyplot as plt
117
+
118
+ query = f"""
119
+ SELECT threshold, recall, false_positive_rate
120
+ FROM ML.ROC_CURVE(MODEL `{PROJECT_ID}.{DATASET_ID}.wide_deep_classifier_income`)
121
+ ORDER BY threshold
122
+ """
123
+ roc = client.query(query).to_dataframe()
124
+
125
+ fig, ax = plt.subplots(figsize=(6, 6))
126
+ ax.plot(roc['false_positive_rate'], roc['recall'], color='#4285F4', linewidth=2, label='ROC curve')
127
+ ax.plot([0, 1], [0, 1], color='gray', linestyle='--', alpha=0.6, label='Random')
128
+ ax.set_xlabel('False Positive Rate')
129
+ ax.set_ylabel('True Positive Rate (Recall)')
130
+ ax.set_title('ROC Curve - Wide & Deep Classifier')
131
+ ax.legend(loc='lower right')
132
+ ax.grid(True, alpha=0.3)
133
+ plt.tight_layout()
134
+ plt.show()
135
+ ```
136
+
137
+ ---
138
+ ## Step 4 — Predict with `ML.PREDICT`
139
+
140
+ ```python
141
+ query = f"""
142
+ SELECT
143
+ predicted_income_bracket,
144
+ predicted_income_bracket_probs
145
+ FROM ML.PREDICT(
146
+ MODEL `{PROJECT_ID}.{DATASET_ID}.wide_deep_classifier_income`,
147
+ (SELECT * FROM `bigquery-public-data.ml_datasets.census_adult_income` LIMIT 10)
148
+ )
149
+ """
150
+ client.query(query).to_dataframe()
151
+ ```
152
+
153
+ ---
154
+ ## Step 5 — Explain predictions
155
+
156
+ - **`ML.EXPLAIN_PREDICT`** — per-row feature attributions (Integrated Gradients)
157
+ - **`ML.GLOBAL_EXPLAIN`** — overall feature importance across the model (requires `enable_global_explain = TRUE`)
158
+
159
+ No coefficients and no split-based importance for this model type, so there is no `ML.WEIGHTS`/`ML.ADVANCED_WEIGHTS` (GLM-only) or `ML.FEATURE_IMPORTANCE` (tree-only) here — same as `models/dnn_classifier` (DNN Classifier).
160
+
161
+ ```python
162
+ query = f"""
163
+ SELECT
164
+ predicted_income_bracket,
165
+ top_feature_attributions
166
+ FROM ML.EXPLAIN_PREDICT(
167
+ MODEL `{PROJECT_ID}.{DATASET_ID}.wide_deep_classifier_income`,
168
+ (SELECT * FROM `bigquery-public-data.ml_datasets.census_adult_income` LIMIT 5),
169
+ STRUCT(5 AS top_k_features)
170
+ )
171
+ """
172
+ client.query(query).to_dataframe()
173
+ ```
174
+
175
+ ```python
176
+ query = f"""
177
+ SELECT *
178
+ FROM ML.GLOBAL_EXPLAIN(MODEL `{PROJECT_ID}.{DATASET_ID}.wide_deep_classifier_income`)
179
+ ORDER BY attribution DESC
180
+ """
181
+ client.query(query).to_dataframe()
182
+ ```
183
+
184
+ ---
185
+ ## Step 6 — Introspect the model
186
+
187
+ `ML.FEATURE_INFO` reports the statistics each feature had during training. `ML.TRAINING_INFO` returns the per-iteration loss curve — verified: `early_stop = TRUE` (the default) stopped training after only 2 iterations here, same pattern as `models/dnn_classifier` (DNN Classifier).
188
+
189
+ ```python
190
+ query = f"""
191
+ SELECT *
192
+ FROM ML.FEATURE_INFO(MODEL `{PROJECT_ID}.{DATASET_ID}.wide_deep_classifier_income`)
193
+ """
194
+ client.query(query).to_dataframe()
195
+ ```
196
+
197
+ ```python
198
+ query = f"""
199
+ SELECT iteration, loss, eval_loss, learning_rate, duration_ms
200
+ FROM ML.TRAINING_INFO(MODEL `{PROJECT_ID}.{DATASET_ID}.wide_deep_classifier_income`)
201
+ ORDER BY iteration
202
+ """
203
+ client.query(query).to_dataframe()
204
+ ```
205
+
206
+ ---
207
+ ## Step 7 — In-model preprocessing with the `TRANSFORM` clause
208
+
209
+ Same best practice as DNN: normalize numeric inputs before gradient-based training. **Verified effect:** final `roc_auc` comes out about the same as Step 1 (~0.89), but this run took 8 iterations to hit the same early-stopping rule (vs. 2 unscaled), with a properly declining loss curve instead of an almost-flat one.
210
+
211
+ ```python
212
+ query = f"""
213
+ CREATE OR REPLACE MODEL `{PROJECT_ID}.{DATASET_ID}.wide_deep_classifier_income_transform`
214
+ TRANSFORM(
215
+ ML.STANDARD_SCALER(age) OVER() AS age,
216
+ ML.STANDARD_SCALER(education_num) OVER() AS education_num,
217
+ ML.STANDARD_SCALER(hours_per_week) OVER() AS hours_per_week,
218
+ workclass, education, marital_status, occupation, relationship, race, sex,
219
+ native_country, income_bracket
220
+ )
221
+ OPTIONS(
222
+ model_type = 'DNN_LINEAR_COMBINED_CLASSIFIER',
223
+ input_label_cols = ['income_bracket'],
224
+ hidden_units = [64, 32],
225
+ activation_fn = 'RELU',
226
+ dropout = 0.15,
227
+ max_iterations = 20,
228
+ early_stop = TRUE,
229
+ auto_class_weights = TRUE,
230
+ data_split_method = 'AUTO_SPLIT',
231
+ enable_global_explain = TRUE
232
+ ) AS
233
+ SELECT
234
+ age, workclass, education, education_num, marital_status, occupation,
235
+ relationship, race, sex, hours_per_week, native_country, income_bracket
236
+ FROM `bigquery-public-data.ml_datasets.census_adult_income`
237
+ """
238
+ client.query(query).result()
239
+ print('Model wide_deep_classifier_income_transform created')
240
+ ```
241
+
242
+ ```python
243
+ # Predict on RAW rows - scaling is applied automatically inside the model
244
+ query = f"""
245
+ SELECT predicted_income_bracket, age
246
+ FROM ML.PREDICT(
247
+ MODEL `{PROJECT_ID}.{DATASET_ID}.wide_deep_classifier_income_transform`,
248
+ (SELECT * FROM `bigquery-public-data.ml_datasets.census_adult_income` LIMIT 5)
249
+ )
250
+ """
251
+ client.query(query).to_dataframe()
252
+ ```
253
+
254
+ ---
255
+ ## Step 8 — Hyperparameter tuning
256
+
257
+ `hidden_units` is only tunable via `HPARAM_CANDIDATES` (same STRUCT-wrapped array syntax as DNN). `dropout` is tunable via `HPARAM_RANGE`.
258
+
259
+ > **GOTCHA (verified, differs from `DNN_CLASSIFIER`/`DNN_REGRESSOR`):** `learn_rate` and `optimizer` are **NOT** tunable for `DNN_LINEAR_COMBINED_*` — both fail immediately with `"Unsupported hyperparameter <name> for model_type DNN_LINEAR_COMBINED_CLASSIFIER"`, even though the general docs summary for DNN-family models lists them as tunable (that applies to plain `DNN_*`, not this type). Confirmed tunable instead: `hidden_units`, `dropout`, `batch_size`, `l1_reg`, `l2_reg`.
260
+
261
+ ```python
262
+ query = f"""
263
+ CREATE OR REPLACE MODEL `{PROJECT_ID}.{DATASET_ID}.wide_deep_classifier_income_tuned`
264
+ OPTIONS(
265
+ model_type = 'DNN_LINEAR_COMBINED_CLASSIFIER',
266
+ input_label_cols = ['income_bracket'],
267
+ auto_class_weights = TRUE,
268
+ num_trials = 4,
269
+ max_parallel_trials = 2,
270
+ hparam_tuning_objectives = ['roc_auc'],
271
+ hidden_units = HPARAM_CANDIDATES([STRUCT([64, 32]), STRUCT([32, 16])]),
272
+ dropout = HPARAM_RANGE(0.0, 0.3)
273
+ ) AS
274
+ SELECT
275
+ age, workclass, education, education_num, marital_status, occupation,
276
+ relationship, race, sex, hours_per_week, native_country, income_bracket
277
+ FROM `bigquery-public-data.ml_datasets.census_adult_income`
278
+ """
279
+ client.query(query).result()
280
+ print('Tuned model created')
281
+ ```
282
+
283
+ ```python
284
+ query = f"""
285
+ SELECT
286
+ trial_id,
287
+ hyperparameters,
288
+ hparam_tuning_evaluation_metrics.roc_auc AS roc_auc,
289
+ is_optimal
290
+ FROM ML.TRIAL_INFO(MODEL `{PROJECT_ID}.{DATASET_ID}.wide_deep_classifier_income_tuned`)
291
+ ORDER BY roc_auc DESC
292
+ """
293
+ client.query(query).to_dataframe()
294
+ ```
295
+
296
+ ---
297
+ ## Examples — `%%bigquery` Magics
298
+
299
+ The same operations using IPython magic commands — write SQL directly in cells without Python string wrapping.
300
+
301
+ - `%%bigquery` — run SQL, display results inline
302
+ - `%%bigquery df` — run SQL, capture results into a pandas DataFrame
303
+
304
+ ### Evaluate with `%%bigquery`
305
+
306
+ ```sql
307
+ %%bigquery --project {PROJECT_ID}
308
+
309
+ SELECT *
310
+ FROM ML.EVALUATE(MODEL `statmike-mlops-349915.bq_ml.wide_deep_classifier_income`)
311
+ ```
312
+
313
+ ---
314
+ ## BigFrames
315
+
316
+ **No first-class wide-and-deep wrapper exists in `bigframes.ml`** (verified against the installed BigFrames API reference: `bigframes.ml.linear_model`, `.ensemble`, `.cluster`, `.decomposition`, `.forecasting`, `.imported`, `.llm` — no `DNNLinearCombinedClassifier`/`Regressor` or equivalent anywhere in the package, same gap as `models/dnn_classifier` (DNN Classifier)/`models/dnn_regressor` (DNN Regressor)). Use the SQL `CREATE MODEL` interface shown above.
@@ -0,0 +1,298 @@
1
+ > **Source (`statmike/bigquery-data-science`):** **Notebook:** [`bigquery-ml/models/wide_and_deep_regressor/wide_and_deep_regressor.ipynb`](https://github.com/statmike/bigquery-data-science/blob/main/bigquery-ml/models/wide_and_deep_regressor/wide_and_deep_regressor.ipynb) · **SQL:** [`bigquery-ml/models/wide_and_deep_regressor/wide_and_deep_regressor.sql`](https://github.com/statmike/bigquery-data-science/blob/main/bigquery-ml/models/wide_and_deep_regressor/wide_and_deep_regressor.sql)
2
+
3
+ # Wide & Deep Regressor — BigQuery ML
4
+
5
+ Train a **regression** model with `CREATE MODEL` (model_type = `DNN_LINEAR_COMBINED_REGRESSOR`) — a jointly-trained combination of a **wide** linear model and a **deep** neural network, trained with TensorFlow inside BigQuery — then walk the full model lifecycle: evaluate, predict, explain, inspect training, apply in-model preprocessing, and tune hyperparameters. No data leaves BigQuery, and no separate training infrastructure is needed.
6
+
7
+ **Lifecycle:** `CREATE MODEL` → `ML.EVALUATE` → `ML.PREDICT` → `ML.EXPLAIN_PREDICT` / `ML.GLOBAL_EXPLAIN` → `ML.FEATURE_INFO` / `ML.TRAINING_INFO` → `TRANSFORM` clause + higher `learn_rate` (the fix) → hyperparameter tuning (`ML.TRIAL_INFO`)
8
+
9
+ **This is the same debugging story as `models/dnn_regressor` (DNN Regressor), verified again for this model type:** with default settings on unscaled features, this model is **badly broken** (`r2_score ~ -27.4`, far worse than predicting the mean) — `early_stop` cuts training to just 2 iterations. Scaling the numeric features alone is known (from the DNN Regressor notebook's detailed walkthrough) to be insufficient; the fix applied directly here is scaling **plus** a much higher `learn_rate` (0.05 vs. the 0.001 default), which reaches `r2_score = 0.79` untuned — and **0.87** after hyperparameter tuning (Step 7), which found `hidden_units=[32,16]` works better than the `[64,32]` default on this small dataset. See `models/dnn_regressor` (DNN Regressor) for the step-by-step diagnosis of *why* scaling alone doesn't work — this notebook applies that lesson rather than re-deriving it.
10
+
11
+ **When to use wide & deep:**
12
+ - Large, sparse categorical features (high-cardinality IDs) common in ranking/recommendation problems
13
+ - You want both memorization of specific feature combinations (wide) and generalization to unseen ones (deep)
14
+ - Structured/tabular data — compare `r2_score` directly against `models/linear_regression` (Linear Regression), `models/boosted_tree_regressor` (Boosted Tree Regressor), `models/random_forest_regressor` (Random Forest Regressor), and `models/dnn_regressor` (DNN Regressor) on the same data
15
+ - For most tabular tasks, try Boosted Tree first — it trains far faster, needs less tuning, and reaches `r2_score ~ 0.98` on this same dataset vs. this notebook's best of ~0.87
16
+
17
+ **Data:** [`bigquery-public-data.ml_datasets.penguins`](https://console.cloud.google.com/marketplace/product/bigquery-public-datasets) — predict `body_mass_g` from physical measurements. **Same data + label as `models/linear_regression` (Linear Regression), `models/boosted_tree_regressor` (Boosted Tree Regressor), `models/random_forest_regressor` (Random Forest Regressor), and `models/dnn_regressor` (DNN Regressor)** — compare `r2_score` across all five techniques. Rows with a NULL label or an invalid `sex` value (`'.'`) are filtered out.
18
+
19
+ **References:** `RESOURCES.md` (Full reference) | [CREATE MODEL (wide-and-deep) docs](https://cloud.google.com/bigquery/docs/reference/standard-sql/bigqueryml-syntax-create-wnd-models) | `setup` (Setup guide)
20
+
21
+ ---
22
+ ## Setup
23
+
24
+ Set your project and location, authenticate, and create a shared dataset for the model.
25
+
26
+ > Wide & deep models train on data already in BigQuery — no connection or remote model required. The model is created with `CREATE MODEL` and stored in your dataset. See the `setup` (Setup Reference) for details.
27
+
28
+ ```python
29
+ PROJECT_ID = 'statmike-mlops-349915' # <-- Replace with your project ID
30
+ LOCATION = 'US' # BigQuery dataset location
31
+ DATASET_ID = 'bq_ml' # Shared dataset across all bigquery-ml notebooks
32
+ ```
33
+
34
+ ### Environment
35
+
36
+ > **Already set up the project environment?** The cell below is a no-op — packages are already in your kernel. See the `setup` (Setup Reference) for details.
37
+ >
38
+ > **Running standalone** (Colab, Colab Enterprise, Vertex AI Workbench)? The cell below installs required packages into your current kernel.
39
+
40
+ ```python
41
+ from google.cloud import bigquery
42
+ import pandas as pd
43
+
44
+ client = bigquery.Client(project=PROJECT_ID)
45
+ pd.set_option('display.max_colwidth', None)
46
+
47
+ # Create the shared dataset (idempotent)
48
+ dataset_ref = bigquery.DatasetReference(PROJECT_ID, DATASET_ID)
49
+ dataset = bigquery.Dataset(dataset_ref)
50
+ dataset.location = LOCATION
51
+ client.create_dataset(dataset, exists_ok=True)
52
+ print(f'Dataset {PROJECT_ID}.{DATASET_ID} ready')
53
+
54
+ # Register %%bigquery cell magic (auto-loaded in Colab, needed elsewhere)
55
+ %load_ext bigquery_magics
56
+ ```
57
+
58
+ ---
59
+ ## Step 1 — Create the model with `CREATE MODEL`
60
+
61
+ `CREATE MODEL` trains and stores the model in your dataset. Same options as `models/wide_and_deep_classifier` (Wide & Deep Classifier) minus `auto_class_weights` (classifier-only).
62
+
63
+ > **GOTCHA (verified — same finding as `models/dnn_regressor` (DNN Regressor), now confirmed for this model type too):** on this small (333-row) dataset, with UNSCALED numeric features and the default `learn_rate = 0.001`, this model is badly broken: `r2_score ~ -27.4` — far worse than just predicting the mean. `early_stop = TRUE` (the default) stops training after only 2 iterations. See Step 6 for the fix (same recipe as DNN Regressor: scale the numeric features AND raise `learn_rate` well above default) — see `models/dnn_regressor` (DNN Regressor) for the detailed diagnostic walkthrough of why scaling alone is not enough; this notebook applies that lesson directly.
64
+
65
+ ```python
66
+ query = f"""
67
+ CREATE OR REPLACE MODEL `{PROJECT_ID}.{DATASET_ID}.wide_deep_regressor_penguins`
68
+ OPTIONS(
69
+ model_type = 'DNN_LINEAR_COMBINED_REGRESSOR',
70
+ input_label_cols = ['body_mass_g'],
71
+ hidden_units = [64, 32],
72
+ activation_fn = 'RELU',
73
+ dropout = 0.15,
74
+ max_iterations = 20,
75
+ early_stop = TRUE,
76
+ data_split_method = 'AUTO_SPLIT',
77
+ enable_global_explain = TRUE
78
+ ) AS
79
+ SELECT
80
+ species, island, culmen_length_mm, culmen_depth_mm, flipper_length_mm, sex, body_mass_g
81
+ FROM `bigquery-public-data.ml_datasets.penguins`
82
+ WHERE body_mass_g IS NOT NULL AND sex IN ('MALE', 'FEMALE')
83
+ """
84
+ client.query(query).result()
85
+ print('Model wide_deep_regressor_penguins created')
86
+ ```
87
+
88
+ ---
89
+ ## Step 2 — Evaluate with `ML.EVALUATE`
90
+
91
+ Expect a strongly negative `r2_score` here — see the Step 1 gotcha. This is the real output of the baseline config.
92
+
93
+ ```python
94
+ query = f"""
95
+ SELECT *
96
+ FROM ML.EVALUATE(MODEL `{PROJECT_ID}.{DATASET_ID}.wide_deep_regressor_penguins`)
97
+ """
98
+ client.query(query).to_dataframe()
99
+ ```
100
+
101
+ ---
102
+ ## Step 3 — Predict with `ML.PREDICT`
103
+
104
+ ```python
105
+ query = f"""
106
+ SELECT
107
+ predicted_body_mass_g,
108
+ species,
109
+ flipper_length_mm
110
+ FROM ML.PREDICT(
111
+ MODEL `{PROJECT_ID}.{DATASET_ID}.wide_deep_regressor_penguins`,
112
+ (SELECT * FROM `bigquery-public-data.ml_datasets.penguins`
113
+ WHERE body_mass_g IS NOT NULL AND sex IN ('MALE', 'FEMALE') LIMIT 10)
114
+ )
115
+ """
116
+ client.query(query).to_dataframe()
117
+ ```
118
+
119
+ ---
120
+ ## Step 4 — Explain predictions
121
+
122
+ - **`ML.EXPLAIN_PREDICT`** — per-row feature attributions (Integrated Gradients)
123
+ - **`ML.GLOBAL_EXPLAIN`** — overall feature importance across the model (requires `enable_global_explain = TRUE`)
124
+
125
+ No `ML.WEIGHTS`/`ML.ADVANCED_WEIGHTS` (GLM-only) or `ML.FEATURE_IMPORTANCE` (tree-only) for this model type — same as `models/dnn_regressor` (DNN Regressor).
126
+
127
+ ```python
128
+ query = f"""
129
+ SELECT
130
+ predicted_body_mass_g,
131
+ top_feature_attributions
132
+ FROM ML.EXPLAIN_PREDICT(
133
+ MODEL `{PROJECT_ID}.{DATASET_ID}.wide_deep_regressor_penguins`,
134
+ (SELECT * FROM `bigquery-public-data.ml_datasets.penguins`
135
+ WHERE body_mass_g IS NOT NULL AND sex IN ('MALE', 'FEMALE') LIMIT 5),
136
+ STRUCT(5 AS top_k_features)
137
+ )
138
+ """
139
+ client.query(query).to_dataframe()
140
+ ```
141
+
142
+ ```python
143
+ query = f"""
144
+ SELECT *
145
+ FROM ML.GLOBAL_EXPLAIN(MODEL `{PROJECT_ID}.{DATASET_ID}.wide_deep_regressor_penguins`)
146
+ ORDER BY attribution DESC
147
+ """
148
+ client.query(query).to_dataframe()
149
+ ```
150
+
151
+ ---
152
+ ## Step 5 — Introspect the model
153
+
154
+ `ML.FEATURE_INFO` reports the statistics each feature had during training. `ML.TRAINING_INFO` returns the per-iteration loss curve — verified: only 2 iterations run before `early_stop` kicks in, same pattern as `models/dnn_regressor` (DNN Regressor).
155
+
156
+ ```python
157
+ query = f"""
158
+ SELECT *
159
+ FROM ML.FEATURE_INFO(MODEL `{PROJECT_ID}.{DATASET_ID}.wide_deep_regressor_penguins`)
160
+ """
161
+ client.query(query).to_dataframe()
162
+ ```
163
+
164
+ ```python
165
+ query = f"""
166
+ SELECT iteration, loss, eval_loss, learning_rate, duration_ms
167
+ FROM ML.TRAINING_INFO(MODEL `{PROJECT_ID}.{DATASET_ID}.wide_deep_regressor_penguins`)
168
+ ORDER BY iteration
169
+ """
170
+ client.query(query).to_dataframe()
171
+ ```
172
+
173
+ ---
174
+ ## Step 6 — The fix: `TRANSFORM` (scaling) + a higher `learn_rate`
175
+
176
+ Same fix as `models/dnn_regressor` (DNN Regressor): scale the numeric features AND raise `learn_rate` to 0.05 (50x the 0.001 default), with more `max_iterations` headroom. Verified: `r2_score` reaches **0.79** with the default `hidden_units=[64,32]` — a real fix (up from -27.4), though notably lower than DNN Regressor's ~0.86 with the equivalent fix on the same data. Step 7's hyperparameter tuning closes most of that gap by finding a better `hidden_units` value for this small dataset.
177
+
178
+ ```python
179
+ query = f"""
180
+ CREATE OR REPLACE MODEL `{PROJECT_ID}.{DATASET_ID}.wide_deep_regressor_penguins_transform`
181
+ TRANSFORM(
182
+ ML.STANDARD_SCALER(culmen_length_mm) OVER() AS culmen_length_mm,
183
+ ML.STANDARD_SCALER(culmen_depth_mm) OVER() AS culmen_depth_mm,
184
+ ML.STANDARD_SCALER(flipper_length_mm) OVER() AS flipper_length_mm,
185
+ species, island, sex, body_mass_g
186
+ )
187
+ OPTIONS(
188
+ model_type = 'DNN_LINEAR_COMBINED_REGRESSOR',
189
+ input_label_cols = ['body_mass_g'],
190
+ hidden_units = [64, 32],
191
+ activation_fn = 'RELU',
192
+ learn_rate = 0.05,
193
+ dropout = 0.15,
194
+ max_iterations = 30,
195
+ early_stop = TRUE,
196
+ data_split_method = 'AUTO_SPLIT',
197
+ enable_global_explain = TRUE
198
+ ) AS
199
+ SELECT
200
+ species, island, culmen_length_mm, culmen_depth_mm, flipper_length_mm, sex, body_mass_g
201
+ FROM `bigquery-public-data.ml_datasets.penguins`
202
+ WHERE body_mass_g IS NOT NULL AND sex IN ('MALE', 'FEMALE')
203
+ """
204
+ client.query(query).result()
205
+ print('Model wide_deep_regressor_penguins_transform created')
206
+ ```
207
+
208
+ ```python
209
+ # Confirm the fix with ML.EVALUATE
210
+ query = f"""
211
+ SELECT *
212
+ FROM ML.EVALUATE(MODEL `{PROJECT_ID}.{DATASET_ID}.wide_deep_regressor_penguins_transform`)
213
+ """
214
+ client.query(query).to_dataframe()
215
+ ```
216
+
217
+ ```python
218
+ # Predict on RAW rows - scaling is applied automatically inside the model
219
+ query = f"""
220
+ SELECT predicted_body_mass_g, species, flipper_length_mm
221
+ FROM ML.PREDICT(
222
+ MODEL `{PROJECT_ID}.{DATASET_ID}.wide_deep_regressor_penguins_transform`,
223
+ (SELECT * FROM `bigquery-public-data.ml_datasets.penguins`
224
+ WHERE body_mass_g IS NOT NULL AND sex IN ('MALE', 'FEMALE') LIMIT 5)
225
+ )
226
+ """
227
+ client.query(query).to_dataframe()
228
+ ```
229
+
230
+ ---
231
+ ## Step 7 — Hyperparameter tuning
232
+
233
+ `learn_rate` and `optimizer` are **NOT** tunable for this model type (verified — see `models/wide_and_deep_classifier` (Wide & Deep Classifier) Step 8 gotcha), so `learn_rate` stays fixed at the known-good 0.05 from Step 6 while tuning `hidden_units` (`HPARAM_CANDIDATES`) and `dropout` (`HPARAM_RANGE`) instead.
234
+
235
+ > **Verified:** all 4 trials succeeded (no failures), and all landed in the good region already — since `learn_rate` is fixed rather than searched, there's no risk here of trials getting stuck in the catastrophic unscaled/low-`learn_rate` region the way `models/dnn_regressor` (DNN Regressor)'s tuning step can.
236
+
237
+ ```python
238
+ query = f"""
239
+ CREATE OR REPLACE MODEL `{PROJECT_ID}.{DATASET_ID}.wide_deep_regressor_penguins_tuned`
240
+ TRANSFORM(
241
+ ML.STANDARD_SCALER(culmen_length_mm) OVER() AS culmen_length_mm,
242
+ ML.STANDARD_SCALER(culmen_depth_mm) OVER() AS culmen_depth_mm,
243
+ ML.STANDARD_SCALER(flipper_length_mm) OVER() AS flipper_length_mm,
244
+ species, island, sex, body_mass_g
245
+ )
246
+ OPTIONS(
247
+ model_type = 'DNN_LINEAR_COMBINED_REGRESSOR',
248
+ input_label_cols = ['body_mass_g'],
249
+ learn_rate = 0.05,
250
+ num_trials = 4,
251
+ max_parallel_trials = 2,
252
+ hparam_tuning_objectives = ['r2_score'],
253
+ hidden_units = HPARAM_CANDIDATES([STRUCT([64, 32]), STRUCT([32, 16])]),
254
+ dropout = HPARAM_RANGE(0.0, 0.3)
255
+ ) AS
256
+ SELECT
257
+ species, island, culmen_length_mm, culmen_depth_mm, flipper_length_mm, sex, body_mass_g
258
+ FROM `bigquery-public-data.ml_datasets.penguins`
259
+ WHERE body_mass_g IS NOT NULL AND sex IN ('MALE', 'FEMALE')
260
+ """
261
+ client.query(query).result()
262
+ print('Tuned model created')
263
+ ```
264
+
265
+ ```python
266
+ query = f"""
267
+ SELECT
268
+ trial_id,
269
+ hyperparameters,
270
+ hparam_tuning_evaluation_metrics.r2_score AS r2_score,
271
+ is_optimal
272
+ FROM ML.TRIAL_INFO(MODEL `{PROJECT_ID}.{DATASET_ID}.wide_deep_regressor_penguins_tuned`)
273
+ ORDER BY r2_score DESC
274
+ """
275
+ client.query(query).to_dataframe()
276
+ ```
277
+
278
+ ---
279
+ ## Examples — `%%bigquery` Magics
280
+
281
+ The same operations using IPython magic commands — write SQL directly in cells without Python string wrapping.
282
+
283
+ - `%%bigquery` — run SQL, display results inline
284
+ - `%%bigquery df` — run SQL, capture results into a pandas DataFrame
285
+
286
+ ### Evaluate with `%%bigquery`
287
+
288
+ ```sql
289
+ %%bigquery --project {PROJECT_ID}
290
+
291
+ SELECT *
292
+ FROM ML.EVALUATE(MODEL `statmike-mlops-349915.bq_ml.wide_deep_regressor_penguins`)
293
+ ```
294
+
295
+ ---
296
+ ## BigFrames
297
+
298
+ **No first-class wide-and-deep wrapper exists in `bigframes.ml`** (verified against the installed BigFrames API reference: `bigframes.ml.linear_model`, `.ensemble`, `.cluster`, `.decomposition`, `.forecasting`, `.imported`, `.llm` — no `DNNLinearCombinedClassifier`/`Regressor` or equivalent anywhere in the package, same gap as `models/dnn_classifier` (DNN Classifier)/`models/dnn_regressor` (DNN Regressor)). Use the SQL `CREATE MODEL` interface shown above.