bigquery-data-science 0.7.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (286) hide show
  1. bigquery_data_science/__init__.py +58 -0
  2. bigquery_data_science/_skills/PLANS.md +76 -0
  3. bigquery_data_science/_skills/bigquery-ai/SKILL.md +66 -0
  4. bigquery_data_science/_skills/bigquery-ai/narrative/ai_agg.md +528 -0
  5. bigquery_data_science/_skills/bigquery-ai/narrative/ai_classify.md +450 -0
  6. bigquery_data_science/_skills/bigquery-ai/narrative/ai_count_tokens.md +180 -0
  7. bigquery_data_science/_skills/bigquery-ai/narrative/ai_detect_anomalies.md +322 -0
  8. bigquery_data_science/_skills/bigquery-ai/narrative/ai_embed.md +425 -0
  9. bigquery_data_science/_skills/bigquery-ai/narrative/ai_evaluate.md +368 -0
  10. bigquery_data_science/_skills/bigquery-ai/narrative/ai_forecast.md +437 -0
  11. bigquery_data_science/_skills/bigquery-ai/narrative/ai_generate.md +693 -0
  12. bigquery_data_science/_skills/bigquery-ai/narrative/ai_generate_bool.md +248 -0
  13. bigquery_data_science/_skills/bigquery-ai/narrative/ai_generate_double.md +254 -0
  14. bigquery_data_science/_skills/bigquery-ai/narrative/ai_generate_embedding.md +402 -0
  15. bigquery_data_science/_skills/bigquery-ai/narrative/ai_generate_int.md +252 -0
  16. bigquery_data_science/_skills/bigquery-ai/narrative/ai_generate_table.md +447 -0
  17. bigquery_data_science/_skills/bigquery-ai/narrative/ai_generate_text.md +356 -0
  18. bigquery_data_science/_skills/bigquery-ai/narrative/ai_if.md +373 -0
  19. bigquery_data_science/_skills/bigquery-ai/narrative/ai_key_drivers.md +331 -0
  20. bigquery_data_science/_skills/bigquery-ai/narrative/ai_parse_document.md +416 -0
  21. bigquery_data_science/_skills/bigquery-ai/narrative/ai_predict.md +559 -0
  22. bigquery_data_science/_skills/bigquery-ai/narrative/ai_score.md +310 -0
  23. bigquery_data_science/_skills/bigquery-ai/narrative/ai_search.md +465 -0
  24. bigquery_data_science/_skills/bigquery-ai/narrative/ai_similarity.md +371 -0
  25. bigquery_data_science/_skills/bigquery-ai/narrative/catalog_search.md +951 -0
  26. bigquery_data_science/_skills/bigquery-ai/narrative/content_analysis.md +284 -0
  27. bigquery_data_science/_skills/bigquery-ai/narrative/content_moderation.md +369 -0
  28. bigquery_data_science/_skills/bigquery-ai/narrative/data_enrichment.md +426 -0
  29. bigquery_data_science/_skills/bigquery-ai/narrative/document_intelligence.md +470 -0
  30. bigquery_data_science/_skills/bigquery-ai/narrative/document_rag.md +453 -0
  31. bigquery_data_science/_skills/bigquery-ai/narrative/image_deduplication.md +770 -0
  32. bigquery_data_science/_skills/bigquery-ai/narrative/log_analysis.md +668 -0
  33. bigquery_data_science/_skills/bigquery-ai/narrative/metric_diagnostics.md +568 -0
  34. bigquery_data_science/_skills/bigquery-ai/narrative/ml_annotate_image.md +680 -0
  35. bigquery_data_science/_skills/bigquery-ai/narrative/ml_generate_embedding.md +217 -0
  36. bigquery_data_science/_skills/bigquery-ai/narrative/ml_generate_text.md +223 -0
  37. bigquery_data_science/_skills/bigquery-ai/narrative/ml_process_document.md +392 -0
  38. bigquery_data_science/_skills/bigquery-ai/narrative/ml_transcribe.md +815 -0
  39. bigquery_data_science/_skills/bigquery-ai/narrative/ml_translate.md +600 -0
  40. bigquery_data_science/_skills/bigquery-ai/narrative/ml_understand_text.md +673 -0
  41. bigquery_data_science/_skills/bigquery-ai/narrative/multimodal_analysis.md +266 -0
  42. bigquery_data_science/_skills/bigquery-ai/narrative/rag_pipeline.md +593 -0
  43. bigquery_data_science/_skills/bigquery-ai/narrative/semantic_search.md +510 -0
  44. bigquery_data_science/_skills/bigquery-ai/narrative/tabular_prediction.md +485 -0
  45. bigquery_data_science/_skills/bigquery-ai/narrative/time_series_intelligence.md +489 -0
  46. bigquery_data_science/_skills/bigquery-ai/narrative/vector_search.md +672 -0
  47. bigquery_data_science/_skills/bigquery-ai/reference/augmented-analytics.md +77 -0
  48. bigquery_data_science/_skills/bigquery-ai/reference/cloud-ai-service-models.md +88 -0
  49. bigquery_data_science/_skills/bigquery-ai/reference/document-processing.md +95 -0
  50. bigquery_data_science/_skills/bigquery-ai/reference/embeddings-and-search.md +166 -0
  51. bigquery_data_science/_skills/bigquery-ai/reference/general-purpose-generation.md +87 -0
  52. bigquery_data_science/_skills/bigquery-ai/reference/managed-functions.md +72 -0
  53. bigquery_data_science/_skills/bigquery-ai/reference/predictive-ai.md +113 -0
  54. bigquery_data_science/_skills/bigquery-ai/reference/workflows.md +56 -0
  55. bigquery_data_science/_skills/bigquery-ai/skill.manifest.json +109 -0
  56. bigquery_data_science/_skills/bigquery-ai/syntax/INDEX.md +71 -0
  57. bigquery_data_science/_skills/bigquery-ai/syntax/aiagg.md +67 -0
  58. bigquery_data_science/_skills/bigquery-ai/syntax/aicausal_effect-covered-in-bigquery-ml.md +17 -0
  59. bigquery_data_science/_skills/bigquery-ai/syntax/aiclassify.md +72 -0
  60. bigquery_data_science/_skills/bigquery-ai/syntax/aicount_tokens.md +53 -0
  61. bigquery_data_science/_skills/bigquery-ai/syntax/aidetect_anomalies.md +63 -0
  62. bigquery_data_science/_skills/bigquery-ai/syntax/aiembed.md +85 -0
  63. bigquery_data_science/_skills/bigquery-ai/syntax/aievaluate.md +114 -0
  64. bigquery_data_science/_skills/bigquery-ai/syntax/aiforecast.md +123 -0
  65. bigquery_data_science/_skills/bigquery-ai/syntax/aigenerate.md +69 -0
  66. bigquery_data_science/_skills/bigquery-ai/syntax/aigenerate_bool.md +53 -0
  67. bigquery_data_science/_skills/bigquery-ai/syntax/aigenerate_double.md +45 -0
  68. bigquery_data_science/_skills/bigquery-ai/syntax/aigenerate_embedding.md +108 -0
  69. bigquery_data_science/_skills/bigquery-ai/syntax/aigenerate_int.md +45 -0
  70. bigquery_data_science/_skills/bigquery-ai/syntax/aigenerate_table.md +64 -0
  71. bigquery_data_science/_skills/bigquery-ai/syntax/aigenerate_text.md +167 -0
  72. bigquery_data_science/_skills/bigquery-ai/syntax/aiif.md +56 -0
  73. bigquery_data_science/_skills/bigquery-ai/syntax/aikey_drivers.md +76 -0
  74. bigquery_data_science/_skills/bigquery-ai/syntax/aiparse_document.md +91 -0
  75. bigquery_data_science/_skills/bigquery-ai/syntax/aipredict.md +74 -0
  76. bigquery_data_science/_skills/bigquery-ai/syntax/aiscore.md +52 -0
  77. bigquery_data_science/_skills/bigquery-ai/syntax/aisearch.md +66 -0
  78. bigquery_data_science/_skills/bigquery-ai/syntax/aisimilarity.md +79 -0
  79. bigquery_data_science/_skills/bigquery-ai/syntax/choosing-between-these-and-the-ai-functions.md +11 -0
  80. bigquery_data_science/_skills/bigquery-ai/syntax/hybrid-search-capability.md +165 -0
  81. bigquery_data_science/_skills/bigquery-ai/syntax/mlannotate_image.md +24 -0
  82. bigquery_data_science/_skills/bigquery-ai/syntax/mlgenerate_embedding.md +34 -0
  83. bigquery_data_science/_skills/bigquery-ai/syntax/mlgenerate_text.md +65 -0
  84. bigquery_data_science/_skills/bigquery-ai/syntax/mlprocess_document.md +125 -0
  85. bigquery_data_science/_skills/bigquery-ai/syntax/mltranscribe.md +26 -0
  86. bigquery_data_science/_skills/bigquery-ai/syntax/mltranslate.md +27 -0
  87. bigquery_data_science/_skills/bigquery-ai/syntax/mlunderstand_text.md +32 -0
  88. bigquery_data_science/_skills/bigquery-ai/syntax/multimodal-input-patterns-by-function.md +128 -0
  89. bigquery_data_science/_skills/bigquery-ai/syntax/object-tables.md +109 -0
  90. bigquery_data_science/_skills/bigquery-ai/syntax/objectref-and-objectrefruntime-schema-reference.md +50 -0
  91. bigquery_data_science/_skills/bigquery-ai/syntax/objfetch_metadata.md +49 -0
  92. bigquery_data_science/_skills/bigquery-ai/syntax/objget_access_url.md +66 -0
  93. bigquery_data_science/_skills/bigquery-ai/syntax/objget_read_url.md +40 -0
  94. bigquery_data_science/_skills/bigquery-ai/syntax/objlist.md +34 -0
  95. bigquery_data_science/_skills/bigquery-ai/syntax/objmake_ref.md +58 -0
  96. bigquery_data_science/_skills/bigquery-ai/syntax/per-row-failures-and-the-retry-pattern.md +19 -0
  97. bigquery_data_science/_skills/bigquery-ai/syntax/setup-apis-connection-roles.md +39 -0
  98. bigquery_data_science/_skills/bigquery-ai/syntax/two-input-shapes.md +22 -0
  99. bigquery_data_science/_skills/bigquery-ai/syntax/vector_search.md +78 -0
  100. bigquery_data_science/_skills/bigquery-ai/syntax/what-the-five-have-in-common.md +27 -0
  101. bigquery_data_science/_skills/bigquery-data-science/SKILL.md +69 -0
  102. bigquery_data_science/_skills/bigquery-data-science/reference/namespace-matrix.md +112 -0
  103. bigquery_data_science/_skills/bigquery-data-science/skill.manifest.json +13 -0
  104. bigquery_data_science/_skills/bigquery-ml/SKILL.md +70 -0
  105. bigquery_data_science/_skills/bigquery-ml/narrative/airflow_with_kfp.md +379 -0
  106. bigquery_data_science/_skills/bigquery-ml/narrative/anomaly_fraud_detection.md +195 -0
  107. bigquery_data_science/_skills/bigquery-ml/narrative/arima_plus.md +635 -0
  108. bigquery_data_science/_skills/bigquery-ml/narrative/arima_plus_xreg.md +453 -0
  109. bigquery_data_science/_skills/bigquery-ml/narrative/autoencoder.md +470 -0
  110. bigquery_data_science/_skills/bigquery-ml/narrative/automl_classifier.md +291 -0
  111. bigquery_data_science/_skills/bigquery-ml/narrative/automl_regressor.md +274 -0
  112. bigquery_data_science/_skills/bigquery-ml/narrative/boosted_tree_classifier.md +449 -0
  113. bigquery_data_science/_skills/bigquery-ml/narrative/boosted_tree_regressor.md +400 -0
  114. bigquery_data_science/_skills/bigquery-ml/narrative/bucketizing.md +201 -0
  115. bigquery_data_science/_skills/bigquery-ml/narrative/causal_effect.md +773 -0
  116. bigquery_data_science/_skills/bigquery-ml/narrative/churn_retention.md +271 -0
  117. bigquery_data_science/_skills/bigquery-ml/narrative/cloud_workflows.md +370 -0
  118. bigquery_data_science/_skills/bigquery-ml/narrative/composer_airflow.md +475 -0
  119. bigquery_data_science/_skills/bigquery-ml/narrative/contribution_analysis.md +346 -0
  120. bigquery_data_science/_skills/bigquery-ml/narrative/cross_validation.md +185 -0
  121. bigquery_data_science/_skills/bigquery-ml/narrative/customer_segmentation.md +233 -0
  122. bigquery_data_science/_skills/bigquery-ml/narrative/dataform.md +372 -0
  123. bigquery_data_science/_skills/bigquery-ml/narrative/dbt.md +283 -0
  124. bigquery_data_science/_skills/bigquery-ml/narrative/difference_in_differences.md +280 -0
  125. bigquery_data_science/_skills/bigquery-ml/narrative/distance.md +195 -0
  126. bigquery_data_science/_skills/bigquery-ml/narrative/dnn_classifier.md +326 -0
  127. bigquery_data_science/_skills/bigquery-ml/narrative/dnn_regressor.md +308 -0
  128. bigquery_data_science/_skills/bigquery-ml/narrative/embeddings_classification.md +426 -0
  129. bigquery_data_science/_skills/bigquery-ml/narrative/encoding.md +238 -0
  130. bigquery_data_science/_skills/bigquery-ml/narrative/ensembling.md +278 -0
  131. bigquery_data_science/_skills/bigquery-ml/narrative/evaluation.md +445 -0
  132. bigquery_data_science/_skills/bigquery-ml/narrative/exploration.md +779 -0
  133. bigquery_data_science/_skills/bigquery-ml/narrative/export.md +321 -0
  134. bigquery_data_science/_skills/bigquery-ml/narrative/feature_engineering.md +280 -0
  135. bigquery_data_science/_skills/bigquery-ml/narrative/feature_store.md +652 -0
  136. bigquery_data_science/_skills/bigquery-ml/narrative/feature_store_workflow.md +426 -0
  137. bigquery_data_science/_skills/bigquery-ml/narrative/ga4_churn_prediction.md +285 -0
  138. bigquery_data_science/_skills/bigquery-ml/narrative/hierarchical_forecasting.md +441 -0
  139. bigquery_data_science/_skills/bigquery-ml/narrative/image.md +791 -0
  140. bigquery_data_science/_skills/bigquery-ml/narrative/imported.md +426 -0
  141. bigquery_data_science/_skills/bigquery-ml/narrative/kmeans.md +348 -0
  142. bigquery_data_science/_skills/bigquery-ml/narrative/linear_regression.md +372 -0
  143. bigquery_data_science/_skills/bigquery-ml/narrative/logistic_regression.md +390 -0
  144. bigquery_data_science/_skills/bigquery-ml/narrative/matrix_factorization.md +404 -0
  145. bigquery_data_science/_skills/bigquery-ml/narrative/model_monitoring.md +609 -0
  146. bigquery_data_science/_skills/bigquery-ml/narrative/pca.md +357 -0
  147. bigquery_data_science/_skills/bigquery-ml/narrative/price_elasticity_dml.md +286 -0
  148. bigquery_data_science/_skills/bigquery-ml/narrative/propensity_score_matching.md +350 -0
  149. bigquery_data_science/_skills/bigquery-ml/narrative/random_forest_classifier.md +445 -0
  150. bigquery_data_science/_skills/bigquery-ml/narrative/random_forest_regressor.md +409 -0
  151. bigquery_data_science/_skills/bigquery-ml/narrative/recommendation.md +281 -0
  152. bigquery_data_science/_skills/bigquery-ml/narrative/regression_based_forecasting.md +491 -0
  153. bigquery_data_science/_skills/bigquery-ml/narrative/remote.md +428 -0
  154. bigquery_data_science/_skills/bigquery-ml/narrative/scalers.md +325 -0
  155. bigquery_data_science/_skills/bigquery-ml/narrative/scheduled_queries.md +284 -0
  156. bigquery_data_science/_skills/bigquery-ml/narrative/sql_scripting.md +243 -0
  157. bigquery_data_science/_skills/bigquery-ml/narrative/survival_analysis.md +363 -0
  158. bigquery_data_science/_skills/bigquery-ml/narrative/synthetic_control.md +221 -0
  159. bigquery_data_science/_skills/bigquery-ml/narrative/text.md +215 -0
  160. bigquery_data_science/_skills/bigquery-ml/narrative/time_series.md +669 -0
  161. bigquery_data_science/_skills/bigquery-ml/narrative/transform_only.md +273 -0
  162. bigquery_data_science/_skills/bigquery-ml/narrative/uplift_cate.md +241 -0
  163. bigquery_data_science/_skills/bigquery-ml/narrative/vertex_kfp.md +360 -0
  164. bigquery_data_science/_skills/bigquery-ml/narrative/wide_and_deep_classifier.md +316 -0
  165. bigquery_data_science/_skills/bigquery-ml/narrative/wide_and_deep_regressor.md +298 -0
  166. bigquery_data_science/_skills/bigquery-ml/reference/classification.md +80 -0
  167. bigquery_data_science/_skills/bigquery-ml/reference/model-management.md +90 -0
  168. bigquery_data_science/_skills/bigquery-ml/reference/preprocessing-functions.md +234 -0
  169. bigquery_data_science/_skills/bigquery-ml/reference/regression.md +79 -0
  170. bigquery_data_science/_skills/bigquery-ml/reference/unsupervised-and-specialized.md +101 -0
  171. bigquery_data_science/_skills/bigquery-ml/reference/workflows-and-pipelines.md +111 -0
  172. bigquery_data_science/_skills/bigquery-ml/skill.manifest.json +177 -0
  173. bigquery_data_science/_skills/bigquery-ml/syntax/1-model_type-catalog.md +28 -0
  174. bigquery_data_science/_skills/bigquery-ml/syntax/2-task-mlevaluate-metric-columns.md +26 -0
  175. bigquery_data_science/_skills/bigquery-ml/syntax/3-capability-matrix-per-model-type.md +21 -0
  176. bigquery_data_science/_skills/bigquery-ml/syntax/4-explainability-weights-functions.md +14 -0
  177. bigquery_data_science/_skills/bigquery-ml/syntax/5-connection-matrix.md +11 -0
  178. bigquery_data_science/_skills/bigquery-ml/syntax/6-which-model-free-function-no-create-model-no-connection.md +15 -0
  179. bigquery_data_science/_skills/bigquery-ml/syntax/7-point-in-time-feature-retrieval-which-of-the-two.md +11 -0
  180. bigquery_data_science/_skills/bigquery-ml/syntax/INDEX.md +111 -0
  181. bigquery_data_science/_skills/bigquery-ml/syntax/aicausal_effect.md +85 -0
  182. bigquery_data_science/_skills/bigquery-ml/syntax/arima_plus-forecasting-lifecycle-functions.md +14 -0
  183. bigquery_data_science/_skills/bigquery-ml/syntax/arima_plus-univariate-time-series-forecasting.md +132 -0
  184. bigquery_data_science/_skills/bigquery-ml/syntax/arima_plus_xreg-multivariate-time-series-with-external-regressors.md +103 -0
  185. bigquery_data_science/_skills/bigquery-ml/syntax/autoencoder.md +102 -0
  186. bigquery_data_science/_skills/bigquery-ml/syntax/automl_classifier-automl_regressor.md +88 -0
  187. bigquery_data_science/_skills/bigquery-ml/syntax/boosted_tree_classifier-boosted_tree_regressor.md +117 -0
  188. bigquery_data_science/_skills/bigquery-ml/syntax/contribution_analysis.md +113 -0
  189. bigquery_data_science/_skills/bigquery-ml/syntax/dnn_classifier-dnn_regressor.md +107 -0
  190. bigquery_data_science/_skills/bigquery-ml/syntax/dnn_linear_combined_classifier-dnn_linear_combined_regressor-wide-and-deep.md +98 -0
  191. bigquery_data_science/_skills/bigquery-ml/syntax/explainability-functions-mlexplain_predict-mlglobal_explain-mlfeature_importance.md +7 -0
  192. bigquery_data_science/_skills/bigquery-ml/syntax/exploratory-data-analysis-mldescribe_data-mlcorrelation.md +11 -0
  193. bigquery_data_science/_skills/bigquery-ml/syntax/export-model.md +92 -0
  194. bigquery_data_science/_skills/bigquery-ml/syntax/feature-engineering-standalone-preprocessing-functions.md +13 -0
  195. bigquery_data_science/_skills/bigquery-ml/syntax/image-preprocessing-functions-mldecode_image-mlresize_image-mlconvert_image_type-mlconvert_color_space.md +63 -0
  196. bigquery_data_science/_skills/bigquery-ml/syntax/imported-models-tensorflow-tensorflow_lite-onnx-xgboost.md +36 -0
  197. bigquery_data_science/_skills/bigquery-ml/syntax/kmeans.md +87 -0
  198. bigquery_data_science/_skills/bigquery-ml/syntax/linear_reg-logistic_reg-generalized-linear-models.md +101 -0
  199. bigquery_data_science/_skills/bigquery-ml/syntax/matrix_factorization.md +162 -0
  200. bigquery_data_science/_skills/bigquery-ml/syntax/mladvanced_weights.md +76 -0
  201. bigquery_data_science/_skills/bigquery-ml/syntax/mlarima_coefficients.md +30 -0
  202. bigquery_data_science/_skills/bigquery-ml/syntax/mlarima_evaluate.md +49 -0
  203. bigquery_data_science/_skills/bigquery-ml/syntax/mlbag_of_words.md +47 -0
  204. bigquery_data_science/_skills/bigquery-ml/syntax/mlbucketize.md +47 -0
  205. bigquery_data_science/_skills/bigquery-ml/syntax/mlcentroids.md +46 -0
  206. bigquery_data_science/_skills/bigquery-ml/syntax/mlconfusion_matrix.md +56 -0
  207. bigquery_data_science/_skills/bigquery-ml/syntax/mlconvert_color_space.md +53 -0
  208. bigquery_data_science/_skills/bigquery-ml/syntax/mlconvert_image_type.md +39 -0
  209. bigquery_data_science/_skills/bigquery-ml/syntax/mlcorrelation.md +78 -0
  210. bigquery_data_science/_skills/bigquery-ml/syntax/mldecode_image.md +44 -0
  211. bigquery_data_science/_skills/bigquery-ml/syntax/mldescribe_data.md +74 -0
  212. bigquery_data_science/_skills/bigquery-ml/syntax/mldetect_anomalies.md +86 -0
  213. bigquery_data_science/_skills/bigquery-ml/syntax/mldistance.md +48 -0
  214. bigquery_data_science/_skills/bigquery-ml/syntax/mlentity_features_at_time.md +56 -0
  215. bigquery_data_science/_skills/bigquery-ml/syntax/mlevaluate.md +72 -0
  216. bigquery_data_science/_skills/bigquery-ml/syntax/mlexplain_forecast.md +50 -0
  217. bigquery_data_science/_skills/bigquery-ml/syntax/mlexplain_predict.md +66 -0
  218. bigquery_data_science/_skills/bigquery-ml/syntax/mlfeature_cross.md +46 -0
  219. bigquery_data_science/_skills/bigquery-ml/syntax/mlfeature_importance.md +47 -0
  220. bigquery_data_science/_skills/bigquery-ml/syntax/mlfeature_info.md +50 -0
  221. bigquery_data_science/_skills/bigquery-ml/syntax/mlfeatures_at_time.md +51 -0
  222. bigquery_data_science/_skills/bigquery-ml/syntax/mlforecast.md +76 -0
  223. bigquery_data_science/_skills/bigquery-ml/syntax/mlgenerate_embedding-from-a-bqml-pca-autoencoder-matrix_factorization-model.md +64 -0
  224. bigquery_data_science/_skills/bigquery-ml/syntax/mlglobal_explain.md +63 -0
  225. bigquery_data_science/_skills/bigquery-ml/syntax/mlhash_bucketize.md +47 -0
  226. bigquery_data_science/_skills/bigquery-ml/syntax/mlholiday_info.md +46 -0
  227. bigquery_data_science/_skills/bigquery-ml/syntax/mlimputer.md +46 -0
  228. bigquery_data_science/_skills/bigquery-ml/syntax/mllp_norm.md +47 -0
  229. bigquery_data_science/_skills/bigquery-ml/syntax/mlmetrics.md +80 -0
  230. bigquery_data_science/_skills/bigquery-ml/syntax/mlngrams.md +37 -0
  231. bigquery_data_science/_skills/bigquery-ml/syntax/mlone_hot_encoder-mlmulti_hot_encoder-mllabel_encoder.md +67 -0
  232. bigquery_data_science/_skills/bigquery-ml/syntax/mlpolynomial_expand.md +47 -0
  233. bigquery_data_science/_skills/bigquery-ml/syntax/mlpredict.md +67 -0
  234. bigquery_data_science/_skills/bigquery-ml/syntax/mlprincipal_component_info.md +42 -0
  235. bigquery_data_science/_skills/bigquery-ml/syntax/mlprincipal_components.md +40 -0
  236. bigquery_data_science/_skills/bigquery-ml/syntax/mlquantile_bucketize.md +43 -0
  237. bigquery_data_science/_skills/bigquery-ml/syntax/mlrecommend.md +60 -0
  238. bigquery_data_science/_skills/bigquery-ml/syntax/mlreconstruction_loss.md +48 -0
  239. bigquery_data_science/_skills/bigquery-ml/syntax/mlresize_image.md +45 -0
  240. bigquery_data_science/_skills/bigquery-ml/syntax/mlroc_curve.md +63 -0
  241. bigquery_data_science/_skills/bigquery-ml/syntax/mltf_idf.md +37 -0
  242. bigquery_data_science/_skills/bigquery-ml/syntax/mltfdv_describe.md +34 -0
  243. bigquery_data_science/_skills/bigquery-ml/syntax/mltfdv_validate.md +48 -0
  244. bigquery_data_science/_skills/bigquery-ml/syntax/mltraining_info.md +78 -0
  245. bigquery_data_science/_skills/bigquery-ml/syntax/mltransform-applying-a-saved-transform.md +52 -0
  246. bigquery_data_science/_skills/bigquery-ml/syntax/mltranspose-not-a-function-pointer.md +32 -0
  247. bigquery_data_science/_skills/bigquery-ml/syntax/mltrial_info.md +60 -0
  248. bigquery_data_science/_skills/bigquery-ml/syntax/mlvalidate_data_drift.md +99 -0
  249. bigquery_data_science/_skills/bigquery-ml/syntax/mlvalidate_data_skew.md +50 -0
  250. bigquery_data_science/_skills/bigquery-ml/syntax/mlweights.md +84 -0
  251. bigquery_data_science/_skills/bigquery-ml/syntax/model-free-time-series-decomposition-mltrend-mlseasonality-mldetect_change_points.md +100 -0
  252. bigquery_data_science/_skills/bigquery-ml/syntax/model-introspection-mlfeature_info-mltraining_info-mltrial_info.md +7 -0
  253. bigquery_data_science/_skills/bigquery-ml/syntax/model-monitoring-data-validation.md +23 -0
  254. bigquery_data_science/_skills/bigquery-ml/syntax/numerical-scalers-mlstandard_scaler-mlmin_max_scaler-mlmax_abs_scaler-mlrobust_scaler-mlnormalizer.md +77 -0
  255. bigquery_data_science/_skills/bigquery-ml/syntax/onnx.md +55 -0
  256. bigquery_data_science/_skills/bigquery-ml/syntax/pca.md +91 -0
  257. bigquery_data_science/_skills/bigquery-ml/syntax/point-in-time-feature-retrieval-mlfeatures_at_time-mlentity_features_at_time.md +25 -0
  258. bigquery_data_science/_skills/bigquery-ml/syntax/quick-comparison.md +14 -0
  259. bigquery_data_science/_skills/bigquery-ml/syntax/random_forest_classifier-random_forest_regressor.md +123 -0
  260. bigquery_data_science/_skills/bigquery-ml/syntax/remote-remote-model-over-a-vertex-ai-endpoint-the-mechanism.md +87 -0
  261. bigquery_data_science/_skills/bigquery-ml/syntax/tensorflow.md +48 -0
  262. bigquery_data_science/_skills/bigquery-ml/syntax/tensorflow_lite.md +29 -0
  263. bigquery_data_science/_skills/bigquery-ml/syntax/text-preprocessing-functions-mlngrams-mltf_idf-mlbag_of_words.md +18 -0
  264. bigquery_data_science/_skills/bigquery-ml/syntax/timesfm-built-in-foundation-forecaster-see-bigquery-ai.md +28 -0
  265. bigquery_data_science/_skills/bigquery-ml/syntax/transform_only.md +91 -0
  266. bigquery_data_science/_skills/bigquery-ml/syntax/xgboost.md +72 -0
  267. bigquery_data_science/_skills/pinned-models.json +28 -0
  268. bigquery_data_science/catalog.py +1107 -0
  269. bigquery_data_science/cli.py +786 -0
  270. bigquery_data_science/installer.py +282 -0
  271. bigquery_data_science/mcp_server.py +317 -0
  272. bigquery_data_science/tooling/__init__.py +15 -0
  273. bigquery_data_science/tooling/convert_notebook.py +176 -0
  274. bigquery_data_science/tooling/link_policy.py +247 -0
  275. bigquery_data_science/tooling/manifest.py +152 -0
  276. bigquery_data_science/tooling/markdown_anchors.py +84 -0
  277. bigquery_data_science/tooling/narrative_drift.py +163 -0
  278. bigquery_data_science/tooling/pinned_models.py +216 -0
  279. bigquery_data_science/tooling/reference_structure.py +85 -0
  280. bigquery_data_science/tooling/sync_syntax.py +407 -0
  281. bigquery_data_science/tooling/validate.py +208 -0
  282. bigquery_data_science-0.7.0.dist-info/METADATA +306 -0
  283. bigquery_data_science-0.7.0.dist-info/RECORD +286 -0
  284. bigquery_data_science-0.7.0.dist-info/WHEEL +4 -0
  285. bigquery_data_science-0.7.0.dist-info/entry_points.txt +5 -0
  286. bigquery_data_science-0.7.0.dist-info/licenses/LICENSE +201 -0
@@ -0,0 +1,404 @@
1
+ > **Source (`statmike/bigquery-data-science`):** **Notebook:** [`bigquery-ml/models/matrix_factorization/matrix_factorization.ipynb`](https://github.com/statmike/bigquery-data-science/blob/main/bigquery-ml/models/matrix_factorization/matrix_factorization.ipynb) · **SQL:** [`bigquery-ml/models/matrix_factorization/matrix_factorization.sql`](https://github.com/statmike/bigquery-data-science/blob/main/bigquery-ml/models/matrix_factorization/matrix_factorization.sql)
2
+
3
+ # Matrix Factorization — BigQuery ML
4
+
5
+ Train a **collaborative-filtering recommendation** model with `CREATE MODEL` (model_type = `MATRIX_FACTORIZATION`) — factorizes a sparse (user, item, rating) matrix into low-dimensional user and item latent factor vectors — then walk the full model lifecycle: evaluate, recommend, inspect factors, generate embeddings, item-item similarity search, introspect features/training, and tune hyperparameters.
6
+
7
+ **Lifecycle:** `CREATE MODEL` → `ML.EVALUATE` → `ML.RECOMMEND` (user→items, item→users) → `ML.WEIGHTS` → `ML.GENERATE_EMBEDDING` → `VECTOR_SEARCH` → `ML.FEATURE_INFO` → `ML.TRAINING_INFO` → hyperparameter tuning (`ML.TRIAL_INFO`)
8
+
9
+ **The one model type in this entire project that needs real infrastructure setup first:**
10
+ - **`MATRIX_FACTORIZATION` cannot train under on-demand (per-byte) pricing** — verified directly: attempting `CREATE MODEL` without a reservation fails immediately with `"Training Matrix Factorization models is not available for on-demand usage."` Every other model type in this project trains fine on-demand; this one requires a BigQuery slot reservation.
11
+ - **The reservation's edition matters too, not just its existence** — verified: a `STANDARD` edition reservation still fails, with a *different* error (`"Using BQML related functionalities is disallowed in STANDARD edition"`). `ENTERPRISE` (or higher) is required for any BQML training, matrix factorization included.
12
+ - This notebook creates a small **autoscaling** BigQuery Editions reservation (`ENTERPRISE`, 0 baseline slots, scales up to 100 only while a job actually runs) in Setup, and deletes it in Cleanup. This is **not** a capacity commitment — no upfront purchase, no minimum term; it bills per-second only for slot-time actually consumed while queries run, and idles at zero cost otherwise.
13
+ - **A newly created reservation assignment needs time to propagate** — verified: `CREATE MODEL` failed on the very first attempt right after creating and assigning a fresh reservation, and succeeded only after waiting.
14
+ - No `TRANSFORM` clause exists for this model type — the training query must produce exactly a (user, item, rating) triple.
15
+
16
+ **Data:** [`bigquery-public-data.google_analytics_sample`](https://console.cloud.google.com/marketplace/product/bigquery-public-datasets) (Google Merchandise Store demo data, July 2017) — an **IMPLICIT** feedback signal built from product-detail-page view counts per (visitor, product) pair: 42,178 users × 320 items across ~1.3M interactions. `feedback_type='IMPLICIT'` fits this data (a behavioral proxy, not an explicit star rating) — see `RESOURCES.md` (RESOURCES.md) for how `EXPLICIT` feedback differs (different options, different `ML.EVALUATE` metrics).
17
+
18
+ **References:** `RESOURCES.md` (Full reference) | [CREATE MODEL (matrix factorization) docs](https://cloud.google.com/bigquery/docs/reference/standard-sql/bigqueryml-syntax-create-matrix-factorization) | [ML.RECOMMEND docs](https://cloud.google.com/bigquery/docs/reference/standard-sql/bigqueryml-syntax-recommend) | `setup` (Setup guide)
19
+
20
+ > **Cost note (measured, not estimated):** this notebook creates a temporary BigQuery Editions reservation to enable training — there's no capacity commitment and no charge while it sits idle, but training under `ENTERPRISE` autoscale bills per slot-hour of actual usage. Building and validating this notebook (base model + tuning job + the BigFrames example below, each retrained at least once) consumed **~6.4 cumulative slot-hours** of real, measured usage — check the [current BigQuery Editions pricing](https://cloud.google.com/bigquery/pricing) to estimate the actual cost for your project/region before running. The reservation is deleted in Cleanup; if a run is interrupted before Cleanup, delete it manually (see Cleanup for the exact commands) to avoid ongoing charges.
21
+
22
+ ---
23
+ ## Setup
24
+
25
+ Set your project and location, authenticate, and create a shared dataset for the model.
26
+
27
+ ```python
28
+ PROJECT_ID = 'statmike-mlops-349915' # <-- Replace with your project ID
29
+ LOCATION = 'US' # BigQuery dataset location
30
+ DATASET_ID = 'bq_ml' # Shared dataset across all bigquery-ml notebooks
31
+ RESERVATION_ID = 'bqml-mf-temp' # Temporary reservation name (Setup creates it, Cleanup removes it)
32
+ ```
33
+
34
+ ### Environment
35
+
36
+ > **Already set up the project environment?** The cell below is a no-op — packages are already in your kernel. See the `setup` (Setup Reference) for details.
37
+ >
38
+ > **Running standalone** (Colab, Colab Enterprise, Vertex AI Workbench)? The cell below installs required packages into your current kernel.
39
+
40
+ ```python
41
+ from google.cloud import bigquery
42
+ import pandas as pd
43
+
44
+ client = bigquery.Client(project=PROJECT_ID)
45
+ pd.set_option('display.max_colwidth', None)
46
+
47
+ # Create the shared dataset (idempotent)
48
+ dataset_ref = bigquery.DatasetReference(PROJECT_ID, DATASET_ID)
49
+ dataset = bigquery.Dataset(dataset_ref)
50
+ dataset.location = LOCATION
51
+ client.create_dataset(dataset, exists_ok=True)
52
+ print(f'Dataset {PROJECT_ID}.{DATASET_ID} ready')
53
+
54
+ # Register %%bigquery cell magic (auto-loaded in Colab, needed elsewhere)
55
+ %load_ext bigquery_magics
56
+ ```
57
+
58
+ ### Create a temporary slot reservation (required for this model type only)
59
+
60
+ `ENTERPRISE` edition, 0 baseline slots, autoscaling up to 100 slots — no capacity commitment, billed per-second only while queries actually run. Assigned to this project for `QUERY`-type jobs (BQML training runs as a `QUERY` job). Waits ~90 seconds afterward for the assignment to actually take effect — a real, verified propagation delay, not an arbitrary pause.
61
+
62
+ **Idempotent:** if this cell is re-run (e.g. after an interrupted prior run whose Cleanup never executed), it detects and reuses the existing reservation/assignment instead of erroring or creating a duplicate — and skips the 90-second wait when reusing an already-propagated assignment.
63
+
64
+ ```python
65
+ import time
66
+ from google.cloud import bigquery_reservation_v1
67
+ from google.api_core.exceptions import AlreadyExists
68
+
69
+ reservation_client = bigquery_reservation_v1.ReservationServiceClient()
70
+ reservation_parent = f'projects/{PROJECT_ID}/locations/{LOCATION}'
71
+ reservation_name = f'{reservation_parent}/reservations/{RESERVATION_ID}'
72
+
73
+ # Idempotent: reuse an existing reservation/assignment (e.g. from an interrupted
74
+ # prior run) instead of erroring or creating a duplicate.
75
+ try:
76
+ reservation = reservation_client.create_reservation(
77
+ parent=reservation_parent,
78
+ reservation_id=RESERVATION_ID,
79
+ reservation=bigquery_reservation_v1.Reservation(
80
+ slot_capacity=0,
81
+ edition=bigquery_reservation_v1.Edition.ENTERPRISE,
82
+ autoscale=bigquery_reservation_v1.Reservation.Autoscale(max_slots=100),
83
+ ignore_idle_slots=True,
84
+ ),
85
+ )
86
+ print(f'Created {reservation.name}')
87
+ except AlreadyExists:
88
+ reservation = reservation_client.get_reservation(name=reservation_name)
89
+ print(f'Reservation already exists — reusing {reservation.name}')
90
+
91
+ existing_assignments = list(reservation_client.list_assignments(parent=reservation.name))
92
+ if existing_assignments:
93
+ assignment = existing_assignments[0]
94
+ print(f'Assignment already exists — reusing {assignment.name}')
95
+ else:
96
+ assignment = reservation_client.create_assignment(
97
+ parent=reservation.name,
98
+ assignment=bigquery_reservation_v1.Assignment(
99
+ job_type=bigquery_reservation_v1.Assignment.JobType.QUERY,
100
+ assignee=f'projects/{PROJECT_ID}',
101
+ ),
102
+ )
103
+ print(f'Created {assignment.name}, waiting ~90s for it to propagate...')
104
+ time.sleep(90)
105
+
106
+ print('Ready to train.')
107
+ ```
108
+
109
+ ---
110
+ ## Step 1 — Create the model with `CREATE MODEL`
111
+
112
+ `feedback_type = 'IMPLICIT'` fits this data — `view_count` is a behavioral proxy signal, not an explicit star rating. `num_factors = 16` is a deliberate, modest choice for a 320-item catalog. There is no `TRANSFORM` clause for this model type — the training `SELECT` must produce exactly a (user, item, rating) triple.
113
+
114
+ ```python
115
+ query = f"""
116
+ CREATE OR REPLACE MODEL `{PROJECT_ID}.{DATASET_ID}.matrix_factorization_ga`
117
+ OPTIONS(
118
+ model_type = 'MATRIX_FACTORIZATION',
119
+ feedback_type = 'IMPLICIT',
120
+ user_col = 'visitor_id',
121
+ item_col = 'product_name',
122
+ rating_col = 'view_count',
123
+ num_factors = 16
124
+ ) AS
125
+ SELECT
126
+ fullVisitorId AS visitor_id,
127
+ product.v2ProductName AS product_name,
128
+ COUNT(*) AS view_count
129
+ FROM `bigquery-public-data.google_analytics_sample.ga_sessions_*`,
130
+ UNNEST(hits) AS hits,
131
+ UNNEST(hits.product) AS product
132
+ WHERE _TABLE_SUFFIX BETWEEN '20170701' AND '20170801'
133
+ AND product.v2ProductName IS NOT NULL AND product.v2ProductName != '(not set)'
134
+ GROUP BY visitor_id, product_name
135
+ """
136
+ client.query(query).result()
137
+ print('Model matrix_factorization_ga created')
138
+ ```
139
+
140
+ ---
141
+ ## Step 2 — Evaluate with `ML.EVALUATE`
142
+
143
+ IMPLICIT feedback gets ranking-style metrics — `mean_average_precision` and `normalized_discounted_cumulative_gain` (both higher-is-better, bounded [0,1]), plus `mean_squared_error` and `average_rank`. `EXPLICIT` models get a different, regression-style metric set instead (see `RESOURCES.md` (RESOURCES.md)) — the two feedback types aren't evaluated the same way.
144
+
145
+ > **Verified:** retraining this exact model (same name, same SQL) does not reproduce `mean_average_precision` exactly — one training run reached `0.873`, another reached `0.860`. Matrix factorization's WALS training shows measurable retraining variance, similar to `models/kmeans` (K-Means)/`RANDOM_FOREST_*` elsewhere in this project, unlike PCA's full determinism or the DNN family's bit-for-bit reproducibility under a fixed name.
146
+
147
+ ```python
148
+ query = f"""
149
+ SELECT *
150
+ FROM ML.EVALUATE(MODEL `{PROJECT_ID}.{DATASET_ID}.matrix_factorization_ga`)
151
+ """
152
+ client.query(query).to_dataframe()
153
+ ```
154
+
155
+ ---
156
+ ## Step 3 — Recommend items for a user with `ML.RECOMMEND`
157
+
158
+ Passing only the user column scores every item for that user.
159
+
160
+ > **GOTCHA (verified):** calling this for a `visitor_id` **not present** in the training data does **not** error — it silently returns a full ranked list. `RESOURCES.md` (RESOURCES.md)'s general cold-start note ("cannot recommend for users or items absent from training data") describes the intent, not the observed runtime behavior: comparing the output for a confirmed real trained user against an obviously fake user ID shows completely different top-5 lists, consistent with the fake user falling back to item-side bias/popularity alone rather than genuine personalization (no user factor was ever learned for it). Don't assume a non-empty `ML.RECOMMEND` result means the input user actually existed in training.
161
+
162
+ ```python
163
+ query = f"""
164
+ SELECT visitor_id, product_name, predicted_view_count_confidence
165
+ FROM ML.RECOMMEND(
166
+ MODEL `{PROJECT_ID}.{DATASET_ID}.matrix_factorization_ga`,
167
+ (SELECT '9790296909741758431' AS visitor_id)
168
+ )
169
+ ORDER BY predicted_view_count_confidence DESC
170
+ LIMIT 5
171
+ """
172
+ client.query(query).to_dataframe()
173
+ ```
174
+
175
+ ---
176
+ ## Step 4 — Recommend users for an item with `ML.RECOMMEND`
177
+
178
+ Passing only the item column scores every user for that item — the symmetric counterpart to Step 3.
179
+
180
+ ```python
181
+ query = f"""
182
+ SELECT visitor_id, product_name, predicted_view_count_confidence
183
+ FROM ML.RECOMMEND(
184
+ MODEL `{PROJECT_ID}.{DATASET_ID}.matrix_factorization_ga`,
185
+ (SELECT 'Google Youth Girl Hoodie' AS product_name)
186
+ )
187
+ ORDER BY predicted_view_count_confidence DESC
188
+ LIMIT 5
189
+ """
190
+ client.query(query).to_dataframe()
191
+ ```
192
+
193
+ ---
194
+ ## Step 5 — Inspect the learned factors with `ML.WEIGHTS`
195
+
196
+ One row per user AND per item (distinguished by `processed_input`), each with a `factor_weights` `ARRAY<STRUCT<factor, weight>>` (length = `num_factors`) and an `intercept` (per-user/per-item bias term).
197
+
198
+ > **GOTCHA (verified):** there is also exactly one extra row with `processed_input=NULL` and `feature='global__INTERCEPT__'` — the single global bias term shared across the whole model, separate from every user's/item's own intercept.
199
+
200
+ ```python
201
+ query = f"""
202
+ SELECT processed_input, COUNT(*) AS n
203
+ FROM ML.WEIGHTS(MODEL `{PROJECT_ID}.{DATASET_ID}.matrix_factorization_ga`)
204
+ GROUP BY processed_input
205
+ """
206
+ client.query(query).to_dataframe()
207
+ ```
208
+
209
+ ---
210
+ ## Step 6 — Factor vectors as embeddings with `ML.GENERATE_EMBEDDING`
211
+
212
+ > **GOTCHA (verified):** for `MATRIX_FACTORIZATION`, `ML.GENERATE_EMBEDDING` takes **only the model** — no input table/query argument at all. Passing one errors immediately: `"Function ML.GENERATE_EMBEDDING for MATRIX_FACTORIZATION models only expects 1 argument but 2 were passed."` This is a genuinely different signature from `models/pca` (PCA)/`models/autoencoder` (Autoencoder) (both require a 2nd input-data argument) — it returns embeddings for **every** user and item in the model in one call, mirroring `ML.WEIGHTS`' `processed_input`/`feature` shape rather than `ML.PREDICT`'s.
213
+ >
214
+ > **Second GOTCHA (verified):** the embedding array has **`num_factors + 1` elements (17 here, not 16)** — one more than `ML.WEIGHTS`' `factor_weights` array for the same model (confirmed 16 elements there, matching `num_factors` exactly). The extra element is almost certainly the per-user/per-item `intercept` appended onto the raw factor vector. Don't assume the embedding array length always equals `num_factors` for this model type.
215
+
216
+ ```python
217
+ query = f"""
218
+ SELECT processed_input, feature, ml_generate_embedding_result
219
+ FROM ML.GENERATE_EMBEDDING(MODEL `{PROJECT_ID}.{DATASET_ID}.matrix_factorization_ga`)
220
+ LIMIT 5
221
+ """
222
+ client.query(query).to_dataframe()
223
+ ```
224
+
225
+ ---
226
+ ## Step 7 — Item-item similarity with `VECTOR_SEARCH`
227
+
228
+ Materialize the item embeddings (Step 6's output filtered to `processed_input='product_name'`) into a real table — `VECTOR_SEARCH` doesn't accept `ML.GENERATE_EMBEDDING` output directly as its base-table argument, the same limitation already verified in `models/autoencoder` (Autoencoder).
229
+
230
+ ```python
231
+ query = f"""
232
+ CREATE OR REPLACE TABLE `{PROJECT_ID}.{DATASET_ID}.matrix_factorization_item_embeddings` AS
233
+ SELECT feature AS product_name, ml_generate_embedding_result AS embedding
234
+ FROM ML.GENERATE_EMBEDDING(MODEL `{PROJECT_ID}.{DATASET_ID}.matrix_factorization_ga`)
235
+ WHERE processed_input = 'product_name'
236
+ """
237
+ client.query(query).result()
238
+ print('Table matrix_factorization_item_embeddings created')
239
+ ```
240
+
241
+ > **Verified:** querying "Google Youth Girl Hoodie" surfaces other youth apparel (youth tees, youth t-shirts) as its nearest neighbors — the learned item factors capture real product-category structure, not just noise.
242
+
243
+ ```python
244
+ query = f"""
245
+ SELECT query.product_name AS query_product, base.product_name AS similar_product, distance
246
+ FROM VECTOR_SEARCH(
247
+ TABLE `{PROJECT_ID}.{DATASET_ID}.matrix_factorization_item_embeddings`, 'embedding',
248
+ (SELECT product_name, embedding
249
+ FROM `{PROJECT_ID}.{DATASET_ID}.matrix_factorization_item_embeddings`
250
+ WHERE product_name = 'Google Youth Girl Hoodie'),
251
+ top_k => 6,
252
+ distance_type => 'COSINE'
253
+ )
254
+ ORDER BY distance
255
+ """
256
+ client.query(query).to_dataframe()
257
+ ```
258
+
259
+ ---
260
+ ## Step 8 — Introspect the training columns with `ML.FEATURE_INFO`
261
+
262
+ ```python
263
+ query = f"""
264
+ SELECT *
265
+ FROM ML.FEATURE_INFO(MODEL `{PROJECT_ID}.{DATASET_ID}.matrix_factorization_ga`)
266
+ """
267
+ client.query(query).to_dataframe()
268
+ ```
269
+
270
+ ---
271
+ ## Step 9 — Training loss curve with `ML.TRAINING_INFO`
272
+
273
+ > **Verified:** `eval_loss` is `NULL` throughout for this model — unlike the supervised model types in this project, `MATRIX_FACTORIZATION`'s `ML.TRAINING_INFO` doesn't populate a held-out `eval_loss` column here.
274
+
275
+ ```python
276
+ query = f"""
277
+ SELECT *
278
+ FROM ML.TRAINING_INFO(MODEL `{PROJECT_ID}.{DATASET_ID}.matrix_factorization_ga`)
279
+ """
280
+ client.query(query).to_dataframe()
281
+ ```
282
+
283
+ ---
284
+ ## Step 10 — Hyperparameter tuning
285
+
286
+ Tunes `num_factors`, `l2_reg`, and `wals_alpha` (IMPLICIT-only) together, maximizing `mean_average_precision` (the IMPLICIT default objective).
287
+
288
+ > **Verified:** tuning improved on Step 2's baseline here — the best trial (`num_factors=26`, `l2_reg=2.88`, `wals_alpha=49.2`) reached `mean_average_precision=0.899`, versus this run's untuned baseline of `0.860`. A separate training of the identical baseline and tuning config reached different specific numbers (baseline `0.873`, best tuned trial `0.915` with `wals_alpha=30.8` instead) — consistent with Step 2's retraining-variance finding, the exact hyperparameter search results aren't bit-for-bit reproducible either. **The qualitative result held in both runs: tuning beat the untuned baseline.** Contrast with `models/autoencoder` (Autoencoder), where an equally-sized 4-trial search failed to beat its untuned baseline at all.
289
+
290
+ ```python
291
+ query = f"""
292
+ CREATE OR REPLACE MODEL `{PROJECT_ID}.{DATASET_ID}.matrix_factorization_ga_tuned`
293
+ OPTIONS(
294
+ model_type = 'MATRIX_FACTORIZATION',
295
+ feedback_type = 'IMPLICIT',
296
+ user_col = 'visitor_id',
297
+ item_col = 'product_name',
298
+ rating_col = 'view_count',
299
+ num_factors = HPARAM_RANGE(8, 32),
300
+ l2_reg = HPARAM_RANGE(0.1, 10.0),
301
+ wals_alpha = HPARAM_RANGE(20.0, 60.0),
302
+ num_trials = 4,
303
+ max_parallel_trials = 2
304
+ ) AS
305
+ SELECT
306
+ fullVisitorId AS visitor_id,
307
+ product.v2ProductName AS product_name,
308
+ COUNT(*) AS view_count
309
+ FROM `bigquery-public-data.google_analytics_sample.ga_sessions_*`,
310
+ UNNEST(hits) AS hits,
311
+ UNNEST(hits.product) AS product
312
+ WHERE _TABLE_SUFFIX BETWEEN '20170701' AND '20170801'
313
+ AND product.v2ProductName IS NOT NULL AND product.v2ProductName != '(not set)'
314
+ GROUP BY visitor_id, product_name
315
+ """
316
+ client.query(query).result()
317
+ print('Tuned model created')
318
+ ```
319
+
320
+ ```python
321
+ query = f"""
322
+ SELECT
323
+ trial_id,
324
+ hyperparameters,
325
+ hparam_tuning_evaluation_metrics.mean_average_precision AS mean_average_precision,
326
+ is_optimal
327
+ FROM ML.TRIAL_INFO(MODEL `{PROJECT_ID}.{DATASET_ID}.matrix_factorization_ga_tuned`)
328
+ ORDER BY mean_average_precision DESC
329
+ """
330
+ client.query(query).to_dataframe()
331
+ ```
332
+
333
+ Calling `ML.EVALUATE` on a tuned model with no extra arguments returns every trial's metrics in one result (one row per `trial_id`) — there's no `STRUCT(trial_id)` 2-argument form for `ML.EVALUATE` the way `ML.RECOMMEND`/`ML.PREDICT` accept one; passing a `STRUCT` as a 2nd argument errors (`"argument 2 must be a relation"`).
334
+
335
+ ```python
336
+ query = f"""
337
+ SELECT *
338
+ FROM ML.EVALUATE(MODEL `{PROJECT_ID}.{DATASET_ID}.matrix_factorization_ga_tuned`)
339
+ """
340
+ client.query(query).to_dataframe()
341
+ ```
342
+
343
+ ---
344
+ ## Examples — `%%bigquery` Magics
345
+
346
+ The same operations using IPython magic commands — write SQL directly in cells without Python string wrapping.
347
+
348
+ - `%%bigquery` — run SQL, display results inline
349
+ - `%%bigquery df` — run SQL, capture results into a pandas DataFrame
350
+
351
+ ### Evaluate with `%%bigquery`
352
+
353
+ ```sql
354
+ %%bigquery --project {PROJECT_ID}
355
+
356
+ SELECT *
357
+ FROM ML.EVALUATE(MODEL `statmike-mlops-349915.bq_ml.matrix_factorization_ga`)
358
+ ```
359
+
360
+ ---
361
+ ## Examples — BigFrames
362
+
363
+ BigFrames provides a scikit-learn-style API (`bigframes.ml`) that trains BigQuery ML models under the hood. **Matrix factorization has a genuine first-class wrapper: `bigframes.ml.decomposition.MatrixFactorization`.**
364
+
365
+ > **GOTCHA (verified against the live installed constructor signature):** `MatrixFactorization(feedback_type=..., num_factors=..., user_col=..., item_col=..., rating_col=..., l2_reg=...)` — there is **no `wals_alpha` parameter**, unlike the SQL `CREATE MODEL` syntax. For `feedback_type='implicit'`, BigFrames trains with BigQuery ML's default `wals_alpha` (40) with no way to override it through this wrapper — use SQL directly if you need to set `wals_alpha`.
366
+
367
+ ```python
368
+ import bigframes.pandas as bpd
369
+
370
+ bpd.close_session() # Reset session to apply project/location settings
371
+ bpd.options.bigquery.project = PROJECT_ID
372
+ bpd.options.bigquery.location = LOCATION
373
+ ```
374
+
375
+ ```python
376
+ from bigframes.ml.decomposition import MatrixFactorization
377
+
378
+ query = f"""
379
+ SELECT
380
+ fullVisitorId AS visitor_id,
381
+ product.v2ProductName AS product_name,
382
+ COUNT(*) AS view_count
383
+ FROM `bigquery-public-data.google_analytics_sample.ga_sessions_*`,
384
+ UNNEST(hits) AS hits,
385
+ UNNEST(hits.product) AS product
386
+ WHERE _TABLE_SUFFIX BETWEEN '20170701' AND '20170801'
387
+ AND product.v2ProductName IS NOT NULL AND product.v2ProductName != '(not set)'
388
+ GROUP BY visitor_id, product_name
389
+ """
390
+ df = bpd.read_gbq(query)
391
+
392
+ # Train (creates a BigQuery ML model behind the scenes)
393
+ model = MatrixFactorization(
394
+ feedback_type='implicit',
395
+ num_factors=16,
396
+ user_col='visitor_id',
397
+ item_col='product_name',
398
+ rating_col='view_count',
399
+ )
400
+ model.fit(df)
401
+
402
+ # Evaluate
403
+ model.score(df).to_pandas()
404
+ ```