bigquery-data-science 0.7.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (286) hide show
  1. bigquery_data_science/__init__.py +58 -0
  2. bigquery_data_science/_skills/PLANS.md +76 -0
  3. bigquery_data_science/_skills/bigquery-ai/SKILL.md +66 -0
  4. bigquery_data_science/_skills/bigquery-ai/narrative/ai_agg.md +528 -0
  5. bigquery_data_science/_skills/bigquery-ai/narrative/ai_classify.md +450 -0
  6. bigquery_data_science/_skills/bigquery-ai/narrative/ai_count_tokens.md +180 -0
  7. bigquery_data_science/_skills/bigquery-ai/narrative/ai_detect_anomalies.md +322 -0
  8. bigquery_data_science/_skills/bigquery-ai/narrative/ai_embed.md +425 -0
  9. bigquery_data_science/_skills/bigquery-ai/narrative/ai_evaluate.md +368 -0
  10. bigquery_data_science/_skills/bigquery-ai/narrative/ai_forecast.md +437 -0
  11. bigquery_data_science/_skills/bigquery-ai/narrative/ai_generate.md +693 -0
  12. bigquery_data_science/_skills/bigquery-ai/narrative/ai_generate_bool.md +248 -0
  13. bigquery_data_science/_skills/bigquery-ai/narrative/ai_generate_double.md +254 -0
  14. bigquery_data_science/_skills/bigquery-ai/narrative/ai_generate_embedding.md +402 -0
  15. bigquery_data_science/_skills/bigquery-ai/narrative/ai_generate_int.md +252 -0
  16. bigquery_data_science/_skills/bigquery-ai/narrative/ai_generate_table.md +447 -0
  17. bigquery_data_science/_skills/bigquery-ai/narrative/ai_generate_text.md +356 -0
  18. bigquery_data_science/_skills/bigquery-ai/narrative/ai_if.md +373 -0
  19. bigquery_data_science/_skills/bigquery-ai/narrative/ai_key_drivers.md +331 -0
  20. bigquery_data_science/_skills/bigquery-ai/narrative/ai_parse_document.md +416 -0
  21. bigquery_data_science/_skills/bigquery-ai/narrative/ai_predict.md +559 -0
  22. bigquery_data_science/_skills/bigquery-ai/narrative/ai_score.md +310 -0
  23. bigquery_data_science/_skills/bigquery-ai/narrative/ai_search.md +465 -0
  24. bigquery_data_science/_skills/bigquery-ai/narrative/ai_similarity.md +371 -0
  25. bigquery_data_science/_skills/bigquery-ai/narrative/catalog_search.md +951 -0
  26. bigquery_data_science/_skills/bigquery-ai/narrative/content_analysis.md +284 -0
  27. bigquery_data_science/_skills/bigquery-ai/narrative/content_moderation.md +369 -0
  28. bigquery_data_science/_skills/bigquery-ai/narrative/data_enrichment.md +426 -0
  29. bigquery_data_science/_skills/bigquery-ai/narrative/document_intelligence.md +470 -0
  30. bigquery_data_science/_skills/bigquery-ai/narrative/document_rag.md +453 -0
  31. bigquery_data_science/_skills/bigquery-ai/narrative/image_deduplication.md +770 -0
  32. bigquery_data_science/_skills/bigquery-ai/narrative/log_analysis.md +668 -0
  33. bigquery_data_science/_skills/bigquery-ai/narrative/metric_diagnostics.md +568 -0
  34. bigquery_data_science/_skills/bigquery-ai/narrative/ml_annotate_image.md +680 -0
  35. bigquery_data_science/_skills/bigquery-ai/narrative/ml_generate_embedding.md +217 -0
  36. bigquery_data_science/_skills/bigquery-ai/narrative/ml_generate_text.md +223 -0
  37. bigquery_data_science/_skills/bigquery-ai/narrative/ml_process_document.md +392 -0
  38. bigquery_data_science/_skills/bigquery-ai/narrative/ml_transcribe.md +815 -0
  39. bigquery_data_science/_skills/bigquery-ai/narrative/ml_translate.md +600 -0
  40. bigquery_data_science/_skills/bigquery-ai/narrative/ml_understand_text.md +673 -0
  41. bigquery_data_science/_skills/bigquery-ai/narrative/multimodal_analysis.md +266 -0
  42. bigquery_data_science/_skills/bigquery-ai/narrative/rag_pipeline.md +593 -0
  43. bigquery_data_science/_skills/bigquery-ai/narrative/semantic_search.md +510 -0
  44. bigquery_data_science/_skills/bigquery-ai/narrative/tabular_prediction.md +485 -0
  45. bigquery_data_science/_skills/bigquery-ai/narrative/time_series_intelligence.md +489 -0
  46. bigquery_data_science/_skills/bigquery-ai/narrative/vector_search.md +672 -0
  47. bigquery_data_science/_skills/bigquery-ai/reference/augmented-analytics.md +77 -0
  48. bigquery_data_science/_skills/bigquery-ai/reference/cloud-ai-service-models.md +88 -0
  49. bigquery_data_science/_skills/bigquery-ai/reference/document-processing.md +95 -0
  50. bigquery_data_science/_skills/bigquery-ai/reference/embeddings-and-search.md +166 -0
  51. bigquery_data_science/_skills/bigquery-ai/reference/general-purpose-generation.md +87 -0
  52. bigquery_data_science/_skills/bigquery-ai/reference/managed-functions.md +72 -0
  53. bigquery_data_science/_skills/bigquery-ai/reference/predictive-ai.md +113 -0
  54. bigquery_data_science/_skills/bigquery-ai/reference/workflows.md +56 -0
  55. bigquery_data_science/_skills/bigquery-ai/skill.manifest.json +109 -0
  56. bigquery_data_science/_skills/bigquery-ai/syntax/INDEX.md +71 -0
  57. bigquery_data_science/_skills/bigquery-ai/syntax/aiagg.md +67 -0
  58. bigquery_data_science/_skills/bigquery-ai/syntax/aicausal_effect-covered-in-bigquery-ml.md +17 -0
  59. bigquery_data_science/_skills/bigquery-ai/syntax/aiclassify.md +72 -0
  60. bigquery_data_science/_skills/bigquery-ai/syntax/aicount_tokens.md +53 -0
  61. bigquery_data_science/_skills/bigquery-ai/syntax/aidetect_anomalies.md +63 -0
  62. bigquery_data_science/_skills/bigquery-ai/syntax/aiembed.md +85 -0
  63. bigquery_data_science/_skills/bigquery-ai/syntax/aievaluate.md +114 -0
  64. bigquery_data_science/_skills/bigquery-ai/syntax/aiforecast.md +123 -0
  65. bigquery_data_science/_skills/bigquery-ai/syntax/aigenerate.md +69 -0
  66. bigquery_data_science/_skills/bigquery-ai/syntax/aigenerate_bool.md +53 -0
  67. bigquery_data_science/_skills/bigquery-ai/syntax/aigenerate_double.md +45 -0
  68. bigquery_data_science/_skills/bigquery-ai/syntax/aigenerate_embedding.md +108 -0
  69. bigquery_data_science/_skills/bigquery-ai/syntax/aigenerate_int.md +45 -0
  70. bigquery_data_science/_skills/bigquery-ai/syntax/aigenerate_table.md +64 -0
  71. bigquery_data_science/_skills/bigquery-ai/syntax/aigenerate_text.md +167 -0
  72. bigquery_data_science/_skills/bigquery-ai/syntax/aiif.md +56 -0
  73. bigquery_data_science/_skills/bigquery-ai/syntax/aikey_drivers.md +76 -0
  74. bigquery_data_science/_skills/bigquery-ai/syntax/aiparse_document.md +91 -0
  75. bigquery_data_science/_skills/bigquery-ai/syntax/aipredict.md +74 -0
  76. bigquery_data_science/_skills/bigquery-ai/syntax/aiscore.md +52 -0
  77. bigquery_data_science/_skills/bigquery-ai/syntax/aisearch.md +66 -0
  78. bigquery_data_science/_skills/bigquery-ai/syntax/aisimilarity.md +79 -0
  79. bigquery_data_science/_skills/bigquery-ai/syntax/choosing-between-these-and-the-ai-functions.md +11 -0
  80. bigquery_data_science/_skills/bigquery-ai/syntax/hybrid-search-capability.md +165 -0
  81. bigquery_data_science/_skills/bigquery-ai/syntax/mlannotate_image.md +24 -0
  82. bigquery_data_science/_skills/bigquery-ai/syntax/mlgenerate_embedding.md +34 -0
  83. bigquery_data_science/_skills/bigquery-ai/syntax/mlgenerate_text.md +65 -0
  84. bigquery_data_science/_skills/bigquery-ai/syntax/mlprocess_document.md +125 -0
  85. bigquery_data_science/_skills/bigquery-ai/syntax/mltranscribe.md +26 -0
  86. bigquery_data_science/_skills/bigquery-ai/syntax/mltranslate.md +27 -0
  87. bigquery_data_science/_skills/bigquery-ai/syntax/mlunderstand_text.md +32 -0
  88. bigquery_data_science/_skills/bigquery-ai/syntax/multimodal-input-patterns-by-function.md +128 -0
  89. bigquery_data_science/_skills/bigquery-ai/syntax/object-tables.md +109 -0
  90. bigquery_data_science/_skills/bigquery-ai/syntax/objectref-and-objectrefruntime-schema-reference.md +50 -0
  91. bigquery_data_science/_skills/bigquery-ai/syntax/objfetch_metadata.md +49 -0
  92. bigquery_data_science/_skills/bigquery-ai/syntax/objget_access_url.md +66 -0
  93. bigquery_data_science/_skills/bigquery-ai/syntax/objget_read_url.md +40 -0
  94. bigquery_data_science/_skills/bigquery-ai/syntax/objlist.md +34 -0
  95. bigquery_data_science/_skills/bigquery-ai/syntax/objmake_ref.md +58 -0
  96. bigquery_data_science/_skills/bigquery-ai/syntax/per-row-failures-and-the-retry-pattern.md +19 -0
  97. bigquery_data_science/_skills/bigquery-ai/syntax/setup-apis-connection-roles.md +39 -0
  98. bigquery_data_science/_skills/bigquery-ai/syntax/two-input-shapes.md +22 -0
  99. bigquery_data_science/_skills/bigquery-ai/syntax/vector_search.md +78 -0
  100. bigquery_data_science/_skills/bigquery-ai/syntax/what-the-five-have-in-common.md +27 -0
  101. bigquery_data_science/_skills/bigquery-data-science/SKILL.md +69 -0
  102. bigquery_data_science/_skills/bigquery-data-science/reference/namespace-matrix.md +112 -0
  103. bigquery_data_science/_skills/bigquery-data-science/skill.manifest.json +13 -0
  104. bigquery_data_science/_skills/bigquery-ml/SKILL.md +70 -0
  105. bigquery_data_science/_skills/bigquery-ml/narrative/airflow_with_kfp.md +379 -0
  106. bigquery_data_science/_skills/bigquery-ml/narrative/anomaly_fraud_detection.md +195 -0
  107. bigquery_data_science/_skills/bigquery-ml/narrative/arima_plus.md +635 -0
  108. bigquery_data_science/_skills/bigquery-ml/narrative/arima_plus_xreg.md +453 -0
  109. bigquery_data_science/_skills/bigquery-ml/narrative/autoencoder.md +470 -0
  110. bigquery_data_science/_skills/bigquery-ml/narrative/automl_classifier.md +291 -0
  111. bigquery_data_science/_skills/bigquery-ml/narrative/automl_regressor.md +274 -0
  112. bigquery_data_science/_skills/bigquery-ml/narrative/boosted_tree_classifier.md +449 -0
  113. bigquery_data_science/_skills/bigquery-ml/narrative/boosted_tree_regressor.md +400 -0
  114. bigquery_data_science/_skills/bigquery-ml/narrative/bucketizing.md +201 -0
  115. bigquery_data_science/_skills/bigquery-ml/narrative/causal_effect.md +773 -0
  116. bigquery_data_science/_skills/bigquery-ml/narrative/churn_retention.md +271 -0
  117. bigquery_data_science/_skills/bigquery-ml/narrative/cloud_workflows.md +370 -0
  118. bigquery_data_science/_skills/bigquery-ml/narrative/composer_airflow.md +475 -0
  119. bigquery_data_science/_skills/bigquery-ml/narrative/contribution_analysis.md +346 -0
  120. bigquery_data_science/_skills/bigquery-ml/narrative/cross_validation.md +185 -0
  121. bigquery_data_science/_skills/bigquery-ml/narrative/customer_segmentation.md +233 -0
  122. bigquery_data_science/_skills/bigquery-ml/narrative/dataform.md +372 -0
  123. bigquery_data_science/_skills/bigquery-ml/narrative/dbt.md +283 -0
  124. bigquery_data_science/_skills/bigquery-ml/narrative/difference_in_differences.md +280 -0
  125. bigquery_data_science/_skills/bigquery-ml/narrative/distance.md +195 -0
  126. bigquery_data_science/_skills/bigquery-ml/narrative/dnn_classifier.md +326 -0
  127. bigquery_data_science/_skills/bigquery-ml/narrative/dnn_regressor.md +308 -0
  128. bigquery_data_science/_skills/bigquery-ml/narrative/embeddings_classification.md +426 -0
  129. bigquery_data_science/_skills/bigquery-ml/narrative/encoding.md +238 -0
  130. bigquery_data_science/_skills/bigquery-ml/narrative/ensembling.md +278 -0
  131. bigquery_data_science/_skills/bigquery-ml/narrative/evaluation.md +445 -0
  132. bigquery_data_science/_skills/bigquery-ml/narrative/exploration.md +779 -0
  133. bigquery_data_science/_skills/bigquery-ml/narrative/export.md +321 -0
  134. bigquery_data_science/_skills/bigquery-ml/narrative/feature_engineering.md +280 -0
  135. bigquery_data_science/_skills/bigquery-ml/narrative/feature_store.md +652 -0
  136. bigquery_data_science/_skills/bigquery-ml/narrative/feature_store_workflow.md +426 -0
  137. bigquery_data_science/_skills/bigquery-ml/narrative/ga4_churn_prediction.md +285 -0
  138. bigquery_data_science/_skills/bigquery-ml/narrative/hierarchical_forecasting.md +441 -0
  139. bigquery_data_science/_skills/bigquery-ml/narrative/image.md +791 -0
  140. bigquery_data_science/_skills/bigquery-ml/narrative/imported.md +426 -0
  141. bigquery_data_science/_skills/bigquery-ml/narrative/kmeans.md +348 -0
  142. bigquery_data_science/_skills/bigquery-ml/narrative/linear_regression.md +372 -0
  143. bigquery_data_science/_skills/bigquery-ml/narrative/logistic_regression.md +390 -0
  144. bigquery_data_science/_skills/bigquery-ml/narrative/matrix_factorization.md +404 -0
  145. bigquery_data_science/_skills/bigquery-ml/narrative/model_monitoring.md +609 -0
  146. bigquery_data_science/_skills/bigquery-ml/narrative/pca.md +357 -0
  147. bigquery_data_science/_skills/bigquery-ml/narrative/price_elasticity_dml.md +286 -0
  148. bigquery_data_science/_skills/bigquery-ml/narrative/propensity_score_matching.md +350 -0
  149. bigquery_data_science/_skills/bigquery-ml/narrative/random_forest_classifier.md +445 -0
  150. bigquery_data_science/_skills/bigquery-ml/narrative/random_forest_regressor.md +409 -0
  151. bigquery_data_science/_skills/bigquery-ml/narrative/recommendation.md +281 -0
  152. bigquery_data_science/_skills/bigquery-ml/narrative/regression_based_forecasting.md +491 -0
  153. bigquery_data_science/_skills/bigquery-ml/narrative/remote.md +428 -0
  154. bigquery_data_science/_skills/bigquery-ml/narrative/scalers.md +325 -0
  155. bigquery_data_science/_skills/bigquery-ml/narrative/scheduled_queries.md +284 -0
  156. bigquery_data_science/_skills/bigquery-ml/narrative/sql_scripting.md +243 -0
  157. bigquery_data_science/_skills/bigquery-ml/narrative/survival_analysis.md +363 -0
  158. bigquery_data_science/_skills/bigquery-ml/narrative/synthetic_control.md +221 -0
  159. bigquery_data_science/_skills/bigquery-ml/narrative/text.md +215 -0
  160. bigquery_data_science/_skills/bigquery-ml/narrative/time_series.md +669 -0
  161. bigquery_data_science/_skills/bigquery-ml/narrative/transform_only.md +273 -0
  162. bigquery_data_science/_skills/bigquery-ml/narrative/uplift_cate.md +241 -0
  163. bigquery_data_science/_skills/bigquery-ml/narrative/vertex_kfp.md +360 -0
  164. bigquery_data_science/_skills/bigquery-ml/narrative/wide_and_deep_classifier.md +316 -0
  165. bigquery_data_science/_skills/bigquery-ml/narrative/wide_and_deep_regressor.md +298 -0
  166. bigquery_data_science/_skills/bigquery-ml/reference/classification.md +80 -0
  167. bigquery_data_science/_skills/bigquery-ml/reference/model-management.md +90 -0
  168. bigquery_data_science/_skills/bigquery-ml/reference/preprocessing-functions.md +234 -0
  169. bigquery_data_science/_skills/bigquery-ml/reference/regression.md +79 -0
  170. bigquery_data_science/_skills/bigquery-ml/reference/unsupervised-and-specialized.md +101 -0
  171. bigquery_data_science/_skills/bigquery-ml/reference/workflows-and-pipelines.md +111 -0
  172. bigquery_data_science/_skills/bigquery-ml/skill.manifest.json +177 -0
  173. bigquery_data_science/_skills/bigquery-ml/syntax/1-model_type-catalog.md +28 -0
  174. bigquery_data_science/_skills/bigquery-ml/syntax/2-task-mlevaluate-metric-columns.md +26 -0
  175. bigquery_data_science/_skills/bigquery-ml/syntax/3-capability-matrix-per-model-type.md +21 -0
  176. bigquery_data_science/_skills/bigquery-ml/syntax/4-explainability-weights-functions.md +14 -0
  177. bigquery_data_science/_skills/bigquery-ml/syntax/5-connection-matrix.md +11 -0
  178. bigquery_data_science/_skills/bigquery-ml/syntax/6-which-model-free-function-no-create-model-no-connection.md +15 -0
  179. bigquery_data_science/_skills/bigquery-ml/syntax/7-point-in-time-feature-retrieval-which-of-the-two.md +11 -0
  180. bigquery_data_science/_skills/bigquery-ml/syntax/INDEX.md +111 -0
  181. bigquery_data_science/_skills/bigquery-ml/syntax/aicausal_effect.md +85 -0
  182. bigquery_data_science/_skills/bigquery-ml/syntax/arima_plus-forecasting-lifecycle-functions.md +14 -0
  183. bigquery_data_science/_skills/bigquery-ml/syntax/arima_plus-univariate-time-series-forecasting.md +132 -0
  184. bigquery_data_science/_skills/bigquery-ml/syntax/arima_plus_xreg-multivariate-time-series-with-external-regressors.md +103 -0
  185. bigquery_data_science/_skills/bigquery-ml/syntax/autoencoder.md +102 -0
  186. bigquery_data_science/_skills/bigquery-ml/syntax/automl_classifier-automl_regressor.md +88 -0
  187. bigquery_data_science/_skills/bigquery-ml/syntax/boosted_tree_classifier-boosted_tree_regressor.md +117 -0
  188. bigquery_data_science/_skills/bigquery-ml/syntax/contribution_analysis.md +113 -0
  189. bigquery_data_science/_skills/bigquery-ml/syntax/dnn_classifier-dnn_regressor.md +107 -0
  190. bigquery_data_science/_skills/bigquery-ml/syntax/dnn_linear_combined_classifier-dnn_linear_combined_regressor-wide-and-deep.md +98 -0
  191. bigquery_data_science/_skills/bigquery-ml/syntax/explainability-functions-mlexplain_predict-mlglobal_explain-mlfeature_importance.md +7 -0
  192. bigquery_data_science/_skills/bigquery-ml/syntax/exploratory-data-analysis-mldescribe_data-mlcorrelation.md +11 -0
  193. bigquery_data_science/_skills/bigquery-ml/syntax/export-model.md +92 -0
  194. bigquery_data_science/_skills/bigquery-ml/syntax/feature-engineering-standalone-preprocessing-functions.md +13 -0
  195. bigquery_data_science/_skills/bigquery-ml/syntax/image-preprocessing-functions-mldecode_image-mlresize_image-mlconvert_image_type-mlconvert_color_space.md +63 -0
  196. bigquery_data_science/_skills/bigquery-ml/syntax/imported-models-tensorflow-tensorflow_lite-onnx-xgboost.md +36 -0
  197. bigquery_data_science/_skills/bigquery-ml/syntax/kmeans.md +87 -0
  198. bigquery_data_science/_skills/bigquery-ml/syntax/linear_reg-logistic_reg-generalized-linear-models.md +101 -0
  199. bigquery_data_science/_skills/bigquery-ml/syntax/matrix_factorization.md +162 -0
  200. bigquery_data_science/_skills/bigquery-ml/syntax/mladvanced_weights.md +76 -0
  201. bigquery_data_science/_skills/bigquery-ml/syntax/mlarima_coefficients.md +30 -0
  202. bigquery_data_science/_skills/bigquery-ml/syntax/mlarima_evaluate.md +49 -0
  203. bigquery_data_science/_skills/bigquery-ml/syntax/mlbag_of_words.md +47 -0
  204. bigquery_data_science/_skills/bigquery-ml/syntax/mlbucketize.md +47 -0
  205. bigquery_data_science/_skills/bigquery-ml/syntax/mlcentroids.md +46 -0
  206. bigquery_data_science/_skills/bigquery-ml/syntax/mlconfusion_matrix.md +56 -0
  207. bigquery_data_science/_skills/bigquery-ml/syntax/mlconvert_color_space.md +53 -0
  208. bigquery_data_science/_skills/bigquery-ml/syntax/mlconvert_image_type.md +39 -0
  209. bigquery_data_science/_skills/bigquery-ml/syntax/mlcorrelation.md +78 -0
  210. bigquery_data_science/_skills/bigquery-ml/syntax/mldecode_image.md +44 -0
  211. bigquery_data_science/_skills/bigquery-ml/syntax/mldescribe_data.md +74 -0
  212. bigquery_data_science/_skills/bigquery-ml/syntax/mldetect_anomalies.md +86 -0
  213. bigquery_data_science/_skills/bigquery-ml/syntax/mldistance.md +48 -0
  214. bigquery_data_science/_skills/bigquery-ml/syntax/mlentity_features_at_time.md +56 -0
  215. bigquery_data_science/_skills/bigquery-ml/syntax/mlevaluate.md +72 -0
  216. bigquery_data_science/_skills/bigquery-ml/syntax/mlexplain_forecast.md +50 -0
  217. bigquery_data_science/_skills/bigquery-ml/syntax/mlexplain_predict.md +66 -0
  218. bigquery_data_science/_skills/bigquery-ml/syntax/mlfeature_cross.md +46 -0
  219. bigquery_data_science/_skills/bigquery-ml/syntax/mlfeature_importance.md +47 -0
  220. bigquery_data_science/_skills/bigquery-ml/syntax/mlfeature_info.md +50 -0
  221. bigquery_data_science/_skills/bigquery-ml/syntax/mlfeatures_at_time.md +51 -0
  222. bigquery_data_science/_skills/bigquery-ml/syntax/mlforecast.md +76 -0
  223. bigquery_data_science/_skills/bigquery-ml/syntax/mlgenerate_embedding-from-a-bqml-pca-autoencoder-matrix_factorization-model.md +64 -0
  224. bigquery_data_science/_skills/bigquery-ml/syntax/mlglobal_explain.md +63 -0
  225. bigquery_data_science/_skills/bigquery-ml/syntax/mlhash_bucketize.md +47 -0
  226. bigquery_data_science/_skills/bigquery-ml/syntax/mlholiday_info.md +46 -0
  227. bigquery_data_science/_skills/bigquery-ml/syntax/mlimputer.md +46 -0
  228. bigquery_data_science/_skills/bigquery-ml/syntax/mllp_norm.md +47 -0
  229. bigquery_data_science/_skills/bigquery-ml/syntax/mlmetrics.md +80 -0
  230. bigquery_data_science/_skills/bigquery-ml/syntax/mlngrams.md +37 -0
  231. bigquery_data_science/_skills/bigquery-ml/syntax/mlone_hot_encoder-mlmulti_hot_encoder-mllabel_encoder.md +67 -0
  232. bigquery_data_science/_skills/bigquery-ml/syntax/mlpolynomial_expand.md +47 -0
  233. bigquery_data_science/_skills/bigquery-ml/syntax/mlpredict.md +67 -0
  234. bigquery_data_science/_skills/bigquery-ml/syntax/mlprincipal_component_info.md +42 -0
  235. bigquery_data_science/_skills/bigquery-ml/syntax/mlprincipal_components.md +40 -0
  236. bigquery_data_science/_skills/bigquery-ml/syntax/mlquantile_bucketize.md +43 -0
  237. bigquery_data_science/_skills/bigquery-ml/syntax/mlrecommend.md +60 -0
  238. bigquery_data_science/_skills/bigquery-ml/syntax/mlreconstruction_loss.md +48 -0
  239. bigquery_data_science/_skills/bigquery-ml/syntax/mlresize_image.md +45 -0
  240. bigquery_data_science/_skills/bigquery-ml/syntax/mlroc_curve.md +63 -0
  241. bigquery_data_science/_skills/bigquery-ml/syntax/mltf_idf.md +37 -0
  242. bigquery_data_science/_skills/bigquery-ml/syntax/mltfdv_describe.md +34 -0
  243. bigquery_data_science/_skills/bigquery-ml/syntax/mltfdv_validate.md +48 -0
  244. bigquery_data_science/_skills/bigquery-ml/syntax/mltraining_info.md +78 -0
  245. bigquery_data_science/_skills/bigquery-ml/syntax/mltransform-applying-a-saved-transform.md +52 -0
  246. bigquery_data_science/_skills/bigquery-ml/syntax/mltranspose-not-a-function-pointer.md +32 -0
  247. bigquery_data_science/_skills/bigquery-ml/syntax/mltrial_info.md +60 -0
  248. bigquery_data_science/_skills/bigquery-ml/syntax/mlvalidate_data_drift.md +99 -0
  249. bigquery_data_science/_skills/bigquery-ml/syntax/mlvalidate_data_skew.md +50 -0
  250. bigquery_data_science/_skills/bigquery-ml/syntax/mlweights.md +84 -0
  251. bigquery_data_science/_skills/bigquery-ml/syntax/model-free-time-series-decomposition-mltrend-mlseasonality-mldetect_change_points.md +100 -0
  252. bigquery_data_science/_skills/bigquery-ml/syntax/model-introspection-mlfeature_info-mltraining_info-mltrial_info.md +7 -0
  253. bigquery_data_science/_skills/bigquery-ml/syntax/model-monitoring-data-validation.md +23 -0
  254. bigquery_data_science/_skills/bigquery-ml/syntax/numerical-scalers-mlstandard_scaler-mlmin_max_scaler-mlmax_abs_scaler-mlrobust_scaler-mlnormalizer.md +77 -0
  255. bigquery_data_science/_skills/bigquery-ml/syntax/onnx.md +55 -0
  256. bigquery_data_science/_skills/bigquery-ml/syntax/pca.md +91 -0
  257. bigquery_data_science/_skills/bigquery-ml/syntax/point-in-time-feature-retrieval-mlfeatures_at_time-mlentity_features_at_time.md +25 -0
  258. bigquery_data_science/_skills/bigquery-ml/syntax/quick-comparison.md +14 -0
  259. bigquery_data_science/_skills/bigquery-ml/syntax/random_forest_classifier-random_forest_regressor.md +123 -0
  260. bigquery_data_science/_skills/bigquery-ml/syntax/remote-remote-model-over-a-vertex-ai-endpoint-the-mechanism.md +87 -0
  261. bigquery_data_science/_skills/bigquery-ml/syntax/tensorflow.md +48 -0
  262. bigquery_data_science/_skills/bigquery-ml/syntax/tensorflow_lite.md +29 -0
  263. bigquery_data_science/_skills/bigquery-ml/syntax/text-preprocessing-functions-mlngrams-mltf_idf-mlbag_of_words.md +18 -0
  264. bigquery_data_science/_skills/bigquery-ml/syntax/timesfm-built-in-foundation-forecaster-see-bigquery-ai.md +28 -0
  265. bigquery_data_science/_skills/bigquery-ml/syntax/transform_only.md +91 -0
  266. bigquery_data_science/_skills/bigquery-ml/syntax/xgboost.md +72 -0
  267. bigquery_data_science/_skills/pinned-models.json +28 -0
  268. bigquery_data_science/catalog.py +1107 -0
  269. bigquery_data_science/cli.py +786 -0
  270. bigquery_data_science/installer.py +282 -0
  271. bigquery_data_science/mcp_server.py +317 -0
  272. bigquery_data_science/tooling/__init__.py +15 -0
  273. bigquery_data_science/tooling/convert_notebook.py +176 -0
  274. bigquery_data_science/tooling/link_policy.py +247 -0
  275. bigquery_data_science/tooling/manifest.py +152 -0
  276. bigquery_data_science/tooling/markdown_anchors.py +84 -0
  277. bigquery_data_science/tooling/narrative_drift.py +163 -0
  278. bigquery_data_science/tooling/pinned_models.py +216 -0
  279. bigquery_data_science/tooling/reference_structure.py +85 -0
  280. bigquery_data_science/tooling/sync_syntax.py +407 -0
  281. bigquery_data_science/tooling/validate.py +208 -0
  282. bigquery_data_science-0.7.0.dist-info/METADATA +306 -0
  283. bigquery_data_science-0.7.0.dist-info/RECORD +286 -0
  284. bigquery_data_science-0.7.0.dist-info/WHEEL +4 -0
  285. bigquery_data_science-0.7.0.dist-info/entry_points.txt +5 -0
  286. bigquery_data_science-0.7.0.dist-info/licenses/LICENSE +201 -0
@@ -0,0 +1,693 @@
1
+ > **Source (`statmike/bigquery-data-science`):** **Notebook:** [`bigquery-ai/functions/ai_generate/ai_generate.ipynb`](https://github.com/statmike/bigquery-data-science/blob/main/bigquery-ai/functions/ai_generate/ai_generate.ipynb) · **SQL:** [`bigquery-ai/functions/ai_generate/ai_generate.sql`](https://github.com/statmike/bigquery-data-science/blob/main/bigquery-ai/functions/ai_generate/ai_generate.sql)
2
+
3
+ # AI.GENERATE — BigQuery AI Functions
4
+
5
+ `AI.GENERATE` is the simplest way to call a Gemini model from SQL. It's a **scalar function** — use it in SELECT, WHERE, or anywhere you'd use a regular function. No model creation required.
6
+
7
+ **What it does:** Sends a prompt to a Gemini model and returns generated text or structured output.
8
+
9
+ **When to use it:**
10
+ - You want to call Gemini from SQL with minimal setup
11
+ - You need free-form text generation or structured output with typed columns
12
+ - You're working with Gemini models only (for Claude/Llama/Mistral, use `functions/ai_generate_text` (`AI.GENERATE_TEXT`))
13
+
14
+ **Alternatives:**
15
+ - `functions/ai_generate_text` (`AI.GENERATE_TEXT`) — table-valued function, supports non-Gemini models, requires `CREATE MODEL`
16
+ - `functions/ai_if` (`AI.IF`), `functions/ai_score` (`AI.SCORE`), `functions/ai_classify` (`AI.CLASSIFY`) — simplified interfaces for common tasks with auto-optimized prompts
17
+ - `functions/ai_generate_bool` (`AI.GENERATE_BOOL`), `functions/ai_generate_double` (`AI.GENERATE_DOUBLE`), `functions/ai_generate_int` (`AI.GENERATE_INT`) — typed scalar variants returning `BOOL`, `FLOAT64`, `INT64`
18
+
19
+ **Multimodal:** Supports document, image, and video input via `reference/unstructured-data-infrastructure.md#objectref-and-objectrefruntime-schema-reference` (ObjectRef). Pass a STRUCT prompt with `ObjectRef` fields to analyze unstructured data from Cloud Storage.
20
+
21
+ **Thinking:** Supports extended reasoning via `model_params` — use `thinking_budget` (Gemini 2.5) or `thinking_level` (Gemini 3.0+) to control reasoning depth.
22
+
23
+ **Featured in:** `workflows/content_analysis` (Content Analysis Pipeline) | `workflows/data_enrichment` (Data Enrichment) | `workflows/rag_pipeline` (RAG Pipeline) | `workflows/document_intelligence` (Document Intelligence) | `workflows/content_moderation` (Content Moderation) | `workflows/multimodal_analysis` (Multimodal Analysis) | `workflows/document_rag` (Document RAG Pipeline) | `workflows/metric_diagnostics` (Metric Diagnostics) | `workflows/tabular_prediction` (Zero-Shot Tabular Prediction)
24
+
25
+ **References:** `RESOURCES.md` (Full syntax reference) | [Official documentation](https://cloud.google.com/bigquery/docs/reference/standard-sql/bigqueryml-syntax-ai-generate) | `setup` (Setup guide)
26
+
27
+ ---
28
+ ## Setup
29
+
30
+ Set your project and location, authenticate, and create a temporary dataset for this notebook.
31
+
32
+ > `AI.GENERATE` doesn't require a connection or model — it uses end-user credentials and defaults to `gemini-2.5-flash`. The [multimodal examples](#examples--multimodal-with-objectref) later in this notebook add a connection for GCS access. See the `setup` (Setup Reference) for details.
33
+
34
+ ```python
35
+ PROJECT_ID = 'statmike-mlops-349915' # <-- Replace with your project ID
36
+ LOCATION = 'US' # BigQuery dataset location
37
+ DATASET_ID = 'bq_ai_functions' # Shared dataset across all notebooks
38
+ CONNECTION_ID = 'bq_ai_functions' # Cloud resource connection (used by multimodal examples)
39
+ BUCKET = PROJECT_ID # GCS bucket (same name as project)
40
+ ```
41
+
42
+ ### Environment
43
+
44
+ > **Already set up the project environment?** The cell below is a no-op — packages are already in your kernel. See the `setup` (Setup Reference) for details.
45
+ >
46
+ > **Running standalone** (Colab, Colab Enterprise, Vertex AI Workbench)? The cell below installs required packages into your current kernel.
47
+
48
+ ```python
49
+ from google.cloud import bigquery
50
+ import pandas as pd
51
+
52
+ client = bigquery.Client(project=PROJECT_ID)
53
+ pd.set_option('display.max_colwidth', None)
54
+
55
+ # Create the shared dataset (idempotent)
56
+ dataset_ref = bigquery.DatasetReference(PROJECT_ID, DATASET_ID)
57
+ dataset = bigquery.Dataset(dataset_ref)
58
+ dataset.location = LOCATION
59
+ client.create_dataset(dataset, exists_ok=True)
60
+ print(f'Dataset {PROJECT_ID}.{DATASET_ID} ready')
61
+
62
+ # Register %%bigquery cell magic (auto-loaded in Colab, needed elsewhere)
63
+ %load_ext bigquery_magics
64
+ ```
65
+
66
+ ### Schema helper — Pydantic to BigQuery
67
+
68
+ For simple schemas, you can write the `output_schema` string directly. For complex schemas with descriptions, array types, and many fields, defining a Pydantic model is cleaner and less error-prone.
69
+
70
+ This helper converts a Pydantic `BaseModel` to the formats that `AI.GENERATE` (SQL string) and BigFrames (dict) expect.
71
+
72
+ ```python
73
+ from pydantic import BaseModel, Field
74
+ from typing import get_origin, get_args
75
+
76
+ # Python type → BigQuery type
77
+ _BQ_TYPES = {str: 'STRING', int: 'INT64', float: 'FLOAT64', bool: 'BOOL'}
78
+
79
+ def _bq_type(annotation) -> str:
80
+ """Map a Python type annotation to a BigQuery type string."""
81
+ if get_origin(annotation) is list:
82
+ inner = _BQ_TYPES[get_args(annotation)[0]]
83
+ return f'ARRAY<{inner}>'
84
+ return _BQ_TYPES[annotation]
85
+
86
+ def bq_schema(model: type[BaseModel]) -> str:
87
+ """Convert a Pydantic model to a BigQuery output_schema string.
88
+
89
+ Supports str, int, float, bool, and list[T].
90
+ Field descriptions become OPTIONS(description = '...').
91
+ """
92
+ fields = []
93
+ for name, info in model.model_fields.items():
94
+ field_str = f'{name} {_bq_type(info.annotation)}'
95
+ if info.description:
96
+ field_str += f" OPTIONS(description = '{info.description}')"
97
+ fields.append(field_str)
98
+ return ', '.join(fields)
99
+
100
+ def bf_schema(model: type[BaseModel]) -> dict[str, str]:
101
+ """Convert a Pydantic model to a BigFrames output_schema dict."""
102
+ return {name: _bq_type(info.annotation) for name, info in model.model_fields.items()}
103
+ ```
104
+
105
+ ---
106
+ ## Examples — SQL
107
+
108
+ Progressive examples from simplest to most advanced. Each cell adds one new concept.
109
+
110
+ ### 1. Simplest possible call
111
+
112
+ `AI.GENERATE` takes a prompt string and returns a STRUCT. Access the generated text with `.result`.
113
+
114
+ ```python
115
+ query = """
116
+ SELECT
117
+ (AI.GENERATE('What is BigQuery?')).result AS answer
118
+ """
119
+ df = client.query(query).to_dataframe()
120
+ print(df.iloc[0]['answer'])
121
+ ```
122
+
123
+ ### 2. Using column values in prompts
124
+
125
+ Use `CONCAT` to build prompts from column values. `AI.GENERATE` runs once per row.
126
+
127
+ ```python
128
+ query = """
129
+ SELECT
130
+ city,
131
+ (AI.GENERATE(
132
+ CONCAT('What country is ', city, ' in? Answer in one word.')
133
+ )).result AS country
134
+ FROM UNNEST(['Tokyo', 'Paris', 'Nairobi', 'Lima', 'Sydney']) AS city
135
+ """
136
+ client.query(query).to_dataframe()
137
+ ```
138
+
139
+ ### 3. Accessing the full response
140
+
141
+ `AI.GENERATE` returns a STRUCT with three fields:
142
+ - `.result` — the generated text (STRING)
143
+ - `.full_response` — the complete API response (JSON)
144
+ - `.status` — error message if failed, empty if successful (STRING)
145
+
146
+ ```python
147
+ import json
148
+
149
+ query = """
150
+ SELECT AI.GENERATE('Explain cloud computing in one sentence.') AS response
151
+ """
152
+ df = client.query(query).to_dataframe()
153
+
154
+ print('result:', df.iloc[0]['response']['result'])
155
+ print('status:', repr(df.iloc[0]['response']['status']))
156
+
157
+ # Explore the full_response JSON structure
158
+ full = json.loads(df.iloc[0]['response']['full_response'])
159
+ print('\nfull_response keys:', list(full.keys()))
160
+ if 'candidates' in full:
161
+ print('candidate keys:', list(full['candidates'][0].keys()))
162
+ if 'usageMetadata' in full:
163
+ print('usage:', full['usageMetadata'])
164
+ ```
165
+
166
+ ### 4. Specifying an endpoint
167
+
168
+ Override the default model (`gemini-2.5-flash`) with any Gemini model.
169
+
170
+ ```python
171
+ query = """
172
+ SELECT
173
+ (AI.GENERATE(
174
+ 'Write a haiku about data warehouses.',
175
+ endpoint => 'gemini-2.5-pro'
176
+ )).result AS haiku
177
+ """
178
+ df = client.query(query).to_dataframe()
179
+ print(df.iloc[0]['haiku'])
180
+ ```
181
+
182
+ ### 5. Structured output with `output_schema`
183
+
184
+ Provide an `output_schema` to get typed columns instead of free-form text. The `result` field is replaced by your custom schema fields.
185
+
186
+ ```python
187
+ query = """
188
+ SELECT
189
+ animal,
190
+ result.habitat,
191
+ result.average_lifespan_years,
192
+ result.is_endangered
193
+ FROM
194
+ UNNEST(['Eagle', 'Salmon', 'Cobra']) AS animal,
195
+ UNNEST([
196
+ AI.GENERATE(
197
+ CONCAT('Give me facts about: ', animal),
198
+ output_schema => 'habitat STRING, average_lifespan_years INT64, is_endangered BOOL'
199
+ )
200
+ ]) AS result
201
+ """
202
+ client.query(query).to_dataframe()
203
+ ```
204
+
205
+ ### 6. Structured output with Pydantic schemas
206
+
207
+ For complex schemas with descriptions and array types, define a Pydantic model and convert it with `bq_schema()`. `Field(description=...)` maps to BigQuery's `OPTIONS(description = '...')`.
208
+
209
+ ```python
210
+ class ProductAnalysis(BaseModel):
211
+ category: str = Field(description='Product category like Electronics, Apparel, Kitchen')
212
+ price_range_usd: str = Field(description='Estimated price range like $50-$100')
213
+ key_features: list[str] = Field(description='Top 3 selling points')
214
+ needs_batteries: bool = Field(description='Whether the product typically requires batteries')
215
+
216
+ # See what the converter produces
217
+ print(bq_schema(ProductAnalysis))
218
+ ```
219
+
220
+ ```python
221
+ output_schema = bq_schema(ProductAnalysis)
222
+
223
+ query = f'''
224
+ SELECT
225
+ product,
226
+ result.category,
227
+ result.price_range_usd,
228
+ result.key_features,
229
+ result.needs_batteries
230
+ FROM
231
+ UNNEST(['laptop', 'running shoes', 'espresso machine']) AS product,
232
+ UNNEST([
233
+ AI.GENERATE(
234
+ CONCAT('Analyze this product for an e-commerce listing: ', product),
235
+ output_schema => """{output_schema}"""
236
+ )
237
+ ]) AS result
238
+ '''
239
+ client.query(query).to_dataframe()
240
+ ```
241
+
242
+ ### 7. Controlling generation with `model_params`
243
+
244
+ `model_params` accepts any field from the [Gemini `generateContent` request body](https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/gemini#request_body) (except `contents`). Common uses: temperature, top_p, max_output_tokens.
245
+
246
+ ```python
247
+ query = """
248
+ SELECT
249
+ (AI.GENERATE(
250
+ 'Invent a creative name for a new coffee shop. Just the name, nothing else.',
251
+ model_params => JSON '{"generation_config": {"temperature": 1.5, "max_output_tokens": 50}}'
252
+ )).result AS creative_name
253
+ """
254
+ df = client.query(query).to_dataframe()
255
+ print(df.iloc[0]['creative_name'])
256
+ ```
257
+
258
+ ### 8. Thinking budget
259
+
260
+ Gemini 2.5 models support "thinking" — internal reasoning before responding. Control the thinking budget (in tokens) to balance quality vs cost.
261
+
262
+ > **Gemini 3.0+:** Use `thinking_level` (`LOW`, `MEDIUM`, `HIGH`) in `thinking_config` instead of `thinking_budget`:
263
+ > ```sql
264
+ > model_params => JSON '{"generation_config": {"thinking_config": {"thinking_level": "LOW"}}}'
265
+ > ```
266
+
267
+ ```python
268
+ query = """
269
+ SELECT
270
+ (AI.GENERATE(
271
+ 'What is the sum of all prime numbers less than 50?',
272
+ model_params => JSON '{"generation_config": {"thinking_config": {"thinking_budget": 2048}}}'
273
+ )).result AS answer
274
+ """
275
+ df = client.query(query).to_dataframe()
276
+ print(df.iloc[0]['answer'])
277
+ ```
278
+
279
+ ### 9. Grounding with Google Search
280
+
281
+ Enable Google Search grounding via the `tools` field in `model_params`. The model searches the web for current information before responding.
282
+
283
+ ```python
284
+ query = """
285
+ SELECT
286
+ (AI.GENERATE(
287
+ 'What were the top 3 news stories today?',
288
+ model_params => JSON '{"tools": [{"googleSearch": {}}]}'
289
+ )).result AS grounded_answer
290
+ """
291
+ df = client.query(query).to_dataframe()
292
+ print(df.iloc[0]['grounded_answer'])
293
+ ```
294
+
295
+ ### 10. Processing table data at scale
296
+
297
+ **Best practice:** When using `LIMIT`, materialize selected rows first to avoid re-evaluating the subquery and incurring extra Vertex AI charges.
298
+
299
+ ```python
300
+ class SentimentResult(BaseModel):
301
+ sentiment: str
302
+ confidence: float
303
+
304
+ output_schema = bq_schema(SentimentResult)
305
+
306
+ query = f"""
307
+ WITH selected_reviews AS (
308
+ SELECT *
309
+ FROM UNNEST([
310
+ STRUCT('r1' AS id, 'The product is amazing, fast delivery!' AS review),
311
+ STRUCT('r2', 'Terrible quality. Broke after one day.'),
312
+ STRUCT('r3', 'Average product, nothing special.'),
313
+ STRUCT('r4', 'Great value for the price. Highly recommend!'),
314
+ STRUCT('r5', 'Worst purchase ever. Do not buy.')
315
+ ])
316
+ )
317
+ SELECT id, review, response.sentiment, response.confidence
318
+ FROM (
319
+ SELECT
320
+ id,
321
+ review,
322
+ AI.GENERATE(
323
+ CONCAT('Classify this review sentiment as positive, negative, or neutral: ', review),
324
+ output_schema => '{output_schema}'
325
+ ) AS response
326
+ FROM selected_reviews
327
+ )
328
+ """
329
+ client.query(query).to_dataframe()
330
+ ```
331
+
332
+ ---
333
+ ## Examples — Multimodal with ObjectRef
334
+
335
+ `AI.GENERATE` can process documents, images, and video stored in Cloud Storage. `OBJ.MAKE_REF` turns a GCS URI into an **ObjectRef** — a pointer the function resolves for you:
336
+
337
+ ```
338
+ OBJ.MAKE_REF(uri, connection) → ObjectRef — pass this straight to the function
339
+ ```
340
+
341
+ Pass `ObjectRef` values in a STRUCT prompt with `prompt` and `object_refs` fields. `OBJ.MAKE_REF` is for files you already know by URI; an object table's `ref` column covers a bucket you keep tracking; and `OBJ.LIST` (GA) — example 14 below — covers the middle case, every file under a wildcard path, with no table at all. `OBJ.FETCH_METADATA` (content type and size) and `OBJ.GET_ACCESS_URL` (a signed `ObjectRefRuntime` URL) still have their uses — displaying an object, delegated access, an explicit TTL — but the AI functions take the `ObjectRef` itself. See the `reference/unstructured-data-infrastructure.md#objectref-and-objectrefruntime-schema-reference` (ObjectRef reference) for details.
342
+
343
+ ### Multimodal setup — connection and sample documents
344
+
345
+ ObjectRef requires a `setup` (Cloud resource connection) to access GCS. The cells below create a connection (if needed) and upload two sample documents.
346
+
347
+ ```python
348
+ import subprocess as _sp
349
+
350
+ # Create the connection (idempotent — succeeds even if it already exists)
351
+ _sp.run(
352
+ ['bq', 'mk', '--connection', '--location', LOCATION,
353
+ '--connection_type', 'CLOUD_RESOURCE',
354
+ '--project_id', PROJECT_ID, CONNECTION_ID],
355
+ capture_output=True, text=True
356
+ )
357
+
358
+ # Get the connection's service account
359
+ import json as _json
360
+ _conn = _sp.run(
361
+ ['bq', 'show', '--connection', '--format=json',
362
+ f'{PROJECT_ID}.{LOCATION}.{CONNECTION_ID}'],
363
+ capture_output=True, text=True
364
+ )
365
+ sa = _json.loads(_conn.stdout)['cloudResource']['serviceAccountId']
366
+ print(f'Connection SA: {sa}')
367
+
368
+ # Grant GCS read access (needed for ObjectRef to read documents)
369
+ _sp.run(
370
+ ['gcloud', 'projects', 'add-iam-policy-binding', PROJECT_ID,
371
+ f'--member=serviceAccount:{sa}', '--role=roles/storage.objectViewer', '--quiet'],
372
+ capture_output=True, text=True
373
+ )
374
+ print('Granted roles/storage.objectViewer')
375
+ ```
376
+
377
+ ```python
378
+ from google.cloud import storage as _storage
379
+ from pathlib import Path
380
+
381
+ _gcs = _storage.Client(project=PROJECT_ID)
382
+ _bucket = _gcs.bucket(BUCKET)
383
+ _prefix = 'bq_ai_functions/ai_generate'
384
+
385
+ # Find the data directory (works from repo checkout or notebook directory)
386
+ _data = Path('../../data/documents')
387
+ if not _data.exists():
388
+ _data = Path('data/documents')
389
+
390
+ # Upload one invoice and one receipt
391
+ for subdir, filename in [('invoices', 'invoice_001.pdf'), ('receipts', 'receipt_001.pdf')]:
392
+ blob = _bucket.blob(f'{_prefix}/{filename}')
393
+ if not blob.exists():
394
+ blob.upload_from_filename(str(_data / subdir / filename))
395
+ print(f'Uploaded {filename} → gs://{BUCKET}/{_prefix}/{filename}')
396
+ else:
397
+ print(f'Already exists: gs://{BUCKET}/{_prefix}/{filename}')
398
+ ```
399
+
400
+ ### 11. Describe a document with ObjectRef
401
+
402
+ `OBJ.MAKE_REF` turns a GCS URI into an `ObjectRef` that `AI.GENERATE` can read. Pass it in a STRUCT with `prompt` (text) and `object_refs` (array of references) fields.
403
+
404
+ ```python
405
+ query = f"""
406
+ SELECT
407
+ (AI.GENERATE(
408
+ STRUCT(
409
+ 'Describe what this document is and summarize its key details.' AS prompt,
410
+ [OBJ.MAKE_REF(
411
+ 'gs://{BUCKET}/bq_ai_functions/ai_generate/invoice_001.pdf',
412
+ '{PROJECT_ID}.{LOCATION}.{CONNECTION_ID}'
413
+ )] AS object_refs
414
+ )
415
+ )).result AS description
416
+ """
417
+ df = client.query(query).to_dataframe()
418
+ print(df.iloc[0]['description'])
419
+ ```
420
+
421
+ ### 12. Extract structured data from a document
422
+
423
+ Combine ObjectRef with `output_schema` to extract typed fields from documents — no Document AI processor needed.
424
+
425
+ ```python
426
+ query = f"""
427
+ SELECT
428
+ result.vendor_name,
429
+ result.invoice_number,
430
+ result.total_amount,
431
+ result.currency,
432
+ result.invoice_date,
433
+ result.line_item_count
434
+ FROM UNNEST([
435
+ AI.GENERATE(
436
+ STRUCT(
437
+ 'Extract the key fields from this invoice.' AS prompt,
438
+ [OBJ.MAKE_REF(
439
+ 'gs://{BUCKET}/bq_ai_functions/ai_generate/invoice_001.pdf',
440
+ '{PROJECT_ID}.{LOCATION}.{CONNECTION_ID}'
441
+ )] AS object_refs
442
+ ),
443
+ output_schema => 'vendor_name STRING, invoice_number STRING, total_amount FLOAT64, currency STRING, invoice_date STRING, line_item_count INT64'
444
+ )
445
+ ]) AS result
446
+ """
447
+ client.query(query).to_dataframe()
448
+ ```
449
+
450
+ ### 13. Process multiple documents
451
+
452
+ Process different document types in a single query by building ObjectRef from a list of URIs.
453
+
454
+ ```python
455
+ query = f"""
456
+ SELECT
457
+ uri,
458
+ result.document_type,
459
+ result.total_amount,
460
+ result.date,
461
+ result.summary
462
+ FROM
463
+ UNNEST([
464
+ 'gs://{BUCKET}/bq_ai_functions/ai_generate/invoice_001.pdf',
465
+ 'gs://{BUCKET}/bq_ai_functions/ai_generate/receipt_001.pdf'
466
+ ]) AS uri,
467
+ UNNEST([
468
+ AI.GENERATE(
469
+ STRUCT(
470
+ 'Identify the document type and extract key details.' AS prompt,
471
+ [OBJ.MAKE_REF(uri, '{PROJECT_ID}.{LOCATION}.{CONNECTION_ID}')] AS object_refs
472
+ ),
473
+ output_schema => 'document_type STRING, total_amount FLOAT64, date STRING, summary STRING'
474
+ )
475
+ ]) AS result
476
+ """
477
+ client.query(query).to_dataframe()
478
+ ```
479
+
480
+ ### 14. Discover documents with `OBJ.LIST`
481
+
482
+ Example 13 hand-built the URI list. `OBJ.LIST` builds it for you: one wildcard path (at most one `*` per segment, no trailing `/`) returns every matching object's metadata — `uri`, `content_type`, `size`, `md5_hash`, `updated`, `generation` — plus a `ref` column that already *is* an `ObjectRef`, so the AI call takes it directly. No object table, no DDL, nothing to clean up.
483
+
484
+ The second argument is the connection. Omit it and the returned references use **direct access** — the objects are read as whoever runs the query, which works on a public bucket and on any bucket where you hold `storage.objectViewer`; pass it, as here, for delegated access through the connection's service account. Use an object table instead when you need a persistent listing that keeps tracking new arrivals.
485
+
486
+ ```python
487
+ query = f"""
488
+ SELECT
489
+ uri,
490
+ content_type,
491
+ size,
492
+ result.document_type,
493
+ result.total_amount,
494
+ result.summary
495
+ FROM
496
+ OBJ.LIST('gs://{BUCKET}/bq_ai_functions/ai_generate/*.pdf', '{PROJECT_ID}.{LOCATION}.{CONNECTION_ID}'),
497
+ UNNEST([
498
+ AI.GENERATE(
499
+ STRUCT(
500
+ 'Identify the document type and extract key details.' AS prompt,
501
+ [ref] AS object_refs
502
+ ),
503
+ output_schema => 'document_type STRING, total_amount FLOAT64, summary STRING'
504
+ )
505
+ ]) AS result
506
+ ORDER BY uri
507
+ """
508
+ client.query(query).to_dataframe()
509
+ ```
510
+
511
+ ---
512
+ ## Examples — `%%bigquery` Magics
513
+
514
+ The same examples using IPython magic commands. Magics let you write SQL directly in notebook cells without Python string wrapping.
515
+
516
+ Key patterns:
517
+ - `%%bigquery` — run SQL, display results inline
518
+ - `%%bigquery df` — run SQL, capture results into a pandas DataFrame
519
+
520
+ ### Basic call with `%%bigquery`
521
+
522
+ ```sql
523
+ %%bigquery --project {PROJECT_ID}
524
+
525
+ SELECT
526
+ city,
527
+ (AI.GENERATE(
528
+ CONCAT('What country is ', city, ' in? Answer in one word.')
529
+ )).result AS country
530
+ FROM UNNEST(['Tokyo', 'Paris', 'Nairobi', 'Lima', 'Sydney']) AS city
531
+ ```
532
+
533
+ ### Capture results to a DataFrame with `%%bigquery df`
534
+
535
+ ```sql
536
+ %%bigquery df_animals --project {PROJECT_ID}
537
+
538
+ SELECT
539
+ animal,
540
+ result.habitat,
541
+ result.average_lifespan_years,
542
+ result.is_endangered
543
+ FROM
544
+ UNNEST(['Eagle', 'Salmon', 'Cobra']) AS animal,
545
+ UNNEST([
546
+ AI.GENERATE(
547
+ CONCAT('Give me facts about: ', animal),
548
+ output_schema => 'habitat STRING, average_lifespan_years INT64, is_endangered BOOL'
549
+ )
550
+ ]) AS result
551
+ ```
552
+
553
+ ```python
554
+ # Now df_animals is a regular pandas DataFrame
555
+ print(f'Shape: {df_animals.shape}')
556
+ print(f'Columns: {list(df_animals.columns)}')
557
+ df_animals
558
+ ```
559
+
560
+ ### Structured output with field descriptions
561
+
562
+ ```sql
563
+ %%bigquery df_products --project {PROJECT_ID}
564
+
565
+ SELECT
566
+ product,
567
+ result.category,
568
+ result.price_range_usd,
569
+ result.key_features,
570
+ result.needs_batteries
571
+ FROM
572
+ UNNEST(['laptop', 'running shoes', 'espresso machine']) AS product,
573
+ UNNEST([
574
+ AI.GENERATE(
575
+ CONCAT('Analyze this product for an e-commerce listing: ', product),
576
+ output_schema => """
577
+ category STRING OPTIONS(description = 'Product category like Electronics, Apparel, Kitchen'),
578
+ price_range_usd STRING OPTIONS(description = 'Estimated price range like $50-$100'),
579
+ key_features ARRAY<STRING> OPTIONS(description = 'Top 3 selling points'),
580
+ needs_batteries BOOL OPTIONS(description = 'Whether the product typically requires batteries')
581
+ """
582
+ )
583
+ ]) AS result
584
+ ```
585
+
586
+ ```python
587
+ df_products
588
+ ```
589
+
590
+ ### Grounding with Google Search
591
+
592
+ ```sql
593
+ %%bigquery --project {PROJECT_ID}
594
+
595
+ SELECT
596
+ (AI.GENERATE(
597
+ 'What were the top 3 news stories today?',
598
+ model_params => JSON '{"tools": [{"googleSearch": {}}]}'
599
+ )).result AS grounded_answer
600
+ ```
601
+
602
+ ---
603
+ ## Examples — BigFrames
604
+
605
+ BigFrames provides a pandas-like interface to BigQuery. The `bigframes.bigquery.ai.generate()` function wraps `AI.GENERATE` for use in Python.
606
+
607
+ **Key difference from SQL:** Prompts use a **tuple pattern** — a tuple of string literals and Series that get concatenated:
608
+ ```python
609
+ bbq.ai.generate(("Summarize: ", df["text"]))
610
+ ```
611
+
612
+ ```python
613
+ import bigframes.pandas as bpd
614
+ import bigframes.bigquery as bbq
615
+
616
+ bpd.options.bigquery.project = PROJECT_ID
617
+ bpd.options.bigquery.location = LOCATION
618
+ ```
619
+
620
+ ### Basic generation
621
+
622
+ ```python
623
+ # Create a BigFrames DataFrame
624
+ df = bpd.DataFrame({'city': ['Tokyo', 'Paris', 'Nairobi', 'Lima', 'Sydney']})
625
+
626
+ # AI.GENERATE with tuple prompt: string literal + Series
627
+ df['response'] = bbq.ai.generate(
628
+ ('What country is ', df['city'], ' in? Answer in one word.')
629
+ )
630
+
631
+ # The result is a struct — access .result with struct accessor
632
+ df['country'] = df['response'].struct.field('result')
633
+ df[['city', 'country']].to_pandas()
634
+ ```
635
+
636
+ ### Structured output with Pydantic + `bf_schema()`
637
+
638
+ BigFrames takes `output_schema` as a `dict[str, str]`. Use `bf_schema()` to convert the same Pydantic model.
639
+
640
+ ```python
641
+ class AnimalFacts(BaseModel):
642
+ habitat: str
643
+ average_lifespan_years: int
644
+ is_endangered: bool
645
+
646
+ df_animals = bpd.DataFrame({'animal': ['Eagle', 'Salmon', 'Cobra']})
647
+
648
+ df_animals['facts'] = bbq.ai.generate(
649
+ ('Give me facts about: ', df_animals['animal']),
650
+ output_schema=bf_schema(AnimalFacts)
651
+ )
652
+
653
+ # Extract structured fields
654
+ df_animals['habitat'] = df_animals['facts'].struct.field('habitat')
655
+ df_animals['lifespan'] = df_animals['facts'].struct.field('average_lifespan_years')
656
+ df_animals['endangered'] = df_animals['facts'].struct.field('is_endangered')
657
+
658
+ df_animals[['animal', 'habitat', 'lifespan', 'endangered']].to_pandas()
659
+ ```
660
+
661
+ ### Complex schema with descriptions
662
+
663
+ ```python
664
+ # Reuse the ProductAnalysis model defined earlier
665
+ # bf_schema() extracts the types (descriptions are SQL-only via bq_schema())
666
+ print(bf_schema(ProductAnalysis))
667
+
668
+ df_products = bpd.DataFrame({'product': ['laptop', 'running shoes', 'espresso machine']})
669
+
670
+ df_products['analysis'] = bbq.ai.generate(
671
+ ('Analyze this product for an e-commerce listing: ', df_products['product']),
672
+ output_schema=bf_schema(ProductAnalysis)
673
+ )
674
+
675
+ df_products['category'] = df_products['analysis'].struct.field('category')
676
+ df_products['price'] = df_products['analysis'].struct.field('price_range_usd')
677
+ df_products['features'] = df_products['analysis'].struct.field('key_features')
678
+
679
+ df_products[['product', 'category', 'price', 'features']].to_pandas()
680
+ ```
681
+
682
+ ### Specifying an endpoint
683
+
684
+ ```python
685
+ df = bpd.DataFrame({'prompt': ['Write a haiku about data warehouses.']})
686
+
687
+ result = bbq.ai.generate(
688
+ df['prompt'],
689
+ endpoint='gemini-2.5-pro'
690
+ )
691
+
692
+ result.struct.field('result').to_pandas()
693
+ ```