sqlmesh 0.213.1.dev1__py3-none-any.whl → 0.227.2.dev4__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (252) hide show
  1. sqlmesh/__init__.py +12 -2
  2. sqlmesh/_version.py +2 -2
  3. sqlmesh/cli/main.py +0 -44
  4. sqlmesh/cli/project_init.py +11 -2
  5. sqlmesh/core/_typing.py +1 -0
  6. sqlmesh/core/audit/definition.py +8 -2
  7. sqlmesh/core/config/__init__.py +1 -1
  8. sqlmesh/core/config/connection.py +17 -5
  9. sqlmesh/core/config/dbt.py +13 -0
  10. sqlmesh/core/config/janitor.py +12 -0
  11. sqlmesh/core/config/loader.py +7 -0
  12. sqlmesh/core/config/model.py +2 -0
  13. sqlmesh/core/config/root.py +3 -0
  14. sqlmesh/core/console.py +81 -3
  15. sqlmesh/core/constants.py +1 -1
  16. sqlmesh/core/context.py +69 -26
  17. sqlmesh/core/dialect.py +3 -0
  18. sqlmesh/core/engine_adapter/_typing.py +2 -0
  19. sqlmesh/core/engine_adapter/base.py +322 -22
  20. sqlmesh/core/engine_adapter/base_postgres.py +17 -1
  21. sqlmesh/core/engine_adapter/bigquery.py +146 -7
  22. sqlmesh/core/engine_adapter/clickhouse.py +17 -13
  23. sqlmesh/core/engine_adapter/databricks.py +33 -2
  24. sqlmesh/core/engine_adapter/fabric.py +10 -29
  25. sqlmesh/core/engine_adapter/mixins.py +142 -48
  26. sqlmesh/core/engine_adapter/mssql.py +15 -4
  27. sqlmesh/core/engine_adapter/mysql.py +2 -2
  28. sqlmesh/core/engine_adapter/postgres.py +9 -3
  29. sqlmesh/core/engine_adapter/redshift.py +4 -0
  30. sqlmesh/core/engine_adapter/risingwave.py +1 -0
  31. sqlmesh/core/engine_adapter/shared.py +6 -0
  32. sqlmesh/core/engine_adapter/snowflake.py +82 -11
  33. sqlmesh/core/engine_adapter/spark.py +14 -10
  34. sqlmesh/core/engine_adapter/trino.py +4 -2
  35. sqlmesh/core/environment.py +2 -0
  36. sqlmesh/core/janitor.py +181 -0
  37. sqlmesh/core/lineage.py +1 -0
  38. sqlmesh/core/linter/definition.py +13 -13
  39. sqlmesh/core/linter/rules/builtin.py +29 -0
  40. sqlmesh/core/macros.py +35 -13
  41. sqlmesh/core/model/common.py +2 -0
  42. sqlmesh/core/model/definition.py +82 -28
  43. sqlmesh/core/model/kind.py +66 -2
  44. sqlmesh/core/model/meta.py +108 -4
  45. sqlmesh/core/node.py +101 -1
  46. sqlmesh/core/plan/builder.py +18 -10
  47. sqlmesh/core/plan/common.py +199 -2
  48. sqlmesh/core/plan/definition.py +25 -6
  49. sqlmesh/core/plan/evaluator.py +75 -113
  50. sqlmesh/core/plan/explainer.py +90 -8
  51. sqlmesh/core/plan/stages.py +42 -21
  52. sqlmesh/core/renderer.py +78 -32
  53. sqlmesh/core/scheduler.py +102 -22
  54. sqlmesh/core/selector.py +137 -9
  55. sqlmesh/core/signal.py +64 -1
  56. sqlmesh/core/snapshot/__init__.py +2 -0
  57. sqlmesh/core/snapshot/definition.py +146 -34
  58. sqlmesh/core/snapshot/evaluator.py +689 -124
  59. sqlmesh/core/state_sync/__init__.py +0 -1
  60. sqlmesh/core/state_sync/base.py +55 -33
  61. sqlmesh/core/state_sync/cache.py +12 -7
  62. sqlmesh/core/state_sync/common.py +216 -111
  63. sqlmesh/core/state_sync/db/environment.py +6 -4
  64. sqlmesh/core/state_sync/db/facade.py +42 -24
  65. sqlmesh/core/state_sync/db/interval.py +27 -7
  66. sqlmesh/core/state_sync/db/migrator.py +34 -16
  67. sqlmesh/core/state_sync/db/snapshot.py +177 -169
  68. sqlmesh/core/table_diff.py +2 -2
  69. sqlmesh/core/test/context.py +2 -0
  70. sqlmesh/core/test/definition.py +14 -9
  71. sqlmesh/dbt/adapter.py +22 -16
  72. sqlmesh/dbt/basemodel.py +75 -56
  73. sqlmesh/dbt/builtin.py +116 -12
  74. sqlmesh/dbt/column.py +17 -5
  75. sqlmesh/dbt/common.py +19 -5
  76. sqlmesh/dbt/context.py +14 -1
  77. sqlmesh/dbt/loader.py +61 -9
  78. sqlmesh/dbt/manifest.py +174 -16
  79. sqlmesh/dbt/model.py +183 -85
  80. sqlmesh/dbt/package.py +16 -1
  81. sqlmesh/dbt/profile.py +3 -3
  82. sqlmesh/dbt/project.py +12 -7
  83. sqlmesh/dbt/seed.py +6 -1
  84. sqlmesh/dbt/source.py +13 -1
  85. sqlmesh/dbt/target.py +25 -6
  86. sqlmesh/dbt/test.py +36 -5
  87. sqlmesh/migrations/v0000_baseline.py +95 -0
  88. sqlmesh/migrations/v0061_mysql_fix_blob_text_type.py +5 -7
  89. sqlmesh/migrations/v0062_add_model_gateway.py +5 -1
  90. sqlmesh/migrations/v0063_change_signals.py +5 -3
  91. sqlmesh/migrations/v0064_join_when_matched_strings.py +5 -3
  92. sqlmesh/migrations/v0065_add_model_optimize.py +5 -1
  93. sqlmesh/migrations/v0066_add_auto_restatements.py +8 -3
  94. sqlmesh/migrations/v0067_add_tsql_date_full_precision.py +5 -1
  95. sqlmesh/migrations/v0068_include_unrendered_query_in_metadata_hash.py +5 -1
  96. sqlmesh/migrations/v0069_update_dev_table_suffix.py +5 -3
  97. sqlmesh/migrations/v0070_include_grains_in_metadata_hash.py +5 -1
  98. sqlmesh/migrations/v0071_add_dev_version_to_intervals.py +9 -5
  99. sqlmesh/migrations/v0072_add_environment_statements.py +5 -3
  100. sqlmesh/migrations/v0073_remove_symbolic_disable_restatement.py +5 -3
  101. sqlmesh/migrations/v0074_add_partition_by_time_column_property.py +5 -1
  102. sqlmesh/migrations/v0075_remove_validate_query.py +5 -3
  103. sqlmesh/migrations/v0076_add_cron_tz.py +5 -1
  104. sqlmesh/migrations/v0077_fix_column_type_hash_calculation.py +5 -1
  105. sqlmesh/migrations/v0078_warn_if_non_migratable_python_env.py +5 -3
  106. sqlmesh/migrations/v0079_add_gateway_managed_property.py +10 -5
  107. sqlmesh/migrations/v0080_add_batch_size_to_scd_type_2_models.py +5 -1
  108. sqlmesh/migrations/v0081_update_partitioned_by.py +5 -3
  109. sqlmesh/migrations/v0082_warn_if_incorrectly_duplicated_statements.py +5 -3
  110. sqlmesh/migrations/v0083_use_sql_for_scd_time_data_type_data_hash.py +5 -1
  111. sqlmesh/migrations/v0084_normalize_quote_when_matched_and_merge_filter.py +5 -1
  112. sqlmesh/migrations/v0085_deterministic_repr.py +5 -3
  113. sqlmesh/migrations/v0086_check_deterministic_bug.py +5 -3
  114. sqlmesh/migrations/v0087_normalize_blueprint_variables.py +5 -3
  115. sqlmesh/migrations/v0088_warn_about_variable_python_env_diffs.py +5 -3
  116. sqlmesh/migrations/v0089_add_virtual_environment_mode.py +5 -1
  117. sqlmesh/migrations/v0090_add_forward_only_column.py +9 -5
  118. sqlmesh/migrations/v0091_on_additive_change.py +5 -1
  119. sqlmesh/migrations/v0092_warn_about_dbt_data_type_diff.py +5 -3
  120. sqlmesh/migrations/v0093_use_raw_sql_in_fingerprint.py +5 -1
  121. sqlmesh/migrations/v0094_add_dev_version_and_fingerprint_columns.py +123 -0
  122. sqlmesh/migrations/v0095_warn_about_dbt_raw_sql_diff.py +49 -0
  123. sqlmesh/migrations/v0096_remove_plan_dags_table.py +13 -0
  124. sqlmesh/migrations/v0097_add_dbt_name_in_node.py +9 -0
  125. sqlmesh/migrations/{v0060_move_audits_to_model.py → v0098_add_dbt_node_info_in_node.py} +33 -16
  126. sqlmesh/migrations/v0099_add_last_altered_to_intervals.py +25 -0
  127. sqlmesh/migrations/v0100_add_grants_and_grants_target_layer.py +9 -0
  128. sqlmesh/utils/__init__.py +8 -1
  129. sqlmesh/utils/cache.py +5 -1
  130. sqlmesh/utils/connection_pool.py +2 -1
  131. sqlmesh/utils/dag.py +65 -10
  132. sqlmesh/utils/date.py +8 -1
  133. sqlmesh/utils/errors.py +8 -0
  134. sqlmesh/utils/jinja.py +54 -4
  135. sqlmesh/utils/pydantic.py +6 -6
  136. sqlmesh/utils/windows.py +13 -3
  137. {sqlmesh-0.213.1.dev1.dist-info → sqlmesh-0.227.2.dev4.dist-info}/METADATA +7 -10
  138. sqlmesh-0.227.2.dev4.dist-info/RECORD +370 -0
  139. sqlmesh_dbt/cli.py +70 -7
  140. sqlmesh_dbt/console.py +14 -6
  141. sqlmesh_dbt/operations.py +103 -24
  142. sqlmesh_dbt/selectors.py +39 -1
  143. web/client/dist/assets/{Audits-Ucsx1GzF.js → Audits-CBiYyyx-.js} +1 -1
  144. web/client/dist/assets/{Banner-BWDzvavM.js → Banner-DSRbUlO5.js} +1 -1
  145. web/client/dist/assets/{ChevronDownIcon-D2VL13Ah.js → ChevronDownIcon-MK_nrjD_.js} +1 -1
  146. web/client/dist/assets/{ChevronRightIcon-DWGYbf1l.js → ChevronRightIcon-CLWtT22Q.js} +1 -1
  147. web/client/dist/assets/{Content-DdHDZM3I.js → Content-BNuGZN5l.js} +1 -1
  148. web/client/dist/assets/{Content-Bikfy8fh.js → Content-CSHJyW0n.js} +1 -1
  149. web/client/dist/assets/{Data-CzAJH7rW.js → Data-C1oRDbLx.js} +1 -1
  150. web/client/dist/assets/{DataCatalog-BJF11g8f.js → DataCatalog-HXyX2-_j.js} +1 -1
  151. web/client/dist/assets/{Editor-s0SBpV2y.js → Editor-BDyfpUuw.js} +1 -1
  152. web/client/dist/assets/{Editor-DgLhgKnm.js → Editor-D0jNItwC.js} +1 -1
  153. web/client/dist/assets/{Errors-D0m0O1d3.js → Errors-BfuFLcPi.js} +1 -1
  154. web/client/dist/assets/{FileExplorer-CEv0vXkt.js → FileExplorer-BR9IE3he.js} +1 -1
  155. web/client/dist/assets/{Footer-BwzXn8Ew.js → Footer-CgBEtiAh.js} +1 -1
  156. web/client/dist/assets/{Header-6heDkEqG.js → Header-DSqR6nSO.js} +1 -1
  157. web/client/dist/assets/{Input-obuJsD6k.js → Input-B-oZ6fGO.js} +1 -1
  158. web/client/dist/assets/Lineage-DYQVwDbD.js +1 -0
  159. web/client/dist/assets/{ListboxShow-HM9_qyrt.js → ListboxShow-BE5-xevs.js} +1 -1
  160. web/client/dist/assets/{ModelLineage-zWdKo0U2.js → ModelLineage-DkIFAYo4.js} +1 -1
  161. web/client/dist/assets/{Models-Bcu66SRz.js → Models-D5dWr8RB.js} +1 -1
  162. web/client/dist/assets/{Page-BWEEQfIt.js → Page-C-XfU5BR.js} +1 -1
  163. web/client/dist/assets/{Plan-C4gXCqlf.js → Plan-ZEuTINBq.js} +1 -1
  164. web/client/dist/assets/{PlusCircleIcon-CVDO651q.js → PlusCircleIcon-DVXAHG8_.js} +1 -1
  165. web/client/dist/assets/{ReportErrors-BT6xFwAr.js → ReportErrors-B7FEPzMB.js} +1 -1
  166. web/client/dist/assets/{Root-ryJoBK4h.js → Root-8aZyhPxF.js} +1 -1
  167. web/client/dist/assets/{SearchList-DB04sPb9.js → SearchList-W_iT2G82.js} +1 -1
  168. web/client/dist/assets/{SelectEnvironment-CUYcXUu6.js → SelectEnvironment-C65jALmO.js} +1 -1
  169. web/client/dist/assets/{SourceList-Doo_9ZGp.js → SourceList-DSLO6nVJ.js} +1 -1
  170. web/client/dist/assets/{SourceListItem-D5Mj7Dly.js → SourceListItem-BHt8d9-I.js} +1 -1
  171. web/client/dist/assets/{SplitPane-qHmkD1qy.js → SplitPane-CViaZmw6.js} +1 -1
  172. web/client/dist/assets/{Tests-DH1Z74ML.js → Tests-DhaVt5t1.js} +1 -1
  173. web/client/dist/assets/{Welcome-DqUJUNMF.js → Welcome-DvpjH-_4.js} +1 -1
  174. web/client/dist/assets/context-BctCsyGb.js +71 -0
  175. web/client/dist/assets/{context-Dr54UHLi.js → context-DFNeGsFF.js} +1 -1
  176. web/client/dist/assets/{editor-DYIP1yQ4.js → editor-CcO28cqd.js} +1 -1
  177. web/client/dist/assets/{file-DarlIDVi.js → file-CvJN3aZO.js} +1 -1
  178. web/client/dist/assets/{floating-ui.react-dom-BH3TFvkM.js → floating-ui.react-dom-CjE-JNW1.js} +1 -1
  179. web/client/dist/assets/{help-Bl8wqaQc.js → help-DuPhjipa.js} +1 -1
  180. web/client/dist/assets/{index-D1sR7wpN.js → index-C-dJH7yZ.js} +1 -1
  181. web/client/dist/assets/{index-O3mjYpnE.js → index-Dj0i1-CA.js} +2 -2
  182. web/client/dist/assets/{plan-CehRrJUG.js → plan-BTRSbjKn.js} +1 -1
  183. web/client/dist/assets/{popover-CqgMRE0G.js → popover-_Sf0yvOI.js} +1 -1
  184. web/client/dist/assets/{project-6gxepOhm.js → project-BvSOI8MY.js} +1 -1
  185. web/client/dist/index.html +1 -1
  186. sqlmesh/integrations/llm.py +0 -56
  187. sqlmesh/migrations/v0001_init.py +0 -60
  188. sqlmesh/migrations/v0002_remove_identify.py +0 -5
  189. sqlmesh/migrations/v0003_move_batch_size.py +0 -34
  190. sqlmesh/migrations/v0004_environmnent_add_finalized_at.py +0 -23
  191. sqlmesh/migrations/v0005_create_seed_table.py +0 -24
  192. sqlmesh/migrations/v0006_change_seed_hash.py +0 -5
  193. sqlmesh/migrations/v0007_env_table_info_to_kind.py +0 -99
  194. sqlmesh/migrations/v0008_create_intervals_table.py +0 -38
  195. sqlmesh/migrations/v0009_remove_pre_post_hooks.py +0 -62
  196. sqlmesh/migrations/v0010_seed_hash_batch_size.py +0 -5
  197. sqlmesh/migrations/v0011_add_model_kind_name.py +0 -63
  198. sqlmesh/migrations/v0012_update_jinja_expressions.py +0 -86
  199. sqlmesh/migrations/v0013_serde_using_model_dialects.py +0 -87
  200. sqlmesh/migrations/v0014_fix_dev_intervals.py +0 -14
  201. sqlmesh/migrations/v0015_environment_add_promoted_snapshot_ids.py +0 -26
  202. sqlmesh/migrations/v0016_fix_windows_path.py +0 -59
  203. sqlmesh/migrations/v0017_fix_windows_seed_path.py +0 -55
  204. sqlmesh/migrations/v0018_rename_snapshot_model_to_node.py +0 -53
  205. sqlmesh/migrations/v0019_add_env_suffix_target.py +0 -28
  206. sqlmesh/migrations/v0020_remove_redundant_attributes_from_dbt_models.py +0 -80
  207. sqlmesh/migrations/v0021_fix_table_properties.py +0 -62
  208. sqlmesh/migrations/v0022_move_project_to_model.py +0 -54
  209. sqlmesh/migrations/v0023_fix_added_models_with_forward_only_parents.py +0 -65
  210. sqlmesh/migrations/v0024_replace_model_kind_name_enum_with_value.py +0 -55
  211. sqlmesh/migrations/v0025_fix_intervals_and_missing_change_category.py +0 -117
  212. sqlmesh/migrations/v0026_remove_dialect_from_seed.py +0 -55
  213. sqlmesh/migrations/v0027_minute_interval_to_five.py +0 -57
  214. sqlmesh/migrations/v0028_add_plan_dags_table.py +0 -29
  215. sqlmesh/migrations/v0029_generate_schema_types_using_dialect.py +0 -69
  216. sqlmesh/migrations/v0030_update_unrestorable_snapshots.py +0 -65
  217. sqlmesh/migrations/v0031_remove_dbt_target_fields.py +0 -65
  218. sqlmesh/migrations/v0032_add_sqlmesh_version.py +0 -25
  219. sqlmesh/migrations/v0033_mysql_fix_blob_text_type.py +0 -45
  220. sqlmesh/migrations/v0034_add_default_catalog.py +0 -367
  221. sqlmesh/migrations/v0035_add_catalog_name_override.py +0 -22
  222. sqlmesh/migrations/v0036_delete_plan_dags_bug_fix.py +0 -14
  223. sqlmesh/migrations/v0037_remove_dbt_is_incremental_macro.py +0 -61
  224. sqlmesh/migrations/v0038_add_expiration_ts_to_snapshot.py +0 -73
  225. sqlmesh/migrations/v0039_include_environment_in_plan_dag_spec.py +0 -68
  226. sqlmesh/migrations/v0040_add_previous_finalized_snapshots.py +0 -26
  227. sqlmesh/migrations/v0041_remove_hash_raw_query_attribute.py +0 -59
  228. sqlmesh/migrations/v0042_trim_indirect_versions.py +0 -66
  229. sqlmesh/migrations/v0043_fix_remove_obsolete_attributes_in_plan_dags.py +0 -61
  230. sqlmesh/migrations/v0044_quote_identifiers_in_model_attributes.py +0 -5
  231. sqlmesh/migrations/v0045_move_gateway_variable.py +0 -70
  232. sqlmesh/migrations/v0046_add_batch_concurrency.py +0 -8
  233. sqlmesh/migrations/v0047_change_scd_string_to_column.py +0 -5
  234. sqlmesh/migrations/v0048_drop_indirect_versions.py +0 -59
  235. sqlmesh/migrations/v0049_replace_identifier_with_version_in_seeds_table.py +0 -57
  236. sqlmesh/migrations/v0050_drop_seeds_table.py +0 -11
  237. sqlmesh/migrations/v0051_rename_column_descriptions.py +0 -65
  238. sqlmesh/migrations/v0052_add_normalize_name_in_environment_naming_info.py +0 -28
  239. sqlmesh/migrations/v0053_custom_model_kind_extra_attributes.py +0 -5
  240. sqlmesh/migrations/v0054_fix_trailing_comments.py +0 -5
  241. sqlmesh/migrations/v0055_add_updated_ts_unpaused_ts_ttl_ms_unrestorable_to_snapshot.py +0 -132
  242. sqlmesh/migrations/v0056_restore_table_indexes.py +0 -118
  243. sqlmesh/migrations/v0057_add_table_format.py +0 -5
  244. sqlmesh/migrations/v0058_add_requirements.py +0 -26
  245. sqlmesh/migrations/v0059_add_physical_version.py +0 -5
  246. sqlmesh-0.213.1.dev1.dist-info/RECORD +0 -421
  247. web/client/dist/assets/Lineage-D0Hgdz2v.js +0 -1
  248. web/client/dist/assets/context-DgX0fp2E.js +0 -68
  249. {sqlmesh-0.213.1.dev1.dist-info → sqlmesh-0.227.2.dev4.dist-info}/WHEEL +0 -0
  250. {sqlmesh-0.213.1.dev1.dist-info → sqlmesh-0.227.2.dev4.dist-info}/entry_points.txt +0 -0
  251. {sqlmesh-0.213.1.dev1.dist-info → sqlmesh-0.227.2.dev4.dist-info}/licenses/LICENSE +0 -0
  252. {sqlmesh-0.213.1.dev1.dist-info → sqlmesh-0.227.2.dev4.dist-info}/top_level.txt +0 -0
@@ -32,12 +32,14 @@ from functools import reduce
32
32
 
33
33
  from sqlglot import exp, select
34
34
  from sqlglot.executor import execute
35
+ from tenacity import retry, stop_after_attempt, wait_exponential, retry_if_not_exception_type
35
36
 
36
37
  from sqlmesh.core import constants as c
37
38
  from sqlmesh.core import dialect as d
38
39
  from sqlmesh.core.audit import Audit, StandaloneAudit
39
40
  from sqlmesh.core.dialect import schema_
40
- from sqlmesh.core.engine_adapter.shared import InsertOverwriteStrategy, DataObjectType
41
+ from sqlmesh.core.engine_adapter.shared import InsertOverwriteStrategy, DataObjectType, DataObject
42
+ from sqlmesh.core.model.meta import GrantsTargetLayer
41
43
  from sqlmesh.core.macros import RuntimeStage
42
44
  from sqlmesh.core.model import (
43
45
  AuditResult,
@@ -49,8 +51,8 @@ from sqlmesh.core.model import (
49
51
  ViewKind,
50
52
  CustomKind,
51
53
  )
52
- from sqlmesh.core.model.kind import _Incremental
53
- from sqlmesh.utils import CompletionStatus
54
+ from sqlmesh.core.model.kind import _Incremental, DbtCustomKind
55
+ from sqlmesh.utils import CompletionStatus, columns_to_types_all_known
54
56
  from sqlmesh.core.schema_diff import (
55
57
  has_drop_alteration,
56
58
  TableAlterOperation,
@@ -66,7 +68,7 @@ from sqlmesh.core.snapshot import (
66
68
  SnapshotTableCleanupTask,
67
69
  )
68
70
  from sqlmesh.core.snapshot.execution_tracker import QueryExecutionTracker
69
- from sqlmesh.utils import random_id, CorrelationId
71
+ from sqlmesh.utils import random_id, CorrelationId, AttributeDict
70
72
  from sqlmesh.utils.concurrency import (
71
73
  concurrent_apply_to_snapshots,
72
74
  concurrent_apply_to_values,
@@ -76,11 +78,13 @@ from sqlmesh.utils.date import TimeLike, now, time_like_to_str
76
78
  from sqlmesh.utils.errors import (
77
79
  ConfigError,
78
80
  DestructiveChangeError,
81
+ MigrationNotSupportedError,
79
82
  SQLMeshError,
80
83
  format_destructive_change_msg,
81
84
  format_additive_change_msg,
82
85
  AdditiveChangeError,
83
86
  )
87
+ from sqlmesh.utils.jinja import MacroReturnVal
84
88
 
85
89
  if sys.version_info >= (3, 12):
86
90
  from importlib import metadata
@@ -304,6 +308,9 @@ class SnapshotEvaluator:
304
308
  ]
305
309
  self._create_schemas(gateway_table_pairs=gateway_table_pairs)
306
310
 
311
+ # Fetch the view data objects for the promoted snapshots to get them cached
312
+ self._get_virtual_data_objects(target_snapshots, environment_naming_info)
313
+
307
314
  deployability_index = deployability_index or DeployabilityIndex.all_deployable()
308
315
  with self.concurrent_context():
309
316
  concurrent_apply_to_snapshots(
@@ -422,50 +429,16 @@ class SnapshotEvaluator:
422
429
  target_snapshots: Target snapshots.
423
430
  deployability_index: Determines snapshots that are deployable / representative in the context of this creation.
424
431
  """
425
- snapshots_with_table_names = defaultdict(set)
426
- tables_by_gateway_and_schema: t.Dict[t.Union[str, None], t.Dict[exp.Table, set[str]]] = (
427
- defaultdict(lambda: defaultdict(set))
432
+ existing_data_objects = self._get_physical_data_objects(
433
+ target_snapshots, deployability_index
428
434
  )
429
-
435
+ snapshots_to_create = []
430
436
  for snapshot in target_snapshots:
431
437
  if not snapshot.is_model or snapshot.is_symbolic:
432
438
  continue
433
- is_deployable = deployability_index.is_deployable(snapshot)
434
- table = exp.to_table(snapshot.table_name(is_deployable), dialect=snapshot.model.dialect)
435
- snapshots_with_table_names[snapshot].add(table.name)
436
- table_schema = d.schema_(table.db, catalog=table.catalog)
437
- tables_by_gateway_and_schema[snapshot.model_gateway][table_schema].add(table.name)
438
-
439
- def _get_data_objects(
440
- schema: exp.Table,
441
- object_names: t.Optional[t.Set[str]] = None,
442
- gateway: t.Optional[str] = None,
443
- ) -> t.Set[str]:
444
- logger.info("Listing data objects in schema %s", schema.sql())
445
- objs = self.get_adapter(gateway).get_data_objects(schema, object_names)
446
- return {obj.name for obj in objs}
447
-
448
- with self.concurrent_context():
449
- existing_objects: t.Set[str] = set()
450
- # A schema can be shared across multiple engines, so we need to group tables by both gateway and schema
451
- for gateway, tables_by_schema in tables_by_gateway_and_schema.items():
452
- objs_for_gateway = {
453
- obj
454
- for objs in concurrent_apply_to_values(
455
- list(tables_by_schema),
456
- lambda s: _get_data_objects(
457
- schema=s, object_names=tables_by_schema.get(s), gateway=gateway
458
- ),
459
- self.ddl_concurrent_tasks,
460
- )
461
- for obj in objs
462
- }
463
- existing_objects.update(objs_for_gateway)
464
-
465
- snapshots_to_create = []
466
- for snapshot, table_names in snapshots_with_table_names.items():
467
- missing_tables = table_names - existing_objects
468
- if missing_tables or (snapshot.is_seed and not snapshot.intervals):
439
+ if snapshot.snapshot_id not in existing_data_objects or (
440
+ snapshot.is_seed and not snapshot.intervals
441
+ ):
469
442
  snapshots_to_create.append(snapshot)
470
443
 
471
444
  return snapshots_to_create
@@ -514,16 +487,25 @@ class SnapshotEvaluator:
514
487
  allow_additive_snapshots: Set of snapshots that are allowed to have additive schema changes.
515
488
  deployability_index: Determines snapshots that are deployable in the context of this evaluation.
516
489
  """
490
+ deployability_index = deployability_index or DeployabilityIndex.all_deployable()
491
+ target_data_objects = self._get_physical_data_objects(target_snapshots, deployability_index)
492
+ if not target_data_objects:
493
+ return
494
+
495
+ if not snapshots:
496
+ snapshots = {s.snapshot_id: s for s in target_snapshots}
497
+
517
498
  allow_destructive_snapshots = allow_destructive_snapshots or set()
518
499
  allow_additive_snapshots = allow_additive_snapshots or set()
519
- deployability_index = deployability_index or DeployabilityIndex.all_deployable()
520
500
  snapshots_by_name = {s.name: s for s in snapshots.values()}
521
501
  with self.concurrent_context():
502
+ # Only migrate snapshots for which there's an existing data object
522
503
  concurrent_apply_to_snapshots(
523
504
  target_snapshots,
524
505
  lambda s: self._migrate_snapshot(
525
506
  s,
526
507
  snapshots_by_name,
508
+ target_data_objects.get(s.snapshot_id),
527
509
  allow_destructive_snapshots,
528
510
  allow_additive_snapshots,
529
511
  self.get_adapter(s.model_gateway),
@@ -543,10 +525,12 @@ class SnapshotEvaluator:
543
525
  target_snapshots: Snapshots to cleanup.
544
526
  on_complete: A callback to call on each successfully deleted database object.
545
527
  """
528
+ target_snapshots = [
529
+ t for t in target_snapshots if t.snapshot.is_model and not t.snapshot.is_symbolic
530
+ ]
546
531
  snapshots_to_dev_table_only = {
547
532
  t.snapshot.snapshot_id: t.dev_table_only for t in target_snapshots
548
533
  }
549
-
550
534
  with self.concurrent_context():
551
535
  concurrent_apply_to_snapshots(
552
536
  [t.snapshot for t in target_snapshots],
@@ -766,33 +750,51 @@ class SnapshotEvaluator:
766
750
  **render_statements_kwargs
767
751
  )
768
752
 
753
+ evaluation_strategy = _evaluation_strategy(snapshot, adapter)
754
+ evaluation_strategy.run_pre_statements(
755
+ snapshot=snapshot,
756
+ render_kwargs={**render_statements_kwargs, "inside_transaction": False},
757
+ )
758
+
769
759
  with (
770
760
  adapter.transaction(),
771
761
  adapter.session(snapshot.model.render_session_properties(**render_statements_kwargs)),
772
762
  ):
773
- adapter.execute(model.render_pre_statements(**render_statements_kwargs))
763
+ evaluation_strategy.run_pre_statements(
764
+ snapshot=snapshot,
765
+ render_kwargs={**render_statements_kwargs, "inside_transaction": True},
766
+ )
774
767
 
775
768
  if not target_table_exists or (model.is_seed and not snapshot.intervals):
769
+ # Only create the empty table if the columns were provided explicitly by the user
770
+ should_create_empty_table = (
771
+ model.kind.is_materialized
772
+ and model.columns_to_types_
773
+ and columns_to_types_all_known(model.columns_to_types_)
774
+ )
775
+ if not should_create_empty_table:
776
+ # Or if the model is self-referential and its query is fully annotated with types
777
+ should_create_empty_table = model.depends_on_self and model.annotated
776
778
  if self._can_clone(snapshot, deployability_index):
777
779
  self._clone_snapshot_in_dev(
778
780
  snapshot=snapshot,
779
781
  snapshots=snapshots,
780
782
  deployability_index=deployability_index,
781
783
  render_kwargs=create_render_kwargs,
782
- rendered_physical_properties=rendered_physical_properties,
784
+ rendered_physical_properties=rendered_physical_properties.copy(),
783
785
  allow_destructive_snapshots=allow_destructive_snapshots,
784
786
  allow_additive_snapshots=allow_additive_snapshots,
785
787
  )
786
788
  runtime_stage = RuntimeStage.EVALUATING
787
789
  target_table_exists = True
788
- elif model.annotated or model.is_seed or model.kind.is_scd_type_2:
790
+ elif should_create_empty_table or model.is_seed or model.kind.is_scd_type_2:
789
791
  self._execute_create(
790
792
  snapshot=snapshot,
791
793
  table_name=target_table_name,
792
794
  is_table_deployable=is_snapshot_deployable,
793
795
  deployability_index=deployability_index,
794
796
  create_render_kwargs=create_render_kwargs,
795
- rendered_physical_properties=rendered_physical_properties,
797
+ rendered_physical_properties=rendered_physical_properties.copy(),
796
798
  dry_run=False,
797
799
  run_pre_post_statements=False,
798
800
  )
@@ -809,6 +811,7 @@ class SnapshotEvaluator:
809
811
  if (
810
812
  snapshot.is_materialized
811
813
  and target_table_exists
814
+ and adapter.wap_enabled
812
815
  and (model.wap_supported or adapter.wap_supported(target_table_name))
813
816
  ):
814
817
  wap_id = random_id()[0:8]
@@ -830,9 +833,17 @@ class SnapshotEvaluator:
830
833
  batch_index=batch_index,
831
834
  )
832
835
 
833
- adapter.execute(model.render_post_statements(**render_statements_kwargs))
836
+ evaluation_strategy.run_post_statements(
837
+ snapshot=snapshot,
838
+ render_kwargs={**render_statements_kwargs, "inside_transaction": True},
839
+ )
840
+
841
+ evaluation_strategy.run_post_statements(
842
+ snapshot=snapshot,
843
+ render_kwargs={**render_statements_kwargs, "inside_transaction": False},
844
+ )
834
845
 
835
- return wap_id
846
+ return wap_id
836
847
 
837
848
  def create_snapshot(
838
849
  self,
@@ -866,6 +877,11 @@ class SnapshotEvaluator:
866
877
  deployability_index=deployability_index,
867
878
  )
868
879
 
880
+ evaluation_strategy = _evaluation_strategy(snapshot, adapter)
881
+ evaluation_strategy.run_pre_statements(
882
+ snapshot=snapshot, render_kwargs={**create_render_kwargs, "inside_transaction": False}
883
+ )
884
+
869
885
  with (
870
886
  adapter.transaction(),
871
887
  adapter.session(snapshot.model.render_session_properties(**create_render_kwargs)),
@@ -883,6 +899,7 @@ class SnapshotEvaluator:
883
899
  rendered_physical_properties=rendered_physical_properties,
884
900
  allow_destructive_snapshots=allow_destructive_snapshots,
885
901
  allow_additive_snapshots=allow_additive_snapshots,
902
+ run_pre_post_statements=True,
886
903
  )
887
904
  else:
888
905
  is_table_deployable = deployability_index.is_deployable(snapshot)
@@ -896,6 +913,10 @@ class SnapshotEvaluator:
896
913
  dry_run=True,
897
914
  )
898
915
 
916
+ evaluation_strategy.run_post_statements(
917
+ snapshot=snapshot, render_kwargs={**create_render_kwargs, "inside_transaction": False}
918
+ )
919
+
899
920
  if on_complete is not None:
900
921
  on_complete(snapshot)
901
922
 
@@ -933,6 +954,7 @@ class SnapshotEvaluator:
933
954
  model = snapshot.model
934
955
  adapter = self.get_adapter(model.gateway)
935
956
  evaluation_strategy = _evaluation_strategy(snapshot, adapter)
957
+ is_snapshot_deployable = deployability_index.is_deployable(snapshot)
936
958
 
937
959
  queries_or_dfs = self._render_snapshot_for_evaluation(
938
960
  snapshot,
@@ -956,6 +978,7 @@ class SnapshotEvaluator:
956
978
  execution_time=execution_time,
957
979
  physical_properties=rendered_physical_properties,
958
980
  render_kwargs=create_render_kwargs,
981
+ is_snapshot_deployable=is_snapshot_deployable,
959
982
  )
960
983
  else:
961
984
  logger.info(
@@ -978,6 +1001,7 @@ class SnapshotEvaluator:
978
1001
  execution_time=execution_time,
979
1002
  physical_properties=rendered_physical_properties,
980
1003
  render_kwargs=create_render_kwargs,
1004
+ is_snapshot_deployable=is_snapshot_deployable,
981
1005
  )
982
1006
 
983
1007
  # DataFrames, unlike SQL expressions, can provide partial results by yielding dataframes. As a result,
@@ -997,6 +1021,11 @@ class SnapshotEvaluator:
997
1021
  ):
998
1022
  import pandas as pd
999
1023
 
1024
+ try:
1025
+ first_query_or_df = next(queries_or_dfs)
1026
+ except StopIteration:
1027
+ return
1028
+
1000
1029
  query_or_df = reduce(
1001
1030
  lambda a, b: (
1002
1031
  pd.concat([a, b], ignore_index=True) # type: ignore
@@ -1004,6 +1033,7 @@ class SnapshotEvaluator:
1004
1033
  else a.union_all(b) # type: ignore
1005
1034
  ), # type: ignore
1006
1035
  queries_or_dfs,
1036
+ first_query_or_df,
1007
1037
  )
1008
1038
  apply(query_or_df, index=0)
1009
1039
  else:
@@ -1042,6 +1072,7 @@ class SnapshotEvaluator:
1042
1072
  rendered_physical_properties: t.Dict[str, exp.Expression],
1043
1073
  allow_destructive_snapshots: t.Set[str],
1044
1074
  allow_additive_snapshots: t.Set[str],
1075
+ run_pre_post_statements: bool = False,
1045
1076
  ) -> None:
1046
1077
  adapter = self.get_adapter(snapshot.model.gateway)
1047
1078
 
@@ -1053,7 +1084,6 @@ class SnapshotEvaluator:
1053
1084
  adapter.clone_table(
1054
1085
  target_table_name,
1055
1086
  snapshot.table_name(),
1056
- replace=True,
1057
1087
  rendered_physical_properties=rendered_physical_properties,
1058
1088
  )
1059
1089
  self._migrate_target_table(
@@ -1065,7 +1095,9 @@ class SnapshotEvaluator:
1065
1095
  rendered_physical_properties=rendered_physical_properties,
1066
1096
  allow_destructive_snapshots=allow_destructive_snapshots,
1067
1097
  allow_additive_snapshots=allow_additive_snapshots,
1098
+ run_pre_post_statements=run_pre_post_statements,
1068
1099
  )
1100
+
1069
1101
  except Exception:
1070
1102
  adapter.drop_table(target_table_name)
1071
1103
  raise
@@ -1074,12 +1106,13 @@ class SnapshotEvaluator:
1074
1106
  self,
1075
1107
  snapshot: Snapshot,
1076
1108
  snapshots: t.Dict[str, Snapshot],
1109
+ target_data_object: t.Optional[DataObject],
1077
1110
  allow_destructive_snapshots: t.Set[str],
1078
1111
  allow_additive_snapshots: t.Set[str],
1079
1112
  adapter: EngineAdapter,
1080
1113
  deployability_index: DeployabilityIndex,
1081
1114
  ) -> None:
1082
- if not snapshot.requires_schema_migration_in_prod:
1115
+ if not snapshot.is_model or snapshot.is_symbolic:
1083
1116
  return
1084
1117
 
1085
1118
  deployability_index = DeployabilityIndex.all_deployable()
@@ -1091,17 +1124,25 @@ class SnapshotEvaluator:
1091
1124
  )
1092
1125
  target_table_name = snapshot.table_name()
1093
1126
 
1127
+ evaluation_strategy = _evaluation_strategy(snapshot, adapter)
1128
+ evaluation_strategy.run_pre_statements(
1129
+ snapshot=snapshot, render_kwargs={**render_kwargs, "inside_transaction": False}
1130
+ )
1131
+
1094
1132
  with (
1095
1133
  adapter.transaction(),
1096
1134
  adapter.session(snapshot.model.render_session_properties(**render_kwargs)),
1097
1135
  ):
1098
- target_data_object = adapter.get_data_object(target_table_name)
1099
1136
  table_exists = target_data_object is not None
1100
1137
  if adapter.drop_data_object_on_type_mismatch(
1101
1138
  target_data_object, _snapshot_to_data_object_type(snapshot)
1102
1139
  ):
1103
1140
  table_exists = False
1104
1141
 
1142
+ rendered_physical_properties = snapshot.model.render_physical_properties(
1143
+ **render_kwargs
1144
+ )
1145
+
1105
1146
  if table_exists:
1106
1147
  self._migrate_target_table(
1107
1148
  target_table_name=target_table_name,
@@ -1109,14 +1150,35 @@ class SnapshotEvaluator:
1109
1150
  snapshots=snapshots,
1110
1151
  deployability_index=deployability_index,
1111
1152
  render_kwargs=render_kwargs,
1112
- rendered_physical_properties=snapshot.model.render_physical_properties(
1113
- **render_kwargs
1114
- ),
1153
+ rendered_physical_properties=rendered_physical_properties,
1115
1154
  allow_destructive_snapshots=allow_destructive_snapshots,
1116
1155
  allow_additive_snapshots=allow_additive_snapshots,
1117
1156
  run_pre_post_statements=True,
1118
1157
  )
1158
+ else:
1159
+ self._execute_create(
1160
+ snapshot=snapshot,
1161
+ table_name=snapshot.table_name(is_deployable=True),
1162
+ is_table_deployable=True,
1163
+ deployability_index=deployability_index,
1164
+ create_render_kwargs=render_kwargs,
1165
+ rendered_physical_properties=rendered_physical_properties,
1166
+ dry_run=True,
1167
+ )
1119
1168
 
1169
+ evaluation_strategy.run_post_statements(
1170
+ snapshot=snapshot, render_kwargs={**render_kwargs, "inside_transaction": False}
1171
+ )
1172
+
1173
+ # Retry in case when the table is migrated concurrently from another plan application
1174
+ @retry(
1175
+ reraise=True,
1176
+ stop=stop_after_attempt(5),
1177
+ wait=wait_exponential(min=1, max=16),
1178
+ retry=retry_if_not_exception_type(
1179
+ (DestructiveChangeError, AdditiveChangeError, MigrationNotSupportedError)
1180
+ ),
1181
+ )
1120
1182
  def _migrate_target_table(
1121
1183
  self,
1122
1184
  target_table_name: str,
@@ -1131,7 +1193,10 @@ class SnapshotEvaluator:
1131
1193
  ) -> None:
1132
1194
  adapter = self.get_adapter(snapshot.model.gateway)
1133
1195
 
1134
- tmp_table_name = f"{target_table_name}_schema_tmp"
1196
+ tmp_table = exp.to_table(target_table_name)
1197
+ tmp_table.this.set("this", f"{tmp_table.name}_schema_tmp")
1198
+ tmp_table_name = tmp_table.sql()
1199
+
1135
1200
  if snapshot.is_materialized:
1136
1201
  self._execute_create(
1137
1202
  snapshot=snapshot,
@@ -1142,6 +1207,7 @@ class SnapshotEvaluator:
1142
1207
  rendered_physical_properties=rendered_physical_properties,
1143
1208
  dry_run=False,
1144
1209
  run_pre_post_statements=run_pre_post_statements,
1210
+ skip_grants=True, # skip grants for tmp table
1145
1211
  )
1146
1212
  try:
1147
1213
  evaluation_strategy = _evaluation_strategy(snapshot, adapter)
@@ -1159,6 +1225,7 @@ class SnapshotEvaluator:
1159
1225
  allow_additive_snapshots=allow_additive_snapshots,
1160
1226
  ignore_destructive=snapshot.model.on_destructive_change.is_ignore,
1161
1227
  ignore_additive=snapshot.model.on_additive_change.is_ignore,
1228
+ deployability_index=deployability_index,
1162
1229
  )
1163
1230
  finally:
1164
1231
  if snapshot.is_materialized:
@@ -1208,6 +1275,7 @@ class SnapshotEvaluator:
1208
1275
  model=snapshot.model,
1209
1276
  environment=environment_naming_info.name,
1210
1277
  snapshots=snapshots,
1278
+ snapshot=snapshot,
1211
1279
  **render_kwargs,
1212
1280
  )
1213
1281
 
@@ -1407,6 +1475,7 @@ class SnapshotEvaluator:
1407
1475
  rendered_physical_properties: t.Dict[str, exp.Expression],
1408
1476
  dry_run: bool,
1409
1477
  run_pre_post_statements: bool = True,
1478
+ skip_grants: bool = False,
1410
1479
  ) -> None:
1411
1480
  adapter = self.get_adapter(snapshot.model.gateway)
1412
1481
  evaluation_strategy = _evaluation_strategy(snapshot, adapter)
@@ -1420,19 +1489,28 @@ class SnapshotEvaluator:
1420
1489
  "table_mapping": {snapshot.name: table_name},
1421
1490
  }
1422
1491
  if run_pre_post_statements:
1423
- adapter.execute(snapshot.model.render_pre_statements(**create_render_kwargs))
1492
+ evaluation_strategy.run_pre_statements(
1493
+ snapshot=snapshot,
1494
+ render_kwargs={**create_render_kwargs, "inside_transaction": True},
1495
+ )
1424
1496
  evaluation_strategy.create(
1425
1497
  table_name=table_name,
1426
1498
  model=snapshot.model,
1427
1499
  is_table_deployable=is_table_deployable,
1500
+ skip_grants=skip_grants,
1428
1501
  render_kwargs=create_render_kwargs,
1429
1502
  is_snapshot_deployable=is_snapshot_deployable,
1430
1503
  is_snapshot_representative=is_snapshot_representative,
1431
1504
  dry_run=dry_run,
1432
1505
  physical_properties=rendered_physical_properties,
1506
+ snapshot=snapshot,
1507
+ deployability_index=deployability_index,
1433
1508
  )
1434
1509
  if run_pre_post_statements:
1435
- adapter.execute(snapshot.model.render_post_statements(**create_render_kwargs))
1510
+ evaluation_strategy.run_post_statements(
1511
+ snapshot=snapshot,
1512
+ render_kwargs={**create_render_kwargs, "inside_transaction": True},
1513
+ )
1436
1514
 
1437
1515
  def _can_clone(self, snapshot: Snapshot, deployability_index: DeployabilityIndex) -> bool:
1438
1516
  adapter = self.get_adapter(snapshot.model.gateway)
@@ -1441,11 +1519,125 @@ class SnapshotEvaluator:
1441
1519
  and snapshot.is_materialized
1442
1520
  and bool(snapshot.previous_versions)
1443
1521
  and adapter.SUPPORTS_CLONING
1444
- # managed models cannot have their schema mutated because theyre based on queries, so clone + alter wont work
1522
+ # managed models cannot have their schema mutated because they're based on queries, so clone + alter won't work
1445
1523
  and not snapshot.is_managed
1446
- # If the deployable table is missing we can't clone it
1524
+ and not snapshot.is_dbt_custom
1447
1525
  and not deployability_index.is_deployable(snapshot)
1526
+ # If the deployable table is missing we can't clone it
1527
+ and adapter.table_exists(snapshot.table_name())
1528
+ )
1529
+
1530
+ def _get_physical_data_objects(
1531
+ self,
1532
+ target_snapshots: t.Iterable[Snapshot],
1533
+ deployability_index: DeployabilityIndex,
1534
+ ) -> t.Dict[SnapshotId, DataObject]:
1535
+ """Returns a dictionary of snapshot IDs to existing data objects of their physical tables.
1536
+
1537
+ Args:
1538
+ target_snapshots: Target snapshots.
1539
+ deployability_index: The deployability index to determine whether to look for a deployable or
1540
+ a non-deployable physical table.
1541
+
1542
+ Returns:
1543
+ A dictionary of snapshot IDs to existing data objects of their physical tables. If the data object
1544
+ for a snapshot is not found, it will not be included in the dictionary.
1545
+ """
1546
+ return self._get_data_objects(
1547
+ target_snapshots,
1548
+ lambda s: exp.to_table(
1549
+ s.table_name(deployability_index.is_deployable(s)), dialect=s.model.dialect
1550
+ ),
1551
+ )
1552
+
1553
+ def _get_virtual_data_objects(
1554
+ self,
1555
+ target_snapshots: t.Iterable[Snapshot],
1556
+ environment_naming_info: EnvironmentNamingInfo,
1557
+ ) -> t.Dict[SnapshotId, DataObject]:
1558
+ """Returns a dictionary of snapshot IDs to existing data objects of their virtual views.
1559
+
1560
+ Args:
1561
+ target_snapshots: Target snapshots.
1562
+ environment_naming_info: The environment naming info of the target virtual environment.
1563
+
1564
+ Returns:
1565
+ A dictionary of snapshot IDs to existing data objects of their virtual views. If the data object
1566
+ for a snapshot is not found, it will not be included in the dictionary.
1567
+ """
1568
+
1569
+ def _get_view_name(s: Snapshot) -> exp.Table:
1570
+ adapter = (
1571
+ self.get_adapter(s.model_gateway)
1572
+ if environment_naming_info.gateway_managed
1573
+ else self.adapter
1574
+ )
1575
+ return exp.to_table(
1576
+ s.qualified_view_name.for_environment(
1577
+ environment_naming_info, dialect=adapter.dialect
1578
+ ),
1579
+ dialect=adapter.dialect,
1580
+ )
1581
+
1582
+ return self._get_data_objects(target_snapshots, _get_view_name)
1583
+
1584
+ def _get_data_objects(
1585
+ self,
1586
+ target_snapshots: t.Iterable[Snapshot],
1587
+ table_name_callable: t.Callable[[Snapshot], exp.Table],
1588
+ ) -> t.Dict[SnapshotId, DataObject]:
1589
+ """Returns a dictionary of snapshot IDs to existing data objects.
1590
+
1591
+ Args:
1592
+ target_snapshots: Target snapshots.
1593
+ table_name_callable: A function that takes a snapshot and returns the table to look for.
1594
+
1595
+ Returns:
1596
+ A dictionary of snapshot IDs to existing data objects. If the data object for a snapshot is not found,
1597
+ it will not be included in the dictionary.
1598
+ """
1599
+ tables_by_gateway_and_schema: t.Dict[t.Union[str, None], t.Dict[exp.Table, set[str]]] = (
1600
+ defaultdict(lambda: defaultdict(set))
1448
1601
  )
1602
+ snapshots_by_table_name: t.Dict[exp.Table, t.Dict[str, Snapshot]] = defaultdict(dict)
1603
+ for snapshot in target_snapshots:
1604
+ if not snapshot.is_model or snapshot.is_symbolic:
1605
+ continue
1606
+ table = table_name_callable(snapshot)
1607
+ table_schema = d.schema_(table.db, catalog=table.catalog)
1608
+ tables_by_gateway_and_schema[snapshot.model_gateway][table_schema].add(table.name)
1609
+ snapshots_by_table_name[table_schema][table.name] = snapshot
1610
+
1611
+ def _get_data_objects_in_schema(
1612
+ schema: exp.Table,
1613
+ object_names: t.Optional[t.Set[str]] = None,
1614
+ gateway: t.Optional[str] = None,
1615
+ ) -> t.List[DataObject]:
1616
+ logger.info("Listing data objects in schema %s", schema.sql())
1617
+ return self.get_adapter(gateway).get_data_objects(
1618
+ schema, object_names, safe_to_cache=True
1619
+ )
1620
+
1621
+ with self.concurrent_context():
1622
+ snapshot_id_to_obj: t.Dict[SnapshotId, DataObject] = {}
1623
+ # A schema can be shared across multiple engines, so we need to group tables by both gateway and schema
1624
+ for gateway, tables_by_schema in tables_by_gateway_and_schema.items():
1625
+ schema_list = list(tables_by_schema.keys())
1626
+ results = concurrent_apply_to_values(
1627
+ schema_list,
1628
+ lambda s: _get_data_objects_in_schema(
1629
+ schema=s, object_names=tables_by_schema.get(s), gateway=gateway
1630
+ ),
1631
+ self.ddl_concurrent_tasks,
1632
+ )
1633
+
1634
+ for schema, objs in zip(schema_list, results):
1635
+ snapshots_by_name = snapshots_by_table_name.get(schema, {})
1636
+ for obj in objs:
1637
+ if obj.name in snapshots_by_name:
1638
+ snapshot_id_to_obj[snapshots_by_name[obj.name].snapshot_id] = obj
1639
+
1640
+ return snapshot_id_to_obj
1449
1641
 
1450
1642
 
1451
1643
  def _evaluation_strategy(snapshot: SnapshotInfoLike, adapter: EngineAdapter) -> EvaluationStrategy:
@@ -1470,6 +1662,19 @@ def _evaluation_strategy(snapshot: SnapshotInfoLike, adapter: EngineAdapter) ->
1470
1662
  klass = ViewStrategy
1471
1663
  elif snapshot.is_scd_type_2:
1472
1664
  klass = SCDType2Strategy
1665
+ elif snapshot.is_dbt_custom:
1666
+ if hasattr(snapshot, "model") and isinstance(
1667
+ (model_kind := snapshot.model.kind), DbtCustomKind
1668
+ ):
1669
+ return DbtCustomMaterializationStrategy(
1670
+ adapter=adapter,
1671
+ materialization_name=model_kind.materialization,
1672
+ materialization_template=model_kind.definition,
1673
+ )
1674
+
1675
+ raise SQLMeshError(
1676
+ f"Expected DbtCustomKind for dbt custom materialization in model '{snapshot.name}'"
1677
+ )
1473
1678
  elif snapshot.is_custom:
1474
1679
  if snapshot.custom_materialization is None:
1475
1680
  raise SQLMeshError(
@@ -1537,6 +1742,7 @@ class EvaluationStrategy(abc.ABC):
1537
1742
  model: Model,
1538
1743
  is_table_deployable: bool,
1539
1744
  render_kwargs: t.Dict[str, t.Any],
1745
+ skip_grants: bool,
1540
1746
  **kwargs: t.Any,
1541
1747
  ) -> None:
1542
1748
  """Creates the target table or view.
@@ -1609,6 +1815,84 @@ class EvaluationStrategy(abc.ABC):
1609
1815
  view_name: The name of the target view in the virtual layer.
1610
1816
  """
1611
1817
 
1818
+ @abc.abstractmethod
1819
+ def run_pre_statements(self, snapshot: Snapshot, render_kwargs: t.Any) -> None:
1820
+ """Executes the snapshot's pre statements.
1821
+
1822
+ Args:
1823
+ snapshot: The target snapshot.
1824
+ render_kwargs: Additional key-value arguments to pass when rendering the statements.
1825
+ """
1826
+
1827
+ @abc.abstractmethod
1828
+ def run_post_statements(self, snapshot: Snapshot, render_kwargs: t.Any) -> None:
1829
+ """Executes the snapshot's post statements.
1830
+
1831
+ Args:
1832
+ snapshot: The target snapshot.
1833
+ render_kwargs: Additional key-value arguments to pass when rendering the statements.
1834
+ """
1835
+
1836
+ def _apply_grants(
1837
+ self,
1838
+ model: Model,
1839
+ table_name: str,
1840
+ target_layer: GrantsTargetLayer,
1841
+ is_snapshot_deployable: bool = False,
1842
+ ) -> None:
1843
+ """Apply grants for a model if grants are configured.
1844
+
1845
+ This method provides consistent grants application across all evaluation strategies.
1846
+ It ensures that whenever a physical database object (table, view, materialized view)
1847
+ is created or modified, the appropriate grants are applied.
1848
+
1849
+ Args:
1850
+ model: The SQLMesh model containing grants configuration
1851
+ table_name: The target table/view name to apply grants to
1852
+ target_layer: The grants application layer (physical or virtual)
1853
+ is_snapshot_deployable: Whether the snapshot is deployable (targeting production)
1854
+ """
1855
+ grants_config = model.grants
1856
+ if grants_config is None:
1857
+ return
1858
+
1859
+ if not self.adapter.SUPPORTS_GRANTS:
1860
+ logger.warning(
1861
+ f"Engine {self.adapter.__class__.__name__} does not support grants. "
1862
+ f"Skipping grants application for model {model.name}"
1863
+ )
1864
+ return
1865
+
1866
+ model_grants_target_layer = model.grants_target_layer
1867
+ deployable_vde_dev_only = (
1868
+ is_snapshot_deployable and model.virtual_environment_mode.is_dev_only
1869
+ )
1870
+
1871
+ # table_type is always a VIEW in the virtual layer unless model is deployable and VDE is dev_only
1872
+ # in which case we fall back to the model's model_grants_table_type
1873
+ if target_layer == GrantsTargetLayer.VIRTUAL and not deployable_vde_dev_only:
1874
+ model_grants_table_type = DataObjectType.VIEW
1875
+ else:
1876
+ model_grants_table_type = model.grants_table_type
1877
+
1878
+ if (
1879
+ model_grants_target_layer.is_all
1880
+ or model_grants_target_layer == target_layer
1881
+ # Always apply grants in production when VDE is dev_only regardless of target_layer
1882
+ # since only physical tables are created in production
1883
+ or deployable_vde_dev_only
1884
+ ):
1885
+ logger.info(f"Applying grants for model {model.name} to table {table_name}")
1886
+ self.adapter.sync_grants_config(
1887
+ exp.to_table(table_name, dialect=self.adapter.dialect),
1888
+ grants_config,
1889
+ model_grants_table_type,
1890
+ )
1891
+ else:
1892
+ logger.debug(
1893
+ f"Skipping grants application for model {model.name} in {target_layer} layer"
1894
+ )
1895
+
1612
1896
 
1613
1897
  class SymbolicStrategy(EvaluationStrategy):
1614
1898
  def insert(
@@ -1638,6 +1922,7 @@ class SymbolicStrategy(EvaluationStrategy):
1638
1922
  model: Model,
1639
1923
  is_table_deployable: bool,
1640
1924
  render_kwargs: t.Dict[str, t.Any],
1925
+ skip_grants: bool,
1641
1926
  **kwargs: t.Any,
1642
1927
  ) -> None:
1643
1928
  pass
@@ -1670,6 +1955,12 @@ class SymbolicStrategy(EvaluationStrategy):
1670
1955
  def demote(self, view_name: str, **kwargs: t.Any) -> None:
1671
1956
  pass
1672
1957
 
1958
+ def run_pre_statements(self, snapshot: Snapshot, render_kwargs: t.Dict[str, t.Any]) -> None:
1959
+ pass
1960
+
1961
+ def run_post_statements(self, snapshot: Snapshot, render_kwargs: t.Dict[str, t.Any]) -> None:
1962
+ pass
1963
+
1673
1964
 
1674
1965
  class EmbeddedStrategy(SymbolicStrategy):
1675
1966
  def promote(
@@ -1713,10 +2004,27 @@ class PromotableStrategy(EvaluationStrategy, abc.ABC):
1713
2004
  view_properties=model.render_virtual_properties(**render_kwargs),
1714
2005
  )
1715
2006
 
2007
+ snapshot = kwargs.get("snapshot")
2008
+ deployability_index = kwargs.get("deployability_index")
2009
+ is_snapshot_deployable = (
2010
+ deployability_index.is_deployable(snapshot)
2011
+ if snapshot and deployability_index
2012
+ else False
2013
+ )
2014
+
2015
+ # Apply grants to the virtual layer (view) after promotion
2016
+ self._apply_grants(model, view_name, GrantsTargetLayer.VIRTUAL, is_snapshot_deployable)
2017
+
1716
2018
  def demote(self, view_name: str, **kwargs: t.Any) -> None:
1717
2019
  logger.info("Dropping view '%s'", view_name)
1718
2020
  self.adapter.drop_view(view_name, cascade=False)
1719
2021
 
2022
+ def run_pre_statements(self, snapshot: Snapshot, render_kwargs: t.Any) -> None:
2023
+ self.adapter.execute(snapshot.model.render_pre_statements(**render_kwargs))
2024
+
2025
+ def run_post_statements(self, snapshot: Snapshot, render_kwargs: t.Any) -> None:
2026
+ self.adapter.execute(snapshot.model.render_post_statements(**render_kwargs))
2027
+
1720
2028
 
1721
2029
  class MaterializableStrategy(PromotableStrategy, abc.ABC):
1722
2030
  def create(
@@ -1725,6 +2033,7 @@ class MaterializableStrategy(PromotableStrategy, abc.ABC):
1725
2033
  model: Model,
1726
2034
  is_table_deployable: bool,
1727
2035
  render_kwargs: t.Dict[str, t.Any],
2036
+ skip_grants: bool,
1728
2037
  **kwargs: t.Any,
1729
2038
  ) -> None:
1730
2039
  ctas_query = model.ctas_query(**render_kwargs)
@@ -1769,6 +2078,13 @@ class MaterializableStrategy(PromotableStrategy, abc.ABC):
1769
2078
  column_descriptions=model.column_descriptions if is_table_deployable else None,
1770
2079
  )
1771
2080
 
2081
+ # Apply grants after table creation (unless explicitly skipped by caller)
2082
+ if not skip_grants:
2083
+ is_snapshot_deployable = kwargs.get("is_snapshot_deployable", False)
2084
+ self._apply_grants(
2085
+ model, table_name, GrantsTargetLayer.PHYSICAL, is_snapshot_deployable
2086
+ )
2087
+
1772
2088
  def migrate(
1773
2089
  self,
1774
2090
  target_table_name: str,
@@ -1794,6 +2110,15 @@ class MaterializableStrategy(PromotableStrategy, abc.ABC):
1794
2110
  )
1795
2111
  self.adapter.alter_table(alter_operations)
1796
2112
 
2113
+ # Apply grants after schema migration
2114
+ deployability_index = kwargs.get("deployability_index")
2115
+ is_snapshot_deployable = (
2116
+ deployability_index.is_deployable(snapshot) if deployability_index else False
2117
+ )
2118
+ self._apply_grants(
2119
+ snapshot.model, target_table_name, GrantsTargetLayer.PHYSICAL, is_snapshot_deployable
2120
+ )
2121
+
1797
2122
  def delete(self, name: str, **kwargs: t.Any) -> None:
1798
2123
  _check_table_db_is_physical_schema(name, kwargs["physical_schema"])
1799
2124
  self.adapter.drop_table(name, cascade=kwargs.pop("cascade", False))
@@ -1805,6 +2130,7 @@ class MaterializableStrategy(PromotableStrategy, abc.ABC):
1805
2130
  name: str,
1806
2131
  query_or_df: QueryOrDF,
1807
2132
  render_kwargs: t.Dict[str, t.Any],
2133
+ skip_grants: bool = False,
1808
2134
  **kwargs: t.Any,
1809
2135
  ) -> None:
1810
2136
  """Replaces the table for the given model.
@@ -1841,6 +2167,11 @@ class MaterializableStrategy(PromotableStrategy, abc.ABC):
1841
2167
  source_columns=source_columns,
1842
2168
  )
1843
2169
 
2170
+ # Apply grants after table replacement (unless explicitly skipped by caller)
2171
+ if not skip_grants:
2172
+ is_snapshot_deployable = kwargs.get("is_snapshot_deployable", False)
2173
+ self._apply_grants(model, name, GrantsTargetLayer.PHYSICAL, is_snapshot_deployable)
2174
+
1844
2175
  def _get_target_and_source_columns(
1845
2176
  self,
1846
2177
  model: Model,
@@ -1862,7 +2193,13 @@ class MaterializableStrategy(PromotableStrategy, abc.ABC):
1862
2193
  if model.on_destructive_change.is_ignore or model.on_additive_change.is_ignore:
1863
2194
  # We need to identify the columns that are only in the source so we create an empty table with
1864
2195
  # the user query to determine that
1865
- with self.adapter.temp_table(model.ctas_query(**render_kwargs)) as temp_table:
2196
+ temp_table_name = exp.table_(
2197
+ "diff",
2198
+ db=model.physical_schema,
2199
+ )
2200
+ with self.adapter.temp_table(
2201
+ model.ctas_query(**render_kwargs), name=temp_table_name
2202
+ ) as temp_table:
1866
2203
  source_columns = list(self.adapter.columns(temp_table))
1867
2204
  else:
1868
2205
  source_columns = None
@@ -2088,6 +2425,7 @@ class SeedStrategy(MaterializableStrategy):
2088
2425
  model: Model,
2089
2426
  is_table_deployable: bool,
2090
2427
  render_kwargs: t.Dict[str, t.Any],
2428
+ skip_grants: bool,
2091
2429
  **kwargs: t.Any,
2092
2430
  ) -> None:
2093
2431
  model = t.cast(SeedModel, model)
@@ -2101,22 +2439,52 @@ class SeedStrategy(MaterializableStrategy):
2101
2439
  )
2102
2440
  return
2103
2441
 
2104
- super().create(table_name, model, is_table_deployable, render_kwargs, **kwargs)
2105
- if is_table_deployable:
2106
- # For seeds we insert data at the time of table creation.
2107
- try:
2108
- for index, df in enumerate(model.render_seed()):
2109
- if index == 0:
2110
- self._replace_query_for_model(
2111
- model, table_name, df, render_kwargs, **kwargs
2112
- )
2113
- else:
2114
- self.adapter.insert_append(
2115
- table_name, df, target_columns_to_types=model.columns_to_types
2116
- )
2117
- except Exception:
2118
- self.adapter.drop_table(table_name)
2119
- raise
2442
+ super().create(
2443
+ table_name,
2444
+ model,
2445
+ is_table_deployable,
2446
+ render_kwargs,
2447
+ skip_grants=True, # Skip grants; they're applied after data insertion
2448
+ **kwargs,
2449
+ )
2450
+ # For seeds we insert data at the time of table creation.
2451
+ try:
2452
+ for index, df in enumerate(model.render_seed()):
2453
+ if index == 0:
2454
+ self._replace_query_for_model(
2455
+ model,
2456
+ table_name,
2457
+ df,
2458
+ render_kwargs,
2459
+ skip_grants=True, # Skip grants; they're applied after data insertion
2460
+ **kwargs,
2461
+ )
2462
+ else:
2463
+ self.adapter.insert_append(
2464
+ table_name, df, target_columns_to_types=model.columns_to_types
2465
+ )
2466
+
2467
+ if not skip_grants:
2468
+ # Apply grants after seed table creation and data insertion
2469
+ is_snapshot_deployable = kwargs.get("is_snapshot_deployable", False)
2470
+ self._apply_grants(
2471
+ model, table_name, GrantsTargetLayer.PHYSICAL, is_snapshot_deployable
2472
+ )
2473
+ except Exception:
2474
+ self.adapter.drop_table(table_name)
2475
+ raise
2476
+
2477
+ def migrate(
2478
+ self,
2479
+ target_table_name: str,
2480
+ source_table_name: str,
2481
+ snapshot: Snapshot,
2482
+ *,
2483
+ ignore_destructive: bool,
2484
+ ignore_additive: bool,
2485
+ **kwargs: t.Any,
2486
+ ) -> None:
2487
+ raise NotImplementedError("Seeds do not support migrations.")
2120
2488
 
2121
2489
  def insert(
2122
2490
  self,
@@ -2149,6 +2517,7 @@ class SCDType2Strategy(IncrementalStrategy):
2149
2517
  model: Model,
2150
2518
  is_table_deployable: bool,
2151
2519
  render_kwargs: t.Dict[str, t.Any],
2520
+ skip_grants: bool,
2152
2521
  **kwargs: t.Any,
2153
2522
  ) -> None:
2154
2523
  assert isinstance(model.kind, (SCDType2ByTimeKind, SCDType2ByColumnKind))
@@ -2178,9 +2547,17 @@ class SCDType2Strategy(IncrementalStrategy):
2178
2547
  model,
2179
2548
  is_table_deployable,
2180
2549
  render_kwargs,
2550
+ skip_grants,
2181
2551
  **kwargs,
2182
2552
  )
2183
2553
 
2554
+ if not skip_grants:
2555
+ # Apply grants after SCD Type 2 table creation
2556
+ is_snapshot_deployable = kwargs.get("is_snapshot_deployable", False)
2557
+ self._apply_grants(
2558
+ model, table_name, GrantsTargetLayer.PHYSICAL, is_snapshot_deployable
2559
+ )
2560
+
2184
2561
  def insert(
2185
2562
  self,
2186
2563
  table_name: str,
@@ -2214,6 +2591,11 @@ class SCDType2Strategy(IncrementalStrategy):
2214
2591
  column_descriptions=model.column_descriptions,
2215
2592
  truncate=is_first_insert,
2216
2593
  source_columns=source_columns,
2594
+ storage_format=model.storage_format,
2595
+ partitioned_by=model.partitioned_by,
2596
+ partition_interval_unit=model.partition_interval_unit,
2597
+ clustered_by=model.clustered_by,
2598
+ table_properties=kwargs.get("physical_properties", model.physical_properties),
2217
2599
  )
2218
2600
  elif isinstance(model.kind, SCDType2ByColumnKind):
2219
2601
  self.adapter.scd_type_2_by_column(
@@ -2232,12 +2614,21 @@ class SCDType2Strategy(IncrementalStrategy):
2232
2614
  column_descriptions=model.column_descriptions,
2233
2615
  truncate=is_first_insert,
2234
2616
  source_columns=source_columns,
2617
+ storage_format=model.storage_format,
2618
+ partitioned_by=model.partitioned_by,
2619
+ partition_interval_unit=model.partition_interval_unit,
2620
+ clustered_by=model.clustered_by,
2621
+ table_properties=kwargs.get("physical_properties", model.physical_properties),
2235
2622
  )
2236
2623
  else:
2237
2624
  raise SQLMeshError(
2238
2625
  f"Unexpected SCD Type 2 kind: {model.kind}. This is not expected and please report this as a bug."
2239
2626
  )
2240
2627
 
2628
+ # Apply grants after SCD Type 2 table recreation
2629
+ is_snapshot_deployable = kwargs.get("is_snapshot_deployable", False)
2630
+ self._apply_grants(model, table_name, GrantsTargetLayer.PHYSICAL, is_snapshot_deployable)
2631
+
2241
2632
  def append(
2242
2633
  self,
2243
2634
  table_name: str,
@@ -2246,51 +2637,14 @@ class SCDType2Strategy(IncrementalStrategy):
2246
2637
  render_kwargs: t.Dict[str, t.Any],
2247
2638
  **kwargs: t.Any,
2248
2639
  ) -> None:
2249
- # Source columns from the underlying table to prevent unintentional table schema changes during restatement of incremental models.
2250
- columns_to_types, source_columns = self._get_target_and_source_columns(
2251
- model,
2640
+ return self.insert(
2252
2641
  table_name,
2642
+ query_or_df,
2643
+ model,
2644
+ is_first_insert=False,
2253
2645
  render_kwargs=render_kwargs,
2254
- force_get_columns_from_target=True,
2646
+ **kwargs,
2255
2647
  )
2256
- if isinstance(model.kind, SCDType2ByTimeKind):
2257
- self.adapter.scd_type_2_by_time(
2258
- target_table=table_name,
2259
- source_table=query_or_df,
2260
- unique_key=model.unique_key,
2261
- valid_from_col=model.kind.valid_from_name,
2262
- valid_to_col=model.kind.valid_to_name,
2263
- updated_at_col=model.kind.updated_at_name,
2264
- invalidate_hard_deletes=model.kind.invalidate_hard_deletes,
2265
- updated_at_as_valid_from=model.kind.updated_at_as_valid_from,
2266
- target_columns_to_types=columns_to_types,
2267
- table_format=model.table_format,
2268
- table_description=model.description,
2269
- column_descriptions=model.column_descriptions,
2270
- source_columns=source_columns,
2271
- **kwargs,
2272
- )
2273
- elif isinstance(model.kind, SCDType2ByColumnKind):
2274
- self.adapter.scd_type_2_by_column(
2275
- target_table=table_name,
2276
- source_table=query_or_df,
2277
- unique_key=model.unique_key,
2278
- valid_from_col=model.kind.valid_from_name,
2279
- valid_to_col=model.kind.valid_to_name,
2280
- check_columns=model.kind.columns,
2281
- target_columns_to_types=columns_to_types,
2282
- table_format=model.table_format,
2283
- invalidate_hard_deletes=model.kind.invalidate_hard_deletes,
2284
- execution_time_as_valid_from=model.kind.execution_time_as_valid_from,
2285
- table_description=model.description,
2286
- column_descriptions=model.column_descriptions,
2287
- source_columns=source_columns,
2288
- **kwargs,
2289
- )
2290
- else:
2291
- raise SQLMeshError(
2292
- f"Unexpected SCD Type 2 kind: {model.kind}. This is not expected and please report this as a bug."
2293
- )
2294
2648
 
2295
2649
 
2296
2650
  class ViewStrategy(PromotableStrategy):
@@ -2331,6 +2685,10 @@ class ViewStrategy(PromotableStrategy):
2331
2685
  column_descriptions=model.column_descriptions,
2332
2686
  )
2333
2687
 
2688
+ # Apply grants after view creation / replacement
2689
+ is_snapshot_deployable = kwargs.get("is_snapshot_deployable", False)
2690
+ self._apply_grants(model, table_name, GrantsTargetLayer.PHYSICAL, is_snapshot_deployable)
2691
+
2334
2692
  def append(
2335
2693
  self,
2336
2694
  table_name: str,
@@ -2347,12 +2705,21 @@ class ViewStrategy(PromotableStrategy):
2347
2705
  model: Model,
2348
2706
  is_table_deployable: bool,
2349
2707
  render_kwargs: t.Dict[str, t.Any],
2708
+ skip_grants: bool,
2350
2709
  **kwargs: t.Any,
2351
2710
  ) -> None:
2711
+ is_snapshot_deployable = kwargs.get("is_snapshot_deployable", False)
2712
+
2352
2713
  if self.adapter.table_exists(table_name):
2353
2714
  # Make sure we don't recreate the view to prevent deletion of downstream views in engines with no late
2354
2715
  # binding support (because of DROP CASCADE).
2355
2716
  logger.info("View '%s' already exists", table_name)
2717
+
2718
+ if not skip_grants:
2719
+ # Always apply grants when present, even if view exists, to handle grants updates
2720
+ self._apply_grants(
2721
+ model, table_name, GrantsTargetLayer.PHYSICAL, is_snapshot_deployable
2722
+ )
2356
2723
  return
2357
2724
 
2358
2725
  logger.info("Creating view '%s'", table_name)
@@ -2376,6 +2743,12 @@ class ViewStrategy(PromotableStrategy):
2376
2743
  column_descriptions=model.column_descriptions if is_table_deployable else None,
2377
2744
  )
2378
2745
 
2746
+ if not skip_grants:
2747
+ # Apply grants after view creation
2748
+ self._apply_grants(
2749
+ model, table_name, GrantsTargetLayer.PHYSICAL, is_snapshot_deployable
2750
+ )
2751
+
2379
2752
  def migrate(
2380
2753
  self,
2381
2754
  target_table_name: str,
@@ -2402,6 +2775,15 @@ class ViewStrategy(PromotableStrategy):
2402
2775
  column_descriptions=model.column_descriptions,
2403
2776
  )
2404
2777
 
2778
+ # Apply grants after view migration
2779
+ deployability_index = kwargs.get("deployability_index")
2780
+ is_snapshot_deployable = (
2781
+ deployability_index.is_deployable(snapshot) if deployability_index else False
2782
+ )
2783
+ self._apply_grants(
2784
+ snapshot.model, target_table_name, GrantsTargetLayer.PHYSICAL, is_snapshot_deployable
2785
+ )
2786
+
2405
2787
  def delete(self, name: str, **kwargs: t.Any) -> None:
2406
2788
  cascade = kwargs.pop("cascade", False)
2407
2789
  try:
@@ -2541,6 +2923,169 @@ def get_custom_materialization_type_or_raise(
2541
2923
  raise SQLMeshError(f"Custom materialization '{name}' not present in the Python environment")
2542
2924
 
2543
2925
 
2926
+ class DbtCustomMaterializationStrategy(MaterializableStrategy):
2927
+ def __init__(
2928
+ self,
2929
+ adapter: EngineAdapter,
2930
+ materialization_name: str,
2931
+ materialization_template: str,
2932
+ ):
2933
+ super().__init__(adapter)
2934
+ self.materialization_name = materialization_name
2935
+ self.materialization_template = materialization_template
2936
+
2937
+ def create(
2938
+ self,
2939
+ table_name: str,
2940
+ model: Model,
2941
+ is_table_deployable: bool,
2942
+ render_kwargs: t.Dict[str, t.Any],
2943
+ skip_grants: bool,
2944
+ **kwargs: t.Any,
2945
+ ) -> None:
2946
+ original_query = model.render_query_or_raise(**render_kwargs)
2947
+ self._execute_materialization(
2948
+ table_name=table_name,
2949
+ query_or_df=original_query.limit(0),
2950
+ model=model,
2951
+ is_first_insert=True,
2952
+ render_kwargs=render_kwargs,
2953
+ create_only=True,
2954
+ **kwargs,
2955
+ )
2956
+
2957
+ # Apply grants after dbt custom materialization table creation
2958
+ if not skip_grants:
2959
+ is_snapshot_deployable = kwargs.get("is_snapshot_deployable", False)
2960
+ self._apply_grants(
2961
+ model, table_name, GrantsTargetLayer.PHYSICAL, is_snapshot_deployable
2962
+ )
2963
+
2964
+ def insert(
2965
+ self,
2966
+ table_name: str,
2967
+ query_or_df: QueryOrDF,
2968
+ model: Model,
2969
+ is_first_insert: bool,
2970
+ render_kwargs: t.Dict[str, t.Any],
2971
+ **kwargs: t.Any,
2972
+ ) -> None:
2973
+ self._execute_materialization(
2974
+ table_name=table_name,
2975
+ query_or_df=query_or_df,
2976
+ model=model,
2977
+ is_first_insert=is_first_insert,
2978
+ render_kwargs=render_kwargs,
2979
+ **kwargs,
2980
+ )
2981
+
2982
+ # Apply grants after custom materialization insert (only on first insert)
2983
+ if is_first_insert:
2984
+ is_snapshot_deployable = kwargs.get("is_snapshot_deployable", False)
2985
+ self._apply_grants(
2986
+ model, table_name, GrantsTargetLayer.PHYSICAL, is_snapshot_deployable
2987
+ )
2988
+
2989
+ def append(
2990
+ self,
2991
+ table_name: str,
2992
+ query_or_df: QueryOrDF,
2993
+ model: Model,
2994
+ render_kwargs: t.Dict[str, t.Any],
2995
+ **kwargs: t.Any,
2996
+ ) -> None:
2997
+ return self.insert(
2998
+ table_name,
2999
+ query_or_df,
3000
+ model,
3001
+ is_first_insert=False,
3002
+ render_kwargs=render_kwargs,
3003
+ **kwargs,
3004
+ )
3005
+
3006
+ def run_pre_statements(self, snapshot: Snapshot, render_kwargs: t.Any) -> None:
3007
+ # in dbt custom materialisations it's up to the user to run the pre hooks inside the transaction
3008
+ if not render_kwargs.get("inside_transaction", True):
3009
+ super().run_pre_statements(
3010
+ snapshot=snapshot,
3011
+ render_kwargs=render_kwargs,
3012
+ )
3013
+
3014
+ def run_post_statements(self, snapshot: Snapshot, render_kwargs: t.Any) -> None:
3015
+ # in dbt custom materialisations it's up to the user to run the post hooks inside the transaction
3016
+ if not render_kwargs.get("inside_transaction", True):
3017
+ super().run_post_statements(
3018
+ snapshot=snapshot,
3019
+ render_kwargs=render_kwargs,
3020
+ )
3021
+
3022
+ def _execute_materialization(
3023
+ self,
3024
+ table_name: str,
3025
+ query_or_df: QueryOrDF,
3026
+ model: Model,
3027
+ is_first_insert: bool,
3028
+ render_kwargs: t.Dict[str, t.Any],
3029
+ create_only: bool = False,
3030
+ **kwargs: t.Any,
3031
+ ) -> None:
3032
+ jinja_macros = model.jinja_macros
3033
+
3034
+ # For vdes we need to use the table, since we don't know the schema/table at parse time
3035
+ parts = exp.to_table(table_name, dialect=self.adapter.dialect)
3036
+
3037
+ existing_globals = jinja_macros.global_objs
3038
+ relation_info = existing_globals.get("this")
3039
+ if isinstance(relation_info, dict):
3040
+ relation_info["database"] = parts.catalog
3041
+ relation_info["identifier"] = parts.name
3042
+ relation_info["name"] = parts.name
3043
+
3044
+ jinja_globals = {
3045
+ **existing_globals,
3046
+ "this": relation_info,
3047
+ "database": parts.catalog,
3048
+ "schema": parts.db,
3049
+ "identifier": parts.name,
3050
+ "target": existing_globals.get("target", {"type": self.adapter.dialect}),
3051
+ "execution_dt": kwargs.get("execution_time"),
3052
+ "engine_adapter": self.adapter,
3053
+ "sql": str(query_or_df),
3054
+ "is_first_insert": is_first_insert,
3055
+ "create_only": create_only,
3056
+ "pre_hooks": [
3057
+ AttributeDict({"sql": s.this.this, "transaction": transaction})
3058
+ for s in model.pre_statements
3059
+ if (transaction := s.args.get("transaction", True))
3060
+ ],
3061
+ "post_hooks": [
3062
+ AttributeDict({"sql": s.this.this, "transaction": transaction})
3063
+ for s in model.post_statements
3064
+ if (transaction := s.args.get("transaction", True))
3065
+ ],
3066
+ "model_instance": model,
3067
+ **kwargs,
3068
+ }
3069
+
3070
+ try:
3071
+ jinja_env = jinja_macros.build_environment(**jinja_globals)
3072
+ template = jinja_env.from_string(self.materialization_template)
3073
+
3074
+ try:
3075
+ template.render()
3076
+ except MacroReturnVal as ret:
3077
+ # this is a successful return from a macro call (dbt uses this list of Relations to update their relation cache)
3078
+ returned_relations = ret.value.get("relations", [])
3079
+ logger.info(
3080
+ f"Materialization {self.materialization_name} returned relations: {returned_relations}"
3081
+ )
3082
+
3083
+ except Exception as e:
3084
+ raise SQLMeshError(
3085
+ f"Failed to execute dbt materialization '{self.materialization_name}': {e}"
3086
+ ) from e
3087
+
3088
+
2544
3089
  class EngineManagedStrategy(MaterializableStrategy):
2545
3090
  def create(
2546
3091
  self,
@@ -2548,6 +3093,7 @@ class EngineManagedStrategy(MaterializableStrategy):
2548
3093
  model: Model,
2549
3094
  is_table_deployable: bool,
2550
3095
  render_kwargs: t.Dict[str, t.Any],
3096
+ skip_grants: bool,
2551
3097
  **kwargs: t.Any,
2552
3098
  ) -> None:
2553
3099
  is_snapshot_deployable: bool = kwargs["is_snapshot_deployable"]
@@ -2566,6 +3112,13 @@ class EngineManagedStrategy(MaterializableStrategy):
2566
3112
  column_descriptions=model.column_descriptions,
2567
3113
  table_format=model.table_format,
2568
3114
  )
3115
+
3116
+ # Apply grants after managed table creation
3117
+ if not skip_grants:
3118
+ self._apply_grants(
3119
+ model, table_name, GrantsTargetLayer.PHYSICAL, is_snapshot_deployable
3120
+ )
3121
+
2569
3122
  elif not is_table_deployable:
2570
3123
  # Only create the dev preview table as a normal table.
2571
3124
  # For the main table, if the snapshot is cant be deployed to prod (eg upstream is forward-only) do nothing.
@@ -2576,6 +3129,7 @@ class EngineManagedStrategy(MaterializableStrategy):
2576
3129
  model=model,
2577
3130
  is_table_deployable=is_table_deployable,
2578
3131
  render_kwargs=render_kwargs,
3132
+ skip_grants=skip_grants,
2579
3133
  **kwargs,
2580
3134
  )
2581
3135
 
@@ -2591,7 +3145,6 @@ class EngineManagedStrategy(MaterializableStrategy):
2591
3145
  deployability_index: DeployabilityIndex = kwargs["deployability_index"]
2592
3146
  snapshot: Snapshot = kwargs["snapshot"]
2593
3147
  is_snapshot_deployable = deployability_index.is_deployable(snapshot)
2594
-
2595
3148
  if is_first_insert and is_snapshot_deployable and not self.adapter.table_exists(table_name):
2596
3149
  self.adapter.create_managed_table(
2597
3150
  table_name=table_name,
@@ -2604,6 +3157,9 @@ class EngineManagedStrategy(MaterializableStrategy):
2604
3157
  column_descriptions=model.column_descriptions,
2605
3158
  table_format=model.table_format,
2606
3159
  )
3160
+ self._apply_grants(
3161
+ model, table_name, GrantsTargetLayer.PHYSICAL, is_snapshot_deployable
3162
+ )
2607
3163
  elif not is_snapshot_deployable:
2608
3164
  # Snapshot isnt deployable; update the preview table instead
2609
3165
  # If the snapshot was deployable, then data would have already been loaded in create() because a managed table would have been created
@@ -2648,10 +3204,19 @@ class EngineManagedStrategy(MaterializableStrategy):
2648
3204
  )
2649
3205
  if len(potential_alter_operations) > 0:
2650
3206
  # this can happen if a user changes a managed model and deliberately overrides a plan to be forward only, eg `sqlmesh plan --forward-only`
2651
- raise SQLMeshError(
3207
+ raise MigrationNotSupportedError(
2652
3208
  f"The schema of the managed model '{target_table_name}' cannot be updated in a forward-only fashion."
2653
3209
  )
2654
3210
 
3211
+ # Apply grants after verifying no schema changes
3212
+ deployability_index = kwargs.get("deployability_index")
3213
+ is_snapshot_deployable = (
3214
+ deployability_index.is_deployable(snapshot) if deployability_index else False
3215
+ )
3216
+ self._apply_grants(
3217
+ snapshot.model, target_table_name, GrantsTargetLayer.PHYSICAL, is_snapshot_deployable
3218
+ )
3219
+
2655
3220
  def delete(self, name: str, **kwargs: t.Any) -> None:
2656
3221
  # a dev preview table is created as a normal table, so it needs to be dropped as a normal table
2657
3222
  _check_table_db_is_physical_schema(name, kwargs["physical_schema"])