sqlmesh 0.217.1.dev1__py3-none-any.whl → 0.227.2.dev4__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (183) hide show
  1. sqlmesh/__init__.py +12 -2
  2. sqlmesh/_version.py +2 -2
  3. sqlmesh/cli/project_init.py +10 -2
  4. sqlmesh/core/_typing.py +1 -0
  5. sqlmesh/core/audit/definition.py +8 -2
  6. sqlmesh/core/config/__init__.py +1 -1
  7. sqlmesh/core/config/connection.py +17 -5
  8. sqlmesh/core/config/dbt.py +13 -0
  9. sqlmesh/core/config/janitor.py +12 -0
  10. sqlmesh/core/config/loader.py +7 -0
  11. sqlmesh/core/config/model.py +2 -0
  12. sqlmesh/core/config/root.py +3 -0
  13. sqlmesh/core/console.py +80 -2
  14. sqlmesh/core/constants.py +1 -1
  15. sqlmesh/core/context.py +61 -25
  16. sqlmesh/core/dialect.py +3 -0
  17. sqlmesh/core/engine_adapter/_typing.py +2 -0
  18. sqlmesh/core/engine_adapter/base.py +322 -22
  19. sqlmesh/core/engine_adapter/base_postgres.py +17 -1
  20. sqlmesh/core/engine_adapter/bigquery.py +146 -7
  21. sqlmesh/core/engine_adapter/clickhouse.py +17 -13
  22. sqlmesh/core/engine_adapter/databricks.py +33 -2
  23. sqlmesh/core/engine_adapter/fabric.py +1 -29
  24. sqlmesh/core/engine_adapter/mixins.py +142 -48
  25. sqlmesh/core/engine_adapter/mssql.py +15 -4
  26. sqlmesh/core/engine_adapter/mysql.py +2 -2
  27. sqlmesh/core/engine_adapter/postgres.py +9 -3
  28. sqlmesh/core/engine_adapter/redshift.py +4 -0
  29. sqlmesh/core/engine_adapter/risingwave.py +1 -0
  30. sqlmesh/core/engine_adapter/shared.py +6 -0
  31. sqlmesh/core/engine_adapter/snowflake.py +82 -11
  32. sqlmesh/core/engine_adapter/spark.py +14 -10
  33. sqlmesh/core/engine_adapter/trino.py +4 -2
  34. sqlmesh/core/janitor.py +181 -0
  35. sqlmesh/core/lineage.py +1 -0
  36. sqlmesh/core/macros.py +35 -13
  37. sqlmesh/core/model/common.py +2 -0
  38. sqlmesh/core/model/definition.py +65 -4
  39. sqlmesh/core/model/kind.py +66 -2
  40. sqlmesh/core/model/meta.py +107 -2
  41. sqlmesh/core/node.py +101 -2
  42. sqlmesh/core/plan/builder.py +15 -10
  43. sqlmesh/core/plan/common.py +196 -2
  44. sqlmesh/core/plan/definition.py +21 -6
  45. sqlmesh/core/plan/evaluator.py +72 -113
  46. sqlmesh/core/plan/explainer.py +90 -8
  47. sqlmesh/core/plan/stages.py +42 -21
  48. sqlmesh/core/renderer.py +26 -18
  49. sqlmesh/core/scheduler.py +60 -19
  50. sqlmesh/core/selector.py +137 -9
  51. sqlmesh/core/signal.py +64 -1
  52. sqlmesh/core/snapshot/__init__.py +1 -0
  53. sqlmesh/core/snapshot/definition.py +109 -25
  54. sqlmesh/core/snapshot/evaluator.py +610 -50
  55. sqlmesh/core/state_sync/__init__.py +0 -1
  56. sqlmesh/core/state_sync/base.py +31 -27
  57. sqlmesh/core/state_sync/cache.py +12 -4
  58. sqlmesh/core/state_sync/common.py +216 -111
  59. sqlmesh/core/state_sync/db/facade.py +30 -15
  60. sqlmesh/core/state_sync/db/interval.py +27 -7
  61. sqlmesh/core/state_sync/db/migrator.py +14 -8
  62. sqlmesh/core/state_sync/db/snapshot.py +119 -87
  63. sqlmesh/core/table_diff.py +2 -2
  64. sqlmesh/core/test/definition.py +14 -9
  65. sqlmesh/dbt/adapter.py +20 -11
  66. sqlmesh/dbt/basemodel.py +52 -41
  67. sqlmesh/dbt/builtin.py +27 -11
  68. sqlmesh/dbt/column.py +17 -5
  69. sqlmesh/dbt/common.py +4 -2
  70. sqlmesh/dbt/context.py +14 -1
  71. sqlmesh/dbt/loader.py +60 -8
  72. sqlmesh/dbt/manifest.py +136 -8
  73. sqlmesh/dbt/model.py +105 -25
  74. sqlmesh/dbt/package.py +16 -1
  75. sqlmesh/dbt/profile.py +3 -3
  76. sqlmesh/dbt/project.py +12 -7
  77. sqlmesh/dbt/seed.py +1 -1
  78. sqlmesh/dbt/source.py +6 -1
  79. sqlmesh/dbt/target.py +25 -6
  80. sqlmesh/dbt/test.py +31 -1
  81. sqlmesh/migrations/v0000_baseline.py +3 -6
  82. sqlmesh/migrations/v0061_mysql_fix_blob_text_type.py +2 -5
  83. sqlmesh/migrations/v0062_add_model_gateway.py +2 -2
  84. sqlmesh/migrations/v0063_change_signals.py +2 -4
  85. sqlmesh/migrations/v0064_join_when_matched_strings.py +2 -4
  86. sqlmesh/migrations/v0065_add_model_optimize.py +2 -2
  87. sqlmesh/migrations/v0066_add_auto_restatements.py +2 -6
  88. sqlmesh/migrations/v0067_add_tsql_date_full_precision.py +2 -2
  89. sqlmesh/migrations/v0068_include_unrendered_query_in_metadata_hash.py +2 -2
  90. sqlmesh/migrations/v0069_update_dev_table_suffix.py +2 -4
  91. sqlmesh/migrations/v0070_include_grains_in_metadata_hash.py +2 -2
  92. sqlmesh/migrations/v0071_add_dev_version_to_intervals.py +2 -6
  93. sqlmesh/migrations/v0072_add_environment_statements.py +2 -4
  94. sqlmesh/migrations/v0073_remove_symbolic_disable_restatement.py +2 -4
  95. sqlmesh/migrations/v0074_add_partition_by_time_column_property.py +2 -2
  96. sqlmesh/migrations/v0075_remove_validate_query.py +2 -4
  97. sqlmesh/migrations/v0076_add_cron_tz.py +2 -2
  98. sqlmesh/migrations/v0077_fix_column_type_hash_calculation.py +2 -2
  99. sqlmesh/migrations/v0078_warn_if_non_migratable_python_env.py +2 -4
  100. sqlmesh/migrations/v0079_add_gateway_managed_property.py +7 -9
  101. sqlmesh/migrations/v0080_add_batch_size_to_scd_type_2_models.py +2 -2
  102. sqlmesh/migrations/v0081_update_partitioned_by.py +2 -4
  103. sqlmesh/migrations/v0082_warn_if_incorrectly_duplicated_statements.py +2 -4
  104. sqlmesh/migrations/v0083_use_sql_for_scd_time_data_type_data_hash.py +2 -2
  105. sqlmesh/migrations/v0084_normalize_quote_when_matched_and_merge_filter.py +2 -2
  106. sqlmesh/migrations/v0085_deterministic_repr.py +2 -4
  107. sqlmesh/migrations/v0086_check_deterministic_bug.py +2 -4
  108. sqlmesh/migrations/v0087_normalize_blueprint_variables.py +2 -4
  109. sqlmesh/migrations/v0088_warn_about_variable_python_env_diffs.py +2 -4
  110. sqlmesh/migrations/v0089_add_virtual_environment_mode.py +2 -2
  111. sqlmesh/migrations/v0090_add_forward_only_column.py +2 -6
  112. sqlmesh/migrations/v0091_on_additive_change.py +2 -2
  113. sqlmesh/migrations/v0092_warn_about_dbt_data_type_diff.py +2 -4
  114. sqlmesh/migrations/v0093_use_raw_sql_in_fingerprint.py +2 -2
  115. sqlmesh/migrations/v0094_add_dev_version_and_fingerprint_columns.py +2 -6
  116. sqlmesh/migrations/v0095_warn_about_dbt_raw_sql_diff.py +2 -4
  117. sqlmesh/migrations/v0096_remove_plan_dags_table.py +2 -4
  118. sqlmesh/migrations/v0097_add_dbt_name_in_node.py +2 -2
  119. sqlmesh/migrations/v0098_add_dbt_node_info_in_node.py +103 -0
  120. sqlmesh/migrations/v0099_add_last_altered_to_intervals.py +25 -0
  121. sqlmesh/migrations/v0100_add_grants_and_grants_target_layer.py +9 -0
  122. sqlmesh/utils/__init__.py +8 -1
  123. sqlmesh/utils/cache.py +5 -1
  124. sqlmesh/utils/date.py +1 -1
  125. sqlmesh/utils/errors.py +4 -0
  126. sqlmesh/utils/jinja.py +25 -2
  127. sqlmesh/utils/pydantic.py +6 -6
  128. sqlmesh/utils/windows.py +13 -3
  129. {sqlmesh-0.217.1.dev1.dist-info → sqlmesh-0.227.2.dev4.dist-info}/METADATA +5 -5
  130. {sqlmesh-0.217.1.dev1.dist-info → sqlmesh-0.227.2.dev4.dist-info}/RECORD +181 -176
  131. sqlmesh_dbt/cli.py +70 -7
  132. sqlmesh_dbt/console.py +14 -6
  133. sqlmesh_dbt/operations.py +103 -24
  134. sqlmesh_dbt/selectors.py +39 -1
  135. web/client/dist/assets/{Audits-Ucsx1GzF.js → Audits-CBiYyyx-.js} +1 -1
  136. web/client/dist/assets/{Banner-BWDzvavM.js → Banner-DSRbUlO5.js} +1 -1
  137. web/client/dist/assets/{ChevronDownIcon-D2VL13Ah.js → ChevronDownIcon-MK_nrjD_.js} +1 -1
  138. web/client/dist/assets/{ChevronRightIcon-DWGYbf1l.js → ChevronRightIcon-CLWtT22Q.js} +1 -1
  139. web/client/dist/assets/{Content-DdHDZM3I.js → Content-BNuGZN5l.js} +1 -1
  140. web/client/dist/assets/{Content-Bikfy8fh.js → Content-CSHJyW0n.js} +1 -1
  141. web/client/dist/assets/{Data-CzAJH7rW.js → Data-C1oRDbLx.js} +1 -1
  142. web/client/dist/assets/{DataCatalog-BJF11g8f.js → DataCatalog-HXyX2-_j.js} +1 -1
  143. web/client/dist/assets/{Editor-s0SBpV2y.js → Editor-BDyfpUuw.js} +1 -1
  144. web/client/dist/assets/{Editor-DgLhgKnm.js → Editor-D0jNItwC.js} +1 -1
  145. web/client/dist/assets/{Errors-D0m0O1d3.js → Errors-BfuFLcPi.js} +1 -1
  146. web/client/dist/assets/{FileExplorer-CEv0vXkt.js → FileExplorer-BR9IE3he.js} +1 -1
  147. web/client/dist/assets/{Footer-BwzXn8Ew.js → Footer-CgBEtiAh.js} +1 -1
  148. web/client/dist/assets/{Header-6heDkEqG.js → Header-DSqR6nSO.js} +1 -1
  149. web/client/dist/assets/{Input-obuJsD6k.js → Input-B-oZ6fGO.js} +1 -1
  150. web/client/dist/assets/Lineage-DYQVwDbD.js +1 -0
  151. web/client/dist/assets/{ListboxShow-HM9_qyrt.js → ListboxShow-BE5-xevs.js} +1 -1
  152. web/client/dist/assets/{ModelLineage-zWdKo0U2.js → ModelLineage-DkIFAYo4.js} +1 -1
  153. web/client/dist/assets/{Models-Bcu66SRz.js → Models-D5dWr8RB.js} +1 -1
  154. web/client/dist/assets/{Page-BWEEQfIt.js → Page-C-XfU5BR.js} +1 -1
  155. web/client/dist/assets/{Plan-C4gXCqlf.js → Plan-ZEuTINBq.js} +1 -1
  156. web/client/dist/assets/{PlusCircleIcon-CVDO651q.js → PlusCircleIcon-DVXAHG8_.js} +1 -1
  157. web/client/dist/assets/{ReportErrors-BT6xFwAr.js → ReportErrors-B7FEPzMB.js} +1 -1
  158. web/client/dist/assets/{Root-ryJoBK4h.js → Root-8aZyhPxF.js} +1 -1
  159. web/client/dist/assets/{SearchList-DB04sPb9.js → SearchList-W_iT2G82.js} +1 -1
  160. web/client/dist/assets/{SelectEnvironment-CUYcXUu6.js → SelectEnvironment-C65jALmO.js} +1 -1
  161. web/client/dist/assets/{SourceList-Doo_9ZGp.js → SourceList-DSLO6nVJ.js} +1 -1
  162. web/client/dist/assets/{SourceListItem-D5Mj7Dly.js → SourceListItem-BHt8d9-I.js} +1 -1
  163. web/client/dist/assets/{SplitPane-qHmkD1qy.js → SplitPane-CViaZmw6.js} +1 -1
  164. web/client/dist/assets/{Tests-DH1Z74ML.js → Tests-DhaVt5t1.js} +1 -1
  165. web/client/dist/assets/{Welcome-DqUJUNMF.js → Welcome-DvpjH-_4.js} +1 -1
  166. web/client/dist/assets/context-BctCsyGb.js +71 -0
  167. web/client/dist/assets/{context-Dr54UHLi.js → context-DFNeGsFF.js} +1 -1
  168. web/client/dist/assets/{editor-DYIP1yQ4.js → editor-CcO28cqd.js} +1 -1
  169. web/client/dist/assets/{file-DarlIDVi.js → file-CvJN3aZO.js} +1 -1
  170. web/client/dist/assets/{floating-ui.react-dom-BH3TFvkM.js → floating-ui.react-dom-CjE-JNW1.js} +1 -1
  171. web/client/dist/assets/{help-Bl8wqaQc.js → help-DuPhjipa.js} +1 -1
  172. web/client/dist/assets/{index-D1sR7wpN.js → index-C-dJH7yZ.js} +1 -1
  173. web/client/dist/assets/{index-O3mjYpnE.js → index-Dj0i1-CA.js} +2 -2
  174. web/client/dist/assets/{plan-CehRrJUG.js → plan-BTRSbjKn.js} +1 -1
  175. web/client/dist/assets/{popover-CqgMRE0G.js → popover-_Sf0yvOI.js} +1 -1
  176. web/client/dist/assets/{project-6gxepOhm.js → project-BvSOI8MY.js} +1 -1
  177. web/client/dist/index.html +1 -1
  178. web/client/dist/assets/Lineage-D0Hgdz2v.js +0 -1
  179. web/client/dist/assets/context-DgX0fp2E.js +0 -68
  180. {sqlmesh-0.217.1.dev1.dist-info → sqlmesh-0.227.2.dev4.dist-info}/WHEEL +0 -0
  181. {sqlmesh-0.217.1.dev1.dist-info → sqlmesh-0.227.2.dev4.dist-info}/entry_points.txt +0 -0
  182. {sqlmesh-0.217.1.dev1.dist-info → sqlmesh-0.227.2.dev4.dist-info}/licenses/LICENSE +0 -0
  183. {sqlmesh-0.217.1.dev1.dist-info → sqlmesh-0.227.2.dev4.dist-info}/top_level.txt +0 -0
sqlmesh/core/macros.py CHANGED
@@ -128,6 +128,17 @@ def _macro_str_replace(text: str) -> str:
128
128
  return f"self.template({text}, locals())"
129
129
 
130
130
 
131
+ class CaseInsensitiveMapping(t.Dict[str, t.Any]):
132
+ def __init__(self, data: t.Dict[str, t.Any]) -> None:
133
+ super().__init__(data)
134
+
135
+ def __getitem__(self, key: str) -> t.Any:
136
+ return super().__getitem__(key.lower())
137
+
138
+ def get(self, key: str, default: t.Any = None, /) -> t.Any:
139
+ return super().get(key.lower(), default)
140
+
141
+
131
142
  class MacroDialect(Python):
132
143
  class Generator(Python.Generator):
133
144
  TRANSFORMS = {
@@ -256,14 +267,18 @@ class MacroEvaluator:
256
267
  changed = True
257
268
  variables = self.variables
258
269
 
259
- if node.name not in self.locals and node.name.lower() not in variables:
270
+ # This makes all variables case-insensitive, e.g. @X is the same as @x. We do this
271
+ # for consistency, since `variables` and `blueprint_variables` are normalized.
272
+ var_name = node.name.lower()
273
+
274
+ if var_name not in self.locals and var_name not in variables:
260
275
  if not isinstance(node.parent, StagedFilePath):
261
276
  raise SQLMeshError(f"Macro variable '{node.name}' is undefined.")
262
277
 
263
278
  return node
264
279
 
265
280
  # Precedence order is locals (e.g. @DEF) > blueprint variables > config variables
266
- value = self.locals.get(node.name, variables.get(node.name.lower()))
281
+ value = self.locals.get(var_name, variables.get(var_name))
267
282
  if isinstance(value, list):
268
283
  return exp.convert(
269
284
  tuple(
@@ -313,11 +328,16 @@ class MacroEvaluator:
313
328
  """
314
329
  # We try to convert all variables into sqlglot expressions because they're going to be converted
315
330
  # into strings; in sql we don't convert strings because that would result in adding quotes
316
- mapping = {
317
- k: convert_sql(v, self.dialect)
331
+ base_mapping = {
332
+ k.lower(): convert_sql(v, self.dialect)
318
333
  for k, v in chain(self.variables.items(), self.locals.items(), local_variables.items())
334
+ if k.lower()
335
+ not in (
336
+ "engine_adapter",
337
+ "snapshot",
338
+ )
319
339
  }
320
- return MacroStrTemplate(str(text)).safe_substitute(mapping)
340
+ return MacroStrTemplate(str(text)).safe_substitute(CaseInsensitiveMapping(base_mapping))
321
341
 
322
342
  def evaluate(self, node: MacroFunc) -> exp.Expression | t.List[exp.Expression] | None:
323
343
  if isinstance(node, MacroDef):
@@ -327,7 +347,9 @@ class MacroEvaluator:
327
347
  args[0] if len(args) == 1 else exp.Tuple(expressions=list(args))
328
348
  )
329
349
  else:
330
- self.locals[node.name] = self.transform(node.expression)
350
+ # Make variables defined through `@DEF` case-insensitive
351
+ self.locals[node.name.lower()] = self.transform(node.expression)
352
+
331
353
  return node
332
354
 
333
355
  if isinstance(node, (MacroSQL, MacroStrReplace)):
@@ -630,7 +652,7 @@ def _norm_var_arg_lambda(
630
652
  ) -> exp.Expression | t.List[exp.Expression] | None:
631
653
  if isinstance(node, (exp.Identifier, exp.Var)):
632
654
  if not isinstance(node.parent, exp.Column):
633
- name = node.name
655
+ name = node.name.lower()
634
656
  if name in args:
635
657
  return args[name].copy()
636
658
  if name in evaluator.locals:
@@ -663,7 +685,7 @@ def _norm_var_arg_lambda(
663
685
  return expressions, lambda args: func.this.transform(
664
686
  substitute,
665
687
  {
666
- expression.name: arg
688
+ expression.name.lower(): arg
667
689
  for expression, arg in zip(
668
690
  func.expressions, args.expressions if isinstance(args, exp.Tuple) else [args]
669
691
  )
@@ -1128,7 +1150,7 @@ def haversine_distance(
1128
1150
  def pivot(
1129
1151
  evaluator: MacroEvaluator,
1130
1152
  column: SQL,
1131
- values: t.List[SQL],
1153
+ values: t.List[exp.Expression],
1132
1154
  alias: bool = True,
1133
1155
  agg: exp.Expression = exp.Literal.string("SUM"),
1134
1156
  cmp: exp.Expression = exp.Literal.string("="),
@@ -1146,10 +1168,10 @@ def pivot(
1146
1168
  >>> from sqlmesh.core.macros import MacroEvaluator
1147
1169
  >>> sql = "SELECT date_day, @PIVOT(status, ['cancelled', 'completed']) FROM rides GROUP BY 1"
1148
1170
  >>> MacroEvaluator().transform(parse_one(sql)).sql()
1149
- 'SELECT date_day, SUM(CASE WHEN status = \\'cancelled\\' THEN 1 ELSE 0 END) AS "\\'cancelled\\'", SUM(CASE WHEN status = \\'completed\\' THEN 1 ELSE 0 END) AS "\\'completed\\'" FROM rides GROUP BY 1'
1171
+ 'SELECT date_day, SUM(CASE WHEN status = \\'cancelled\\' THEN 1 ELSE 0 END) AS "cancelled", SUM(CASE WHEN status = \\'completed\\' THEN 1 ELSE 0 END) AS "completed" FROM rides GROUP BY 1'
1150
1172
  >>> sql = "SELECT @PIVOT(a, ['v'], then_value := tv, suffix := '_sfx', quote := FALSE)"
1151
1173
  >>> MacroEvaluator(dialect="bigquery").transform(parse_one(sql)).sql("bigquery")
1152
- "SELECT SUM(CASE WHEN a = 'v' THEN tv ELSE 0 END) AS `v_sfx`"
1174
+ "SELECT SUM(CASE WHEN a = 'v' THEN tv ELSE 0 END) AS v_sfx"
1153
1175
  """
1154
1176
  aggregates: t.List[exp.Expression] = []
1155
1177
  for value in values:
@@ -1157,12 +1179,12 @@ def pivot(
1157
1179
  if distinct:
1158
1180
  proj += "DISTINCT "
1159
1181
 
1160
- proj += f"CASE WHEN {column} {cmp.name} {value} THEN {then_value} ELSE {else_value} END) "
1182
+ proj += f"CASE WHEN {column} {cmp.name} {value.sql(evaluator.dialect)} THEN {then_value} ELSE {else_value} END) "
1161
1183
  node = evaluator.parse_one(proj)
1162
1184
 
1163
1185
  if alias:
1164
1186
  node = node.as_(
1165
- f"{prefix.name}{value}{suffix.name}",
1187
+ f"{prefix.name}{value.name}{suffix.name}",
1166
1188
  quoted=quote,
1167
1189
  copy=False,
1168
1190
  dialect=evaluator.dialect,
@@ -641,6 +641,7 @@ properties_validator: t.Callable = field_validator(
641
641
  "physical_properties_",
642
642
  "virtual_properties_",
643
643
  "materialization_properties_",
644
+ "grants_",
644
645
  mode="before",
645
646
  check_fields=False,
646
647
  )(parse_properties)
@@ -662,6 +663,7 @@ depends_on_validator: t.Callable = field_validator(
662
663
 
663
664
  class ParsableSql(PydanticModel):
664
665
  sql: str
666
+ transaction: t.Optional[bool] = None
665
667
 
666
668
  _parsed: t.Optional[exp.Expression] = None
667
669
  _parsed_dialect: t.Optional[str] = None
@@ -67,6 +67,7 @@ if t.TYPE_CHECKING:
67
67
  from sqlmesh.core.context import ExecutionContext
68
68
  from sqlmesh.core.engine_adapter import EngineAdapter
69
69
  from sqlmesh.core.engine_adapter._typing import QueryOrDF
70
+ from sqlmesh.core.engine_adapter.shared import DataObjectType
70
71
  from sqlmesh.core.linter.rule import Rule
71
72
  from sqlmesh.core.snapshot import DeployabilityIndex, Node, Snapshot
72
73
  from sqlmesh.utils.jinja import MacroReference
@@ -362,6 +363,7 @@ class _Model(ModelMeta, frozen=True):
362
363
  expand: t.Iterable[str] = tuple(),
363
364
  deployability_index: t.Optional[DeployabilityIndex] = None,
364
365
  engine_adapter: t.Optional[EngineAdapter] = None,
366
+ inside_transaction: t.Optional[bool] = True,
365
367
  **kwargs: t.Any,
366
368
  ) -> t.List[exp.Expression]:
367
369
  """Renders pre-statements for a model.
@@ -383,7 +385,11 @@ class _Model(ModelMeta, frozen=True):
383
385
  The list of rendered expressions.
384
386
  """
385
387
  return self._render_statements(
386
- self.pre_statements,
388
+ [
389
+ stmt
390
+ for stmt in self.pre_statements
391
+ if stmt.args.get("transaction", True) == inside_transaction
392
+ ],
387
393
  start=start,
388
394
  end=end,
389
395
  execution_time=execution_time,
@@ -404,6 +410,7 @@ class _Model(ModelMeta, frozen=True):
404
410
  expand: t.Iterable[str] = tuple(),
405
411
  deployability_index: t.Optional[DeployabilityIndex] = None,
406
412
  engine_adapter: t.Optional[EngineAdapter] = None,
413
+ inside_transaction: t.Optional[bool] = True,
407
414
  **kwargs: t.Any,
408
415
  ) -> t.List[exp.Expression]:
409
416
  """Renders post-statements for a model.
@@ -419,13 +426,18 @@ class _Model(ModelMeta, frozen=True):
419
426
  that depend on materialized tables. Model definitions are inlined and can thus be run end to
420
427
  end on the fly.
421
428
  deployability_index: Determines snapshots that are deployable in the context of this render.
429
+ inside_transaction: Whether to render hooks with transaction=True (inside) or transaction=False (outside).
422
430
  kwargs: Additional kwargs to pass to the renderer.
423
431
 
424
432
  Returns:
425
433
  The list of rendered expressions.
426
434
  """
427
435
  return self._render_statements(
428
- self.post_statements,
436
+ [
437
+ stmt
438
+ for stmt in self.post_statements
439
+ if stmt.args.get("transaction", True) == inside_transaction
440
+ ],
429
441
  start=start,
430
442
  end=end,
431
443
  execution_time=execution_time,
@@ -566,6 +578,8 @@ class _Model(ModelMeta, frozen=True):
566
578
  result = []
567
579
  for v in value:
568
580
  parsed = v.parse(self.dialect)
581
+ if getattr(v, "transaction", None) is not None:
582
+ parsed.set("transaction", v.transaction)
569
583
  if not isinstance(parsed, exp.Semicolon):
570
584
  result.append(parsed)
571
585
  return result
@@ -1186,6 +1200,8 @@ class _Model(ModelMeta, frozen=True):
1186
1200
  gen(self.session_properties_) if self.session_properties_ else None,
1187
1201
  *[gen(g) for g in self.grains],
1188
1202
  *self._audit_metadata_hash_values(),
1203
+ json.dumps(self.grants, sort_keys=True) if self.grants else None,
1204
+ self.grants_target_layer,
1189
1205
  ]
1190
1206
 
1191
1207
  for key, value in (self.virtual_properties or {}).items():
@@ -1197,6 +1213,9 @@ class _Model(ModelMeta, frozen=True):
1197
1213
  for k, v in sorted(args.items()):
1198
1214
  metadata.append(f"{k}:{gen(v)}")
1199
1215
 
1216
+ if self.dbt_node_info:
1217
+ metadata.append(self.dbt_node_info.json(sort_keys=True))
1218
+
1200
1219
  metadata.extend(self._additional_metadata)
1201
1220
 
1202
1221
  self._metadata_hash = hash_data(metadata)
@@ -1207,6 +1226,24 @@ class _Model(ModelMeta, frozen=True):
1207
1226
  """Return True if this is a model node"""
1208
1227
  return True
1209
1228
 
1229
+ @property
1230
+ def grants_table_type(self) -> DataObjectType:
1231
+ """Get the table type for grants application (TABLE, VIEW, MATERIALIZED_VIEW).
1232
+
1233
+ Returns:
1234
+ The DataObjectType that should be used when applying grants to this model.
1235
+ """
1236
+ from sqlmesh.core.engine_adapter.shared import DataObjectType
1237
+
1238
+ if self.kind.is_view:
1239
+ if hasattr(self.kind, "materialized") and getattr(self.kind, "materialized", False):
1240
+ return DataObjectType.MATERIALIZED_VIEW
1241
+ return DataObjectType.VIEW
1242
+ if self.kind.is_managed:
1243
+ return DataObjectType.MANAGED_TABLE
1244
+ # All other materialized models are tables
1245
+ return DataObjectType.TABLE
1246
+
1210
1247
  @property
1211
1248
  def _additional_metadata(self) -> t.List[str]:
1212
1249
  additional_metadata = []
@@ -1820,6 +1857,12 @@ class SeedModel(_Model):
1820
1857
  for column_name, column_hash in self.column_hashes.items():
1821
1858
  data.append(column_name)
1822
1859
  data.append(column_hash)
1860
+
1861
+ # Include grants in data hash for seed models to force recreation on grant changes
1862
+ # since seed models don't support migration
1863
+ data.append(json.dumps(self.grants, sort_keys=True) if self.grants else "")
1864
+ data.append(self.grants_target_layer)
1865
+
1823
1866
  return data
1824
1867
 
1825
1868
 
@@ -2562,9 +2605,17 @@ def _create_model(
2562
2605
  if statement_field in kwargs:
2563
2606
  # Macros extracted from these statements need to be treated as metadata only
2564
2607
  is_metadata = statement_field == "on_virtual_update"
2565
- statements.extend((stmt, is_metadata) for stmt in kwargs[statement_field])
2608
+ for stmt in kwargs[statement_field]:
2609
+ # Extract the expression if it's ParsableSql already
2610
+ expr = stmt.parse(dialect) if isinstance(stmt, ParsableSql) else stmt
2611
+ statements.append((expr, is_metadata))
2566
2612
  kwargs[statement_field] = [
2567
- ParsableSql.from_parsed_expression(stmt, dialect, use_meta_sql=use_original_sql)
2613
+ # this to retain the transaction information
2614
+ stmt
2615
+ if isinstance(stmt, ParsableSql)
2616
+ else ParsableSql.from_parsed_expression(
2617
+ stmt, dialect, use_meta_sql=use_original_sql
2618
+ )
2568
2619
  for stmt in kwargs[statement_field]
2569
2620
  ]
2570
2621
 
@@ -2866,6 +2917,13 @@ def render_meta_fields(
2866
2917
  for key, value in field_value.items():
2867
2918
  if key in RUNTIME_RENDERED_MODEL_FIELDS:
2868
2919
  rendered_dict[key] = parse_strings_with_macro_refs(value, dialect)
2920
+ elif (
2921
+ # don't parse kind auto_restatement_cron="@..." kwargs (e.g. @daily) into MacroVar
2922
+ key == "auto_restatement_cron"
2923
+ and isinstance(value, str)
2924
+ and value.lower() in CRON_SHORTCUTS
2925
+ ):
2926
+ rendered_dict[key] = value
2869
2927
  elif (rendered := render_field_value(value)) is not None:
2870
2928
  rendered_dict[key] = rendered
2871
2929
 
@@ -3012,6 +3070,9 @@ META_FIELD_CONVERTER: t.Dict[str, t.Callable] = {
3012
3070
  "formatting": str,
3013
3071
  "optimize_query": str,
3014
3072
  "virtual_environment_mode": lambda value: exp.Literal.string(value.value),
3073
+ "dbt_node_info_": lambda value: value.to_expression(),
3074
+ "grants_": lambda value: value,
3075
+ "grants_target_layer": lambda value: exp.Literal.string(value.value),
3015
3076
  }
3016
3077
 
3017
3078
 
@@ -23,7 +23,7 @@ from sqlmesh.utils.pydantic import (
23
23
  PydanticModel,
24
24
  SQLGlotBool,
25
25
  SQLGlotColumn,
26
- SQLGlotListOfColumnsOrStar,
26
+ SQLGlotListOfFieldsOrStar,
27
27
  SQLGlotListOfFields,
28
28
  SQLGlotPositiveInt,
29
29
  SQLGlotString,
@@ -119,6 +119,10 @@ class ModelKindMixin:
119
119
  def is_managed(self) -> bool:
120
120
  return self.model_kind_name == ModelKindName.MANAGED
121
121
 
122
+ @property
123
+ def is_dbt_custom(self) -> bool:
124
+ return self.model_kind_name == ModelKindName.DBT_CUSTOM
125
+
122
126
  @property
123
127
  def is_symbolic(self) -> bool:
124
128
  """A symbolic model is one that doesn't execute at all."""
@@ -150,6 +154,11 @@ class ModelKindMixin:
150
154
  def supports_python_models(self) -> bool:
151
155
  return True
152
156
 
157
+ @property
158
+ def supports_grants(self) -> bool:
159
+ """Whether this model kind supports grants configuration."""
160
+ return self.is_materialized or self.is_view
161
+
153
162
 
154
163
  class ModelKindName(str, ModelKindMixin, Enum):
155
164
  """The kind of model, determining how this data is computed and stored in the warehouse."""
@@ -170,6 +179,7 @@ class ModelKindName(str, ModelKindMixin, Enum):
170
179
  EXTERNAL = "EXTERNAL"
171
180
  CUSTOM = "CUSTOM"
172
181
  MANAGED = "MANAGED"
182
+ DBT_CUSTOM = "DBT_CUSTOM"
173
183
 
174
184
  @property
175
185
  def model_kind_name(self) -> t.Optional[ModelKindName]:
@@ -842,7 +852,7 @@ class SCDType2ByTimeKind(_SCDType2Kind):
842
852
 
843
853
  class SCDType2ByColumnKind(_SCDType2Kind):
844
854
  name: t.Literal[ModelKindName.SCD_TYPE_2_BY_COLUMN] = ModelKindName.SCD_TYPE_2_BY_COLUMN
845
- columns: SQLGlotListOfColumnsOrStar
855
+ columns: SQLGlotListOfFieldsOrStar
846
856
  execution_time_as_valid_from: SQLGlotBool = False
847
857
  updated_at_name: t.Optional[SQLGlotColumn] = None
848
858
 
@@ -887,6 +897,46 @@ class ManagedKind(_ModelKind):
887
897
  return False
888
898
 
889
899
 
900
+ class DbtCustomKind(_ModelKind):
901
+ name: t.Literal[ModelKindName.DBT_CUSTOM] = ModelKindName.DBT_CUSTOM
902
+ materialization: str
903
+ adapter: str = "default"
904
+ definition: str
905
+ dialect: t.Optional[str] = Field(None, validate_default=True)
906
+
907
+ _dialect_validator = kind_dialect_validator
908
+
909
+ @field_validator("materialization", "adapter", "definition", mode="before")
910
+ @classmethod
911
+ def _validate_fields(cls, v: t.Any) -> str:
912
+ return validate_string(v)
913
+
914
+ @property
915
+ def data_hash_values(self) -> t.List[t.Optional[str]]:
916
+ return [
917
+ *super().data_hash_values,
918
+ self.materialization,
919
+ self.definition,
920
+ self.adapter,
921
+ self.dialect,
922
+ ]
923
+
924
+ def to_expression(
925
+ self, expressions: t.Optional[t.List[exp.Expression]] = None, **kwargs: t.Any
926
+ ) -> d.ModelKind:
927
+ return super().to_expression(
928
+ expressions=[
929
+ *(expressions or []),
930
+ *_properties(
931
+ {
932
+ "materialization": exp.Literal.string(self.materialization),
933
+ "adapter": exp.Literal.string(self.adapter),
934
+ }
935
+ ),
936
+ ],
937
+ )
938
+
939
+
890
940
  class EmbeddedKind(_ModelKind):
891
941
  name: t.Literal[ModelKindName.EMBEDDED] = ModelKindName.EMBEDDED
892
942
 
@@ -992,6 +1042,7 @@ ModelKind = t.Annotated[
992
1042
  SCDType2ByColumnKind,
993
1043
  CustomKind,
994
1044
  ManagedKind,
1045
+ DbtCustomKind,
995
1046
  ],
996
1047
  Field(discriminator="name"),
997
1048
  ]
@@ -1011,6 +1062,7 @@ MODEL_KIND_NAME_TO_TYPE: t.Dict[str, t.Type[ModelKind]] = {
1011
1062
  ModelKindName.SCD_TYPE_2_BY_COLUMN: SCDType2ByColumnKind,
1012
1063
  ModelKindName.CUSTOM: CustomKind,
1013
1064
  ModelKindName.MANAGED: ManagedKind,
1065
+ ModelKindName.DBT_CUSTOM: DbtCustomKind,
1014
1066
  }
1015
1067
 
1016
1068
 
@@ -1053,6 +1105,18 @@ def create_model_kind(v: t.Any, dialect: str, defaults: t.Dict[str, t.Any]) -> M
1053
1105
  ):
1054
1106
  props[on_change_property] = defaults.get(on_change_property)
1055
1107
 
1108
+ # only pass the batch_concurrency user default to models inheriting from _IncrementalBy
1109
+ # that don't explicitly set it in the model definition, but ignore subclasses of _IncrementalBy
1110
+ # that hardcode a specific batch_concurrency
1111
+ if issubclass(kind_type, _IncrementalBy):
1112
+ BATCH_CONCURRENCY: t.Final = "batch_concurrency"
1113
+ if (
1114
+ props.get(BATCH_CONCURRENCY) is None
1115
+ and defaults.get(BATCH_CONCURRENCY) is not None
1116
+ and kind_type.all_field_infos()[BATCH_CONCURRENCY].default is None
1117
+ ):
1118
+ props[BATCH_CONCURRENCY] = defaults.get(BATCH_CONCURRENCY)
1119
+
1056
1120
  if kind_type == CustomKind:
1057
1121
  # load the custom materialization class and check if it uses a custom kind type
1058
1122
  from sqlmesh.core.snapshot.evaluator import get_custom_materialization_type
@@ -1,6 +1,7 @@
1
1
  from __future__ import annotations
2
2
 
3
3
  import typing as t
4
+ from enum import Enum
4
5
  from functools import cached_property
5
6
  from typing_extensions import Self
6
7
 
@@ -13,6 +14,7 @@ from sqlmesh.core import dialect as d
13
14
  from sqlmesh.core.config.common import VirtualEnvironmentMode
14
15
  from sqlmesh.core.config.linter import LinterConfig
15
16
  from sqlmesh.core.dialect import normalize_model_name
17
+ from sqlmesh.utils import classproperty
16
18
  from sqlmesh.core.model.common import (
17
19
  bool_validator,
18
20
  default_catalog_validator,
@@ -46,10 +48,41 @@ from sqlmesh.utils.pydantic import (
46
48
 
47
49
  if t.TYPE_CHECKING:
48
50
  from sqlmesh.core._typing import CustomMaterializationProperties, SessionProperties
51
+ from sqlmesh.core.engine_adapter._typing import GrantsConfig
49
52
 
50
53
  FunctionCall = t.Tuple[str, t.Dict[str, exp.Expression]]
51
54
 
52
55
 
56
+ class GrantsTargetLayer(str, Enum):
57
+ """Target layer(s) where grants should be applied."""
58
+
59
+ ALL = "all"
60
+ PHYSICAL = "physical"
61
+ VIRTUAL = "virtual"
62
+
63
+ @classproperty
64
+ def default(cls) -> "GrantsTargetLayer":
65
+ return GrantsTargetLayer.VIRTUAL
66
+
67
+ @property
68
+ def is_all(self) -> bool:
69
+ return self == GrantsTargetLayer.ALL
70
+
71
+ @property
72
+ def is_physical(self) -> bool:
73
+ return self == GrantsTargetLayer.PHYSICAL
74
+
75
+ @property
76
+ def is_virtual(self) -> bool:
77
+ return self == GrantsTargetLayer.VIRTUAL
78
+
79
+ def __str__(self) -> str:
80
+ return self.name
81
+
82
+ def __repr__(self) -> str:
83
+ return str(self)
84
+
85
+
53
86
  class ModelMeta(_Node):
54
87
  """Metadata for models which can be defined in SQL."""
55
88
 
@@ -85,6 +118,8 @@ class ModelMeta(_Node):
85
118
  )
86
119
  formatting: t.Optional[bool] = Field(default=None, exclude=True)
87
120
  virtual_environment_mode: VirtualEnvironmentMode = VirtualEnvironmentMode.default
121
+ grants_: t.Optional[exp.Tuple] = Field(default=None, alias="grants")
122
+ grants_target_layer: GrantsTargetLayer = GrantsTargetLayer.default
88
123
 
89
124
  _bool_validator = bool_validator
90
125
  _model_kind_validator = model_kind_validator
@@ -247,11 +282,15 @@ class ModelMeta(_Node):
247
282
 
248
283
  columns_to_types = info.data.get("columns_to_types_")
249
284
  if columns_to_types:
250
- for column_name in col_descriptions:
285
+ from sqlmesh.core.console import get_console
286
+
287
+ console = get_console()
288
+ for column_name in list(col_descriptions):
251
289
  if column_name not in columns_to_types:
252
- raise ConfigError(
290
+ console.log_warning(
253
291
  f"In model '{info.data['name']}', a description is provided for column '{column_name}' but it is not a column in the model."
254
292
  )
293
+ del col_descriptions[column_name]
255
294
 
256
295
  return col_descriptions
257
296
 
@@ -283,6 +322,14 @@ class ModelMeta(_Node):
283
322
  def ignored_rules_validator(cls, vs: t.Any) -> t.Any:
284
323
  return LinterConfig._validate_rules(vs)
285
324
 
325
+ @field_validator("grants_target_layer", mode="before")
326
+ def _grants_target_layer_validator(cls, v: t.Any) -> t.Any:
327
+ if isinstance(v, exp.Identifier):
328
+ return v.this
329
+ if isinstance(v, exp.Literal) and v.is_string:
330
+ return v.this
331
+ return v
332
+
286
333
  @field_validator("session_properties_", mode="before")
287
334
  def session_properties_validator(cls, v: t.Any, info: ValidationInfo) -> t.Any:
288
335
  # use the generic properties validator to parse the session properties
@@ -390,6 +437,10 @@ class ModelMeta(_Node):
390
437
  f"Model {self.name} has `storage_format` set to a table format '{storage_format}' which is deprecated. Please use the `table_format` property instead."
391
438
  )
392
439
 
440
+ # Validate grants configuration for model kind support
441
+ if self.grants is not None and not kind.supports_grants:
442
+ raise ValueError(f"grants cannot be set for {kind.name} models")
443
+
393
444
  return self
394
445
 
395
446
  @property
@@ -461,6 +512,30 @@ class ModelMeta(_Node):
461
512
  return self.kind.materialization_properties
462
513
  return {}
463
514
 
515
+ @cached_property
516
+ def grants(self) -> t.Optional[GrantsConfig]:
517
+ """A dictionary of grants mapping permission names to lists of grantees."""
518
+
519
+ if self.grants_ is None:
520
+ return None
521
+
522
+ if not self.grants_.expressions:
523
+ return {}
524
+
525
+ grants_dict = {}
526
+ for eq_expr in self.grants_.expressions:
527
+ try:
528
+ permission_name = self._validate_config_expression(eq_expr.left)
529
+ grantee_list = self._validate_nested_config_values(eq_expr.expression)
530
+ grants_dict[permission_name] = grantee_list
531
+ except ConfigError as e:
532
+ permission_name = (
533
+ eq_expr.left.name if hasattr(eq_expr.left, "name") else str(eq_expr.left)
534
+ )
535
+ raise ConfigError(f"Invalid grants configuration for '{permission_name}': {e}")
536
+
537
+ return grants_dict if grants_dict else None
538
+
464
539
  @property
465
540
  def all_references(self) -> t.List[Reference]:
466
541
  """All references including grains."""
@@ -525,3 +600,33 @@ class ModelMeta(_Node):
525
600
  @property
526
601
  def ignored_rules(self) -> t.Set[str]:
527
602
  return self.ignored_rules_ or set()
603
+
604
+ def _validate_config_expression(self, expr: exp.Expression) -> str:
605
+ if isinstance(expr, (d.MacroFunc, d.MacroVar)):
606
+ raise ConfigError(f"Unresolved macro: {expr.sql(dialect=self.dialect)}")
607
+
608
+ if isinstance(expr, exp.Null):
609
+ raise ConfigError("NULL value")
610
+
611
+ if isinstance(expr, exp.Literal):
612
+ return str(expr.this).strip()
613
+ if isinstance(expr, (exp.Column, exp.Identifier)):
614
+ return expr.name
615
+ return expr.sql(dialect=self.dialect).strip()
616
+
617
+ def _validate_nested_config_values(self, value_expr: exp.Expression) -> t.List[str]:
618
+ result = []
619
+
620
+ def flatten_expr(expr: exp.Expression) -> None:
621
+ if isinstance(expr, exp.Array):
622
+ for elem in expr.expressions:
623
+ flatten_expr(elem)
624
+ elif isinstance(expr, (exp.Tuple, exp.Paren)):
625
+ expressions = [expr.unnest()] if isinstance(expr, exp.Paren) else expr.expressions
626
+ for elem in expressions:
627
+ flatten_expr(elem)
628
+ else:
629
+ result.append(self._validate_config_expression(expr))
630
+
631
+ flatten_expr(value_expr)
632
+ return result