sqlmesh 0.217.1.dev1__py3-none-any.whl → 0.227.2.dev4__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (183) hide show
  1. sqlmesh/__init__.py +12 -2
  2. sqlmesh/_version.py +2 -2
  3. sqlmesh/cli/project_init.py +10 -2
  4. sqlmesh/core/_typing.py +1 -0
  5. sqlmesh/core/audit/definition.py +8 -2
  6. sqlmesh/core/config/__init__.py +1 -1
  7. sqlmesh/core/config/connection.py +17 -5
  8. sqlmesh/core/config/dbt.py +13 -0
  9. sqlmesh/core/config/janitor.py +12 -0
  10. sqlmesh/core/config/loader.py +7 -0
  11. sqlmesh/core/config/model.py +2 -0
  12. sqlmesh/core/config/root.py +3 -0
  13. sqlmesh/core/console.py +80 -2
  14. sqlmesh/core/constants.py +1 -1
  15. sqlmesh/core/context.py +61 -25
  16. sqlmesh/core/dialect.py +3 -0
  17. sqlmesh/core/engine_adapter/_typing.py +2 -0
  18. sqlmesh/core/engine_adapter/base.py +322 -22
  19. sqlmesh/core/engine_adapter/base_postgres.py +17 -1
  20. sqlmesh/core/engine_adapter/bigquery.py +146 -7
  21. sqlmesh/core/engine_adapter/clickhouse.py +17 -13
  22. sqlmesh/core/engine_adapter/databricks.py +33 -2
  23. sqlmesh/core/engine_adapter/fabric.py +1 -29
  24. sqlmesh/core/engine_adapter/mixins.py +142 -48
  25. sqlmesh/core/engine_adapter/mssql.py +15 -4
  26. sqlmesh/core/engine_adapter/mysql.py +2 -2
  27. sqlmesh/core/engine_adapter/postgres.py +9 -3
  28. sqlmesh/core/engine_adapter/redshift.py +4 -0
  29. sqlmesh/core/engine_adapter/risingwave.py +1 -0
  30. sqlmesh/core/engine_adapter/shared.py +6 -0
  31. sqlmesh/core/engine_adapter/snowflake.py +82 -11
  32. sqlmesh/core/engine_adapter/spark.py +14 -10
  33. sqlmesh/core/engine_adapter/trino.py +4 -2
  34. sqlmesh/core/janitor.py +181 -0
  35. sqlmesh/core/lineage.py +1 -0
  36. sqlmesh/core/macros.py +35 -13
  37. sqlmesh/core/model/common.py +2 -0
  38. sqlmesh/core/model/definition.py +65 -4
  39. sqlmesh/core/model/kind.py +66 -2
  40. sqlmesh/core/model/meta.py +107 -2
  41. sqlmesh/core/node.py +101 -2
  42. sqlmesh/core/plan/builder.py +15 -10
  43. sqlmesh/core/plan/common.py +196 -2
  44. sqlmesh/core/plan/definition.py +21 -6
  45. sqlmesh/core/plan/evaluator.py +72 -113
  46. sqlmesh/core/plan/explainer.py +90 -8
  47. sqlmesh/core/plan/stages.py +42 -21
  48. sqlmesh/core/renderer.py +26 -18
  49. sqlmesh/core/scheduler.py +60 -19
  50. sqlmesh/core/selector.py +137 -9
  51. sqlmesh/core/signal.py +64 -1
  52. sqlmesh/core/snapshot/__init__.py +1 -0
  53. sqlmesh/core/snapshot/definition.py +109 -25
  54. sqlmesh/core/snapshot/evaluator.py +610 -50
  55. sqlmesh/core/state_sync/__init__.py +0 -1
  56. sqlmesh/core/state_sync/base.py +31 -27
  57. sqlmesh/core/state_sync/cache.py +12 -4
  58. sqlmesh/core/state_sync/common.py +216 -111
  59. sqlmesh/core/state_sync/db/facade.py +30 -15
  60. sqlmesh/core/state_sync/db/interval.py +27 -7
  61. sqlmesh/core/state_sync/db/migrator.py +14 -8
  62. sqlmesh/core/state_sync/db/snapshot.py +119 -87
  63. sqlmesh/core/table_diff.py +2 -2
  64. sqlmesh/core/test/definition.py +14 -9
  65. sqlmesh/dbt/adapter.py +20 -11
  66. sqlmesh/dbt/basemodel.py +52 -41
  67. sqlmesh/dbt/builtin.py +27 -11
  68. sqlmesh/dbt/column.py +17 -5
  69. sqlmesh/dbt/common.py +4 -2
  70. sqlmesh/dbt/context.py +14 -1
  71. sqlmesh/dbt/loader.py +60 -8
  72. sqlmesh/dbt/manifest.py +136 -8
  73. sqlmesh/dbt/model.py +105 -25
  74. sqlmesh/dbt/package.py +16 -1
  75. sqlmesh/dbt/profile.py +3 -3
  76. sqlmesh/dbt/project.py +12 -7
  77. sqlmesh/dbt/seed.py +1 -1
  78. sqlmesh/dbt/source.py +6 -1
  79. sqlmesh/dbt/target.py +25 -6
  80. sqlmesh/dbt/test.py +31 -1
  81. sqlmesh/migrations/v0000_baseline.py +3 -6
  82. sqlmesh/migrations/v0061_mysql_fix_blob_text_type.py +2 -5
  83. sqlmesh/migrations/v0062_add_model_gateway.py +2 -2
  84. sqlmesh/migrations/v0063_change_signals.py +2 -4
  85. sqlmesh/migrations/v0064_join_when_matched_strings.py +2 -4
  86. sqlmesh/migrations/v0065_add_model_optimize.py +2 -2
  87. sqlmesh/migrations/v0066_add_auto_restatements.py +2 -6
  88. sqlmesh/migrations/v0067_add_tsql_date_full_precision.py +2 -2
  89. sqlmesh/migrations/v0068_include_unrendered_query_in_metadata_hash.py +2 -2
  90. sqlmesh/migrations/v0069_update_dev_table_suffix.py +2 -4
  91. sqlmesh/migrations/v0070_include_grains_in_metadata_hash.py +2 -2
  92. sqlmesh/migrations/v0071_add_dev_version_to_intervals.py +2 -6
  93. sqlmesh/migrations/v0072_add_environment_statements.py +2 -4
  94. sqlmesh/migrations/v0073_remove_symbolic_disable_restatement.py +2 -4
  95. sqlmesh/migrations/v0074_add_partition_by_time_column_property.py +2 -2
  96. sqlmesh/migrations/v0075_remove_validate_query.py +2 -4
  97. sqlmesh/migrations/v0076_add_cron_tz.py +2 -2
  98. sqlmesh/migrations/v0077_fix_column_type_hash_calculation.py +2 -2
  99. sqlmesh/migrations/v0078_warn_if_non_migratable_python_env.py +2 -4
  100. sqlmesh/migrations/v0079_add_gateway_managed_property.py +7 -9
  101. sqlmesh/migrations/v0080_add_batch_size_to_scd_type_2_models.py +2 -2
  102. sqlmesh/migrations/v0081_update_partitioned_by.py +2 -4
  103. sqlmesh/migrations/v0082_warn_if_incorrectly_duplicated_statements.py +2 -4
  104. sqlmesh/migrations/v0083_use_sql_for_scd_time_data_type_data_hash.py +2 -2
  105. sqlmesh/migrations/v0084_normalize_quote_when_matched_and_merge_filter.py +2 -2
  106. sqlmesh/migrations/v0085_deterministic_repr.py +2 -4
  107. sqlmesh/migrations/v0086_check_deterministic_bug.py +2 -4
  108. sqlmesh/migrations/v0087_normalize_blueprint_variables.py +2 -4
  109. sqlmesh/migrations/v0088_warn_about_variable_python_env_diffs.py +2 -4
  110. sqlmesh/migrations/v0089_add_virtual_environment_mode.py +2 -2
  111. sqlmesh/migrations/v0090_add_forward_only_column.py +2 -6
  112. sqlmesh/migrations/v0091_on_additive_change.py +2 -2
  113. sqlmesh/migrations/v0092_warn_about_dbt_data_type_diff.py +2 -4
  114. sqlmesh/migrations/v0093_use_raw_sql_in_fingerprint.py +2 -2
  115. sqlmesh/migrations/v0094_add_dev_version_and_fingerprint_columns.py +2 -6
  116. sqlmesh/migrations/v0095_warn_about_dbt_raw_sql_diff.py +2 -4
  117. sqlmesh/migrations/v0096_remove_plan_dags_table.py +2 -4
  118. sqlmesh/migrations/v0097_add_dbt_name_in_node.py +2 -2
  119. sqlmesh/migrations/v0098_add_dbt_node_info_in_node.py +103 -0
  120. sqlmesh/migrations/v0099_add_last_altered_to_intervals.py +25 -0
  121. sqlmesh/migrations/v0100_add_grants_and_grants_target_layer.py +9 -0
  122. sqlmesh/utils/__init__.py +8 -1
  123. sqlmesh/utils/cache.py +5 -1
  124. sqlmesh/utils/date.py +1 -1
  125. sqlmesh/utils/errors.py +4 -0
  126. sqlmesh/utils/jinja.py +25 -2
  127. sqlmesh/utils/pydantic.py +6 -6
  128. sqlmesh/utils/windows.py +13 -3
  129. {sqlmesh-0.217.1.dev1.dist-info → sqlmesh-0.227.2.dev4.dist-info}/METADATA +5 -5
  130. {sqlmesh-0.217.1.dev1.dist-info → sqlmesh-0.227.2.dev4.dist-info}/RECORD +181 -176
  131. sqlmesh_dbt/cli.py +70 -7
  132. sqlmesh_dbt/console.py +14 -6
  133. sqlmesh_dbt/operations.py +103 -24
  134. sqlmesh_dbt/selectors.py +39 -1
  135. web/client/dist/assets/{Audits-Ucsx1GzF.js → Audits-CBiYyyx-.js} +1 -1
  136. web/client/dist/assets/{Banner-BWDzvavM.js → Banner-DSRbUlO5.js} +1 -1
  137. web/client/dist/assets/{ChevronDownIcon-D2VL13Ah.js → ChevronDownIcon-MK_nrjD_.js} +1 -1
  138. web/client/dist/assets/{ChevronRightIcon-DWGYbf1l.js → ChevronRightIcon-CLWtT22Q.js} +1 -1
  139. web/client/dist/assets/{Content-DdHDZM3I.js → Content-BNuGZN5l.js} +1 -1
  140. web/client/dist/assets/{Content-Bikfy8fh.js → Content-CSHJyW0n.js} +1 -1
  141. web/client/dist/assets/{Data-CzAJH7rW.js → Data-C1oRDbLx.js} +1 -1
  142. web/client/dist/assets/{DataCatalog-BJF11g8f.js → DataCatalog-HXyX2-_j.js} +1 -1
  143. web/client/dist/assets/{Editor-s0SBpV2y.js → Editor-BDyfpUuw.js} +1 -1
  144. web/client/dist/assets/{Editor-DgLhgKnm.js → Editor-D0jNItwC.js} +1 -1
  145. web/client/dist/assets/{Errors-D0m0O1d3.js → Errors-BfuFLcPi.js} +1 -1
  146. web/client/dist/assets/{FileExplorer-CEv0vXkt.js → FileExplorer-BR9IE3he.js} +1 -1
  147. web/client/dist/assets/{Footer-BwzXn8Ew.js → Footer-CgBEtiAh.js} +1 -1
  148. web/client/dist/assets/{Header-6heDkEqG.js → Header-DSqR6nSO.js} +1 -1
  149. web/client/dist/assets/{Input-obuJsD6k.js → Input-B-oZ6fGO.js} +1 -1
  150. web/client/dist/assets/Lineage-DYQVwDbD.js +1 -0
  151. web/client/dist/assets/{ListboxShow-HM9_qyrt.js → ListboxShow-BE5-xevs.js} +1 -1
  152. web/client/dist/assets/{ModelLineage-zWdKo0U2.js → ModelLineage-DkIFAYo4.js} +1 -1
  153. web/client/dist/assets/{Models-Bcu66SRz.js → Models-D5dWr8RB.js} +1 -1
  154. web/client/dist/assets/{Page-BWEEQfIt.js → Page-C-XfU5BR.js} +1 -1
  155. web/client/dist/assets/{Plan-C4gXCqlf.js → Plan-ZEuTINBq.js} +1 -1
  156. web/client/dist/assets/{PlusCircleIcon-CVDO651q.js → PlusCircleIcon-DVXAHG8_.js} +1 -1
  157. web/client/dist/assets/{ReportErrors-BT6xFwAr.js → ReportErrors-B7FEPzMB.js} +1 -1
  158. web/client/dist/assets/{Root-ryJoBK4h.js → Root-8aZyhPxF.js} +1 -1
  159. web/client/dist/assets/{SearchList-DB04sPb9.js → SearchList-W_iT2G82.js} +1 -1
  160. web/client/dist/assets/{SelectEnvironment-CUYcXUu6.js → SelectEnvironment-C65jALmO.js} +1 -1
  161. web/client/dist/assets/{SourceList-Doo_9ZGp.js → SourceList-DSLO6nVJ.js} +1 -1
  162. web/client/dist/assets/{SourceListItem-D5Mj7Dly.js → SourceListItem-BHt8d9-I.js} +1 -1
  163. web/client/dist/assets/{SplitPane-qHmkD1qy.js → SplitPane-CViaZmw6.js} +1 -1
  164. web/client/dist/assets/{Tests-DH1Z74ML.js → Tests-DhaVt5t1.js} +1 -1
  165. web/client/dist/assets/{Welcome-DqUJUNMF.js → Welcome-DvpjH-_4.js} +1 -1
  166. web/client/dist/assets/context-BctCsyGb.js +71 -0
  167. web/client/dist/assets/{context-Dr54UHLi.js → context-DFNeGsFF.js} +1 -1
  168. web/client/dist/assets/{editor-DYIP1yQ4.js → editor-CcO28cqd.js} +1 -1
  169. web/client/dist/assets/{file-DarlIDVi.js → file-CvJN3aZO.js} +1 -1
  170. web/client/dist/assets/{floating-ui.react-dom-BH3TFvkM.js → floating-ui.react-dom-CjE-JNW1.js} +1 -1
  171. web/client/dist/assets/{help-Bl8wqaQc.js → help-DuPhjipa.js} +1 -1
  172. web/client/dist/assets/{index-D1sR7wpN.js → index-C-dJH7yZ.js} +1 -1
  173. web/client/dist/assets/{index-O3mjYpnE.js → index-Dj0i1-CA.js} +2 -2
  174. web/client/dist/assets/{plan-CehRrJUG.js → plan-BTRSbjKn.js} +1 -1
  175. web/client/dist/assets/{popover-CqgMRE0G.js → popover-_Sf0yvOI.js} +1 -1
  176. web/client/dist/assets/{project-6gxepOhm.js → project-BvSOI8MY.js} +1 -1
  177. web/client/dist/index.html +1 -1
  178. web/client/dist/assets/Lineage-D0Hgdz2v.js +0 -1
  179. web/client/dist/assets/context-DgX0fp2E.js +0 -68
  180. {sqlmesh-0.217.1.dev1.dist-info → sqlmesh-0.227.2.dev4.dist-info}/WHEEL +0 -0
  181. {sqlmesh-0.217.1.dev1.dist-info → sqlmesh-0.227.2.dev4.dist-info}/entry_points.txt +0 -0
  182. {sqlmesh-0.217.1.dev1.dist-info → sqlmesh-0.227.2.dev4.dist-info}/licenses/LICENSE +0 -0
  183. {sqlmesh-0.217.1.dev1.dist-info → sqlmesh-0.227.2.dev4.dist-info}/top_level.txt +0 -0
@@ -8,9 +8,10 @@ from sqlglot import exp, parse_one
8
8
  from sqlglot.transforms import remove_precision_parameterized_types
9
9
 
10
10
  from sqlmesh.core.dialect import to_schema
11
+ from sqlmesh.core.engine_adapter.base import _get_data_object_cache_key
11
12
  from sqlmesh.core.engine_adapter.mixins import (
12
- InsertOverwriteWithMergeMixin,
13
13
  ClusteredByMixin,
14
+ GrantsFromInfoSchemaMixin,
14
15
  RowDiffMixin,
15
16
  TableAlterClusterByOperation,
16
17
  )
@@ -20,6 +21,7 @@ from sqlmesh.core.engine_adapter.shared import (
20
21
  DataObjectType,
21
22
  SourceQuery,
22
23
  set_catalog,
24
+ InsertOverwriteStrategy,
23
25
  )
24
26
  from sqlmesh.core.node import IntervalUnit
25
27
  from sqlmesh.core.schema_diff import TableAlterOperation, NestedSupport
@@ -39,7 +41,7 @@ if t.TYPE_CHECKING:
39
41
  from google.cloud.bigquery.table import Table as BigQueryTable
40
42
 
41
43
  from sqlmesh.core._typing import SchemaName, SessionProperties, TableName
42
- from sqlmesh.core.engine_adapter._typing import BigframeSession, DF, Query
44
+ from sqlmesh.core.engine_adapter._typing import BigframeSession, DCL, DF, GrantsConfig, Query
43
45
  from sqlmesh.core.engine_adapter.base import QueryOrDF
44
46
 
45
47
 
@@ -54,7 +56,7 @@ NestedFieldsDict = t.Dict[str, t.List[NestedField]]
54
56
 
55
57
 
56
58
  @set_catalog()
57
- class BigQueryEngineAdapter(InsertOverwriteWithMergeMixin, ClusteredByMixin, RowDiffMixin):
59
+ class BigQueryEngineAdapter(ClusteredByMixin, RowDiffMixin, GrantsFromInfoSchemaMixin):
58
60
  """
59
61
  BigQuery Engine Adapter using the `google-cloud-bigquery` library's DB API.
60
62
  """
@@ -64,10 +66,16 @@ class BigQueryEngineAdapter(InsertOverwriteWithMergeMixin, ClusteredByMixin, Row
64
66
  SUPPORTS_TRANSACTIONS = False
65
67
  SUPPORTS_MATERIALIZED_VIEWS = True
66
68
  SUPPORTS_CLONING = True
69
+ SUPPORTS_GRANTS = True
70
+ CURRENT_USER_OR_ROLE_EXPRESSION: exp.Expression = exp.func("session_user")
71
+ SUPPORTS_MULTIPLE_GRANT_PRINCIPALS = True
72
+ USE_CATALOG_IN_GRANTS = True
73
+ GRANT_INFORMATION_SCHEMA_TABLE_NAME = "OBJECT_PRIVILEGES"
67
74
  MAX_TABLE_COMMENT_LENGTH = 1024
68
75
  MAX_COLUMN_COMMENT_LENGTH = 1024
69
76
  SUPPORTS_QUERY_EXECUTION_TRACKING = True
70
77
  SUPPORTED_DROP_CASCADE_OBJECT_KINDS = ["SCHEMA"]
78
+ INSERT_OVERWRITE_STRATEGY = InsertOverwriteStrategy.MERGE
71
79
 
72
80
  SCHEMA_DIFFER_KWARGS = {
73
81
  "compatible_types": {
@@ -168,17 +176,18 @@ class BigQueryEngineAdapter(InsertOverwriteWithMergeMixin, ClusteredByMixin, Row
168
176
  )
169
177
 
170
178
  def query_factory() -> Query:
171
- if bigframes_pd and isinstance(df, bigframes_pd.DataFrame):
172
- df.to_gbq(
179
+ ordered_df = df[list(source_columns_to_types)]
180
+ if bigframes_pd and isinstance(ordered_df, bigframes_pd.DataFrame):
181
+ ordered_df.to_gbq(
173
182
  f"{temp_bq_table.project}.{temp_bq_table.dataset_id}.{temp_bq_table.table_id}",
174
183
  if_exists="replace",
175
184
  )
176
185
  elif not self.table_exists(temp_table):
177
186
  # Make mypy happy
178
- assert isinstance(df, pd.DataFrame)
187
+ assert isinstance(ordered_df, pd.DataFrame)
179
188
  self._db_call(self.client.create_table, table=temp_bq_table, exists_ok=False)
180
189
  result = self.__load_pandas_to_table(
181
- temp_bq_table, df, source_columns_to_types, replace=False
190
+ temp_bq_table, ordered_df, source_columns_to_types, replace=False
182
191
  )
183
192
  if result.errors:
184
193
  raise SQLMeshError(result.errors)
@@ -742,6 +751,12 @@ class BigQueryEngineAdapter(InsertOverwriteWithMergeMixin, ClusteredByMixin, Row
742
751
  )
743
752
 
744
753
  def table_exists(self, table_name: TableName) -> bool:
754
+ table = exp.to_table(table_name)
755
+ data_object_cache_key = _get_data_object_cache_key(table.catalog, table.db, table.name)
756
+ if data_object_cache_key in self._data_object_cache:
757
+ logger.debug("Table existence cache hit: %s", data_object_cache_key)
758
+ return self._data_object_cache[data_object_cache_key] is not None
759
+
745
760
  try:
746
761
  from google.cloud.exceptions import NotFound
747
762
  except ModuleNotFoundError:
@@ -753,6 +768,28 @@ class BigQueryEngineAdapter(InsertOverwriteWithMergeMixin, ClusteredByMixin, Row
753
768
  except NotFound:
754
769
  return False
755
770
 
771
+ def get_table_last_modified_ts(self, table_names: t.List[TableName]) -> t.List[int]:
772
+ from sqlmesh.utils.date import to_timestamp
773
+
774
+ datasets_to_tables: t.DefaultDict[str, t.List[str]] = defaultdict(list)
775
+ for table_name in table_names:
776
+ table = exp.to_table(table_name)
777
+ datasets_to_tables[table.db].append(table.name)
778
+
779
+ results = []
780
+
781
+ for dataset, tables in datasets_to_tables.items():
782
+ query = (
783
+ f"SELECT TIMESTAMP_MILLIS(last_modified_time) FROM `{dataset}.__TABLES__` WHERE "
784
+ )
785
+ for i, table_name in enumerate(tables):
786
+ query += f"TABLE_ID = '{table_name}'"
787
+ if i < len(tables) - 1:
788
+ query += " OR "
789
+ results.extend(self.fetchall(query))
790
+
791
+ return [to_timestamp(row[0]) for row in results]
792
+
756
793
  def _get_table(self, table_name: TableName) -> BigQueryTable:
757
794
  """
758
795
  Returns a BigQueryTable object for the given table name.
@@ -1295,6 +1332,108 @@ class BigQueryEngineAdapter(InsertOverwriteWithMergeMixin, ClusteredByMixin, Row
1295
1332
  def _session_id(self, value: t.Any) -> None:
1296
1333
  self._connection_pool.set_attribute("session_id", value)
1297
1334
 
1335
+ def _get_current_schema(self) -> str:
1336
+ raise NotImplementedError("BigQuery does not support current schema")
1337
+
1338
+ def _get_bq_dataset_location(self, project: str, dataset: str) -> str:
1339
+ return self._db_call(self.client.get_dataset, dataset_ref=f"{project}.{dataset}").location
1340
+
1341
+ def _get_grant_expression(self, table: exp.Table) -> exp.Expression:
1342
+ if not table.db:
1343
+ raise ValueError(
1344
+ f"Table {table.sql(dialect=self.dialect)} does not have a schema (dataset)"
1345
+ )
1346
+ project = table.catalog or self.get_current_catalog()
1347
+ if not project:
1348
+ raise ValueError(
1349
+ f"Table {table.sql(dialect=self.dialect)} does not have a catalog (project)"
1350
+ )
1351
+
1352
+ dataset = table.db
1353
+ table_name = table.name
1354
+ location = self._get_bq_dataset_location(project, dataset)
1355
+
1356
+ # https://cloud.google.com/bigquery/docs/information-schema-object-privileges
1357
+ # OBJECT_PRIVILEGES is a project-level INFORMATION_SCHEMA view with regional qualifier
1358
+ object_privileges_table = exp.to_table(
1359
+ f"`{project}`.`region-{location}`.INFORMATION_SCHEMA.{self.GRANT_INFORMATION_SCHEMA_TABLE_NAME}",
1360
+ dialect=self.dialect,
1361
+ )
1362
+ return (
1363
+ exp.select("privilege_type", "grantee")
1364
+ .from_(object_privileges_table)
1365
+ .where(
1366
+ exp.and_(
1367
+ exp.column("object_schema").eq(exp.Literal.string(dataset)),
1368
+ exp.column("object_name").eq(exp.Literal.string(table_name)),
1369
+ # Filter out current_user
1370
+ # BigQuery grantees format: "user:email" or "group:name"
1371
+ exp.func("split", exp.column("grantee"), exp.Literal.string(":"))[
1372
+ exp.func("OFFSET", exp.Literal.number("1"))
1373
+ ].neq(self.CURRENT_USER_OR_ROLE_EXPRESSION),
1374
+ )
1375
+ )
1376
+ )
1377
+
1378
+ @staticmethod
1379
+ def _grant_object_kind(table_type: DataObjectType) -> str:
1380
+ if table_type == DataObjectType.VIEW:
1381
+ return "VIEW"
1382
+ if table_type == DataObjectType.MATERIALIZED_VIEW:
1383
+ # We actually need to use "MATERIALIZED VIEW" here even though it's not listed
1384
+ # as a supported resource_type in the BigQuery DCL doc:
1385
+ # https://cloud.google.com/bigquery/docs/reference/standard-sql/data-control-language
1386
+ return "MATERIALIZED VIEW"
1387
+ return "TABLE"
1388
+
1389
+ def _dcl_grants_config_expr(
1390
+ self,
1391
+ dcl_cmd: t.Type[DCL],
1392
+ table: exp.Table,
1393
+ grants_config: GrantsConfig,
1394
+ table_type: DataObjectType = DataObjectType.TABLE,
1395
+ ) -> t.List[exp.Expression]:
1396
+ expressions: t.List[exp.Expression] = []
1397
+ if not grants_config:
1398
+ return expressions
1399
+
1400
+ # https://cloud.google.com/bigquery/docs/reference/standard-sql/data-control-language
1401
+
1402
+ def normalize_principal(p: str) -> str:
1403
+ if ":" not in p:
1404
+ raise ValueError(f"Principal '{p}' missing a prefix label")
1405
+
1406
+ # allUsers and allAuthenticatedUsers special groups that are cas-sensitive and must start with "specialGroup:"
1407
+ if p.endswith("allUsers") or p.endswith("allAuthenticatedUsers"):
1408
+ if not p.startswith("specialGroup:"):
1409
+ raise ValueError(
1410
+ f"Special group principal '{p}' must start with 'specialGroup:' prefix label"
1411
+ )
1412
+ return p
1413
+
1414
+ label, principal = p.split(":", 1)
1415
+ # always lowercase principals
1416
+ return f"{label}:{principal.lower()}"
1417
+
1418
+ object_kind = self._grant_object_kind(table_type)
1419
+ for privilege, principals in grants_config.items():
1420
+ if not principals:
1421
+ continue
1422
+
1423
+ noramlized_principals = [exp.Literal.string(normalize_principal(p)) for p in principals]
1424
+ args: t.Dict[str, t.Any] = {
1425
+ "privileges": [exp.GrantPrivilege(this=exp.to_identifier(privilege, quoted=True))],
1426
+ "securable": table.copy(),
1427
+ "principals": noramlized_principals,
1428
+ }
1429
+
1430
+ if object_kind:
1431
+ args["kind"] = exp.Var(this=object_kind)
1432
+
1433
+ expressions.append(dcl_cmd(**args)) # type: ignore[arg-type]
1434
+
1435
+ return expressions
1436
+
1298
1437
 
1299
1438
  class _ErrorCounter:
1300
1439
  """
@@ -112,8 +112,9 @@ class ClickhouseEngineAdapter(EngineAdapterWithIndexSupport, LogicalMergeMixin):
112
112
  storage_format=exp.var("MergeTree"),
113
113
  **kwargs,
114
114
  )
115
+ ordered_df = df[list(source_columns_to_types)]
115
116
 
116
- self.cursor.client.insert_df(temp_table.sql(dialect=self.dialect), df=df)
117
+ self.cursor.client.insert_df(temp_table.sql(dialect=self.dialect), df=ordered_df)
117
118
 
118
119
  return exp.select(*self._casted_columns(target_columns_to_types, source_columns)).from_(
119
120
  temp_table
@@ -223,7 +224,7 @@ class ClickhouseEngineAdapter(EngineAdapterWithIndexSupport, LogicalMergeMixin):
223
224
  target_columns_to_types = target_columns_to_types or self.columns(target_table)
224
225
 
225
226
  temp_table = self._get_temp_table(target_table)
226
- self._create_table_like(temp_table, target_table)
227
+ self.create_table_like(temp_table, target_table)
227
228
 
228
229
  # REPLACE BY KEY: extract kwargs if present
229
230
  dynamic_key = kwargs.get("dynamic_key")
@@ -455,7 +456,11 @@ class ClickhouseEngineAdapter(EngineAdapterWithIndexSupport, LogicalMergeMixin):
455
456
  )
456
457
 
457
458
  def _create_table_like(
458
- self, target_table_name: TableName, source_table_name: TableName
459
+ self,
460
+ target_table_name: TableName,
461
+ source_table_name: TableName,
462
+ exists: bool,
463
+ **kwargs: t.Any,
459
464
  ) -> None:
460
465
  """Create table with identical structure as source table"""
461
466
  self.execute(
@@ -631,16 +636,15 @@ class ClickhouseEngineAdapter(EngineAdapterWithIndexSupport, LogicalMergeMixin):
631
636
  kind: What kind of object to drop. Defaults to TABLE
632
637
  **drop_args: Any extra arguments to set on the Drop expression
633
638
  """
634
- self.execute(
635
- exp.Drop(
636
- this=exp.to_table(name),
637
- kind=kind,
638
- exists=exists,
639
- cluster=exp.OnCluster(this=exp.to_identifier(self.cluster))
640
- if self.engine_run_mode.is_cluster
641
- else None,
642
- **drop_args,
643
- )
639
+ super()._drop_object(
640
+ name=name,
641
+ exists=exists,
642
+ kind=kind,
643
+ cascade=cascade,
644
+ cluster=exp.OnCluster(this=exp.to_identifier(self.cluster))
645
+ if self.engine_run_mode.is_cluster
646
+ else None,
647
+ **drop_args,
644
648
  )
645
649
 
646
650
  def _build_partitioned_by_exp(
@@ -5,7 +5,9 @@ import typing as t
5
5
  from functools import partial
6
6
 
7
7
  from sqlglot import exp
8
+
8
9
  from sqlmesh.core.dialect import to_schema
10
+ from sqlmesh.core.engine_adapter.mixins import GrantsFromInfoSchemaMixin
9
11
  from sqlmesh.core.engine_adapter.shared import (
10
12
  CatalogSupport,
11
13
  DataObject,
@@ -28,12 +30,16 @@ if t.TYPE_CHECKING:
28
30
  logger = logging.getLogger(__name__)
29
31
 
30
32
 
31
- class DatabricksEngineAdapter(SparkEngineAdapter):
33
+ class DatabricksEngineAdapter(SparkEngineAdapter, GrantsFromInfoSchemaMixin):
32
34
  DIALECT = "databricks"
33
35
  INSERT_OVERWRITE_STRATEGY = InsertOverwriteStrategy.REPLACE_WHERE
34
36
  SUPPORTS_CLONING = True
35
37
  SUPPORTS_MATERIALIZED_VIEWS = True
36
38
  SUPPORTS_MATERIALIZED_VIEW_SCHEMA = True
39
+ SUPPORTS_GRANTS = True
40
+ USE_CATALOG_IN_GRANTS = True
41
+ # Spark has this set to false for compatibility when mixing with Trino but that isn't a concern with Databricks
42
+ QUOTE_IDENTIFIERS_IN_VIEWS = True
37
43
  SCHEMA_DIFFER_KWARGS = {
38
44
  "support_positional_add": True,
39
45
  "nested_support": NestedSupport.ALL,
@@ -149,6 +155,28 @@ class DatabricksEngineAdapter(SparkEngineAdapter):
149
155
  def catalog_support(self) -> CatalogSupport:
150
156
  return CatalogSupport.FULL_SUPPORT
151
157
 
158
+ @staticmethod
159
+ def _grant_object_kind(table_type: DataObjectType) -> str:
160
+ if table_type == DataObjectType.VIEW:
161
+ return "VIEW"
162
+ if table_type == DataObjectType.MATERIALIZED_VIEW:
163
+ return "MATERIALIZED VIEW"
164
+ return "TABLE"
165
+
166
+ def _get_grant_expression(self, table: exp.Table) -> exp.Expression:
167
+ # We only care about explicitly granted privileges and not inherited ones
168
+ # if this is removed you would see grants inherited from the catalog get returned
169
+ expression = super()._get_grant_expression(table)
170
+ expression.args["where"].set(
171
+ "this",
172
+ exp.and_(
173
+ expression.args["where"].this,
174
+ exp.column("inherited_from").eq(exp.Literal.string("NONE")),
175
+ wrap=False,
176
+ ),
177
+ )
178
+ return expression
179
+
152
180
  def _begin_session(self, properties: SessionProperties) -> t.Any:
153
181
  """Begin a new session."""
154
182
  # Align the different possible connectors to a single catalog
@@ -266,7 +294,9 @@ class DatabricksEngineAdapter(SparkEngineAdapter):
266
294
  exp.column("table_catalog").as_("catalog"),
267
295
  exp.case(exp.column("table_type"))
268
296
  .when(exp.Literal.string("VIEW"), exp.Literal.string("view"))
269
- .when(exp.Literal.string("MATERIALIZED_VIEW"), exp.Literal.string("view"))
297
+ .when(
298
+ exp.Literal.string("MATERIALIZED_VIEW"), exp.Literal.string("materialized_view")
299
+ )
270
300
  .else_(exp.Literal.string("table"))
271
301
  .as_("type"),
272
302
  )
@@ -297,6 +327,7 @@ class DatabricksEngineAdapter(SparkEngineAdapter):
297
327
  target_table_name: TableName,
298
328
  source_table_name: TableName,
299
329
  replace: bool = False,
330
+ exists: bool = True,
300
331
  clone_kwargs: t.Optional[t.Dict[str, t.Any]] = None,
301
332
  **kwargs: t.Any,
302
333
  ) -> None:
@@ -10,23 +10,15 @@ from tenacity import retry, stop_after_attempt, wait_exponential, retry_if_resul
10
10
  from sqlmesh.core.engine_adapter.mssql import MSSQLEngineAdapter
11
11
  from sqlmesh.core.engine_adapter.shared import (
12
12
  InsertOverwriteStrategy,
13
- SourceQuery,
14
13
  )
15
- from sqlmesh.core.engine_adapter.base import EngineAdapter
16
14
  from sqlmesh.utils.errors import SQLMeshError
17
15
  from sqlmesh.utils.connection_pool import ConnectionPool
18
16
 
19
17
 
20
- if t.TYPE_CHECKING:
21
- from sqlmesh.core._typing import TableName
22
-
23
-
24
- from sqlmesh.core.engine_adapter.mixins import LogicalMergeMixin
25
-
26
18
  logger = logging.getLogger(__name__)
27
19
 
28
20
 
29
- class FabricEngineAdapter(LogicalMergeMixin, MSSQLEngineAdapter):
21
+ class FabricEngineAdapter(MSSQLEngineAdapter):
30
22
  """
31
23
  Adapter for Microsoft Fabric.
32
24
  """
@@ -58,26 +50,6 @@ class FabricEngineAdapter(LogicalMergeMixin, MSSQLEngineAdapter):
58
50
  def _target_catalog(self, value: t.Optional[str]) -> None:
59
51
  self._connection_pool.set_attribute("target_catalog", value)
60
52
 
61
- def _insert_overwrite_by_condition(
62
- self,
63
- table_name: TableName,
64
- source_queries: t.List[SourceQuery],
65
- target_columns_to_types: t.Optional[t.Dict[str, exp.DataType]] = None,
66
- where: t.Optional[exp.Condition] = None,
67
- insert_overwrite_strategy_override: t.Optional[InsertOverwriteStrategy] = None,
68
- **kwargs: t.Any,
69
- ) -> None:
70
- # Override to avoid MERGE statement which isn't fully supported in Fabric
71
- return EngineAdapter._insert_overwrite_by_condition(
72
- self,
73
- table_name=table_name,
74
- source_queries=source_queries,
75
- target_columns_to_types=target_columns_to_types,
76
- where=where,
77
- insert_overwrite_strategy_override=InsertOverwriteStrategy.DELETE_INSERT,
78
- **kwargs,
79
- )
80
-
81
53
  @property
82
54
  def api_client(self) -> FabricHttpClient:
83
55
  # the requests Session is not guaranteed to be threadsafe
@@ -7,9 +7,10 @@ from dataclasses import dataclass
7
7
 
8
8
  from sqlglot import exp, parse_one
9
9
  from sqlglot.helper import seq_get
10
+ from sqlglot.optimizer.normalize_identifiers import normalize_identifiers
10
11
 
11
12
  from sqlmesh.core.engine_adapter.base import EngineAdapter
12
- from sqlmesh.core.engine_adapter.shared import InsertOverwriteStrategy, SourceQuery
13
+ from sqlmesh.core.engine_adapter.shared import DataObjectType
13
14
  from sqlmesh.core.node import IntervalUnit
14
15
  from sqlmesh.core.dialect import schema_
15
16
  from sqlmesh.core.schema_diff import TableAlterOperation
@@ -17,7 +18,12 @@ from sqlmesh.utils.errors import SQLMeshError
17
18
 
18
19
  if t.TYPE_CHECKING:
19
20
  from sqlmesh.core._typing import TableName
20
- from sqlmesh.core.engine_adapter._typing import DF
21
+ from sqlmesh.core.engine_adapter._typing import (
22
+ DCL,
23
+ DF,
24
+ GrantsConfig,
25
+ QueryOrDF,
26
+ )
21
27
  from sqlmesh.core.engine_adapter.base import QueryOrDF
22
28
 
23
29
  logger = logging.getLogger(__name__)
@@ -75,52 +81,6 @@ class PandasNativeFetchDFSupportMixin(EngineAdapter):
75
81
  return df
76
82
 
77
83
 
78
- class InsertOverwriteWithMergeMixin(EngineAdapter):
79
- def _insert_overwrite_by_condition(
80
- self,
81
- table_name: TableName,
82
- source_queries: t.List[SourceQuery],
83
- target_columns_to_types: t.Optional[t.Dict[str, exp.DataType]] = None,
84
- where: t.Optional[exp.Condition] = None,
85
- insert_overwrite_strategy_override: t.Optional[InsertOverwriteStrategy] = None,
86
- **kwargs: t.Any,
87
- ) -> None:
88
- """
89
- Some engines do not support `INSERT OVERWRITE` but instead support
90
- doing an "INSERT OVERWRITE" using a Merge expression but with the
91
- predicate being `False`.
92
- """
93
- target_columns_to_types = target_columns_to_types or self.columns(table_name)
94
- for source_query in source_queries:
95
- with source_query as query:
96
- query = self._order_projections_and_filter(
97
- query, target_columns_to_types, where=where
98
- )
99
- columns = [exp.column(col) for col in target_columns_to_types]
100
- when_not_matched_by_source = exp.When(
101
- matched=False,
102
- source=True,
103
- condition=where,
104
- then=exp.Delete(),
105
- )
106
- when_not_matched_by_target = exp.When(
107
- matched=False,
108
- source=False,
109
- then=exp.Insert(
110
- this=exp.Tuple(expressions=columns),
111
- expression=exp.Tuple(expressions=columns),
112
- ),
113
- )
114
- self._merge(
115
- target_table=table_name,
116
- query=query,
117
- on=exp.false(),
118
- whens=exp.Whens(
119
- expressions=[when_not_matched_by_source, when_not_matched_by_target]
120
- ),
121
- )
122
-
123
-
124
84
  class HiveMetastoreTablePropertiesMixin(EngineAdapter):
125
85
  MAX_TABLE_COMMENT_LENGTH = 4000
126
86
  MAX_COLUMN_COMMENT_LENGTH = 4000
@@ -595,3 +555,137 @@ class RowDiffMixin(EngineAdapter):
595
555
 
596
556
  def _normalize_boolean_value(self, expr: exp.Expression) -> exp.Expression:
597
557
  return exp.cast(expr, "INT")
558
+
559
+
560
+ class GrantsFromInfoSchemaMixin(EngineAdapter):
561
+ CURRENT_USER_OR_ROLE_EXPRESSION: exp.Expression = exp.func("current_user")
562
+ SUPPORTS_MULTIPLE_GRANT_PRINCIPALS = False
563
+ USE_CATALOG_IN_GRANTS = False
564
+ GRANT_INFORMATION_SCHEMA_TABLE_NAME = "table_privileges"
565
+
566
+ @staticmethod
567
+ @abc.abstractmethod
568
+ def _grant_object_kind(table_type: DataObjectType) -> t.Optional[str]:
569
+ pass
570
+
571
+ @abc.abstractmethod
572
+ def _get_current_schema(self) -> str:
573
+ pass
574
+
575
+ def _dcl_grants_config_expr(
576
+ self,
577
+ dcl_cmd: t.Type[DCL],
578
+ table: exp.Table,
579
+ grants_config: GrantsConfig,
580
+ table_type: DataObjectType = DataObjectType.TABLE,
581
+ ) -> t.List[exp.Expression]:
582
+ expressions: t.List[exp.Expression] = []
583
+ if not grants_config:
584
+ return expressions
585
+
586
+ object_kind = self._grant_object_kind(table_type)
587
+ for privilege, principals in grants_config.items():
588
+ args: t.Dict[str, t.Any] = {
589
+ "privileges": [exp.GrantPrivilege(this=exp.Var(this=privilege))],
590
+ "securable": table.copy(),
591
+ }
592
+ if object_kind:
593
+ args["kind"] = exp.Var(this=object_kind)
594
+ if self.SUPPORTS_MULTIPLE_GRANT_PRINCIPALS:
595
+ args["principals"] = [
596
+ normalize_identifiers(
597
+ parse_one(principal, into=exp.GrantPrincipal, dialect=self.dialect),
598
+ dialect=self.dialect,
599
+ )
600
+ for principal in principals
601
+ ]
602
+ expressions.append(dcl_cmd(**args)) # type: ignore[arg-type]
603
+ else:
604
+ for principal in principals:
605
+ args["principals"] = [
606
+ normalize_identifiers(
607
+ parse_one(principal, into=exp.GrantPrincipal, dialect=self.dialect),
608
+ dialect=self.dialect,
609
+ )
610
+ ]
611
+ expressions.append(dcl_cmd(**args)) # type: ignore[arg-type]
612
+
613
+ return expressions
614
+
615
+ def _apply_grants_config_expr(
616
+ self,
617
+ table: exp.Table,
618
+ grants_config: GrantsConfig,
619
+ table_type: DataObjectType = DataObjectType.TABLE,
620
+ ) -> t.List[exp.Expression]:
621
+ return self._dcl_grants_config_expr(exp.Grant, table, grants_config, table_type)
622
+
623
+ def _revoke_grants_config_expr(
624
+ self,
625
+ table: exp.Table,
626
+ grants_config: GrantsConfig,
627
+ table_type: DataObjectType = DataObjectType.TABLE,
628
+ ) -> t.List[exp.Expression]:
629
+ return self._dcl_grants_config_expr(exp.Revoke, table, grants_config, table_type)
630
+
631
+ def _get_grant_expression(self, table: exp.Table) -> exp.Expression:
632
+ schema_identifier = table.args.get("db") or normalize_identifiers(
633
+ exp.to_identifier(self._get_current_schema(), quoted=True), dialect=self.dialect
634
+ )
635
+ schema_name = schema_identifier.this
636
+ table_name = table.args.get("this").this # type: ignore
637
+
638
+ grant_conditions = [
639
+ exp.column("table_schema").eq(exp.Literal.string(schema_name)),
640
+ exp.column("table_name").eq(exp.Literal.string(table_name)),
641
+ exp.column("grantor").eq(self.CURRENT_USER_OR_ROLE_EXPRESSION),
642
+ exp.column("grantee").neq(self.CURRENT_USER_OR_ROLE_EXPRESSION),
643
+ ]
644
+
645
+ info_schema_table = normalize_identifiers(
646
+ exp.table_(self.GRANT_INFORMATION_SCHEMA_TABLE_NAME, db="information_schema"),
647
+ dialect=self.dialect,
648
+ )
649
+ if self.USE_CATALOG_IN_GRANTS:
650
+ catalog_identifier = table.args.get("catalog")
651
+ if not catalog_identifier:
652
+ catalog_name = self.get_current_catalog()
653
+ if not catalog_name:
654
+ raise SQLMeshError(
655
+ "Current catalog could not be determined for fetching grants. This is unexpected."
656
+ )
657
+ catalog_identifier = normalize_identifiers(
658
+ exp.to_identifier(catalog_name, quoted=True), dialect=self.dialect
659
+ )
660
+ catalog_name = catalog_identifier.this
661
+ info_schema_table.set("catalog", catalog_identifier.copy())
662
+ grant_conditions.insert(
663
+ 0, exp.column("table_catalog").eq(exp.Literal.string(catalog_name))
664
+ )
665
+
666
+ return (
667
+ exp.select("privilege_type", "grantee")
668
+ .from_(info_schema_table)
669
+ .where(exp.and_(*grant_conditions))
670
+ )
671
+
672
+ def _get_current_grants_config(self, table: exp.Table) -> GrantsConfig:
673
+ grant_expr = self._get_grant_expression(table)
674
+
675
+ results = self.fetchall(grant_expr)
676
+
677
+ grants_dict: GrantsConfig = {}
678
+ for privilege_raw, grantee_raw in results:
679
+ if privilege_raw is None or grantee_raw is None:
680
+ continue
681
+
682
+ privilege = str(privilege_raw)
683
+ grantee = str(grantee_raw)
684
+ if not privilege or not grantee:
685
+ continue
686
+
687
+ grantees = grants_dict.setdefault(privilege, [])
688
+ if grantee not in grantees:
689
+ grantees.append(grantee)
690
+
691
+ return grants_dict