sqlmesh 0.213.1.dev1__py3-none-any.whl → 0.227.2.dev4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sqlmesh/__init__.py +12 -2
- sqlmesh/_version.py +2 -2
- sqlmesh/cli/main.py +0 -44
- sqlmesh/cli/project_init.py +11 -2
- sqlmesh/core/_typing.py +1 -0
- sqlmesh/core/audit/definition.py +8 -2
- sqlmesh/core/config/__init__.py +1 -1
- sqlmesh/core/config/connection.py +17 -5
- sqlmesh/core/config/dbt.py +13 -0
- sqlmesh/core/config/janitor.py +12 -0
- sqlmesh/core/config/loader.py +7 -0
- sqlmesh/core/config/model.py +2 -0
- sqlmesh/core/config/root.py +3 -0
- sqlmesh/core/console.py +81 -3
- sqlmesh/core/constants.py +1 -1
- sqlmesh/core/context.py +69 -26
- sqlmesh/core/dialect.py +3 -0
- sqlmesh/core/engine_adapter/_typing.py +2 -0
- sqlmesh/core/engine_adapter/base.py +322 -22
- sqlmesh/core/engine_adapter/base_postgres.py +17 -1
- sqlmesh/core/engine_adapter/bigquery.py +146 -7
- sqlmesh/core/engine_adapter/clickhouse.py +17 -13
- sqlmesh/core/engine_adapter/databricks.py +33 -2
- sqlmesh/core/engine_adapter/fabric.py +10 -29
- sqlmesh/core/engine_adapter/mixins.py +142 -48
- sqlmesh/core/engine_adapter/mssql.py +15 -4
- sqlmesh/core/engine_adapter/mysql.py +2 -2
- sqlmesh/core/engine_adapter/postgres.py +9 -3
- sqlmesh/core/engine_adapter/redshift.py +4 -0
- sqlmesh/core/engine_adapter/risingwave.py +1 -0
- sqlmesh/core/engine_adapter/shared.py +6 -0
- sqlmesh/core/engine_adapter/snowflake.py +82 -11
- sqlmesh/core/engine_adapter/spark.py +14 -10
- sqlmesh/core/engine_adapter/trino.py +4 -2
- sqlmesh/core/environment.py +2 -0
- sqlmesh/core/janitor.py +181 -0
- sqlmesh/core/lineage.py +1 -0
- sqlmesh/core/linter/definition.py +13 -13
- sqlmesh/core/linter/rules/builtin.py +29 -0
- sqlmesh/core/macros.py +35 -13
- sqlmesh/core/model/common.py +2 -0
- sqlmesh/core/model/definition.py +82 -28
- sqlmesh/core/model/kind.py +66 -2
- sqlmesh/core/model/meta.py +108 -4
- sqlmesh/core/node.py +101 -1
- sqlmesh/core/plan/builder.py +18 -10
- sqlmesh/core/plan/common.py +199 -2
- sqlmesh/core/plan/definition.py +25 -6
- sqlmesh/core/plan/evaluator.py +75 -113
- sqlmesh/core/plan/explainer.py +90 -8
- sqlmesh/core/plan/stages.py +42 -21
- sqlmesh/core/renderer.py +78 -32
- sqlmesh/core/scheduler.py +102 -22
- sqlmesh/core/selector.py +137 -9
- sqlmesh/core/signal.py +64 -1
- sqlmesh/core/snapshot/__init__.py +2 -0
- sqlmesh/core/snapshot/definition.py +146 -34
- sqlmesh/core/snapshot/evaluator.py +689 -124
- sqlmesh/core/state_sync/__init__.py +0 -1
- sqlmesh/core/state_sync/base.py +55 -33
- sqlmesh/core/state_sync/cache.py +12 -7
- sqlmesh/core/state_sync/common.py +216 -111
- sqlmesh/core/state_sync/db/environment.py +6 -4
- sqlmesh/core/state_sync/db/facade.py +42 -24
- sqlmesh/core/state_sync/db/interval.py +27 -7
- sqlmesh/core/state_sync/db/migrator.py +34 -16
- sqlmesh/core/state_sync/db/snapshot.py +177 -169
- sqlmesh/core/table_diff.py +2 -2
- sqlmesh/core/test/context.py +2 -0
- sqlmesh/core/test/definition.py +14 -9
- sqlmesh/dbt/adapter.py +22 -16
- sqlmesh/dbt/basemodel.py +75 -56
- sqlmesh/dbt/builtin.py +116 -12
- sqlmesh/dbt/column.py +17 -5
- sqlmesh/dbt/common.py +19 -5
- sqlmesh/dbt/context.py +14 -1
- sqlmesh/dbt/loader.py +61 -9
- sqlmesh/dbt/manifest.py +174 -16
- sqlmesh/dbt/model.py +183 -85
- sqlmesh/dbt/package.py +16 -1
- sqlmesh/dbt/profile.py +3 -3
- sqlmesh/dbt/project.py +12 -7
- sqlmesh/dbt/seed.py +6 -1
- sqlmesh/dbt/source.py +13 -1
- sqlmesh/dbt/target.py +25 -6
- sqlmesh/dbt/test.py +36 -5
- sqlmesh/migrations/v0000_baseline.py +95 -0
- sqlmesh/migrations/v0061_mysql_fix_blob_text_type.py +5 -7
- sqlmesh/migrations/v0062_add_model_gateway.py +5 -1
- sqlmesh/migrations/v0063_change_signals.py +5 -3
- sqlmesh/migrations/v0064_join_when_matched_strings.py +5 -3
- sqlmesh/migrations/v0065_add_model_optimize.py +5 -1
- sqlmesh/migrations/v0066_add_auto_restatements.py +8 -3
- sqlmesh/migrations/v0067_add_tsql_date_full_precision.py +5 -1
- sqlmesh/migrations/v0068_include_unrendered_query_in_metadata_hash.py +5 -1
- sqlmesh/migrations/v0069_update_dev_table_suffix.py +5 -3
- sqlmesh/migrations/v0070_include_grains_in_metadata_hash.py +5 -1
- sqlmesh/migrations/v0071_add_dev_version_to_intervals.py +9 -5
- sqlmesh/migrations/v0072_add_environment_statements.py +5 -3
- sqlmesh/migrations/v0073_remove_symbolic_disable_restatement.py +5 -3
- sqlmesh/migrations/v0074_add_partition_by_time_column_property.py +5 -1
- sqlmesh/migrations/v0075_remove_validate_query.py +5 -3
- sqlmesh/migrations/v0076_add_cron_tz.py +5 -1
- sqlmesh/migrations/v0077_fix_column_type_hash_calculation.py +5 -1
- sqlmesh/migrations/v0078_warn_if_non_migratable_python_env.py +5 -3
- sqlmesh/migrations/v0079_add_gateway_managed_property.py +10 -5
- sqlmesh/migrations/v0080_add_batch_size_to_scd_type_2_models.py +5 -1
- sqlmesh/migrations/v0081_update_partitioned_by.py +5 -3
- sqlmesh/migrations/v0082_warn_if_incorrectly_duplicated_statements.py +5 -3
- sqlmesh/migrations/v0083_use_sql_for_scd_time_data_type_data_hash.py +5 -1
- sqlmesh/migrations/v0084_normalize_quote_when_matched_and_merge_filter.py +5 -1
- sqlmesh/migrations/v0085_deterministic_repr.py +5 -3
- sqlmesh/migrations/v0086_check_deterministic_bug.py +5 -3
- sqlmesh/migrations/v0087_normalize_blueprint_variables.py +5 -3
- sqlmesh/migrations/v0088_warn_about_variable_python_env_diffs.py +5 -3
- sqlmesh/migrations/v0089_add_virtual_environment_mode.py +5 -1
- sqlmesh/migrations/v0090_add_forward_only_column.py +9 -5
- sqlmesh/migrations/v0091_on_additive_change.py +5 -1
- sqlmesh/migrations/v0092_warn_about_dbt_data_type_diff.py +5 -3
- sqlmesh/migrations/v0093_use_raw_sql_in_fingerprint.py +5 -1
- sqlmesh/migrations/v0094_add_dev_version_and_fingerprint_columns.py +123 -0
- sqlmesh/migrations/v0095_warn_about_dbt_raw_sql_diff.py +49 -0
- sqlmesh/migrations/v0096_remove_plan_dags_table.py +13 -0
- sqlmesh/migrations/v0097_add_dbt_name_in_node.py +9 -0
- sqlmesh/migrations/{v0060_move_audits_to_model.py → v0098_add_dbt_node_info_in_node.py} +33 -16
- sqlmesh/migrations/v0099_add_last_altered_to_intervals.py +25 -0
- sqlmesh/migrations/v0100_add_grants_and_grants_target_layer.py +9 -0
- sqlmesh/utils/__init__.py +8 -1
- sqlmesh/utils/cache.py +5 -1
- sqlmesh/utils/connection_pool.py +2 -1
- sqlmesh/utils/dag.py +65 -10
- sqlmesh/utils/date.py +8 -1
- sqlmesh/utils/errors.py +8 -0
- sqlmesh/utils/jinja.py +54 -4
- sqlmesh/utils/pydantic.py +6 -6
- sqlmesh/utils/windows.py +13 -3
- {sqlmesh-0.213.1.dev1.dist-info → sqlmesh-0.227.2.dev4.dist-info}/METADATA +7 -10
- sqlmesh-0.227.2.dev4.dist-info/RECORD +370 -0
- sqlmesh_dbt/cli.py +70 -7
- sqlmesh_dbt/console.py +14 -6
- sqlmesh_dbt/operations.py +103 -24
- sqlmesh_dbt/selectors.py +39 -1
- web/client/dist/assets/{Audits-Ucsx1GzF.js → Audits-CBiYyyx-.js} +1 -1
- web/client/dist/assets/{Banner-BWDzvavM.js → Banner-DSRbUlO5.js} +1 -1
- web/client/dist/assets/{ChevronDownIcon-D2VL13Ah.js → ChevronDownIcon-MK_nrjD_.js} +1 -1
- web/client/dist/assets/{ChevronRightIcon-DWGYbf1l.js → ChevronRightIcon-CLWtT22Q.js} +1 -1
- web/client/dist/assets/{Content-DdHDZM3I.js → Content-BNuGZN5l.js} +1 -1
- web/client/dist/assets/{Content-Bikfy8fh.js → Content-CSHJyW0n.js} +1 -1
- web/client/dist/assets/{Data-CzAJH7rW.js → Data-C1oRDbLx.js} +1 -1
- web/client/dist/assets/{DataCatalog-BJF11g8f.js → DataCatalog-HXyX2-_j.js} +1 -1
- web/client/dist/assets/{Editor-s0SBpV2y.js → Editor-BDyfpUuw.js} +1 -1
- web/client/dist/assets/{Editor-DgLhgKnm.js → Editor-D0jNItwC.js} +1 -1
- web/client/dist/assets/{Errors-D0m0O1d3.js → Errors-BfuFLcPi.js} +1 -1
- web/client/dist/assets/{FileExplorer-CEv0vXkt.js → FileExplorer-BR9IE3he.js} +1 -1
- web/client/dist/assets/{Footer-BwzXn8Ew.js → Footer-CgBEtiAh.js} +1 -1
- web/client/dist/assets/{Header-6heDkEqG.js → Header-DSqR6nSO.js} +1 -1
- web/client/dist/assets/{Input-obuJsD6k.js → Input-B-oZ6fGO.js} +1 -1
- web/client/dist/assets/Lineage-DYQVwDbD.js +1 -0
- web/client/dist/assets/{ListboxShow-HM9_qyrt.js → ListboxShow-BE5-xevs.js} +1 -1
- web/client/dist/assets/{ModelLineage-zWdKo0U2.js → ModelLineage-DkIFAYo4.js} +1 -1
- web/client/dist/assets/{Models-Bcu66SRz.js → Models-D5dWr8RB.js} +1 -1
- web/client/dist/assets/{Page-BWEEQfIt.js → Page-C-XfU5BR.js} +1 -1
- web/client/dist/assets/{Plan-C4gXCqlf.js → Plan-ZEuTINBq.js} +1 -1
- web/client/dist/assets/{PlusCircleIcon-CVDO651q.js → PlusCircleIcon-DVXAHG8_.js} +1 -1
- web/client/dist/assets/{ReportErrors-BT6xFwAr.js → ReportErrors-B7FEPzMB.js} +1 -1
- web/client/dist/assets/{Root-ryJoBK4h.js → Root-8aZyhPxF.js} +1 -1
- web/client/dist/assets/{SearchList-DB04sPb9.js → SearchList-W_iT2G82.js} +1 -1
- web/client/dist/assets/{SelectEnvironment-CUYcXUu6.js → SelectEnvironment-C65jALmO.js} +1 -1
- web/client/dist/assets/{SourceList-Doo_9ZGp.js → SourceList-DSLO6nVJ.js} +1 -1
- web/client/dist/assets/{SourceListItem-D5Mj7Dly.js → SourceListItem-BHt8d9-I.js} +1 -1
- web/client/dist/assets/{SplitPane-qHmkD1qy.js → SplitPane-CViaZmw6.js} +1 -1
- web/client/dist/assets/{Tests-DH1Z74ML.js → Tests-DhaVt5t1.js} +1 -1
- web/client/dist/assets/{Welcome-DqUJUNMF.js → Welcome-DvpjH-_4.js} +1 -1
- web/client/dist/assets/context-BctCsyGb.js +71 -0
- web/client/dist/assets/{context-Dr54UHLi.js → context-DFNeGsFF.js} +1 -1
- web/client/dist/assets/{editor-DYIP1yQ4.js → editor-CcO28cqd.js} +1 -1
- web/client/dist/assets/{file-DarlIDVi.js → file-CvJN3aZO.js} +1 -1
- web/client/dist/assets/{floating-ui.react-dom-BH3TFvkM.js → floating-ui.react-dom-CjE-JNW1.js} +1 -1
- web/client/dist/assets/{help-Bl8wqaQc.js → help-DuPhjipa.js} +1 -1
- web/client/dist/assets/{index-D1sR7wpN.js → index-C-dJH7yZ.js} +1 -1
- web/client/dist/assets/{index-O3mjYpnE.js → index-Dj0i1-CA.js} +2 -2
- web/client/dist/assets/{plan-CehRrJUG.js → plan-BTRSbjKn.js} +1 -1
- web/client/dist/assets/{popover-CqgMRE0G.js → popover-_Sf0yvOI.js} +1 -1
- web/client/dist/assets/{project-6gxepOhm.js → project-BvSOI8MY.js} +1 -1
- web/client/dist/index.html +1 -1
- sqlmesh/integrations/llm.py +0 -56
- sqlmesh/migrations/v0001_init.py +0 -60
- sqlmesh/migrations/v0002_remove_identify.py +0 -5
- sqlmesh/migrations/v0003_move_batch_size.py +0 -34
- sqlmesh/migrations/v0004_environmnent_add_finalized_at.py +0 -23
- sqlmesh/migrations/v0005_create_seed_table.py +0 -24
- sqlmesh/migrations/v0006_change_seed_hash.py +0 -5
- sqlmesh/migrations/v0007_env_table_info_to_kind.py +0 -99
- sqlmesh/migrations/v0008_create_intervals_table.py +0 -38
- sqlmesh/migrations/v0009_remove_pre_post_hooks.py +0 -62
- sqlmesh/migrations/v0010_seed_hash_batch_size.py +0 -5
- sqlmesh/migrations/v0011_add_model_kind_name.py +0 -63
- sqlmesh/migrations/v0012_update_jinja_expressions.py +0 -86
- sqlmesh/migrations/v0013_serde_using_model_dialects.py +0 -87
- sqlmesh/migrations/v0014_fix_dev_intervals.py +0 -14
- sqlmesh/migrations/v0015_environment_add_promoted_snapshot_ids.py +0 -26
- sqlmesh/migrations/v0016_fix_windows_path.py +0 -59
- sqlmesh/migrations/v0017_fix_windows_seed_path.py +0 -55
- sqlmesh/migrations/v0018_rename_snapshot_model_to_node.py +0 -53
- sqlmesh/migrations/v0019_add_env_suffix_target.py +0 -28
- sqlmesh/migrations/v0020_remove_redundant_attributes_from_dbt_models.py +0 -80
- sqlmesh/migrations/v0021_fix_table_properties.py +0 -62
- sqlmesh/migrations/v0022_move_project_to_model.py +0 -54
- sqlmesh/migrations/v0023_fix_added_models_with_forward_only_parents.py +0 -65
- sqlmesh/migrations/v0024_replace_model_kind_name_enum_with_value.py +0 -55
- sqlmesh/migrations/v0025_fix_intervals_and_missing_change_category.py +0 -117
- sqlmesh/migrations/v0026_remove_dialect_from_seed.py +0 -55
- sqlmesh/migrations/v0027_minute_interval_to_five.py +0 -57
- sqlmesh/migrations/v0028_add_plan_dags_table.py +0 -29
- sqlmesh/migrations/v0029_generate_schema_types_using_dialect.py +0 -69
- sqlmesh/migrations/v0030_update_unrestorable_snapshots.py +0 -65
- sqlmesh/migrations/v0031_remove_dbt_target_fields.py +0 -65
- sqlmesh/migrations/v0032_add_sqlmesh_version.py +0 -25
- sqlmesh/migrations/v0033_mysql_fix_blob_text_type.py +0 -45
- sqlmesh/migrations/v0034_add_default_catalog.py +0 -367
- sqlmesh/migrations/v0035_add_catalog_name_override.py +0 -22
- sqlmesh/migrations/v0036_delete_plan_dags_bug_fix.py +0 -14
- sqlmesh/migrations/v0037_remove_dbt_is_incremental_macro.py +0 -61
- sqlmesh/migrations/v0038_add_expiration_ts_to_snapshot.py +0 -73
- sqlmesh/migrations/v0039_include_environment_in_plan_dag_spec.py +0 -68
- sqlmesh/migrations/v0040_add_previous_finalized_snapshots.py +0 -26
- sqlmesh/migrations/v0041_remove_hash_raw_query_attribute.py +0 -59
- sqlmesh/migrations/v0042_trim_indirect_versions.py +0 -66
- sqlmesh/migrations/v0043_fix_remove_obsolete_attributes_in_plan_dags.py +0 -61
- sqlmesh/migrations/v0044_quote_identifiers_in_model_attributes.py +0 -5
- sqlmesh/migrations/v0045_move_gateway_variable.py +0 -70
- sqlmesh/migrations/v0046_add_batch_concurrency.py +0 -8
- sqlmesh/migrations/v0047_change_scd_string_to_column.py +0 -5
- sqlmesh/migrations/v0048_drop_indirect_versions.py +0 -59
- sqlmesh/migrations/v0049_replace_identifier_with_version_in_seeds_table.py +0 -57
- sqlmesh/migrations/v0050_drop_seeds_table.py +0 -11
- sqlmesh/migrations/v0051_rename_column_descriptions.py +0 -65
- sqlmesh/migrations/v0052_add_normalize_name_in_environment_naming_info.py +0 -28
- sqlmesh/migrations/v0053_custom_model_kind_extra_attributes.py +0 -5
- sqlmesh/migrations/v0054_fix_trailing_comments.py +0 -5
- sqlmesh/migrations/v0055_add_updated_ts_unpaused_ts_ttl_ms_unrestorable_to_snapshot.py +0 -132
- sqlmesh/migrations/v0056_restore_table_indexes.py +0 -118
- sqlmesh/migrations/v0057_add_table_format.py +0 -5
- sqlmesh/migrations/v0058_add_requirements.py +0 -26
- sqlmesh/migrations/v0059_add_physical_version.py +0 -5
- sqlmesh-0.213.1.dev1.dist-info/RECORD +0 -421
- web/client/dist/assets/Lineage-D0Hgdz2v.js +0 -1
- web/client/dist/assets/context-DgX0fp2E.js +0 -68
- {sqlmesh-0.213.1.dev1.dist-info → sqlmesh-0.227.2.dev4.dist-info}/WHEEL +0 -0
- {sqlmesh-0.213.1.dev1.dist-info → sqlmesh-0.227.2.dev4.dist-info}/entry_points.txt +0 -0
- {sqlmesh-0.213.1.dev1.dist-info → sqlmesh-0.227.2.dev4.dist-info}/licenses/LICENSE +0 -0
- {sqlmesh-0.213.1.dev1.dist-info → sqlmesh-0.227.2.dev4.dist-info}/top_level.txt +0 -0
|
@@ -8,9 +8,10 @@ from sqlglot import exp, parse_one
|
|
|
8
8
|
from sqlglot.transforms import remove_precision_parameterized_types
|
|
9
9
|
|
|
10
10
|
from sqlmesh.core.dialect import to_schema
|
|
11
|
+
from sqlmesh.core.engine_adapter.base import _get_data_object_cache_key
|
|
11
12
|
from sqlmesh.core.engine_adapter.mixins import (
|
|
12
|
-
InsertOverwriteWithMergeMixin,
|
|
13
13
|
ClusteredByMixin,
|
|
14
|
+
GrantsFromInfoSchemaMixin,
|
|
14
15
|
RowDiffMixin,
|
|
15
16
|
TableAlterClusterByOperation,
|
|
16
17
|
)
|
|
@@ -20,6 +21,7 @@ from sqlmesh.core.engine_adapter.shared import (
|
|
|
20
21
|
DataObjectType,
|
|
21
22
|
SourceQuery,
|
|
22
23
|
set_catalog,
|
|
24
|
+
InsertOverwriteStrategy,
|
|
23
25
|
)
|
|
24
26
|
from sqlmesh.core.node import IntervalUnit
|
|
25
27
|
from sqlmesh.core.schema_diff import TableAlterOperation, NestedSupport
|
|
@@ -39,7 +41,7 @@ if t.TYPE_CHECKING:
|
|
|
39
41
|
from google.cloud.bigquery.table import Table as BigQueryTable
|
|
40
42
|
|
|
41
43
|
from sqlmesh.core._typing import SchemaName, SessionProperties, TableName
|
|
42
|
-
from sqlmesh.core.engine_adapter._typing import BigframeSession, DF, Query
|
|
44
|
+
from sqlmesh.core.engine_adapter._typing import BigframeSession, DCL, DF, GrantsConfig, Query
|
|
43
45
|
from sqlmesh.core.engine_adapter.base import QueryOrDF
|
|
44
46
|
|
|
45
47
|
|
|
@@ -54,7 +56,7 @@ NestedFieldsDict = t.Dict[str, t.List[NestedField]]
|
|
|
54
56
|
|
|
55
57
|
|
|
56
58
|
@set_catalog()
|
|
57
|
-
class BigQueryEngineAdapter(
|
|
59
|
+
class BigQueryEngineAdapter(ClusteredByMixin, RowDiffMixin, GrantsFromInfoSchemaMixin):
|
|
58
60
|
"""
|
|
59
61
|
BigQuery Engine Adapter using the `google-cloud-bigquery` library's DB API.
|
|
60
62
|
"""
|
|
@@ -64,10 +66,16 @@ class BigQueryEngineAdapter(InsertOverwriteWithMergeMixin, ClusteredByMixin, Row
|
|
|
64
66
|
SUPPORTS_TRANSACTIONS = False
|
|
65
67
|
SUPPORTS_MATERIALIZED_VIEWS = True
|
|
66
68
|
SUPPORTS_CLONING = True
|
|
69
|
+
SUPPORTS_GRANTS = True
|
|
70
|
+
CURRENT_USER_OR_ROLE_EXPRESSION: exp.Expression = exp.func("session_user")
|
|
71
|
+
SUPPORTS_MULTIPLE_GRANT_PRINCIPALS = True
|
|
72
|
+
USE_CATALOG_IN_GRANTS = True
|
|
73
|
+
GRANT_INFORMATION_SCHEMA_TABLE_NAME = "OBJECT_PRIVILEGES"
|
|
67
74
|
MAX_TABLE_COMMENT_LENGTH = 1024
|
|
68
75
|
MAX_COLUMN_COMMENT_LENGTH = 1024
|
|
69
76
|
SUPPORTS_QUERY_EXECUTION_TRACKING = True
|
|
70
77
|
SUPPORTED_DROP_CASCADE_OBJECT_KINDS = ["SCHEMA"]
|
|
78
|
+
INSERT_OVERWRITE_STRATEGY = InsertOverwriteStrategy.MERGE
|
|
71
79
|
|
|
72
80
|
SCHEMA_DIFFER_KWARGS = {
|
|
73
81
|
"compatible_types": {
|
|
@@ -168,17 +176,18 @@ class BigQueryEngineAdapter(InsertOverwriteWithMergeMixin, ClusteredByMixin, Row
|
|
|
168
176
|
)
|
|
169
177
|
|
|
170
178
|
def query_factory() -> Query:
|
|
171
|
-
|
|
172
|
-
|
|
179
|
+
ordered_df = df[list(source_columns_to_types)]
|
|
180
|
+
if bigframes_pd and isinstance(ordered_df, bigframes_pd.DataFrame):
|
|
181
|
+
ordered_df.to_gbq(
|
|
173
182
|
f"{temp_bq_table.project}.{temp_bq_table.dataset_id}.{temp_bq_table.table_id}",
|
|
174
183
|
if_exists="replace",
|
|
175
184
|
)
|
|
176
185
|
elif not self.table_exists(temp_table):
|
|
177
186
|
# Make mypy happy
|
|
178
|
-
assert isinstance(
|
|
187
|
+
assert isinstance(ordered_df, pd.DataFrame)
|
|
179
188
|
self._db_call(self.client.create_table, table=temp_bq_table, exists_ok=False)
|
|
180
189
|
result = self.__load_pandas_to_table(
|
|
181
|
-
temp_bq_table,
|
|
190
|
+
temp_bq_table, ordered_df, source_columns_to_types, replace=False
|
|
182
191
|
)
|
|
183
192
|
if result.errors:
|
|
184
193
|
raise SQLMeshError(result.errors)
|
|
@@ -742,6 +751,12 @@ class BigQueryEngineAdapter(InsertOverwriteWithMergeMixin, ClusteredByMixin, Row
|
|
|
742
751
|
)
|
|
743
752
|
|
|
744
753
|
def table_exists(self, table_name: TableName) -> bool:
|
|
754
|
+
table = exp.to_table(table_name)
|
|
755
|
+
data_object_cache_key = _get_data_object_cache_key(table.catalog, table.db, table.name)
|
|
756
|
+
if data_object_cache_key in self._data_object_cache:
|
|
757
|
+
logger.debug("Table existence cache hit: %s", data_object_cache_key)
|
|
758
|
+
return self._data_object_cache[data_object_cache_key] is not None
|
|
759
|
+
|
|
745
760
|
try:
|
|
746
761
|
from google.cloud.exceptions import NotFound
|
|
747
762
|
except ModuleNotFoundError:
|
|
@@ -753,6 +768,28 @@ class BigQueryEngineAdapter(InsertOverwriteWithMergeMixin, ClusteredByMixin, Row
|
|
|
753
768
|
except NotFound:
|
|
754
769
|
return False
|
|
755
770
|
|
|
771
|
+
def get_table_last_modified_ts(self, table_names: t.List[TableName]) -> t.List[int]:
|
|
772
|
+
from sqlmesh.utils.date import to_timestamp
|
|
773
|
+
|
|
774
|
+
datasets_to_tables: t.DefaultDict[str, t.List[str]] = defaultdict(list)
|
|
775
|
+
for table_name in table_names:
|
|
776
|
+
table = exp.to_table(table_name)
|
|
777
|
+
datasets_to_tables[table.db].append(table.name)
|
|
778
|
+
|
|
779
|
+
results = []
|
|
780
|
+
|
|
781
|
+
for dataset, tables in datasets_to_tables.items():
|
|
782
|
+
query = (
|
|
783
|
+
f"SELECT TIMESTAMP_MILLIS(last_modified_time) FROM `{dataset}.__TABLES__` WHERE "
|
|
784
|
+
)
|
|
785
|
+
for i, table_name in enumerate(tables):
|
|
786
|
+
query += f"TABLE_ID = '{table_name}'"
|
|
787
|
+
if i < len(tables) - 1:
|
|
788
|
+
query += " OR "
|
|
789
|
+
results.extend(self.fetchall(query))
|
|
790
|
+
|
|
791
|
+
return [to_timestamp(row[0]) for row in results]
|
|
792
|
+
|
|
756
793
|
def _get_table(self, table_name: TableName) -> BigQueryTable:
|
|
757
794
|
"""
|
|
758
795
|
Returns a BigQueryTable object for the given table name.
|
|
@@ -1295,6 +1332,108 @@ class BigQueryEngineAdapter(InsertOverwriteWithMergeMixin, ClusteredByMixin, Row
|
|
|
1295
1332
|
def _session_id(self, value: t.Any) -> None:
|
|
1296
1333
|
self._connection_pool.set_attribute("session_id", value)
|
|
1297
1334
|
|
|
1335
|
+
def _get_current_schema(self) -> str:
|
|
1336
|
+
raise NotImplementedError("BigQuery does not support current schema")
|
|
1337
|
+
|
|
1338
|
+
def _get_bq_dataset_location(self, project: str, dataset: str) -> str:
|
|
1339
|
+
return self._db_call(self.client.get_dataset, dataset_ref=f"{project}.{dataset}").location
|
|
1340
|
+
|
|
1341
|
+
def _get_grant_expression(self, table: exp.Table) -> exp.Expression:
|
|
1342
|
+
if not table.db:
|
|
1343
|
+
raise ValueError(
|
|
1344
|
+
f"Table {table.sql(dialect=self.dialect)} does not have a schema (dataset)"
|
|
1345
|
+
)
|
|
1346
|
+
project = table.catalog or self.get_current_catalog()
|
|
1347
|
+
if not project:
|
|
1348
|
+
raise ValueError(
|
|
1349
|
+
f"Table {table.sql(dialect=self.dialect)} does not have a catalog (project)"
|
|
1350
|
+
)
|
|
1351
|
+
|
|
1352
|
+
dataset = table.db
|
|
1353
|
+
table_name = table.name
|
|
1354
|
+
location = self._get_bq_dataset_location(project, dataset)
|
|
1355
|
+
|
|
1356
|
+
# https://cloud.google.com/bigquery/docs/information-schema-object-privileges
|
|
1357
|
+
# OBJECT_PRIVILEGES is a project-level INFORMATION_SCHEMA view with regional qualifier
|
|
1358
|
+
object_privileges_table = exp.to_table(
|
|
1359
|
+
f"`{project}`.`region-{location}`.INFORMATION_SCHEMA.{self.GRANT_INFORMATION_SCHEMA_TABLE_NAME}",
|
|
1360
|
+
dialect=self.dialect,
|
|
1361
|
+
)
|
|
1362
|
+
return (
|
|
1363
|
+
exp.select("privilege_type", "grantee")
|
|
1364
|
+
.from_(object_privileges_table)
|
|
1365
|
+
.where(
|
|
1366
|
+
exp.and_(
|
|
1367
|
+
exp.column("object_schema").eq(exp.Literal.string(dataset)),
|
|
1368
|
+
exp.column("object_name").eq(exp.Literal.string(table_name)),
|
|
1369
|
+
# Filter out current_user
|
|
1370
|
+
# BigQuery grantees format: "user:email" or "group:name"
|
|
1371
|
+
exp.func("split", exp.column("grantee"), exp.Literal.string(":"))[
|
|
1372
|
+
exp.func("OFFSET", exp.Literal.number("1"))
|
|
1373
|
+
].neq(self.CURRENT_USER_OR_ROLE_EXPRESSION),
|
|
1374
|
+
)
|
|
1375
|
+
)
|
|
1376
|
+
)
|
|
1377
|
+
|
|
1378
|
+
@staticmethod
|
|
1379
|
+
def _grant_object_kind(table_type: DataObjectType) -> str:
|
|
1380
|
+
if table_type == DataObjectType.VIEW:
|
|
1381
|
+
return "VIEW"
|
|
1382
|
+
if table_type == DataObjectType.MATERIALIZED_VIEW:
|
|
1383
|
+
# We actually need to use "MATERIALIZED VIEW" here even though it's not listed
|
|
1384
|
+
# as a supported resource_type in the BigQuery DCL doc:
|
|
1385
|
+
# https://cloud.google.com/bigquery/docs/reference/standard-sql/data-control-language
|
|
1386
|
+
return "MATERIALIZED VIEW"
|
|
1387
|
+
return "TABLE"
|
|
1388
|
+
|
|
1389
|
+
def _dcl_grants_config_expr(
|
|
1390
|
+
self,
|
|
1391
|
+
dcl_cmd: t.Type[DCL],
|
|
1392
|
+
table: exp.Table,
|
|
1393
|
+
grants_config: GrantsConfig,
|
|
1394
|
+
table_type: DataObjectType = DataObjectType.TABLE,
|
|
1395
|
+
) -> t.List[exp.Expression]:
|
|
1396
|
+
expressions: t.List[exp.Expression] = []
|
|
1397
|
+
if not grants_config:
|
|
1398
|
+
return expressions
|
|
1399
|
+
|
|
1400
|
+
# https://cloud.google.com/bigquery/docs/reference/standard-sql/data-control-language
|
|
1401
|
+
|
|
1402
|
+
def normalize_principal(p: str) -> str:
|
|
1403
|
+
if ":" not in p:
|
|
1404
|
+
raise ValueError(f"Principal '{p}' missing a prefix label")
|
|
1405
|
+
|
|
1406
|
+
# allUsers and allAuthenticatedUsers special groups that are cas-sensitive and must start with "specialGroup:"
|
|
1407
|
+
if p.endswith("allUsers") or p.endswith("allAuthenticatedUsers"):
|
|
1408
|
+
if not p.startswith("specialGroup:"):
|
|
1409
|
+
raise ValueError(
|
|
1410
|
+
f"Special group principal '{p}' must start with 'specialGroup:' prefix label"
|
|
1411
|
+
)
|
|
1412
|
+
return p
|
|
1413
|
+
|
|
1414
|
+
label, principal = p.split(":", 1)
|
|
1415
|
+
# always lowercase principals
|
|
1416
|
+
return f"{label}:{principal.lower()}"
|
|
1417
|
+
|
|
1418
|
+
object_kind = self._grant_object_kind(table_type)
|
|
1419
|
+
for privilege, principals in grants_config.items():
|
|
1420
|
+
if not principals:
|
|
1421
|
+
continue
|
|
1422
|
+
|
|
1423
|
+
noramlized_principals = [exp.Literal.string(normalize_principal(p)) for p in principals]
|
|
1424
|
+
args: t.Dict[str, t.Any] = {
|
|
1425
|
+
"privileges": [exp.GrantPrivilege(this=exp.to_identifier(privilege, quoted=True))],
|
|
1426
|
+
"securable": table.copy(),
|
|
1427
|
+
"principals": noramlized_principals,
|
|
1428
|
+
}
|
|
1429
|
+
|
|
1430
|
+
if object_kind:
|
|
1431
|
+
args["kind"] = exp.Var(this=object_kind)
|
|
1432
|
+
|
|
1433
|
+
expressions.append(dcl_cmd(**args)) # type: ignore[arg-type]
|
|
1434
|
+
|
|
1435
|
+
return expressions
|
|
1436
|
+
|
|
1298
1437
|
|
|
1299
1438
|
class _ErrorCounter:
|
|
1300
1439
|
"""
|
|
@@ -112,8 +112,9 @@ class ClickhouseEngineAdapter(EngineAdapterWithIndexSupport, LogicalMergeMixin):
|
|
|
112
112
|
storage_format=exp.var("MergeTree"),
|
|
113
113
|
**kwargs,
|
|
114
114
|
)
|
|
115
|
+
ordered_df = df[list(source_columns_to_types)]
|
|
115
116
|
|
|
116
|
-
self.cursor.client.insert_df(temp_table.sql(dialect=self.dialect), df=
|
|
117
|
+
self.cursor.client.insert_df(temp_table.sql(dialect=self.dialect), df=ordered_df)
|
|
117
118
|
|
|
118
119
|
return exp.select(*self._casted_columns(target_columns_to_types, source_columns)).from_(
|
|
119
120
|
temp_table
|
|
@@ -223,7 +224,7 @@ class ClickhouseEngineAdapter(EngineAdapterWithIndexSupport, LogicalMergeMixin):
|
|
|
223
224
|
target_columns_to_types = target_columns_to_types or self.columns(target_table)
|
|
224
225
|
|
|
225
226
|
temp_table = self._get_temp_table(target_table)
|
|
226
|
-
self.
|
|
227
|
+
self.create_table_like(temp_table, target_table)
|
|
227
228
|
|
|
228
229
|
# REPLACE BY KEY: extract kwargs if present
|
|
229
230
|
dynamic_key = kwargs.get("dynamic_key")
|
|
@@ -455,7 +456,11 @@ class ClickhouseEngineAdapter(EngineAdapterWithIndexSupport, LogicalMergeMixin):
|
|
|
455
456
|
)
|
|
456
457
|
|
|
457
458
|
def _create_table_like(
|
|
458
|
-
self,
|
|
459
|
+
self,
|
|
460
|
+
target_table_name: TableName,
|
|
461
|
+
source_table_name: TableName,
|
|
462
|
+
exists: bool,
|
|
463
|
+
**kwargs: t.Any,
|
|
459
464
|
) -> None:
|
|
460
465
|
"""Create table with identical structure as source table"""
|
|
461
466
|
self.execute(
|
|
@@ -631,16 +636,15 @@ class ClickhouseEngineAdapter(EngineAdapterWithIndexSupport, LogicalMergeMixin):
|
|
|
631
636
|
kind: What kind of object to drop. Defaults to TABLE
|
|
632
637
|
**drop_args: Any extra arguments to set on the Drop expression
|
|
633
638
|
"""
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
)
|
|
639
|
+
super()._drop_object(
|
|
640
|
+
name=name,
|
|
641
|
+
exists=exists,
|
|
642
|
+
kind=kind,
|
|
643
|
+
cascade=cascade,
|
|
644
|
+
cluster=exp.OnCluster(this=exp.to_identifier(self.cluster))
|
|
645
|
+
if self.engine_run_mode.is_cluster
|
|
646
|
+
else None,
|
|
647
|
+
**drop_args,
|
|
644
648
|
)
|
|
645
649
|
|
|
646
650
|
def _build_partitioned_by_exp(
|
|
@@ -5,7 +5,9 @@ import typing as t
|
|
|
5
5
|
from functools import partial
|
|
6
6
|
|
|
7
7
|
from sqlglot import exp
|
|
8
|
+
|
|
8
9
|
from sqlmesh.core.dialect import to_schema
|
|
10
|
+
from sqlmesh.core.engine_adapter.mixins import GrantsFromInfoSchemaMixin
|
|
9
11
|
from sqlmesh.core.engine_adapter.shared import (
|
|
10
12
|
CatalogSupport,
|
|
11
13
|
DataObject,
|
|
@@ -28,12 +30,16 @@ if t.TYPE_CHECKING:
|
|
|
28
30
|
logger = logging.getLogger(__name__)
|
|
29
31
|
|
|
30
32
|
|
|
31
|
-
class DatabricksEngineAdapter(SparkEngineAdapter):
|
|
33
|
+
class DatabricksEngineAdapter(SparkEngineAdapter, GrantsFromInfoSchemaMixin):
|
|
32
34
|
DIALECT = "databricks"
|
|
33
35
|
INSERT_OVERWRITE_STRATEGY = InsertOverwriteStrategy.REPLACE_WHERE
|
|
34
36
|
SUPPORTS_CLONING = True
|
|
35
37
|
SUPPORTS_MATERIALIZED_VIEWS = True
|
|
36
38
|
SUPPORTS_MATERIALIZED_VIEW_SCHEMA = True
|
|
39
|
+
SUPPORTS_GRANTS = True
|
|
40
|
+
USE_CATALOG_IN_GRANTS = True
|
|
41
|
+
# Spark has this set to false for compatibility when mixing with Trino but that isn't a concern with Databricks
|
|
42
|
+
QUOTE_IDENTIFIERS_IN_VIEWS = True
|
|
37
43
|
SCHEMA_DIFFER_KWARGS = {
|
|
38
44
|
"support_positional_add": True,
|
|
39
45
|
"nested_support": NestedSupport.ALL,
|
|
@@ -149,6 +155,28 @@ class DatabricksEngineAdapter(SparkEngineAdapter):
|
|
|
149
155
|
def catalog_support(self) -> CatalogSupport:
|
|
150
156
|
return CatalogSupport.FULL_SUPPORT
|
|
151
157
|
|
|
158
|
+
@staticmethod
|
|
159
|
+
def _grant_object_kind(table_type: DataObjectType) -> str:
|
|
160
|
+
if table_type == DataObjectType.VIEW:
|
|
161
|
+
return "VIEW"
|
|
162
|
+
if table_type == DataObjectType.MATERIALIZED_VIEW:
|
|
163
|
+
return "MATERIALIZED VIEW"
|
|
164
|
+
return "TABLE"
|
|
165
|
+
|
|
166
|
+
def _get_grant_expression(self, table: exp.Table) -> exp.Expression:
|
|
167
|
+
# We only care about explicitly granted privileges and not inherited ones
|
|
168
|
+
# if this is removed you would see grants inherited from the catalog get returned
|
|
169
|
+
expression = super()._get_grant_expression(table)
|
|
170
|
+
expression.args["where"].set(
|
|
171
|
+
"this",
|
|
172
|
+
exp.and_(
|
|
173
|
+
expression.args["where"].this,
|
|
174
|
+
exp.column("inherited_from").eq(exp.Literal.string("NONE")),
|
|
175
|
+
wrap=False,
|
|
176
|
+
),
|
|
177
|
+
)
|
|
178
|
+
return expression
|
|
179
|
+
|
|
152
180
|
def _begin_session(self, properties: SessionProperties) -> t.Any:
|
|
153
181
|
"""Begin a new session."""
|
|
154
182
|
# Align the different possible connectors to a single catalog
|
|
@@ -266,7 +294,9 @@ class DatabricksEngineAdapter(SparkEngineAdapter):
|
|
|
266
294
|
exp.column("table_catalog").as_("catalog"),
|
|
267
295
|
exp.case(exp.column("table_type"))
|
|
268
296
|
.when(exp.Literal.string("VIEW"), exp.Literal.string("view"))
|
|
269
|
-
.when(
|
|
297
|
+
.when(
|
|
298
|
+
exp.Literal.string("MATERIALIZED_VIEW"), exp.Literal.string("materialized_view")
|
|
299
|
+
)
|
|
270
300
|
.else_(exp.Literal.string("table"))
|
|
271
301
|
.as_("type"),
|
|
272
302
|
)
|
|
@@ -297,6 +327,7 @@ class DatabricksEngineAdapter(SparkEngineAdapter):
|
|
|
297
327
|
target_table_name: TableName,
|
|
298
328
|
source_table_name: TableName,
|
|
299
329
|
replace: bool = False,
|
|
330
|
+
exists: bool = True,
|
|
300
331
|
clone_kwargs: t.Optional[t.Dict[str, t.Any]] = None,
|
|
301
332
|
**kwargs: t.Any,
|
|
302
333
|
) -> None:
|
|
@@ -10,23 +10,15 @@ from tenacity import retry, stop_after_attempt, wait_exponential, retry_if_resul
|
|
|
10
10
|
from sqlmesh.core.engine_adapter.mssql import MSSQLEngineAdapter
|
|
11
11
|
from sqlmesh.core.engine_adapter.shared import (
|
|
12
12
|
InsertOverwriteStrategy,
|
|
13
|
-
SourceQuery,
|
|
14
13
|
)
|
|
15
|
-
from sqlmesh.core.engine_adapter.base import EngineAdapter
|
|
16
14
|
from sqlmesh.utils.errors import SQLMeshError
|
|
17
15
|
from sqlmesh.utils.connection_pool import ConnectionPool
|
|
18
16
|
|
|
19
17
|
|
|
20
|
-
if t.TYPE_CHECKING:
|
|
21
|
-
from sqlmesh.core._typing import TableName
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
from sqlmesh.core.engine_adapter.mixins import LogicalMergeMixin
|
|
25
|
-
|
|
26
18
|
logger = logging.getLogger(__name__)
|
|
27
19
|
|
|
28
20
|
|
|
29
|
-
class FabricEngineAdapter(
|
|
21
|
+
class FabricEngineAdapter(MSSQLEngineAdapter):
|
|
30
22
|
"""
|
|
31
23
|
Adapter for Microsoft Fabric.
|
|
32
24
|
"""
|
|
@@ -58,26 +50,6 @@ class FabricEngineAdapter(LogicalMergeMixin, MSSQLEngineAdapter):
|
|
|
58
50
|
def _target_catalog(self, value: t.Optional[str]) -> None:
|
|
59
51
|
self._connection_pool.set_attribute("target_catalog", value)
|
|
60
52
|
|
|
61
|
-
def _insert_overwrite_by_condition(
|
|
62
|
-
self,
|
|
63
|
-
table_name: TableName,
|
|
64
|
-
source_queries: t.List[SourceQuery],
|
|
65
|
-
target_columns_to_types: t.Optional[t.Dict[str, exp.DataType]] = None,
|
|
66
|
-
where: t.Optional[exp.Condition] = None,
|
|
67
|
-
insert_overwrite_strategy_override: t.Optional[InsertOverwriteStrategy] = None,
|
|
68
|
-
**kwargs: t.Any,
|
|
69
|
-
) -> None:
|
|
70
|
-
# Override to avoid MERGE statement which isn't fully supported in Fabric
|
|
71
|
-
return EngineAdapter._insert_overwrite_by_condition(
|
|
72
|
-
self,
|
|
73
|
-
table_name=table_name,
|
|
74
|
-
source_queries=source_queries,
|
|
75
|
-
target_columns_to_types=target_columns_to_types,
|
|
76
|
-
where=where,
|
|
77
|
-
insert_overwrite_strategy_override=InsertOverwriteStrategy.DELETE_INSERT,
|
|
78
|
-
**kwargs,
|
|
79
|
-
)
|
|
80
|
-
|
|
81
53
|
@property
|
|
82
54
|
def api_client(self) -> FabricHttpClient:
|
|
83
55
|
# the requests Session is not guaranteed to be threadsafe
|
|
@@ -121,10 +93,19 @@ class FabricEngineAdapter(LogicalMergeMixin, MSSQLEngineAdapter):
|
|
|
121
93
|
def _drop_catalog(self, catalog_name: exp.Identifier) -> None:
|
|
122
94
|
"""Drop a catalog (warehouse) in Microsoft Fabric via REST API."""
|
|
123
95
|
warehouse_name = catalog_name.sql(dialect=self.dialect, identify=False)
|
|
96
|
+
current_catalog = self.get_current_catalog()
|
|
124
97
|
|
|
125
98
|
logger.info(f"Deleting Fabric warehouse: {warehouse_name}")
|
|
126
99
|
self.api_client.delete_warehouse(warehouse_name)
|
|
127
100
|
|
|
101
|
+
if warehouse_name == current_catalog:
|
|
102
|
+
# Somewhere around 2025-09-08, Fabric started validating the "Database=" connection argument and throwing 'Authentication failed' if the database doesnt exist
|
|
103
|
+
# In addition, set_current_catalog() is implemented using a threadlocal variable "target_catalog"
|
|
104
|
+
# So, when we drop a warehouse, and there are still threads with "target_catalog" set to reference it, any operations on those threads
|
|
105
|
+
# that use an either use an existing connection pointing to this warehouse or trigger a new connection
|
|
106
|
+
# will fail with an 'Authentication Failed' error unless we close all connections here, which also clears all the threadlocal data
|
|
107
|
+
self.close()
|
|
108
|
+
|
|
128
109
|
def set_current_catalog(self, catalog_name: str) -> None:
|
|
129
110
|
"""
|
|
130
111
|
Set the current catalog for Microsoft Fabric connections.
|
|
@@ -7,9 +7,10 @@ from dataclasses import dataclass
|
|
|
7
7
|
|
|
8
8
|
from sqlglot import exp, parse_one
|
|
9
9
|
from sqlglot.helper import seq_get
|
|
10
|
+
from sqlglot.optimizer.normalize_identifiers import normalize_identifiers
|
|
10
11
|
|
|
11
12
|
from sqlmesh.core.engine_adapter.base import EngineAdapter
|
|
12
|
-
from sqlmesh.core.engine_adapter.shared import
|
|
13
|
+
from sqlmesh.core.engine_adapter.shared import DataObjectType
|
|
13
14
|
from sqlmesh.core.node import IntervalUnit
|
|
14
15
|
from sqlmesh.core.dialect import schema_
|
|
15
16
|
from sqlmesh.core.schema_diff import TableAlterOperation
|
|
@@ -17,7 +18,12 @@ from sqlmesh.utils.errors import SQLMeshError
|
|
|
17
18
|
|
|
18
19
|
if t.TYPE_CHECKING:
|
|
19
20
|
from sqlmesh.core._typing import TableName
|
|
20
|
-
from sqlmesh.core.engine_adapter._typing import
|
|
21
|
+
from sqlmesh.core.engine_adapter._typing import (
|
|
22
|
+
DCL,
|
|
23
|
+
DF,
|
|
24
|
+
GrantsConfig,
|
|
25
|
+
QueryOrDF,
|
|
26
|
+
)
|
|
21
27
|
from sqlmesh.core.engine_adapter.base import QueryOrDF
|
|
22
28
|
|
|
23
29
|
logger = logging.getLogger(__name__)
|
|
@@ -75,52 +81,6 @@ class PandasNativeFetchDFSupportMixin(EngineAdapter):
|
|
|
75
81
|
return df
|
|
76
82
|
|
|
77
83
|
|
|
78
|
-
class InsertOverwriteWithMergeMixin(EngineAdapter):
|
|
79
|
-
def _insert_overwrite_by_condition(
|
|
80
|
-
self,
|
|
81
|
-
table_name: TableName,
|
|
82
|
-
source_queries: t.List[SourceQuery],
|
|
83
|
-
target_columns_to_types: t.Optional[t.Dict[str, exp.DataType]] = None,
|
|
84
|
-
where: t.Optional[exp.Condition] = None,
|
|
85
|
-
insert_overwrite_strategy_override: t.Optional[InsertOverwriteStrategy] = None,
|
|
86
|
-
**kwargs: t.Any,
|
|
87
|
-
) -> None:
|
|
88
|
-
"""
|
|
89
|
-
Some engines do not support `INSERT OVERWRITE` but instead support
|
|
90
|
-
doing an "INSERT OVERWRITE" using a Merge expression but with the
|
|
91
|
-
predicate being `False`.
|
|
92
|
-
"""
|
|
93
|
-
target_columns_to_types = target_columns_to_types or self.columns(table_name)
|
|
94
|
-
for source_query in source_queries:
|
|
95
|
-
with source_query as query:
|
|
96
|
-
query = self._order_projections_and_filter(
|
|
97
|
-
query, target_columns_to_types, where=where
|
|
98
|
-
)
|
|
99
|
-
columns = [exp.column(col) for col in target_columns_to_types]
|
|
100
|
-
when_not_matched_by_source = exp.When(
|
|
101
|
-
matched=False,
|
|
102
|
-
source=True,
|
|
103
|
-
condition=where,
|
|
104
|
-
then=exp.Delete(),
|
|
105
|
-
)
|
|
106
|
-
when_not_matched_by_target = exp.When(
|
|
107
|
-
matched=False,
|
|
108
|
-
source=False,
|
|
109
|
-
then=exp.Insert(
|
|
110
|
-
this=exp.Tuple(expressions=columns),
|
|
111
|
-
expression=exp.Tuple(expressions=columns),
|
|
112
|
-
),
|
|
113
|
-
)
|
|
114
|
-
self._merge(
|
|
115
|
-
target_table=table_name,
|
|
116
|
-
query=query,
|
|
117
|
-
on=exp.false(),
|
|
118
|
-
whens=exp.Whens(
|
|
119
|
-
expressions=[when_not_matched_by_source, when_not_matched_by_target]
|
|
120
|
-
),
|
|
121
|
-
)
|
|
122
|
-
|
|
123
|
-
|
|
124
84
|
class HiveMetastoreTablePropertiesMixin(EngineAdapter):
|
|
125
85
|
MAX_TABLE_COMMENT_LENGTH = 4000
|
|
126
86
|
MAX_COLUMN_COMMENT_LENGTH = 4000
|
|
@@ -595,3 +555,137 @@ class RowDiffMixin(EngineAdapter):
|
|
|
595
555
|
|
|
596
556
|
def _normalize_boolean_value(self, expr: exp.Expression) -> exp.Expression:
|
|
597
557
|
return exp.cast(expr, "INT")
|
|
558
|
+
|
|
559
|
+
|
|
560
|
+
class GrantsFromInfoSchemaMixin(EngineAdapter):
|
|
561
|
+
CURRENT_USER_OR_ROLE_EXPRESSION: exp.Expression = exp.func("current_user")
|
|
562
|
+
SUPPORTS_MULTIPLE_GRANT_PRINCIPALS = False
|
|
563
|
+
USE_CATALOG_IN_GRANTS = False
|
|
564
|
+
GRANT_INFORMATION_SCHEMA_TABLE_NAME = "table_privileges"
|
|
565
|
+
|
|
566
|
+
@staticmethod
|
|
567
|
+
@abc.abstractmethod
|
|
568
|
+
def _grant_object_kind(table_type: DataObjectType) -> t.Optional[str]:
|
|
569
|
+
pass
|
|
570
|
+
|
|
571
|
+
@abc.abstractmethod
|
|
572
|
+
def _get_current_schema(self) -> str:
|
|
573
|
+
pass
|
|
574
|
+
|
|
575
|
+
def _dcl_grants_config_expr(
|
|
576
|
+
self,
|
|
577
|
+
dcl_cmd: t.Type[DCL],
|
|
578
|
+
table: exp.Table,
|
|
579
|
+
grants_config: GrantsConfig,
|
|
580
|
+
table_type: DataObjectType = DataObjectType.TABLE,
|
|
581
|
+
) -> t.List[exp.Expression]:
|
|
582
|
+
expressions: t.List[exp.Expression] = []
|
|
583
|
+
if not grants_config:
|
|
584
|
+
return expressions
|
|
585
|
+
|
|
586
|
+
object_kind = self._grant_object_kind(table_type)
|
|
587
|
+
for privilege, principals in grants_config.items():
|
|
588
|
+
args: t.Dict[str, t.Any] = {
|
|
589
|
+
"privileges": [exp.GrantPrivilege(this=exp.Var(this=privilege))],
|
|
590
|
+
"securable": table.copy(),
|
|
591
|
+
}
|
|
592
|
+
if object_kind:
|
|
593
|
+
args["kind"] = exp.Var(this=object_kind)
|
|
594
|
+
if self.SUPPORTS_MULTIPLE_GRANT_PRINCIPALS:
|
|
595
|
+
args["principals"] = [
|
|
596
|
+
normalize_identifiers(
|
|
597
|
+
parse_one(principal, into=exp.GrantPrincipal, dialect=self.dialect),
|
|
598
|
+
dialect=self.dialect,
|
|
599
|
+
)
|
|
600
|
+
for principal in principals
|
|
601
|
+
]
|
|
602
|
+
expressions.append(dcl_cmd(**args)) # type: ignore[arg-type]
|
|
603
|
+
else:
|
|
604
|
+
for principal in principals:
|
|
605
|
+
args["principals"] = [
|
|
606
|
+
normalize_identifiers(
|
|
607
|
+
parse_one(principal, into=exp.GrantPrincipal, dialect=self.dialect),
|
|
608
|
+
dialect=self.dialect,
|
|
609
|
+
)
|
|
610
|
+
]
|
|
611
|
+
expressions.append(dcl_cmd(**args)) # type: ignore[arg-type]
|
|
612
|
+
|
|
613
|
+
return expressions
|
|
614
|
+
|
|
615
|
+
def _apply_grants_config_expr(
|
|
616
|
+
self,
|
|
617
|
+
table: exp.Table,
|
|
618
|
+
grants_config: GrantsConfig,
|
|
619
|
+
table_type: DataObjectType = DataObjectType.TABLE,
|
|
620
|
+
) -> t.List[exp.Expression]:
|
|
621
|
+
return self._dcl_grants_config_expr(exp.Grant, table, grants_config, table_type)
|
|
622
|
+
|
|
623
|
+
def _revoke_grants_config_expr(
|
|
624
|
+
self,
|
|
625
|
+
table: exp.Table,
|
|
626
|
+
grants_config: GrantsConfig,
|
|
627
|
+
table_type: DataObjectType = DataObjectType.TABLE,
|
|
628
|
+
) -> t.List[exp.Expression]:
|
|
629
|
+
return self._dcl_grants_config_expr(exp.Revoke, table, grants_config, table_type)
|
|
630
|
+
|
|
631
|
+
def _get_grant_expression(self, table: exp.Table) -> exp.Expression:
|
|
632
|
+
schema_identifier = table.args.get("db") or normalize_identifiers(
|
|
633
|
+
exp.to_identifier(self._get_current_schema(), quoted=True), dialect=self.dialect
|
|
634
|
+
)
|
|
635
|
+
schema_name = schema_identifier.this
|
|
636
|
+
table_name = table.args.get("this").this # type: ignore
|
|
637
|
+
|
|
638
|
+
grant_conditions = [
|
|
639
|
+
exp.column("table_schema").eq(exp.Literal.string(schema_name)),
|
|
640
|
+
exp.column("table_name").eq(exp.Literal.string(table_name)),
|
|
641
|
+
exp.column("grantor").eq(self.CURRENT_USER_OR_ROLE_EXPRESSION),
|
|
642
|
+
exp.column("grantee").neq(self.CURRENT_USER_OR_ROLE_EXPRESSION),
|
|
643
|
+
]
|
|
644
|
+
|
|
645
|
+
info_schema_table = normalize_identifiers(
|
|
646
|
+
exp.table_(self.GRANT_INFORMATION_SCHEMA_TABLE_NAME, db="information_schema"),
|
|
647
|
+
dialect=self.dialect,
|
|
648
|
+
)
|
|
649
|
+
if self.USE_CATALOG_IN_GRANTS:
|
|
650
|
+
catalog_identifier = table.args.get("catalog")
|
|
651
|
+
if not catalog_identifier:
|
|
652
|
+
catalog_name = self.get_current_catalog()
|
|
653
|
+
if not catalog_name:
|
|
654
|
+
raise SQLMeshError(
|
|
655
|
+
"Current catalog could not be determined for fetching grants. This is unexpected."
|
|
656
|
+
)
|
|
657
|
+
catalog_identifier = normalize_identifiers(
|
|
658
|
+
exp.to_identifier(catalog_name, quoted=True), dialect=self.dialect
|
|
659
|
+
)
|
|
660
|
+
catalog_name = catalog_identifier.this
|
|
661
|
+
info_schema_table.set("catalog", catalog_identifier.copy())
|
|
662
|
+
grant_conditions.insert(
|
|
663
|
+
0, exp.column("table_catalog").eq(exp.Literal.string(catalog_name))
|
|
664
|
+
)
|
|
665
|
+
|
|
666
|
+
return (
|
|
667
|
+
exp.select("privilege_type", "grantee")
|
|
668
|
+
.from_(info_schema_table)
|
|
669
|
+
.where(exp.and_(*grant_conditions))
|
|
670
|
+
)
|
|
671
|
+
|
|
672
|
+
def _get_current_grants_config(self, table: exp.Table) -> GrantsConfig:
|
|
673
|
+
grant_expr = self._get_grant_expression(table)
|
|
674
|
+
|
|
675
|
+
results = self.fetchall(grant_expr)
|
|
676
|
+
|
|
677
|
+
grants_dict: GrantsConfig = {}
|
|
678
|
+
for privilege_raw, grantee_raw in results:
|
|
679
|
+
if privilege_raw is None or grantee_raw is None:
|
|
680
|
+
continue
|
|
681
|
+
|
|
682
|
+
privilege = str(privilege_raw)
|
|
683
|
+
grantee = str(grantee_raw)
|
|
684
|
+
if not privilege or not grantee:
|
|
685
|
+
continue
|
|
686
|
+
|
|
687
|
+
grantees = grants_dict.setdefault(privilege, [])
|
|
688
|
+
if grantee not in grantees:
|
|
689
|
+
grantees.append(grantee)
|
|
690
|
+
|
|
691
|
+
return grants_dict
|