sqlmesh 0.213.1.dev1__py3-none-any.whl → 0.227.2.dev4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sqlmesh/__init__.py +12 -2
- sqlmesh/_version.py +2 -2
- sqlmesh/cli/main.py +0 -44
- sqlmesh/cli/project_init.py +11 -2
- sqlmesh/core/_typing.py +1 -0
- sqlmesh/core/audit/definition.py +8 -2
- sqlmesh/core/config/__init__.py +1 -1
- sqlmesh/core/config/connection.py +17 -5
- sqlmesh/core/config/dbt.py +13 -0
- sqlmesh/core/config/janitor.py +12 -0
- sqlmesh/core/config/loader.py +7 -0
- sqlmesh/core/config/model.py +2 -0
- sqlmesh/core/config/root.py +3 -0
- sqlmesh/core/console.py +81 -3
- sqlmesh/core/constants.py +1 -1
- sqlmesh/core/context.py +69 -26
- sqlmesh/core/dialect.py +3 -0
- sqlmesh/core/engine_adapter/_typing.py +2 -0
- sqlmesh/core/engine_adapter/base.py +322 -22
- sqlmesh/core/engine_adapter/base_postgres.py +17 -1
- sqlmesh/core/engine_adapter/bigquery.py +146 -7
- sqlmesh/core/engine_adapter/clickhouse.py +17 -13
- sqlmesh/core/engine_adapter/databricks.py +33 -2
- sqlmesh/core/engine_adapter/fabric.py +10 -29
- sqlmesh/core/engine_adapter/mixins.py +142 -48
- sqlmesh/core/engine_adapter/mssql.py +15 -4
- sqlmesh/core/engine_adapter/mysql.py +2 -2
- sqlmesh/core/engine_adapter/postgres.py +9 -3
- sqlmesh/core/engine_adapter/redshift.py +4 -0
- sqlmesh/core/engine_adapter/risingwave.py +1 -0
- sqlmesh/core/engine_adapter/shared.py +6 -0
- sqlmesh/core/engine_adapter/snowflake.py +82 -11
- sqlmesh/core/engine_adapter/spark.py +14 -10
- sqlmesh/core/engine_adapter/trino.py +4 -2
- sqlmesh/core/environment.py +2 -0
- sqlmesh/core/janitor.py +181 -0
- sqlmesh/core/lineage.py +1 -0
- sqlmesh/core/linter/definition.py +13 -13
- sqlmesh/core/linter/rules/builtin.py +29 -0
- sqlmesh/core/macros.py +35 -13
- sqlmesh/core/model/common.py +2 -0
- sqlmesh/core/model/definition.py +82 -28
- sqlmesh/core/model/kind.py +66 -2
- sqlmesh/core/model/meta.py +108 -4
- sqlmesh/core/node.py +101 -1
- sqlmesh/core/plan/builder.py +18 -10
- sqlmesh/core/plan/common.py +199 -2
- sqlmesh/core/plan/definition.py +25 -6
- sqlmesh/core/plan/evaluator.py +75 -113
- sqlmesh/core/plan/explainer.py +90 -8
- sqlmesh/core/plan/stages.py +42 -21
- sqlmesh/core/renderer.py +78 -32
- sqlmesh/core/scheduler.py +102 -22
- sqlmesh/core/selector.py +137 -9
- sqlmesh/core/signal.py +64 -1
- sqlmesh/core/snapshot/__init__.py +2 -0
- sqlmesh/core/snapshot/definition.py +146 -34
- sqlmesh/core/snapshot/evaluator.py +689 -124
- sqlmesh/core/state_sync/__init__.py +0 -1
- sqlmesh/core/state_sync/base.py +55 -33
- sqlmesh/core/state_sync/cache.py +12 -7
- sqlmesh/core/state_sync/common.py +216 -111
- sqlmesh/core/state_sync/db/environment.py +6 -4
- sqlmesh/core/state_sync/db/facade.py +42 -24
- sqlmesh/core/state_sync/db/interval.py +27 -7
- sqlmesh/core/state_sync/db/migrator.py +34 -16
- sqlmesh/core/state_sync/db/snapshot.py +177 -169
- sqlmesh/core/table_diff.py +2 -2
- sqlmesh/core/test/context.py +2 -0
- sqlmesh/core/test/definition.py +14 -9
- sqlmesh/dbt/adapter.py +22 -16
- sqlmesh/dbt/basemodel.py +75 -56
- sqlmesh/dbt/builtin.py +116 -12
- sqlmesh/dbt/column.py +17 -5
- sqlmesh/dbt/common.py +19 -5
- sqlmesh/dbt/context.py +14 -1
- sqlmesh/dbt/loader.py +61 -9
- sqlmesh/dbt/manifest.py +174 -16
- sqlmesh/dbt/model.py +183 -85
- sqlmesh/dbt/package.py +16 -1
- sqlmesh/dbt/profile.py +3 -3
- sqlmesh/dbt/project.py +12 -7
- sqlmesh/dbt/seed.py +6 -1
- sqlmesh/dbt/source.py +13 -1
- sqlmesh/dbt/target.py +25 -6
- sqlmesh/dbt/test.py +36 -5
- sqlmesh/migrations/v0000_baseline.py +95 -0
- sqlmesh/migrations/v0061_mysql_fix_blob_text_type.py +5 -7
- sqlmesh/migrations/v0062_add_model_gateway.py +5 -1
- sqlmesh/migrations/v0063_change_signals.py +5 -3
- sqlmesh/migrations/v0064_join_when_matched_strings.py +5 -3
- sqlmesh/migrations/v0065_add_model_optimize.py +5 -1
- sqlmesh/migrations/v0066_add_auto_restatements.py +8 -3
- sqlmesh/migrations/v0067_add_tsql_date_full_precision.py +5 -1
- sqlmesh/migrations/v0068_include_unrendered_query_in_metadata_hash.py +5 -1
- sqlmesh/migrations/v0069_update_dev_table_suffix.py +5 -3
- sqlmesh/migrations/v0070_include_grains_in_metadata_hash.py +5 -1
- sqlmesh/migrations/v0071_add_dev_version_to_intervals.py +9 -5
- sqlmesh/migrations/v0072_add_environment_statements.py +5 -3
- sqlmesh/migrations/v0073_remove_symbolic_disable_restatement.py +5 -3
- sqlmesh/migrations/v0074_add_partition_by_time_column_property.py +5 -1
- sqlmesh/migrations/v0075_remove_validate_query.py +5 -3
- sqlmesh/migrations/v0076_add_cron_tz.py +5 -1
- sqlmesh/migrations/v0077_fix_column_type_hash_calculation.py +5 -1
- sqlmesh/migrations/v0078_warn_if_non_migratable_python_env.py +5 -3
- sqlmesh/migrations/v0079_add_gateway_managed_property.py +10 -5
- sqlmesh/migrations/v0080_add_batch_size_to_scd_type_2_models.py +5 -1
- sqlmesh/migrations/v0081_update_partitioned_by.py +5 -3
- sqlmesh/migrations/v0082_warn_if_incorrectly_duplicated_statements.py +5 -3
- sqlmesh/migrations/v0083_use_sql_for_scd_time_data_type_data_hash.py +5 -1
- sqlmesh/migrations/v0084_normalize_quote_when_matched_and_merge_filter.py +5 -1
- sqlmesh/migrations/v0085_deterministic_repr.py +5 -3
- sqlmesh/migrations/v0086_check_deterministic_bug.py +5 -3
- sqlmesh/migrations/v0087_normalize_blueprint_variables.py +5 -3
- sqlmesh/migrations/v0088_warn_about_variable_python_env_diffs.py +5 -3
- sqlmesh/migrations/v0089_add_virtual_environment_mode.py +5 -1
- sqlmesh/migrations/v0090_add_forward_only_column.py +9 -5
- sqlmesh/migrations/v0091_on_additive_change.py +5 -1
- sqlmesh/migrations/v0092_warn_about_dbt_data_type_diff.py +5 -3
- sqlmesh/migrations/v0093_use_raw_sql_in_fingerprint.py +5 -1
- sqlmesh/migrations/v0094_add_dev_version_and_fingerprint_columns.py +123 -0
- sqlmesh/migrations/v0095_warn_about_dbt_raw_sql_diff.py +49 -0
- sqlmesh/migrations/v0096_remove_plan_dags_table.py +13 -0
- sqlmesh/migrations/v0097_add_dbt_name_in_node.py +9 -0
- sqlmesh/migrations/{v0060_move_audits_to_model.py → v0098_add_dbt_node_info_in_node.py} +33 -16
- sqlmesh/migrations/v0099_add_last_altered_to_intervals.py +25 -0
- sqlmesh/migrations/v0100_add_grants_and_grants_target_layer.py +9 -0
- sqlmesh/utils/__init__.py +8 -1
- sqlmesh/utils/cache.py +5 -1
- sqlmesh/utils/connection_pool.py +2 -1
- sqlmesh/utils/dag.py +65 -10
- sqlmesh/utils/date.py +8 -1
- sqlmesh/utils/errors.py +8 -0
- sqlmesh/utils/jinja.py +54 -4
- sqlmesh/utils/pydantic.py +6 -6
- sqlmesh/utils/windows.py +13 -3
- {sqlmesh-0.213.1.dev1.dist-info → sqlmesh-0.227.2.dev4.dist-info}/METADATA +7 -10
- sqlmesh-0.227.2.dev4.dist-info/RECORD +370 -0
- sqlmesh_dbt/cli.py +70 -7
- sqlmesh_dbt/console.py +14 -6
- sqlmesh_dbt/operations.py +103 -24
- sqlmesh_dbt/selectors.py +39 -1
- web/client/dist/assets/{Audits-Ucsx1GzF.js → Audits-CBiYyyx-.js} +1 -1
- web/client/dist/assets/{Banner-BWDzvavM.js → Banner-DSRbUlO5.js} +1 -1
- web/client/dist/assets/{ChevronDownIcon-D2VL13Ah.js → ChevronDownIcon-MK_nrjD_.js} +1 -1
- web/client/dist/assets/{ChevronRightIcon-DWGYbf1l.js → ChevronRightIcon-CLWtT22Q.js} +1 -1
- web/client/dist/assets/{Content-DdHDZM3I.js → Content-BNuGZN5l.js} +1 -1
- web/client/dist/assets/{Content-Bikfy8fh.js → Content-CSHJyW0n.js} +1 -1
- web/client/dist/assets/{Data-CzAJH7rW.js → Data-C1oRDbLx.js} +1 -1
- web/client/dist/assets/{DataCatalog-BJF11g8f.js → DataCatalog-HXyX2-_j.js} +1 -1
- web/client/dist/assets/{Editor-s0SBpV2y.js → Editor-BDyfpUuw.js} +1 -1
- web/client/dist/assets/{Editor-DgLhgKnm.js → Editor-D0jNItwC.js} +1 -1
- web/client/dist/assets/{Errors-D0m0O1d3.js → Errors-BfuFLcPi.js} +1 -1
- web/client/dist/assets/{FileExplorer-CEv0vXkt.js → FileExplorer-BR9IE3he.js} +1 -1
- web/client/dist/assets/{Footer-BwzXn8Ew.js → Footer-CgBEtiAh.js} +1 -1
- web/client/dist/assets/{Header-6heDkEqG.js → Header-DSqR6nSO.js} +1 -1
- web/client/dist/assets/{Input-obuJsD6k.js → Input-B-oZ6fGO.js} +1 -1
- web/client/dist/assets/Lineage-DYQVwDbD.js +1 -0
- web/client/dist/assets/{ListboxShow-HM9_qyrt.js → ListboxShow-BE5-xevs.js} +1 -1
- web/client/dist/assets/{ModelLineage-zWdKo0U2.js → ModelLineage-DkIFAYo4.js} +1 -1
- web/client/dist/assets/{Models-Bcu66SRz.js → Models-D5dWr8RB.js} +1 -1
- web/client/dist/assets/{Page-BWEEQfIt.js → Page-C-XfU5BR.js} +1 -1
- web/client/dist/assets/{Plan-C4gXCqlf.js → Plan-ZEuTINBq.js} +1 -1
- web/client/dist/assets/{PlusCircleIcon-CVDO651q.js → PlusCircleIcon-DVXAHG8_.js} +1 -1
- web/client/dist/assets/{ReportErrors-BT6xFwAr.js → ReportErrors-B7FEPzMB.js} +1 -1
- web/client/dist/assets/{Root-ryJoBK4h.js → Root-8aZyhPxF.js} +1 -1
- web/client/dist/assets/{SearchList-DB04sPb9.js → SearchList-W_iT2G82.js} +1 -1
- web/client/dist/assets/{SelectEnvironment-CUYcXUu6.js → SelectEnvironment-C65jALmO.js} +1 -1
- web/client/dist/assets/{SourceList-Doo_9ZGp.js → SourceList-DSLO6nVJ.js} +1 -1
- web/client/dist/assets/{SourceListItem-D5Mj7Dly.js → SourceListItem-BHt8d9-I.js} +1 -1
- web/client/dist/assets/{SplitPane-qHmkD1qy.js → SplitPane-CViaZmw6.js} +1 -1
- web/client/dist/assets/{Tests-DH1Z74ML.js → Tests-DhaVt5t1.js} +1 -1
- web/client/dist/assets/{Welcome-DqUJUNMF.js → Welcome-DvpjH-_4.js} +1 -1
- web/client/dist/assets/context-BctCsyGb.js +71 -0
- web/client/dist/assets/{context-Dr54UHLi.js → context-DFNeGsFF.js} +1 -1
- web/client/dist/assets/{editor-DYIP1yQ4.js → editor-CcO28cqd.js} +1 -1
- web/client/dist/assets/{file-DarlIDVi.js → file-CvJN3aZO.js} +1 -1
- web/client/dist/assets/{floating-ui.react-dom-BH3TFvkM.js → floating-ui.react-dom-CjE-JNW1.js} +1 -1
- web/client/dist/assets/{help-Bl8wqaQc.js → help-DuPhjipa.js} +1 -1
- web/client/dist/assets/{index-D1sR7wpN.js → index-C-dJH7yZ.js} +1 -1
- web/client/dist/assets/{index-O3mjYpnE.js → index-Dj0i1-CA.js} +2 -2
- web/client/dist/assets/{plan-CehRrJUG.js → plan-BTRSbjKn.js} +1 -1
- web/client/dist/assets/{popover-CqgMRE0G.js → popover-_Sf0yvOI.js} +1 -1
- web/client/dist/assets/{project-6gxepOhm.js → project-BvSOI8MY.js} +1 -1
- web/client/dist/index.html +1 -1
- sqlmesh/integrations/llm.py +0 -56
- sqlmesh/migrations/v0001_init.py +0 -60
- sqlmesh/migrations/v0002_remove_identify.py +0 -5
- sqlmesh/migrations/v0003_move_batch_size.py +0 -34
- sqlmesh/migrations/v0004_environmnent_add_finalized_at.py +0 -23
- sqlmesh/migrations/v0005_create_seed_table.py +0 -24
- sqlmesh/migrations/v0006_change_seed_hash.py +0 -5
- sqlmesh/migrations/v0007_env_table_info_to_kind.py +0 -99
- sqlmesh/migrations/v0008_create_intervals_table.py +0 -38
- sqlmesh/migrations/v0009_remove_pre_post_hooks.py +0 -62
- sqlmesh/migrations/v0010_seed_hash_batch_size.py +0 -5
- sqlmesh/migrations/v0011_add_model_kind_name.py +0 -63
- sqlmesh/migrations/v0012_update_jinja_expressions.py +0 -86
- sqlmesh/migrations/v0013_serde_using_model_dialects.py +0 -87
- sqlmesh/migrations/v0014_fix_dev_intervals.py +0 -14
- sqlmesh/migrations/v0015_environment_add_promoted_snapshot_ids.py +0 -26
- sqlmesh/migrations/v0016_fix_windows_path.py +0 -59
- sqlmesh/migrations/v0017_fix_windows_seed_path.py +0 -55
- sqlmesh/migrations/v0018_rename_snapshot_model_to_node.py +0 -53
- sqlmesh/migrations/v0019_add_env_suffix_target.py +0 -28
- sqlmesh/migrations/v0020_remove_redundant_attributes_from_dbt_models.py +0 -80
- sqlmesh/migrations/v0021_fix_table_properties.py +0 -62
- sqlmesh/migrations/v0022_move_project_to_model.py +0 -54
- sqlmesh/migrations/v0023_fix_added_models_with_forward_only_parents.py +0 -65
- sqlmesh/migrations/v0024_replace_model_kind_name_enum_with_value.py +0 -55
- sqlmesh/migrations/v0025_fix_intervals_and_missing_change_category.py +0 -117
- sqlmesh/migrations/v0026_remove_dialect_from_seed.py +0 -55
- sqlmesh/migrations/v0027_minute_interval_to_five.py +0 -57
- sqlmesh/migrations/v0028_add_plan_dags_table.py +0 -29
- sqlmesh/migrations/v0029_generate_schema_types_using_dialect.py +0 -69
- sqlmesh/migrations/v0030_update_unrestorable_snapshots.py +0 -65
- sqlmesh/migrations/v0031_remove_dbt_target_fields.py +0 -65
- sqlmesh/migrations/v0032_add_sqlmesh_version.py +0 -25
- sqlmesh/migrations/v0033_mysql_fix_blob_text_type.py +0 -45
- sqlmesh/migrations/v0034_add_default_catalog.py +0 -367
- sqlmesh/migrations/v0035_add_catalog_name_override.py +0 -22
- sqlmesh/migrations/v0036_delete_plan_dags_bug_fix.py +0 -14
- sqlmesh/migrations/v0037_remove_dbt_is_incremental_macro.py +0 -61
- sqlmesh/migrations/v0038_add_expiration_ts_to_snapshot.py +0 -73
- sqlmesh/migrations/v0039_include_environment_in_plan_dag_spec.py +0 -68
- sqlmesh/migrations/v0040_add_previous_finalized_snapshots.py +0 -26
- sqlmesh/migrations/v0041_remove_hash_raw_query_attribute.py +0 -59
- sqlmesh/migrations/v0042_trim_indirect_versions.py +0 -66
- sqlmesh/migrations/v0043_fix_remove_obsolete_attributes_in_plan_dags.py +0 -61
- sqlmesh/migrations/v0044_quote_identifiers_in_model_attributes.py +0 -5
- sqlmesh/migrations/v0045_move_gateway_variable.py +0 -70
- sqlmesh/migrations/v0046_add_batch_concurrency.py +0 -8
- sqlmesh/migrations/v0047_change_scd_string_to_column.py +0 -5
- sqlmesh/migrations/v0048_drop_indirect_versions.py +0 -59
- sqlmesh/migrations/v0049_replace_identifier_with_version_in_seeds_table.py +0 -57
- sqlmesh/migrations/v0050_drop_seeds_table.py +0 -11
- sqlmesh/migrations/v0051_rename_column_descriptions.py +0 -65
- sqlmesh/migrations/v0052_add_normalize_name_in_environment_naming_info.py +0 -28
- sqlmesh/migrations/v0053_custom_model_kind_extra_attributes.py +0 -5
- sqlmesh/migrations/v0054_fix_trailing_comments.py +0 -5
- sqlmesh/migrations/v0055_add_updated_ts_unpaused_ts_ttl_ms_unrestorable_to_snapshot.py +0 -132
- sqlmesh/migrations/v0056_restore_table_indexes.py +0 -118
- sqlmesh/migrations/v0057_add_table_format.py +0 -5
- sqlmesh/migrations/v0058_add_requirements.py +0 -26
- sqlmesh/migrations/v0059_add_physical_version.py +0 -5
- sqlmesh-0.213.1.dev1.dist-info/RECORD +0 -421
- web/client/dist/assets/Lineage-D0Hgdz2v.js +0 -1
- web/client/dist/assets/context-DgX0fp2E.js +0 -68
- {sqlmesh-0.213.1.dev1.dist-info → sqlmesh-0.227.2.dev4.dist-info}/WHEEL +0 -0
- {sqlmesh-0.213.1.dev1.dist-info → sqlmesh-0.227.2.dev4.dist-info}/entry_points.txt +0 -0
- {sqlmesh-0.213.1.dev1.dist-info → sqlmesh-0.227.2.dev4.dist-info}/licenses/LICENSE +0 -0
- {sqlmesh-0.213.1.dev1.dist-info → sqlmesh-0.227.2.dev4.dist-info}/top_level.txt +0 -0
|
@@ -32,12 +32,14 @@ from functools import reduce
|
|
|
32
32
|
|
|
33
33
|
from sqlglot import exp, select
|
|
34
34
|
from sqlglot.executor import execute
|
|
35
|
+
from tenacity import retry, stop_after_attempt, wait_exponential, retry_if_not_exception_type
|
|
35
36
|
|
|
36
37
|
from sqlmesh.core import constants as c
|
|
37
38
|
from sqlmesh.core import dialect as d
|
|
38
39
|
from sqlmesh.core.audit import Audit, StandaloneAudit
|
|
39
40
|
from sqlmesh.core.dialect import schema_
|
|
40
|
-
from sqlmesh.core.engine_adapter.shared import InsertOverwriteStrategy, DataObjectType
|
|
41
|
+
from sqlmesh.core.engine_adapter.shared import InsertOverwriteStrategy, DataObjectType, DataObject
|
|
42
|
+
from sqlmesh.core.model.meta import GrantsTargetLayer
|
|
41
43
|
from sqlmesh.core.macros import RuntimeStage
|
|
42
44
|
from sqlmesh.core.model import (
|
|
43
45
|
AuditResult,
|
|
@@ -49,8 +51,8 @@ from sqlmesh.core.model import (
|
|
|
49
51
|
ViewKind,
|
|
50
52
|
CustomKind,
|
|
51
53
|
)
|
|
52
|
-
from sqlmesh.core.model.kind import _Incremental
|
|
53
|
-
from sqlmesh.utils import CompletionStatus
|
|
54
|
+
from sqlmesh.core.model.kind import _Incremental, DbtCustomKind
|
|
55
|
+
from sqlmesh.utils import CompletionStatus, columns_to_types_all_known
|
|
54
56
|
from sqlmesh.core.schema_diff import (
|
|
55
57
|
has_drop_alteration,
|
|
56
58
|
TableAlterOperation,
|
|
@@ -66,7 +68,7 @@ from sqlmesh.core.snapshot import (
|
|
|
66
68
|
SnapshotTableCleanupTask,
|
|
67
69
|
)
|
|
68
70
|
from sqlmesh.core.snapshot.execution_tracker import QueryExecutionTracker
|
|
69
|
-
from sqlmesh.utils import random_id, CorrelationId
|
|
71
|
+
from sqlmesh.utils import random_id, CorrelationId, AttributeDict
|
|
70
72
|
from sqlmesh.utils.concurrency import (
|
|
71
73
|
concurrent_apply_to_snapshots,
|
|
72
74
|
concurrent_apply_to_values,
|
|
@@ -76,11 +78,13 @@ from sqlmesh.utils.date import TimeLike, now, time_like_to_str
|
|
|
76
78
|
from sqlmesh.utils.errors import (
|
|
77
79
|
ConfigError,
|
|
78
80
|
DestructiveChangeError,
|
|
81
|
+
MigrationNotSupportedError,
|
|
79
82
|
SQLMeshError,
|
|
80
83
|
format_destructive_change_msg,
|
|
81
84
|
format_additive_change_msg,
|
|
82
85
|
AdditiveChangeError,
|
|
83
86
|
)
|
|
87
|
+
from sqlmesh.utils.jinja import MacroReturnVal
|
|
84
88
|
|
|
85
89
|
if sys.version_info >= (3, 12):
|
|
86
90
|
from importlib import metadata
|
|
@@ -304,6 +308,9 @@ class SnapshotEvaluator:
|
|
|
304
308
|
]
|
|
305
309
|
self._create_schemas(gateway_table_pairs=gateway_table_pairs)
|
|
306
310
|
|
|
311
|
+
# Fetch the view data objects for the promoted snapshots to get them cached
|
|
312
|
+
self._get_virtual_data_objects(target_snapshots, environment_naming_info)
|
|
313
|
+
|
|
307
314
|
deployability_index = deployability_index or DeployabilityIndex.all_deployable()
|
|
308
315
|
with self.concurrent_context():
|
|
309
316
|
concurrent_apply_to_snapshots(
|
|
@@ -422,50 +429,16 @@ class SnapshotEvaluator:
|
|
|
422
429
|
target_snapshots: Target snapshots.
|
|
423
430
|
deployability_index: Determines snapshots that are deployable / representative in the context of this creation.
|
|
424
431
|
"""
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
defaultdict(lambda: defaultdict(set))
|
|
432
|
+
existing_data_objects = self._get_physical_data_objects(
|
|
433
|
+
target_snapshots, deployability_index
|
|
428
434
|
)
|
|
429
|
-
|
|
435
|
+
snapshots_to_create = []
|
|
430
436
|
for snapshot in target_snapshots:
|
|
431
437
|
if not snapshot.is_model or snapshot.is_symbolic:
|
|
432
438
|
continue
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
table_schema = d.schema_(table.db, catalog=table.catalog)
|
|
437
|
-
tables_by_gateway_and_schema[snapshot.model_gateway][table_schema].add(table.name)
|
|
438
|
-
|
|
439
|
-
def _get_data_objects(
|
|
440
|
-
schema: exp.Table,
|
|
441
|
-
object_names: t.Optional[t.Set[str]] = None,
|
|
442
|
-
gateway: t.Optional[str] = None,
|
|
443
|
-
) -> t.Set[str]:
|
|
444
|
-
logger.info("Listing data objects in schema %s", schema.sql())
|
|
445
|
-
objs = self.get_adapter(gateway).get_data_objects(schema, object_names)
|
|
446
|
-
return {obj.name for obj in objs}
|
|
447
|
-
|
|
448
|
-
with self.concurrent_context():
|
|
449
|
-
existing_objects: t.Set[str] = set()
|
|
450
|
-
# A schema can be shared across multiple engines, so we need to group tables by both gateway and schema
|
|
451
|
-
for gateway, tables_by_schema in tables_by_gateway_and_schema.items():
|
|
452
|
-
objs_for_gateway = {
|
|
453
|
-
obj
|
|
454
|
-
for objs in concurrent_apply_to_values(
|
|
455
|
-
list(tables_by_schema),
|
|
456
|
-
lambda s: _get_data_objects(
|
|
457
|
-
schema=s, object_names=tables_by_schema.get(s), gateway=gateway
|
|
458
|
-
),
|
|
459
|
-
self.ddl_concurrent_tasks,
|
|
460
|
-
)
|
|
461
|
-
for obj in objs
|
|
462
|
-
}
|
|
463
|
-
existing_objects.update(objs_for_gateway)
|
|
464
|
-
|
|
465
|
-
snapshots_to_create = []
|
|
466
|
-
for snapshot, table_names in snapshots_with_table_names.items():
|
|
467
|
-
missing_tables = table_names - existing_objects
|
|
468
|
-
if missing_tables or (snapshot.is_seed and not snapshot.intervals):
|
|
439
|
+
if snapshot.snapshot_id not in existing_data_objects or (
|
|
440
|
+
snapshot.is_seed and not snapshot.intervals
|
|
441
|
+
):
|
|
469
442
|
snapshots_to_create.append(snapshot)
|
|
470
443
|
|
|
471
444
|
return snapshots_to_create
|
|
@@ -514,16 +487,25 @@ class SnapshotEvaluator:
|
|
|
514
487
|
allow_additive_snapshots: Set of snapshots that are allowed to have additive schema changes.
|
|
515
488
|
deployability_index: Determines snapshots that are deployable in the context of this evaluation.
|
|
516
489
|
"""
|
|
490
|
+
deployability_index = deployability_index or DeployabilityIndex.all_deployable()
|
|
491
|
+
target_data_objects = self._get_physical_data_objects(target_snapshots, deployability_index)
|
|
492
|
+
if not target_data_objects:
|
|
493
|
+
return
|
|
494
|
+
|
|
495
|
+
if not snapshots:
|
|
496
|
+
snapshots = {s.snapshot_id: s for s in target_snapshots}
|
|
497
|
+
|
|
517
498
|
allow_destructive_snapshots = allow_destructive_snapshots or set()
|
|
518
499
|
allow_additive_snapshots = allow_additive_snapshots or set()
|
|
519
|
-
deployability_index = deployability_index or DeployabilityIndex.all_deployable()
|
|
520
500
|
snapshots_by_name = {s.name: s for s in snapshots.values()}
|
|
521
501
|
with self.concurrent_context():
|
|
502
|
+
# Only migrate snapshots for which there's an existing data object
|
|
522
503
|
concurrent_apply_to_snapshots(
|
|
523
504
|
target_snapshots,
|
|
524
505
|
lambda s: self._migrate_snapshot(
|
|
525
506
|
s,
|
|
526
507
|
snapshots_by_name,
|
|
508
|
+
target_data_objects.get(s.snapshot_id),
|
|
527
509
|
allow_destructive_snapshots,
|
|
528
510
|
allow_additive_snapshots,
|
|
529
511
|
self.get_adapter(s.model_gateway),
|
|
@@ -543,10 +525,12 @@ class SnapshotEvaluator:
|
|
|
543
525
|
target_snapshots: Snapshots to cleanup.
|
|
544
526
|
on_complete: A callback to call on each successfully deleted database object.
|
|
545
527
|
"""
|
|
528
|
+
target_snapshots = [
|
|
529
|
+
t for t in target_snapshots if t.snapshot.is_model and not t.snapshot.is_symbolic
|
|
530
|
+
]
|
|
546
531
|
snapshots_to_dev_table_only = {
|
|
547
532
|
t.snapshot.snapshot_id: t.dev_table_only for t in target_snapshots
|
|
548
533
|
}
|
|
549
|
-
|
|
550
534
|
with self.concurrent_context():
|
|
551
535
|
concurrent_apply_to_snapshots(
|
|
552
536
|
[t.snapshot for t in target_snapshots],
|
|
@@ -766,33 +750,51 @@ class SnapshotEvaluator:
|
|
|
766
750
|
**render_statements_kwargs
|
|
767
751
|
)
|
|
768
752
|
|
|
753
|
+
evaluation_strategy = _evaluation_strategy(snapshot, adapter)
|
|
754
|
+
evaluation_strategy.run_pre_statements(
|
|
755
|
+
snapshot=snapshot,
|
|
756
|
+
render_kwargs={**render_statements_kwargs, "inside_transaction": False},
|
|
757
|
+
)
|
|
758
|
+
|
|
769
759
|
with (
|
|
770
760
|
adapter.transaction(),
|
|
771
761
|
adapter.session(snapshot.model.render_session_properties(**render_statements_kwargs)),
|
|
772
762
|
):
|
|
773
|
-
|
|
763
|
+
evaluation_strategy.run_pre_statements(
|
|
764
|
+
snapshot=snapshot,
|
|
765
|
+
render_kwargs={**render_statements_kwargs, "inside_transaction": True},
|
|
766
|
+
)
|
|
774
767
|
|
|
775
768
|
if not target_table_exists or (model.is_seed and not snapshot.intervals):
|
|
769
|
+
# Only create the empty table if the columns were provided explicitly by the user
|
|
770
|
+
should_create_empty_table = (
|
|
771
|
+
model.kind.is_materialized
|
|
772
|
+
and model.columns_to_types_
|
|
773
|
+
and columns_to_types_all_known(model.columns_to_types_)
|
|
774
|
+
)
|
|
775
|
+
if not should_create_empty_table:
|
|
776
|
+
# Or if the model is self-referential and its query is fully annotated with types
|
|
777
|
+
should_create_empty_table = model.depends_on_self and model.annotated
|
|
776
778
|
if self._can_clone(snapshot, deployability_index):
|
|
777
779
|
self._clone_snapshot_in_dev(
|
|
778
780
|
snapshot=snapshot,
|
|
779
781
|
snapshots=snapshots,
|
|
780
782
|
deployability_index=deployability_index,
|
|
781
783
|
render_kwargs=create_render_kwargs,
|
|
782
|
-
rendered_physical_properties=rendered_physical_properties,
|
|
784
|
+
rendered_physical_properties=rendered_physical_properties.copy(),
|
|
783
785
|
allow_destructive_snapshots=allow_destructive_snapshots,
|
|
784
786
|
allow_additive_snapshots=allow_additive_snapshots,
|
|
785
787
|
)
|
|
786
788
|
runtime_stage = RuntimeStage.EVALUATING
|
|
787
789
|
target_table_exists = True
|
|
788
|
-
elif
|
|
790
|
+
elif should_create_empty_table or model.is_seed or model.kind.is_scd_type_2:
|
|
789
791
|
self._execute_create(
|
|
790
792
|
snapshot=snapshot,
|
|
791
793
|
table_name=target_table_name,
|
|
792
794
|
is_table_deployable=is_snapshot_deployable,
|
|
793
795
|
deployability_index=deployability_index,
|
|
794
796
|
create_render_kwargs=create_render_kwargs,
|
|
795
|
-
rendered_physical_properties=rendered_physical_properties,
|
|
797
|
+
rendered_physical_properties=rendered_physical_properties.copy(),
|
|
796
798
|
dry_run=False,
|
|
797
799
|
run_pre_post_statements=False,
|
|
798
800
|
)
|
|
@@ -809,6 +811,7 @@ class SnapshotEvaluator:
|
|
|
809
811
|
if (
|
|
810
812
|
snapshot.is_materialized
|
|
811
813
|
and target_table_exists
|
|
814
|
+
and adapter.wap_enabled
|
|
812
815
|
and (model.wap_supported or adapter.wap_supported(target_table_name))
|
|
813
816
|
):
|
|
814
817
|
wap_id = random_id()[0:8]
|
|
@@ -830,9 +833,17 @@ class SnapshotEvaluator:
|
|
|
830
833
|
batch_index=batch_index,
|
|
831
834
|
)
|
|
832
835
|
|
|
833
|
-
|
|
836
|
+
evaluation_strategy.run_post_statements(
|
|
837
|
+
snapshot=snapshot,
|
|
838
|
+
render_kwargs={**render_statements_kwargs, "inside_transaction": True},
|
|
839
|
+
)
|
|
840
|
+
|
|
841
|
+
evaluation_strategy.run_post_statements(
|
|
842
|
+
snapshot=snapshot,
|
|
843
|
+
render_kwargs={**render_statements_kwargs, "inside_transaction": False},
|
|
844
|
+
)
|
|
834
845
|
|
|
835
|
-
|
|
846
|
+
return wap_id
|
|
836
847
|
|
|
837
848
|
def create_snapshot(
|
|
838
849
|
self,
|
|
@@ -866,6 +877,11 @@ class SnapshotEvaluator:
|
|
|
866
877
|
deployability_index=deployability_index,
|
|
867
878
|
)
|
|
868
879
|
|
|
880
|
+
evaluation_strategy = _evaluation_strategy(snapshot, adapter)
|
|
881
|
+
evaluation_strategy.run_pre_statements(
|
|
882
|
+
snapshot=snapshot, render_kwargs={**create_render_kwargs, "inside_transaction": False}
|
|
883
|
+
)
|
|
884
|
+
|
|
869
885
|
with (
|
|
870
886
|
adapter.transaction(),
|
|
871
887
|
adapter.session(snapshot.model.render_session_properties(**create_render_kwargs)),
|
|
@@ -883,6 +899,7 @@ class SnapshotEvaluator:
|
|
|
883
899
|
rendered_physical_properties=rendered_physical_properties,
|
|
884
900
|
allow_destructive_snapshots=allow_destructive_snapshots,
|
|
885
901
|
allow_additive_snapshots=allow_additive_snapshots,
|
|
902
|
+
run_pre_post_statements=True,
|
|
886
903
|
)
|
|
887
904
|
else:
|
|
888
905
|
is_table_deployable = deployability_index.is_deployable(snapshot)
|
|
@@ -896,6 +913,10 @@ class SnapshotEvaluator:
|
|
|
896
913
|
dry_run=True,
|
|
897
914
|
)
|
|
898
915
|
|
|
916
|
+
evaluation_strategy.run_post_statements(
|
|
917
|
+
snapshot=snapshot, render_kwargs={**create_render_kwargs, "inside_transaction": False}
|
|
918
|
+
)
|
|
919
|
+
|
|
899
920
|
if on_complete is not None:
|
|
900
921
|
on_complete(snapshot)
|
|
901
922
|
|
|
@@ -933,6 +954,7 @@ class SnapshotEvaluator:
|
|
|
933
954
|
model = snapshot.model
|
|
934
955
|
adapter = self.get_adapter(model.gateway)
|
|
935
956
|
evaluation_strategy = _evaluation_strategy(snapshot, adapter)
|
|
957
|
+
is_snapshot_deployable = deployability_index.is_deployable(snapshot)
|
|
936
958
|
|
|
937
959
|
queries_or_dfs = self._render_snapshot_for_evaluation(
|
|
938
960
|
snapshot,
|
|
@@ -956,6 +978,7 @@ class SnapshotEvaluator:
|
|
|
956
978
|
execution_time=execution_time,
|
|
957
979
|
physical_properties=rendered_physical_properties,
|
|
958
980
|
render_kwargs=create_render_kwargs,
|
|
981
|
+
is_snapshot_deployable=is_snapshot_deployable,
|
|
959
982
|
)
|
|
960
983
|
else:
|
|
961
984
|
logger.info(
|
|
@@ -978,6 +1001,7 @@ class SnapshotEvaluator:
|
|
|
978
1001
|
execution_time=execution_time,
|
|
979
1002
|
physical_properties=rendered_physical_properties,
|
|
980
1003
|
render_kwargs=create_render_kwargs,
|
|
1004
|
+
is_snapshot_deployable=is_snapshot_deployable,
|
|
981
1005
|
)
|
|
982
1006
|
|
|
983
1007
|
# DataFrames, unlike SQL expressions, can provide partial results by yielding dataframes. As a result,
|
|
@@ -997,6 +1021,11 @@ class SnapshotEvaluator:
|
|
|
997
1021
|
):
|
|
998
1022
|
import pandas as pd
|
|
999
1023
|
|
|
1024
|
+
try:
|
|
1025
|
+
first_query_or_df = next(queries_or_dfs)
|
|
1026
|
+
except StopIteration:
|
|
1027
|
+
return
|
|
1028
|
+
|
|
1000
1029
|
query_or_df = reduce(
|
|
1001
1030
|
lambda a, b: (
|
|
1002
1031
|
pd.concat([a, b], ignore_index=True) # type: ignore
|
|
@@ -1004,6 +1033,7 @@ class SnapshotEvaluator:
|
|
|
1004
1033
|
else a.union_all(b) # type: ignore
|
|
1005
1034
|
), # type: ignore
|
|
1006
1035
|
queries_or_dfs,
|
|
1036
|
+
first_query_or_df,
|
|
1007
1037
|
)
|
|
1008
1038
|
apply(query_or_df, index=0)
|
|
1009
1039
|
else:
|
|
@@ -1042,6 +1072,7 @@ class SnapshotEvaluator:
|
|
|
1042
1072
|
rendered_physical_properties: t.Dict[str, exp.Expression],
|
|
1043
1073
|
allow_destructive_snapshots: t.Set[str],
|
|
1044
1074
|
allow_additive_snapshots: t.Set[str],
|
|
1075
|
+
run_pre_post_statements: bool = False,
|
|
1045
1076
|
) -> None:
|
|
1046
1077
|
adapter = self.get_adapter(snapshot.model.gateway)
|
|
1047
1078
|
|
|
@@ -1053,7 +1084,6 @@ class SnapshotEvaluator:
|
|
|
1053
1084
|
adapter.clone_table(
|
|
1054
1085
|
target_table_name,
|
|
1055
1086
|
snapshot.table_name(),
|
|
1056
|
-
replace=True,
|
|
1057
1087
|
rendered_physical_properties=rendered_physical_properties,
|
|
1058
1088
|
)
|
|
1059
1089
|
self._migrate_target_table(
|
|
@@ -1065,7 +1095,9 @@ class SnapshotEvaluator:
|
|
|
1065
1095
|
rendered_physical_properties=rendered_physical_properties,
|
|
1066
1096
|
allow_destructive_snapshots=allow_destructive_snapshots,
|
|
1067
1097
|
allow_additive_snapshots=allow_additive_snapshots,
|
|
1098
|
+
run_pre_post_statements=run_pre_post_statements,
|
|
1068
1099
|
)
|
|
1100
|
+
|
|
1069
1101
|
except Exception:
|
|
1070
1102
|
adapter.drop_table(target_table_name)
|
|
1071
1103
|
raise
|
|
@@ -1074,12 +1106,13 @@ class SnapshotEvaluator:
|
|
|
1074
1106
|
self,
|
|
1075
1107
|
snapshot: Snapshot,
|
|
1076
1108
|
snapshots: t.Dict[str, Snapshot],
|
|
1109
|
+
target_data_object: t.Optional[DataObject],
|
|
1077
1110
|
allow_destructive_snapshots: t.Set[str],
|
|
1078
1111
|
allow_additive_snapshots: t.Set[str],
|
|
1079
1112
|
adapter: EngineAdapter,
|
|
1080
1113
|
deployability_index: DeployabilityIndex,
|
|
1081
1114
|
) -> None:
|
|
1082
|
-
if not snapshot.
|
|
1115
|
+
if not snapshot.is_model or snapshot.is_symbolic:
|
|
1083
1116
|
return
|
|
1084
1117
|
|
|
1085
1118
|
deployability_index = DeployabilityIndex.all_deployable()
|
|
@@ -1091,17 +1124,25 @@ class SnapshotEvaluator:
|
|
|
1091
1124
|
)
|
|
1092
1125
|
target_table_name = snapshot.table_name()
|
|
1093
1126
|
|
|
1127
|
+
evaluation_strategy = _evaluation_strategy(snapshot, adapter)
|
|
1128
|
+
evaluation_strategy.run_pre_statements(
|
|
1129
|
+
snapshot=snapshot, render_kwargs={**render_kwargs, "inside_transaction": False}
|
|
1130
|
+
)
|
|
1131
|
+
|
|
1094
1132
|
with (
|
|
1095
1133
|
adapter.transaction(),
|
|
1096
1134
|
adapter.session(snapshot.model.render_session_properties(**render_kwargs)),
|
|
1097
1135
|
):
|
|
1098
|
-
target_data_object = adapter.get_data_object(target_table_name)
|
|
1099
1136
|
table_exists = target_data_object is not None
|
|
1100
1137
|
if adapter.drop_data_object_on_type_mismatch(
|
|
1101
1138
|
target_data_object, _snapshot_to_data_object_type(snapshot)
|
|
1102
1139
|
):
|
|
1103
1140
|
table_exists = False
|
|
1104
1141
|
|
|
1142
|
+
rendered_physical_properties = snapshot.model.render_physical_properties(
|
|
1143
|
+
**render_kwargs
|
|
1144
|
+
)
|
|
1145
|
+
|
|
1105
1146
|
if table_exists:
|
|
1106
1147
|
self._migrate_target_table(
|
|
1107
1148
|
target_table_name=target_table_name,
|
|
@@ -1109,14 +1150,35 @@ class SnapshotEvaluator:
|
|
|
1109
1150
|
snapshots=snapshots,
|
|
1110
1151
|
deployability_index=deployability_index,
|
|
1111
1152
|
render_kwargs=render_kwargs,
|
|
1112
|
-
rendered_physical_properties=
|
|
1113
|
-
**render_kwargs
|
|
1114
|
-
),
|
|
1153
|
+
rendered_physical_properties=rendered_physical_properties,
|
|
1115
1154
|
allow_destructive_snapshots=allow_destructive_snapshots,
|
|
1116
1155
|
allow_additive_snapshots=allow_additive_snapshots,
|
|
1117
1156
|
run_pre_post_statements=True,
|
|
1118
1157
|
)
|
|
1158
|
+
else:
|
|
1159
|
+
self._execute_create(
|
|
1160
|
+
snapshot=snapshot,
|
|
1161
|
+
table_name=snapshot.table_name(is_deployable=True),
|
|
1162
|
+
is_table_deployable=True,
|
|
1163
|
+
deployability_index=deployability_index,
|
|
1164
|
+
create_render_kwargs=render_kwargs,
|
|
1165
|
+
rendered_physical_properties=rendered_physical_properties,
|
|
1166
|
+
dry_run=True,
|
|
1167
|
+
)
|
|
1119
1168
|
|
|
1169
|
+
evaluation_strategy.run_post_statements(
|
|
1170
|
+
snapshot=snapshot, render_kwargs={**render_kwargs, "inside_transaction": False}
|
|
1171
|
+
)
|
|
1172
|
+
|
|
1173
|
+
# Retry in case when the table is migrated concurrently from another plan application
|
|
1174
|
+
@retry(
|
|
1175
|
+
reraise=True,
|
|
1176
|
+
stop=stop_after_attempt(5),
|
|
1177
|
+
wait=wait_exponential(min=1, max=16),
|
|
1178
|
+
retry=retry_if_not_exception_type(
|
|
1179
|
+
(DestructiveChangeError, AdditiveChangeError, MigrationNotSupportedError)
|
|
1180
|
+
),
|
|
1181
|
+
)
|
|
1120
1182
|
def _migrate_target_table(
|
|
1121
1183
|
self,
|
|
1122
1184
|
target_table_name: str,
|
|
@@ -1131,7 +1193,10 @@ class SnapshotEvaluator:
|
|
|
1131
1193
|
) -> None:
|
|
1132
1194
|
adapter = self.get_adapter(snapshot.model.gateway)
|
|
1133
1195
|
|
|
1134
|
-
|
|
1196
|
+
tmp_table = exp.to_table(target_table_name)
|
|
1197
|
+
tmp_table.this.set("this", f"{tmp_table.name}_schema_tmp")
|
|
1198
|
+
tmp_table_name = tmp_table.sql()
|
|
1199
|
+
|
|
1135
1200
|
if snapshot.is_materialized:
|
|
1136
1201
|
self._execute_create(
|
|
1137
1202
|
snapshot=snapshot,
|
|
@@ -1142,6 +1207,7 @@ class SnapshotEvaluator:
|
|
|
1142
1207
|
rendered_physical_properties=rendered_physical_properties,
|
|
1143
1208
|
dry_run=False,
|
|
1144
1209
|
run_pre_post_statements=run_pre_post_statements,
|
|
1210
|
+
skip_grants=True, # skip grants for tmp table
|
|
1145
1211
|
)
|
|
1146
1212
|
try:
|
|
1147
1213
|
evaluation_strategy = _evaluation_strategy(snapshot, adapter)
|
|
@@ -1159,6 +1225,7 @@ class SnapshotEvaluator:
|
|
|
1159
1225
|
allow_additive_snapshots=allow_additive_snapshots,
|
|
1160
1226
|
ignore_destructive=snapshot.model.on_destructive_change.is_ignore,
|
|
1161
1227
|
ignore_additive=snapshot.model.on_additive_change.is_ignore,
|
|
1228
|
+
deployability_index=deployability_index,
|
|
1162
1229
|
)
|
|
1163
1230
|
finally:
|
|
1164
1231
|
if snapshot.is_materialized:
|
|
@@ -1208,6 +1275,7 @@ class SnapshotEvaluator:
|
|
|
1208
1275
|
model=snapshot.model,
|
|
1209
1276
|
environment=environment_naming_info.name,
|
|
1210
1277
|
snapshots=snapshots,
|
|
1278
|
+
snapshot=snapshot,
|
|
1211
1279
|
**render_kwargs,
|
|
1212
1280
|
)
|
|
1213
1281
|
|
|
@@ -1407,6 +1475,7 @@ class SnapshotEvaluator:
|
|
|
1407
1475
|
rendered_physical_properties: t.Dict[str, exp.Expression],
|
|
1408
1476
|
dry_run: bool,
|
|
1409
1477
|
run_pre_post_statements: bool = True,
|
|
1478
|
+
skip_grants: bool = False,
|
|
1410
1479
|
) -> None:
|
|
1411
1480
|
adapter = self.get_adapter(snapshot.model.gateway)
|
|
1412
1481
|
evaluation_strategy = _evaluation_strategy(snapshot, adapter)
|
|
@@ -1420,19 +1489,28 @@ class SnapshotEvaluator:
|
|
|
1420
1489
|
"table_mapping": {snapshot.name: table_name},
|
|
1421
1490
|
}
|
|
1422
1491
|
if run_pre_post_statements:
|
|
1423
|
-
|
|
1492
|
+
evaluation_strategy.run_pre_statements(
|
|
1493
|
+
snapshot=snapshot,
|
|
1494
|
+
render_kwargs={**create_render_kwargs, "inside_transaction": True},
|
|
1495
|
+
)
|
|
1424
1496
|
evaluation_strategy.create(
|
|
1425
1497
|
table_name=table_name,
|
|
1426
1498
|
model=snapshot.model,
|
|
1427
1499
|
is_table_deployable=is_table_deployable,
|
|
1500
|
+
skip_grants=skip_grants,
|
|
1428
1501
|
render_kwargs=create_render_kwargs,
|
|
1429
1502
|
is_snapshot_deployable=is_snapshot_deployable,
|
|
1430
1503
|
is_snapshot_representative=is_snapshot_representative,
|
|
1431
1504
|
dry_run=dry_run,
|
|
1432
1505
|
physical_properties=rendered_physical_properties,
|
|
1506
|
+
snapshot=snapshot,
|
|
1507
|
+
deployability_index=deployability_index,
|
|
1433
1508
|
)
|
|
1434
1509
|
if run_pre_post_statements:
|
|
1435
|
-
|
|
1510
|
+
evaluation_strategy.run_post_statements(
|
|
1511
|
+
snapshot=snapshot,
|
|
1512
|
+
render_kwargs={**create_render_kwargs, "inside_transaction": True},
|
|
1513
|
+
)
|
|
1436
1514
|
|
|
1437
1515
|
def _can_clone(self, snapshot: Snapshot, deployability_index: DeployabilityIndex) -> bool:
|
|
1438
1516
|
adapter = self.get_adapter(snapshot.model.gateway)
|
|
@@ -1441,11 +1519,125 @@ class SnapshotEvaluator:
|
|
|
1441
1519
|
and snapshot.is_materialized
|
|
1442
1520
|
and bool(snapshot.previous_versions)
|
|
1443
1521
|
and adapter.SUPPORTS_CLONING
|
|
1444
|
-
# managed models cannot have their schema mutated because
|
|
1522
|
+
# managed models cannot have their schema mutated because they're based on queries, so clone + alter won't work
|
|
1445
1523
|
and not snapshot.is_managed
|
|
1446
|
-
|
|
1524
|
+
and not snapshot.is_dbt_custom
|
|
1447
1525
|
and not deployability_index.is_deployable(snapshot)
|
|
1526
|
+
# If the deployable table is missing we can't clone it
|
|
1527
|
+
and adapter.table_exists(snapshot.table_name())
|
|
1528
|
+
)
|
|
1529
|
+
|
|
1530
|
+
def _get_physical_data_objects(
|
|
1531
|
+
self,
|
|
1532
|
+
target_snapshots: t.Iterable[Snapshot],
|
|
1533
|
+
deployability_index: DeployabilityIndex,
|
|
1534
|
+
) -> t.Dict[SnapshotId, DataObject]:
|
|
1535
|
+
"""Returns a dictionary of snapshot IDs to existing data objects of their physical tables.
|
|
1536
|
+
|
|
1537
|
+
Args:
|
|
1538
|
+
target_snapshots: Target snapshots.
|
|
1539
|
+
deployability_index: The deployability index to determine whether to look for a deployable or
|
|
1540
|
+
a non-deployable physical table.
|
|
1541
|
+
|
|
1542
|
+
Returns:
|
|
1543
|
+
A dictionary of snapshot IDs to existing data objects of their physical tables. If the data object
|
|
1544
|
+
for a snapshot is not found, it will not be included in the dictionary.
|
|
1545
|
+
"""
|
|
1546
|
+
return self._get_data_objects(
|
|
1547
|
+
target_snapshots,
|
|
1548
|
+
lambda s: exp.to_table(
|
|
1549
|
+
s.table_name(deployability_index.is_deployable(s)), dialect=s.model.dialect
|
|
1550
|
+
),
|
|
1551
|
+
)
|
|
1552
|
+
|
|
1553
|
+
def _get_virtual_data_objects(
|
|
1554
|
+
self,
|
|
1555
|
+
target_snapshots: t.Iterable[Snapshot],
|
|
1556
|
+
environment_naming_info: EnvironmentNamingInfo,
|
|
1557
|
+
) -> t.Dict[SnapshotId, DataObject]:
|
|
1558
|
+
"""Returns a dictionary of snapshot IDs to existing data objects of their virtual views.
|
|
1559
|
+
|
|
1560
|
+
Args:
|
|
1561
|
+
target_snapshots: Target snapshots.
|
|
1562
|
+
environment_naming_info: The environment naming info of the target virtual environment.
|
|
1563
|
+
|
|
1564
|
+
Returns:
|
|
1565
|
+
A dictionary of snapshot IDs to existing data objects of their virtual views. If the data object
|
|
1566
|
+
for a snapshot is not found, it will not be included in the dictionary.
|
|
1567
|
+
"""
|
|
1568
|
+
|
|
1569
|
+
def _get_view_name(s: Snapshot) -> exp.Table:
|
|
1570
|
+
adapter = (
|
|
1571
|
+
self.get_adapter(s.model_gateway)
|
|
1572
|
+
if environment_naming_info.gateway_managed
|
|
1573
|
+
else self.adapter
|
|
1574
|
+
)
|
|
1575
|
+
return exp.to_table(
|
|
1576
|
+
s.qualified_view_name.for_environment(
|
|
1577
|
+
environment_naming_info, dialect=adapter.dialect
|
|
1578
|
+
),
|
|
1579
|
+
dialect=adapter.dialect,
|
|
1580
|
+
)
|
|
1581
|
+
|
|
1582
|
+
return self._get_data_objects(target_snapshots, _get_view_name)
|
|
1583
|
+
|
|
1584
|
+
def _get_data_objects(
|
|
1585
|
+
self,
|
|
1586
|
+
target_snapshots: t.Iterable[Snapshot],
|
|
1587
|
+
table_name_callable: t.Callable[[Snapshot], exp.Table],
|
|
1588
|
+
) -> t.Dict[SnapshotId, DataObject]:
|
|
1589
|
+
"""Returns a dictionary of snapshot IDs to existing data objects.
|
|
1590
|
+
|
|
1591
|
+
Args:
|
|
1592
|
+
target_snapshots: Target snapshots.
|
|
1593
|
+
table_name_callable: A function that takes a snapshot and returns the table to look for.
|
|
1594
|
+
|
|
1595
|
+
Returns:
|
|
1596
|
+
A dictionary of snapshot IDs to existing data objects. If the data object for a snapshot is not found,
|
|
1597
|
+
it will not be included in the dictionary.
|
|
1598
|
+
"""
|
|
1599
|
+
tables_by_gateway_and_schema: t.Dict[t.Union[str, None], t.Dict[exp.Table, set[str]]] = (
|
|
1600
|
+
defaultdict(lambda: defaultdict(set))
|
|
1448
1601
|
)
|
|
1602
|
+
snapshots_by_table_name: t.Dict[exp.Table, t.Dict[str, Snapshot]] = defaultdict(dict)
|
|
1603
|
+
for snapshot in target_snapshots:
|
|
1604
|
+
if not snapshot.is_model or snapshot.is_symbolic:
|
|
1605
|
+
continue
|
|
1606
|
+
table = table_name_callable(snapshot)
|
|
1607
|
+
table_schema = d.schema_(table.db, catalog=table.catalog)
|
|
1608
|
+
tables_by_gateway_and_schema[snapshot.model_gateway][table_schema].add(table.name)
|
|
1609
|
+
snapshots_by_table_name[table_schema][table.name] = snapshot
|
|
1610
|
+
|
|
1611
|
+
def _get_data_objects_in_schema(
|
|
1612
|
+
schema: exp.Table,
|
|
1613
|
+
object_names: t.Optional[t.Set[str]] = None,
|
|
1614
|
+
gateway: t.Optional[str] = None,
|
|
1615
|
+
) -> t.List[DataObject]:
|
|
1616
|
+
logger.info("Listing data objects in schema %s", schema.sql())
|
|
1617
|
+
return self.get_adapter(gateway).get_data_objects(
|
|
1618
|
+
schema, object_names, safe_to_cache=True
|
|
1619
|
+
)
|
|
1620
|
+
|
|
1621
|
+
with self.concurrent_context():
|
|
1622
|
+
snapshot_id_to_obj: t.Dict[SnapshotId, DataObject] = {}
|
|
1623
|
+
# A schema can be shared across multiple engines, so we need to group tables by both gateway and schema
|
|
1624
|
+
for gateway, tables_by_schema in tables_by_gateway_and_schema.items():
|
|
1625
|
+
schema_list = list(tables_by_schema.keys())
|
|
1626
|
+
results = concurrent_apply_to_values(
|
|
1627
|
+
schema_list,
|
|
1628
|
+
lambda s: _get_data_objects_in_schema(
|
|
1629
|
+
schema=s, object_names=tables_by_schema.get(s), gateway=gateway
|
|
1630
|
+
),
|
|
1631
|
+
self.ddl_concurrent_tasks,
|
|
1632
|
+
)
|
|
1633
|
+
|
|
1634
|
+
for schema, objs in zip(schema_list, results):
|
|
1635
|
+
snapshots_by_name = snapshots_by_table_name.get(schema, {})
|
|
1636
|
+
for obj in objs:
|
|
1637
|
+
if obj.name in snapshots_by_name:
|
|
1638
|
+
snapshot_id_to_obj[snapshots_by_name[obj.name].snapshot_id] = obj
|
|
1639
|
+
|
|
1640
|
+
return snapshot_id_to_obj
|
|
1449
1641
|
|
|
1450
1642
|
|
|
1451
1643
|
def _evaluation_strategy(snapshot: SnapshotInfoLike, adapter: EngineAdapter) -> EvaluationStrategy:
|
|
@@ -1470,6 +1662,19 @@ def _evaluation_strategy(snapshot: SnapshotInfoLike, adapter: EngineAdapter) ->
|
|
|
1470
1662
|
klass = ViewStrategy
|
|
1471
1663
|
elif snapshot.is_scd_type_2:
|
|
1472
1664
|
klass = SCDType2Strategy
|
|
1665
|
+
elif snapshot.is_dbt_custom:
|
|
1666
|
+
if hasattr(snapshot, "model") and isinstance(
|
|
1667
|
+
(model_kind := snapshot.model.kind), DbtCustomKind
|
|
1668
|
+
):
|
|
1669
|
+
return DbtCustomMaterializationStrategy(
|
|
1670
|
+
adapter=adapter,
|
|
1671
|
+
materialization_name=model_kind.materialization,
|
|
1672
|
+
materialization_template=model_kind.definition,
|
|
1673
|
+
)
|
|
1674
|
+
|
|
1675
|
+
raise SQLMeshError(
|
|
1676
|
+
f"Expected DbtCustomKind for dbt custom materialization in model '{snapshot.name}'"
|
|
1677
|
+
)
|
|
1473
1678
|
elif snapshot.is_custom:
|
|
1474
1679
|
if snapshot.custom_materialization is None:
|
|
1475
1680
|
raise SQLMeshError(
|
|
@@ -1537,6 +1742,7 @@ class EvaluationStrategy(abc.ABC):
|
|
|
1537
1742
|
model: Model,
|
|
1538
1743
|
is_table_deployable: bool,
|
|
1539
1744
|
render_kwargs: t.Dict[str, t.Any],
|
|
1745
|
+
skip_grants: bool,
|
|
1540
1746
|
**kwargs: t.Any,
|
|
1541
1747
|
) -> None:
|
|
1542
1748
|
"""Creates the target table or view.
|
|
@@ -1609,6 +1815,84 @@ class EvaluationStrategy(abc.ABC):
|
|
|
1609
1815
|
view_name: The name of the target view in the virtual layer.
|
|
1610
1816
|
"""
|
|
1611
1817
|
|
|
1818
|
+
@abc.abstractmethod
|
|
1819
|
+
def run_pre_statements(self, snapshot: Snapshot, render_kwargs: t.Any) -> None:
|
|
1820
|
+
"""Executes the snapshot's pre statements.
|
|
1821
|
+
|
|
1822
|
+
Args:
|
|
1823
|
+
snapshot: The target snapshot.
|
|
1824
|
+
render_kwargs: Additional key-value arguments to pass when rendering the statements.
|
|
1825
|
+
"""
|
|
1826
|
+
|
|
1827
|
+
@abc.abstractmethod
|
|
1828
|
+
def run_post_statements(self, snapshot: Snapshot, render_kwargs: t.Any) -> None:
|
|
1829
|
+
"""Executes the snapshot's post statements.
|
|
1830
|
+
|
|
1831
|
+
Args:
|
|
1832
|
+
snapshot: The target snapshot.
|
|
1833
|
+
render_kwargs: Additional key-value arguments to pass when rendering the statements.
|
|
1834
|
+
"""
|
|
1835
|
+
|
|
1836
|
+
def _apply_grants(
|
|
1837
|
+
self,
|
|
1838
|
+
model: Model,
|
|
1839
|
+
table_name: str,
|
|
1840
|
+
target_layer: GrantsTargetLayer,
|
|
1841
|
+
is_snapshot_deployable: bool = False,
|
|
1842
|
+
) -> None:
|
|
1843
|
+
"""Apply grants for a model if grants are configured.
|
|
1844
|
+
|
|
1845
|
+
This method provides consistent grants application across all evaluation strategies.
|
|
1846
|
+
It ensures that whenever a physical database object (table, view, materialized view)
|
|
1847
|
+
is created or modified, the appropriate grants are applied.
|
|
1848
|
+
|
|
1849
|
+
Args:
|
|
1850
|
+
model: The SQLMesh model containing grants configuration
|
|
1851
|
+
table_name: The target table/view name to apply grants to
|
|
1852
|
+
target_layer: The grants application layer (physical or virtual)
|
|
1853
|
+
is_snapshot_deployable: Whether the snapshot is deployable (targeting production)
|
|
1854
|
+
"""
|
|
1855
|
+
grants_config = model.grants
|
|
1856
|
+
if grants_config is None:
|
|
1857
|
+
return
|
|
1858
|
+
|
|
1859
|
+
if not self.adapter.SUPPORTS_GRANTS:
|
|
1860
|
+
logger.warning(
|
|
1861
|
+
f"Engine {self.adapter.__class__.__name__} does not support grants. "
|
|
1862
|
+
f"Skipping grants application for model {model.name}"
|
|
1863
|
+
)
|
|
1864
|
+
return
|
|
1865
|
+
|
|
1866
|
+
model_grants_target_layer = model.grants_target_layer
|
|
1867
|
+
deployable_vde_dev_only = (
|
|
1868
|
+
is_snapshot_deployable and model.virtual_environment_mode.is_dev_only
|
|
1869
|
+
)
|
|
1870
|
+
|
|
1871
|
+
# table_type is always a VIEW in the virtual layer unless model is deployable and VDE is dev_only
|
|
1872
|
+
# in which case we fall back to the model's model_grants_table_type
|
|
1873
|
+
if target_layer == GrantsTargetLayer.VIRTUAL and not deployable_vde_dev_only:
|
|
1874
|
+
model_grants_table_type = DataObjectType.VIEW
|
|
1875
|
+
else:
|
|
1876
|
+
model_grants_table_type = model.grants_table_type
|
|
1877
|
+
|
|
1878
|
+
if (
|
|
1879
|
+
model_grants_target_layer.is_all
|
|
1880
|
+
or model_grants_target_layer == target_layer
|
|
1881
|
+
# Always apply grants in production when VDE is dev_only regardless of target_layer
|
|
1882
|
+
# since only physical tables are created in production
|
|
1883
|
+
or deployable_vde_dev_only
|
|
1884
|
+
):
|
|
1885
|
+
logger.info(f"Applying grants for model {model.name} to table {table_name}")
|
|
1886
|
+
self.adapter.sync_grants_config(
|
|
1887
|
+
exp.to_table(table_name, dialect=self.adapter.dialect),
|
|
1888
|
+
grants_config,
|
|
1889
|
+
model_grants_table_type,
|
|
1890
|
+
)
|
|
1891
|
+
else:
|
|
1892
|
+
logger.debug(
|
|
1893
|
+
f"Skipping grants application for model {model.name} in {target_layer} layer"
|
|
1894
|
+
)
|
|
1895
|
+
|
|
1612
1896
|
|
|
1613
1897
|
class SymbolicStrategy(EvaluationStrategy):
|
|
1614
1898
|
def insert(
|
|
@@ -1638,6 +1922,7 @@ class SymbolicStrategy(EvaluationStrategy):
|
|
|
1638
1922
|
model: Model,
|
|
1639
1923
|
is_table_deployable: bool,
|
|
1640
1924
|
render_kwargs: t.Dict[str, t.Any],
|
|
1925
|
+
skip_grants: bool,
|
|
1641
1926
|
**kwargs: t.Any,
|
|
1642
1927
|
) -> None:
|
|
1643
1928
|
pass
|
|
@@ -1670,6 +1955,12 @@ class SymbolicStrategy(EvaluationStrategy):
|
|
|
1670
1955
|
def demote(self, view_name: str, **kwargs: t.Any) -> None:
|
|
1671
1956
|
pass
|
|
1672
1957
|
|
|
1958
|
+
def run_pre_statements(self, snapshot: Snapshot, render_kwargs: t.Dict[str, t.Any]) -> None:
|
|
1959
|
+
pass
|
|
1960
|
+
|
|
1961
|
+
def run_post_statements(self, snapshot: Snapshot, render_kwargs: t.Dict[str, t.Any]) -> None:
|
|
1962
|
+
pass
|
|
1963
|
+
|
|
1673
1964
|
|
|
1674
1965
|
class EmbeddedStrategy(SymbolicStrategy):
|
|
1675
1966
|
def promote(
|
|
@@ -1713,10 +2004,27 @@ class PromotableStrategy(EvaluationStrategy, abc.ABC):
|
|
|
1713
2004
|
view_properties=model.render_virtual_properties(**render_kwargs),
|
|
1714
2005
|
)
|
|
1715
2006
|
|
|
2007
|
+
snapshot = kwargs.get("snapshot")
|
|
2008
|
+
deployability_index = kwargs.get("deployability_index")
|
|
2009
|
+
is_snapshot_deployable = (
|
|
2010
|
+
deployability_index.is_deployable(snapshot)
|
|
2011
|
+
if snapshot and deployability_index
|
|
2012
|
+
else False
|
|
2013
|
+
)
|
|
2014
|
+
|
|
2015
|
+
# Apply grants to the virtual layer (view) after promotion
|
|
2016
|
+
self._apply_grants(model, view_name, GrantsTargetLayer.VIRTUAL, is_snapshot_deployable)
|
|
2017
|
+
|
|
1716
2018
|
def demote(self, view_name: str, **kwargs: t.Any) -> None:
|
|
1717
2019
|
logger.info("Dropping view '%s'", view_name)
|
|
1718
2020
|
self.adapter.drop_view(view_name, cascade=False)
|
|
1719
2021
|
|
|
2022
|
+
def run_pre_statements(self, snapshot: Snapshot, render_kwargs: t.Any) -> None:
|
|
2023
|
+
self.adapter.execute(snapshot.model.render_pre_statements(**render_kwargs))
|
|
2024
|
+
|
|
2025
|
+
def run_post_statements(self, snapshot: Snapshot, render_kwargs: t.Any) -> None:
|
|
2026
|
+
self.adapter.execute(snapshot.model.render_post_statements(**render_kwargs))
|
|
2027
|
+
|
|
1720
2028
|
|
|
1721
2029
|
class MaterializableStrategy(PromotableStrategy, abc.ABC):
|
|
1722
2030
|
def create(
|
|
@@ -1725,6 +2033,7 @@ class MaterializableStrategy(PromotableStrategy, abc.ABC):
|
|
|
1725
2033
|
model: Model,
|
|
1726
2034
|
is_table_deployable: bool,
|
|
1727
2035
|
render_kwargs: t.Dict[str, t.Any],
|
|
2036
|
+
skip_grants: bool,
|
|
1728
2037
|
**kwargs: t.Any,
|
|
1729
2038
|
) -> None:
|
|
1730
2039
|
ctas_query = model.ctas_query(**render_kwargs)
|
|
@@ -1769,6 +2078,13 @@ class MaterializableStrategy(PromotableStrategy, abc.ABC):
|
|
|
1769
2078
|
column_descriptions=model.column_descriptions if is_table_deployable else None,
|
|
1770
2079
|
)
|
|
1771
2080
|
|
|
2081
|
+
# Apply grants after table creation (unless explicitly skipped by caller)
|
|
2082
|
+
if not skip_grants:
|
|
2083
|
+
is_snapshot_deployable = kwargs.get("is_snapshot_deployable", False)
|
|
2084
|
+
self._apply_grants(
|
|
2085
|
+
model, table_name, GrantsTargetLayer.PHYSICAL, is_snapshot_deployable
|
|
2086
|
+
)
|
|
2087
|
+
|
|
1772
2088
|
def migrate(
|
|
1773
2089
|
self,
|
|
1774
2090
|
target_table_name: str,
|
|
@@ -1794,6 +2110,15 @@ class MaterializableStrategy(PromotableStrategy, abc.ABC):
|
|
|
1794
2110
|
)
|
|
1795
2111
|
self.adapter.alter_table(alter_operations)
|
|
1796
2112
|
|
|
2113
|
+
# Apply grants after schema migration
|
|
2114
|
+
deployability_index = kwargs.get("deployability_index")
|
|
2115
|
+
is_snapshot_deployable = (
|
|
2116
|
+
deployability_index.is_deployable(snapshot) if deployability_index else False
|
|
2117
|
+
)
|
|
2118
|
+
self._apply_grants(
|
|
2119
|
+
snapshot.model, target_table_name, GrantsTargetLayer.PHYSICAL, is_snapshot_deployable
|
|
2120
|
+
)
|
|
2121
|
+
|
|
1797
2122
|
def delete(self, name: str, **kwargs: t.Any) -> None:
|
|
1798
2123
|
_check_table_db_is_physical_schema(name, kwargs["physical_schema"])
|
|
1799
2124
|
self.adapter.drop_table(name, cascade=kwargs.pop("cascade", False))
|
|
@@ -1805,6 +2130,7 @@ class MaterializableStrategy(PromotableStrategy, abc.ABC):
|
|
|
1805
2130
|
name: str,
|
|
1806
2131
|
query_or_df: QueryOrDF,
|
|
1807
2132
|
render_kwargs: t.Dict[str, t.Any],
|
|
2133
|
+
skip_grants: bool = False,
|
|
1808
2134
|
**kwargs: t.Any,
|
|
1809
2135
|
) -> None:
|
|
1810
2136
|
"""Replaces the table for the given model.
|
|
@@ -1841,6 +2167,11 @@ class MaterializableStrategy(PromotableStrategy, abc.ABC):
|
|
|
1841
2167
|
source_columns=source_columns,
|
|
1842
2168
|
)
|
|
1843
2169
|
|
|
2170
|
+
# Apply grants after table replacement (unless explicitly skipped by caller)
|
|
2171
|
+
if not skip_grants:
|
|
2172
|
+
is_snapshot_deployable = kwargs.get("is_snapshot_deployable", False)
|
|
2173
|
+
self._apply_grants(model, name, GrantsTargetLayer.PHYSICAL, is_snapshot_deployable)
|
|
2174
|
+
|
|
1844
2175
|
def _get_target_and_source_columns(
|
|
1845
2176
|
self,
|
|
1846
2177
|
model: Model,
|
|
@@ -1862,7 +2193,13 @@ class MaterializableStrategy(PromotableStrategy, abc.ABC):
|
|
|
1862
2193
|
if model.on_destructive_change.is_ignore or model.on_additive_change.is_ignore:
|
|
1863
2194
|
# We need to identify the columns that are only in the source so we create an empty table with
|
|
1864
2195
|
# the user query to determine that
|
|
1865
|
-
|
|
2196
|
+
temp_table_name = exp.table_(
|
|
2197
|
+
"diff",
|
|
2198
|
+
db=model.physical_schema,
|
|
2199
|
+
)
|
|
2200
|
+
with self.adapter.temp_table(
|
|
2201
|
+
model.ctas_query(**render_kwargs), name=temp_table_name
|
|
2202
|
+
) as temp_table:
|
|
1866
2203
|
source_columns = list(self.adapter.columns(temp_table))
|
|
1867
2204
|
else:
|
|
1868
2205
|
source_columns = None
|
|
@@ -2088,6 +2425,7 @@ class SeedStrategy(MaterializableStrategy):
|
|
|
2088
2425
|
model: Model,
|
|
2089
2426
|
is_table_deployable: bool,
|
|
2090
2427
|
render_kwargs: t.Dict[str, t.Any],
|
|
2428
|
+
skip_grants: bool,
|
|
2091
2429
|
**kwargs: t.Any,
|
|
2092
2430
|
) -> None:
|
|
2093
2431
|
model = t.cast(SeedModel, model)
|
|
@@ -2101,22 +2439,52 @@ class SeedStrategy(MaterializableStrategy):
|
|
|
2101
2439
|
)
|
|
2102
2440
|
return
|
|
2103
2441
|
|
|
2104
|
-
super().create(
|
|
2105
|
-
|
|
2106
|
-
|
|
2107
|
-
|
|
2108
|
-
|
|
2109
|
-
|
|
2110
|
-
|
|
2111
|
-
|
|
2112
|
-
|
|
2113
|
-
|
|
2114
|
-
|
|
2115
|
-
|
|
2116
|
-
|
|
2117
|
-
|
|
2118
|
-
|
|
2119
|
-
|
|
2442
|
+
super().create(
|
|
2443
|
+
table_name,
|
|
2444
|
+
model,
|
|
2445
|
+
is_table_deployable,
|
|
2446
|
+
render_kwargs,
|
|
2447
|
+
skip_grants=True, # Skip grants; they're applied after data insertion
|
|
2448
|
+
**kwargs,
|
|
2449
|
+
)
|
|
2450
|
+
# For seeds we insert data at the time of table creation.
|
|
2451
|
+
try:
|
|
2452
|
+
for index, df in enumerate(model.render_seed()):
|
|
2453
|
+
if index == 0:
|
|
2454
|
+
self._replace_query_for_model(
|
|
2455
|
+
model,
|
|
2456
|
+
table_name,
|
|
2457
|
+
df,
|
|
2458
|
+
render_kwargs,
|
|
2459
|
+
skip_grants=True, # Skip grants; they're applied after data insertion
|
|
2460
|
+
**kwargs,
|
|
2461
|
+
)
|
|
2462
|
+
else:
|
|
2463
|
+
self.adapter.insert_append(
|
|
2464
|
+
table_name, df, target_columns_to_types=model.columns_to_types
|
|
2465
|
+
)
|
|
2466
|
+
|
|
2467
|
+
if not skip_grants:
|
|
2468
|
+
# Apply grants after seed table creation and data insertion
|
|
2469
|
+
is_snapshot_deployable = kwargs.get("is_snapshot_deployable", False)
|
|
2470
|
+
self._apply_grants(
|
|
2471
|
+
model, table_name, GrantsTargetLayer.PHYSICAL, is_snapshot_deployable
|
|
2472
|
+
)
|
|
2473
|
+
except Exception:
|
|
2474
|
+
self.adapter.drop_table(table_name)
|
|
2475
|
+
raise
|
|
2476
|
+
|
|
2477
|
+
def migrate(
|
|
2478
|
+
self,
|
|
2479
|
+
target_table_name: str,
|
|
2480
|
+
source_table_name: str,
|
|
2481
|
+
snapshot: Snapshot,
|
|
2482
|
+
*,
|
|
2483
|
+
ignore_destructive: bool,
|
|
2484
|
+
ignore_additive: bool,
|
|
2485
|
+
**kwargs: t.Any,
|
|
2486
|
+
) -> None:
|
|
2487
|
+
raise NotImplementedError("Seeds do not support migrations.")
|
|
2120
2488
|
|
|
2121
2489
|
def insert(
|
|
2122
2490
|
self,
|
|
@@ -2149,6 +2517,7 @@ class SCDType2Strategy(IncrementalStrategy):
|
|
|
2149
2517
|
model: Model,
|
|
2150
2518
|
is_table_deployable: bool,
|
|
2151
2519
|
render_kwargs: t.Dict[str, t.Any],
|
|
2520
|
+
skip_grants: bool,
|
|
2152
2521
|
**kwargs: t.Any,
|
|
2153
2522
|
) -> None:
|
|
2154
2523
|
assert isinstance(model.kind, (SCDType2ByTimeKind, SCDType2ByColumnKind))
|
|
@@ -2178,9 +2547,17 @@ class SCDType2Strategy(IncrementalStrategy):
|
|
|
2178
2547
|
model,
|
|
2179
2548
|
is_table_deployable,
|
|
2180
2549
|
render_kwargs,
|
|
2550
|
+
skip_grants,
|
|
2181
2551
|
**kwargs,
|
|
2182
2552
|
)
|
|
2183
2553
|
|
|
2554
|
+
if not skip_grants:
|
|
2555
|
+
# Apply grants after SCD Type 2 table creation
|
|
2556
|
+
is_snapshot_deployable = kwargs.get("is_snapshot_deployable", False)
|
|
2557
|
+
self._apply_grants(
|
|
2558
|
+
model, table_name, GrantsTargetLayer.PHYSICAL, is_snapshot_deployable
|
|
2559
|
+
)
|
|
2560
|
+
|
|
2184
2561
|
def insert(
|
|
2185
2562
|
self,
|
|
2186
2563
|
table_name: str,
|
|
@@ -2214,6 +2591,11 @@ class SCDType2Strategy(IncrementalStrategy):
|
|
|
2214
2591
|
column_descriptions=model.column_descriptions,
|
|
2215
2592
|
truncate=is_first_insert,
|
|
2216
2593
|
source_columns=source_columns,
|
|
2594
|
+
storage_format=model.storage_format,
|
|
2595
|
+
partitioned_by=model.partitioned_by,
|
|
2596
|
+
partition_interval_unit=model.partition_interval_unit,
|
|
2597
|
+
clustered_by=model.clustered_by,
|
|
2598
|
+
table_properties=kwargs.get("physical_properties", model.physical_properties),
|
|
2217
2599
|
)
|
|
2218
2600
|
elif isinstance(model.kind, SCDType2ByColumnKind):
|
|
2219
2601
|
self.adapter.scd_type_2_by_column(
|
|
@@ -2232,12 +2614,21 @@ class SCDType2Strategy(IncrementalStrategy):
|
|
|
2232
2614
|
column_descriptions=model.column_descriptions,
|
|
2233
2615
|
truncate=is_first_insert,
|
|
2234
2616
|
source_columns=source_columns,
|
|
2617
|
+
storage_format=model.storage_format,
|
|
2618
|
+
partitioned_by=model.partitioned_by,
|
|
2619
|
+
partition_interval_unit=model.partition_interval_unit,
|
|
2620
|
+
clustered_by=model.clustered_by,
|
|
2621
|
+
table_properties=kwargs.get("physical_properties", model.physical_properties),
|
|
2235
2622
|
)
|
|
2236
2623
|
else:
|
|
2237
2624
|
raise SQLMeshError(
|
|
2238
2625
|
f"Unexpected SCD Type 2 kind: {model.kind}. This is not expected and please report this as a bug."
|
|
2239
2626
|
)
|
|
2240
2627
|
|
|
2628
|
+
# Apply grants after SCD Type 2 table recreation
|
|
2629
|
+
is_snapshot_deployable = kwargs.get("is_snapshot_deployable", False)
|
|
2630
|
+
self._apply_grants(model, table_name, GrantsTargetLayer.PHYSICAL, is_snapshot_deployable)
|
|
2631
|
+
|
|
2241
2632
|
def append(
|
|
2242
2633
|
self,
|
|
2243
2634
|
table_name: str,
|
|
@@ -2246,51 +2637,14 @@ class SCDType2Strategy(IncrementalStrategy):
|
|
|
2246
2637
|
render_kwargs: t.Dict[str, t.Any],
|
|
2247
2638
|
**kwargs: t.Any,
|
|
2248
2639
|
) -> None:
|
|
2249
|
-
|
|
2250
|
-
columns_to_types, source_columns = self._get_target_and_source_columns(
|
|
2251
|
-
model,
|
|
2640
|
+
return self.insert(
|
|
2252
2641
|
table_name,
|
|
2642
|
+
query_or_df,
|
|
2643
|
+
model,
|
|
2644
|
+
is_first_insert=False,
|
|
2253
2645
|
render_kwargs=render_kwargs,
|
|
2254
|
-
|
|
2646
|
+
**kwargs,
|
|
2255
2647
|
)
|
|
2256
|
-
if isinstance(model.kind, SCDType2ByTimeKind):
|
|
2257
|
-
self.adapter.scd_type_2_by_time(
|
|
2258
|
-
target_table=table_name,
|
|
2259
|
-
source_table=query_or_df,
|
|
2260
|
-
unique_key=model.unique_key,
|
|
2261
|
-
valid_from_col=model.kind.valid_from_name,
|
|
2262
|
-
valid_to_col=model.kind.valid_to_name,
|
|
2263
|
-
updated_at_col=model.kind.updated_at_name,
|
|
2264
|
-
invalidate_hard_deletes=model.kind.invalidate_hard_deletes,
|
|
2265
|
-
updated_at_as_valid_from=model.kind.updated_at_as_valid_from,
|
|
2266
|
-
target_columns_to_types=columns_to_types,
|
|
2267
|
-
table_format=model.table_format,
|
|
2268
|
-
table_description=model.description,
|
|
2269
|
-
column_descriptions=model.column_descriptions,
|
|
2270
|
-
source_columns=source_columns,
|
|
2271
|
-
**kwargs,
|
|
2272
|
-
)
|
|
2273
|
-
elif isinstance(model.kind, SCDType2ByColumnKind):
|
|
2274
|
-
self.adapter.scd_type_2_by_column(
|
|
2275
|
-
target_table=table_name,
|
|
2276
|
-
source_table=query_or_df,
|
|
2277
|
-
unique_key=model.unique_key,
|
|
2278
|
-
valid_from_col=model.kind.valid_from_name,
|
|
2279
|
-
valid_to_col=model.kind.valid_to_name,
|
|
2280
|
-
check_columns=model.kind.columns,
|
|
2281
|
-
target_columns_to_types=columns_to_types,
|
|
2282
|
-
table_format=model.table_format,
|
|
2283
|
-
invalidate_hard_deletes=model.kind.invalidate_hard_deletes,
|
|
2284
|
-
execution_time_as_valid_from=model.kind.execution_time_as_valid_from,
|
|
2285
|
-
table_description=model.description,
|
|
2286
|
-
column_descriptions=model.column_descriptions,
|
|
2287
|
-
source_columns=source_columns,
|
|
2288
|
-
**kwargs,
|
|
2289
|
-
)
|
|
2290
|
-
else:
|
|
2291
|
-
raise SQLMeshError(
|
|
2292
|
-
f"Unexpected SCD Type 2 kind: {model.kind}. This is not expected and please report this as a bug."
|
|
2293
|
-
)
|
|
2294
2648
|
|
|
2295
2649
|
|
|
2296
2650
|
class ViewStrategy(PromotableStrategy):
|
|
@@ -2331,6 +2685,10 @@ class ViewStrategy(PromotableStrategy):
|
|
|
2331
2685
|
column_descriptions=model.column_descriptions,
|
|
2332
2686
|
)
|
|
2333
2687
|
|
|
2688
|
+
# Apply grants after view creation / replacement
|
|
2689
|
+
is_snapshot_deployable = kwargs.get("is_snapshot_deployable", False)
|
|
2690
|
+
self._apply_grants(model, table_name, GrantsTargetLayer.PHYSICAL, is_snapshot_deployable)
|
|
2691
|
+
|
|
2334
2692
|
def append(
|
|
2335
2693
|
self,
|
|
2336
2694
|
table_name: str,
|
|
@@ -2347,12 +2705,21 @@ class ViewStrategy(PromotableStrategy):
|
|
|
2347
2705
|
model: Model,
|
|
2348
2706
|
is_table_deployable: bool,
|
|
2349
2707
|
render_kwargs: t.Dict[str, t.Any],
|
|
2708
|
+
skip_grants: bool,
|
|
2350
2709
|
**kwargs: t.Any,
|
|
2351
2710
|
) -> None:
|
|
2711
|
+
is_snapshot_deployable = kwargs.get("is_snapshot_deployable", False)
|
|
2712
|
+
|
|
2352
2713
|
if self.adapter.table_exists(table_name):
|
|
2353
2714
|
# Make sure we don't recreate the view to prevent deletion of downstream views in engines with no late
|
|
2354
2715
|
# binding support (because of DROP CASCADE).
|
|
2355
2716
|
logger.info("View '%s' already exists", table_name)
|
|
2717
|
+
|
|
2718
|
+
if not skip_grants:
|
|
2719
|
+
# Always apply grants when present, even if view exists, to handle grants updates
|
|
2720
|
+
self._apply_grants(
|
|
2721
|
+
model, table_name, GrantsTargetLayer.PHYSICAL, is_snapshot_deployable
|
|
2722
|
+
)
|
|
2356
2723
|
return
|
|
2357
2724
|
|
|
2358
2725
|
logger.info("Creating view '%s'", table_name)
|
|
@@ -2376,6 +2743,12 @@ class ViewStrategy(PromotableStrategy):
|
|
|
2376
2743
|
column_descriptions=model.column_descriptions if is_table_deployable else None,
|
|
2377
2744
|
)
|
|
2378
2745
|
|
|
2746
|
+
if not skip_grants:
|
|
2747
|
+
# Apply grants after view creation
|
|
2748
|
+
self._apply_grants(
|
|
2749
|
+
model, table_name, GrantsTargetLayer.PHYSICAL, is_snapshot_deployable
|
|
2750
|
+
)
|
|
2751
|
+
|
|
2379
2752
|
def migrate(
|
|
2380
2753
|
self,
|
|
2381
2754
|
target_table_name: str,
|
|
@@ -2402,6 +2775,15 @@ class ViewStrategy(PromotableStrategy):
|
|
|
2402
2775
|
column_descriptions=model.column_descriptions,
|
|
2403
2776
|
)
|
|
2404
2777
|
|
|
2778
|
+
# Apply grants after view migration
|
|
2779
|
+
deployability_index = kwargs.get("deployability_index")
|
|
2780
|
+
is_snapshot_deployable = (
|
|
2781
|
+
deployability_index.is_deployable(snapshot) if deployability_index else False
|
|
2782
|
+
)
|
|
2783
|
+
self._apply_grants(
|
|
2784
|
+
snapshot.model, target_table_name, GrantsTargetLayer.PHYSICAL, is_snapshot_deployable
|
|
2785
|
+
)
|
|
2786
|
+
|
|
2405
2787
|
def delete(self, name: str, **kwargs: t.Any) -> None:
|
|
2406
2788
|
cascade = kwargs.pop("cascade", False)
|
|
2407
2789
|
try:
|
|
@@ -2541,6 +2923,169 @@ def get_custom_materialization_type_or_raise(
|
|
|
2541
2923
|
raise SQLMeshError(f"Custom materialization '{name}' not present in the Python environment")
|
|
2542
2924
|
|
|
2543
2925
|
|
|
2926
|
+
class DbtCustomMaterializationStrategy(MaterializableStrategy):
|
|
2927
|
+
def __init__(
|
|
2928
|
+
self,
|
|
2929
|
+
adapter: EngineAdapter,
|
|
2930
|
+
materialization_name: str,
|
|
2931
|
+
materialization_template: str,
|
|
2932
|
+
):
|
|
2933
|
+
super().__init__(adapter)
|
|
2934
|
+
self.materialization_name = materialization_name
|
|
2935
|
+
self.materialization_template = materialization_template
|
|
2936
|
+
|
|
2937
|
+
def create(
|
|
2938
|
+
self,
|
|
2939
|
+
table_name: str,
|
|
2940
|
+
model: Model,
|
|
2941
|
+
is_table_deployable: bool,
|
|
2942
|
+
render_kwargs: t.Dict[str, t.Any],
|
|
2943
|
+
skip_grants: bool,
|
|
2944
|
+
**kwargs: t.Any,
|
|
2945
|
+
) -> None:
|
|
2946
|
+
original_query = model.render_query_or_raise(**render_kwargs)
|
|
2947
|
+
self._execute_materialization(
|
|
2948
|
+
table_name=table_name,
|
|
2949
|
+
query_or_df=original_query.limit(0),
|
|
2950
|
+
model=model,
|
|
2951
|
+
is_first_insert=True,
|
|
2952
|
+
render_kwargs=render_kwargs,
|
|
2953
|
+
create_only=True,
|
|
2954
|
+
**kwargs,
|
|
2955
|
+
)
|
|
2956
|
+
|
|
2957
|
+
# Apply grants after dbt custom materialization table creation
|
|
2958
|
+
if not skip_grants:
|
|
2959
|
+
is_snapshot_deployable = kwargs.get("is_snapshot_deployable", False)
|
|
2960
|
+
self._apply_grants(
|
|
2961
|
+
model, table_name, GrantsTargetLayer.PHYSICAL, is_snapshot_deployable
|
|
2962
|
+
)
|
|
2963
|
+
|
|
2964
|
+
def insert(
|
|
2965
|
+
self,
|
|
2966
|
+
table_name: str,
|
|
2967
|
+
query_or_df: QueryOrDF,
|
|
2968
|
+
model: Model,
|
|
2969
|
+
is_first_insert: bool,
|
|
2970
|
+
render_kwargs: t.Dict[str, t.Any],
|
|
2971
|
+
**kwargs: t.Any,
|
|
2972
|
+
) -> None:
|
|
2973
|
+
self._execute_materialization(
|
|
2974
|
+
table_name=table_name,
|
|
2975
|
+
query_or_df=query_or_df,
|
|
2976
|
+
model=model,
|
|
2977
|
+
is_first_insert=is_first_insert,
|
|
2978
|
+
render_kwargs=render_kwargs,
|
|
2979
|
+
**kwargs,
|
|
2980
|
+
)
|
|
2981
|
+
|
|
2982
|
+
# Apply grants after custom materialization insert (only on first insert)
|
|
2983
|
+
if is_first_insert:
|
|
2984
|
+
is_snapshot_deployable = kwargs.get("is_snapshot_deployable", False)
|
|
2985
|
+
self._apply_grants(
|
|
2986
|
+
model, table_name, GrantsTargetLayer.PHYSICAL, is_snapshot_deployable
|
|
2987
|
+
)
|
|
2988
|
+
|
|
2989
|
+
def append(
|
|
2990
|
+
self,
|
|
2991
|
+
table_name: str,
|
|
2992
|
+
query_or_df: QueryOrDF,
|
|
2993
|
+
model: Model,
|
|
2994
|
+
render_kwargs: t.Dict[str, t.Any],
|
|
2995
|
+
**kwargs: t.Any,
|
|
2996
|
+
) -> None:
|
|
2997
|
+
return self.insert(
|
|
2998
|
+
table_name,
|
|
2999
|
+
query_or_df,
|
|
3000
|
+
model,
|
|
3001
|
+
is_first_insert=False,
|
|
3002
|
+
render_kwargs=render_kwargs,
|
|
3003
|
+
**kwargs,
|
|
3004
|
+
)
|
|
3005
|
+
|
|
3006
|
+
def run_pre_statements(self, snapshot: Snapshot, render_kwargs: t.Any) -> None:
|
|
3007
|
+
# in dbt custom materialisations it's up to the user to run the pre hooks inside the transaction
|
|
3008
|
+
if not render_kwargs.get("inside_transaction", True):
|
|
3009
|
+
super().run_pre_statements(
|
|
3010
|
+
snapshot=snapshot,
|
|
3011
|
+
render_kwargs=render_kwargs,
|
|
3012
|
+
)
|
|
3013
|
+
|
|
3014
|
+
def run_post_statements(self, snapshot: Snapshot, render_kwargs: t.Any) -> None:
|
|
3015
|
+
# in dbt custom materialisations it's up to the user to run the post hooks inside the transaction
|
|
3016
|
+
if not render_kwargs.get("inside_transaction", True):
|
|
3017
|
+
super().run_post_statements(
|
|
3018
|
+
snapshot=snapshot,
|
|
3019
|
+
render_kwargs=render_kwargs,
|
|
3020
|
+
)
|
|
3021
|
+
|
|
3022
|
+
def _execute_materialization(
|
|
3023
|
+
self,
|
|
3024
|
+
table_name: str,
|
|
3025
|
+
query_or_df: QueryOrDF,
|
|
3026
|
+
model: Model,
|
|
3027
|
+
is_first_insert: bool,
|
|
3028
|
+
render_kwargs: t.Dict[str, t.Any],
|
|
3029
|
+
create_only: bool = False,
|
|
3030
|
+
**kwargs: t.Any,
|
|
3031
|
+
) -> None:
|
|
3032
|
+
jinja_macros = model.jinja_macros
|
|
3033
|
+
|
|
3034
|
+
# For vdes we need to use the table, since we don't know the schema/table at parse time
|
|
3035
|
+
parts = exp.to_table(table_name, dialect=self.adapter.dialect)
|
|
3036
|
+
|
|
3037
|
+
existing_globals = jinja_macros.global_objs
|
|
3038
|
+
relation_info = existing_globals.get("this")
|
|
3039
|
+
if isinstance(relation_info, dict):
|
|
3040
|
+
relation_info["database"] = parts.catalog
|
|
3041
|
+
relation_info["identifier"] = parts.name
|
|
3042
|
+
relation_info["name"] = parts.name
|
|
3043
|
+
|
|
3044
|
+
jinja_globals = {
|
|
3045
|
+
**existing_globals,
|
|
3046
|
+
"this": relation_info,
|
|
3047
|
+
"database": parts.catalog,
|
|
3048
|
+
"schema": parts.db,
|
|
3049
|
+
"identifier": parts.name,
|
|
3050
|
+
"target": existing_globals.get("target", {"type": self.adapter.dialect}),
|
|
3051
|
+
"execution_dt": kwargs.get("execution_time"),
|
|
3052
|
+
"engine_adapter": self.adapter,
|
|
3053
|
+
"sql": str(query_or_df),
|
|
3054
|
+
"is_first_insert": is_first_insert,
|
|
3055
|
+
"create_only": create_only,
|
|
3056
|
+
"pre_hooks": [
|
|
3057
|
+
AttributeDict({"sql": s.this.this, "transaction": transaction})
|
|
3058
|
+
for s in model.pre_statements
|
|
3059
|
+
if (transaction := s.args.get("transaction", True))
|
|
3060
|
+
],
|
|
3061
|
+
"post_hooks": [
|
|
3062
|
+
AttributeDict({"sql": s.this.this, "transaction": transaction})
|
|
3063
|
+
for s in model.post_statements
|
|
3064
|
+
if (transaction := s.args.get("transaction", True))
|
|
3065
|
+
],
|
|
3066
|
+
"model_instance": model,
|
|
3067
|
+
**kwargs,
|
|
3068
|
+
}
|
|
3069
|
+
|
|
3070
|
+
try:
|
|
3071
|
+
jinja_env = jinja_macros.build_environment(**jinja_globals)
|
|
3072
|
+
template = jinja_env.from_string(self.materialization_template)
|
|
3073
|
+
|
|
3074
|
+
try:
|
|
3075
|
+
template.render()
|
|
3076
|
+
except MacroReturnVal as ret:
|
|
3077
|
+
# this is a successful return from a macro call (dbt uses this list of Relations to update their relation cache)
|
|
3078
|
+
returned_relations = ret.value.get("relations", [])
|
|
3079
|
+
logger.info(
|
|
3080
|
+
f"Materialization {self.materialization_name} returned relations: {returned_relations}"
|
|
3081
|
+
)
|
|
3082
|
+
|
|
3083
|
+
except Exception as e:
|
|
3084
|
+
raise SQLMeshError(
|
|
3085
|
+
f"Failed to execute dbt materialization '{self.materialization_name}': {e}"
|
|
3086
|
+
) from e
|
|
3087
|
+
|
|
3088
|
+
|
|
2544
3089
|
class EngineManagedStrategy(MaterializableStrategy):
|
|
2545
3090
|
def create(
|
|
2546
3091
|
self,
|
|
@@ -2548,6 +3093,7 @@ class EngineManagedStrategy(MaterializableStrategy):
|
|
|
2548
3093
|
model: Model,
|
|
2549
3094
|
is_table_deployable: bool,
|
|
2550
3095
|
render_kwargs: t.Dict[str, t.Any],
|
|
3096
|
+
skip_grants: bool,
|
|
2551
3097
|
**kwargs: t.Any,
|
|
2552
3098
|
) -> None:
|
|
2553
3099
|
is_snapshot_deployable: bool = kwargs["is_snapshot_deployable"]
|
|
@@ -2566,6 +3112,13 @@ class EngineManagedStrategy(MaterializableStrategy):
|
|
|
2566
3112
|
column_descriptions=model.column_descriptions,
|
|
2567
3113
|
table_format=model.table_format,
|
|
2568
3114
|
)
|
|
3115
|
+
|
|
3116
|
+
# Apply grants after managed table creation
|
|
3117
|
+
if not skip_grants:
|
|
3118
|
+
self._apply_grants(
|
|
3119
|
+
model, table_name, GrantsTargetLayer.PHYSICAL, is_snapshot_deployable
|
|
3120
|
+
)
|
|
3121
|
+
|
|
2569
3122
|
elif not is_table_deployable:
|
|
2570
3123
|
# Only create the dev preview table as a normal table.
|
|
2571
3124
|
# For the main table, if the snapshot is cant be deployed to prod (eg upstream is forward-only) do nothing.
|
|
@@ -2576,6 +3129,7 @@ class EngineManagedStrategy(MaterializableStrategy):
|
|
|
2576
3129
|
model=model,
|
|
2577
3130
|
is_table_deployable=is_table_deployable,
|
|
2578
3131
|
render_kwargs=render_kwargs,
|
|
3132
|
+
skip_grants=skip_grants,
|
|
2579
3133
|
**kwargs,
|
|
2580
3134
|
)
|
|
2581
3135
|
|
|
@@ -2591,7 +3145,6 @@ class EngineManagedStrategy(MaterializableStrategy):
|
|
|
2591
3145
|
deployability_index: DeployabilityIndex = kwargs["deployability_index"]
|
|
2592
3146
|
snapshot: Snapshot = kwargs["snapshot"]
|
|
2593
3147
|
is_snapshot_deployable = deployability_index.is_deployable(snapshot)
|
|
2594
|
-
|
|
2595
3148
|
if is_first_insert and is_snapshot_deployable and not self.adapter.table_exists(table_name):
|
|
2596
3149
|
self.adapter.create_managed_table(
|
|
2597
3150
|
table_name=table_name,
|
|
@@ -2604,6 +3157,9 @@ class EngineManagedStrategy(MaterializableStrategy):
|
|
|
2604
3157
|
column_descriptions=model.column_descriptions,
|
|
2605
3158
|
table_format=model.table_format,
|
|
2606
3159
|
)
|
|
3160
|
+
self._apply_grants(
|
|
3161
|
+
model, table_name, GrantsTargetLayer.PHYSICAL, is_snapshot_deployable
|
|
3162
|
+
)
|
|
2607
3163
|
elif not is_snapshot_deployable:
|
|
2608
3164
|
# Snapshot isnt deployable; update the preview table instead
|
|
2609
3165
|
# If the snapshot was deployable, then data would have already been loaded in create() because a managed table would have been created
|
|
@@ -2648,10 +3204,19 @@ class EngineManagedStrategy(MaterializableStrategy):
|
|
|
2648
3204
|
)
|
|
2649
3205
|
if len(potential_alter_operations) > 0:
|
|
2650
3206
|
# this can happen if a user changes a managed model and deliberately overrides a plan to be forward only, eg `sqlmesh plan --forward-only`
|
|
2651
|
-
raise
|
|
3207
|
+
raise MigrationNotSupportedError(
|
|
2652
3208
|
f"The schema of the managed model '{target_table_name}' cannot be updated in a forward-only fashion."
|
|
2653
3209
|
)
|
|
2654
3210
|
|
|
3211
|
+
# Apply grants after verifying no schema changes
|
|
3212
|
+
deployability_index = kwargs.get("deployability_index")
|
|
3213
|
+
is_snapshot_deployable = (
|
|
3214
|
+
deployability_index.is_deployable(snapshot) if deployability_index else False
|
|
3215
|
+
)
|
|
3216
|
+
self._apply_grants(
|
|
3217
|
+
snapshot.model, target_table_name, GrantsTargetLayer.PHYSICAL, is_snapshot_deployable
|
|
3218
|
+
)
|
|
3219
|
+
|
|
2655
3220
|
def delete(self, name: str, **kwargs: t.Any) -> None:
|
|
2656
3221
|
# a dev preview table is created as a normal table, so it needs to be dropped as a normal table
|
|
2657
3222
|
_check_table_db_is_physical_schema(name, kwargs["physical_schema"])
|