sqlmesh 0.213.1.dev1__py3-none-any.whl → 0.227.2.dev4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sqlmesh/__init__.py +12 -2
- sqlmesh/_version.py +2 -2
- sqlmesh/cli/main.py +0 -44
- sqlmesh/cli/project_init.py +11 -2
- sqlmesh/core/_typing.py +1 -0
- sqlmesh/core/audit/definition.py +8 -2
- sqlmesh/core/config/__init__.py +1 -1
- sqlmesh/core/config/connection.py +17 -5
- sqlmesh/core/config/dbt.py +13 -0
- sqlmesh/core/config/janitor.py +12 -0
- sqlmesh/core/config/loader.py +7 -0
- sqlmesh/core/config/model.py +2 -0
- sqlmesh/core/config/root.py +3 -0
- sqlmesh/core/console.py +81 -3
- sqlmesh/core/constants.py +1 -1
- sqlmesh/core/context.py +69 -26
- sqlmesh/core/dialect.py +3 -0
- sqlmesh/core/engine_adapter/_typing.py +2 -0
- sqlmesh/core/engine_adapter/base.py +322 -22
- sqlmesh/core/engine_adapter/base_postgres.py +17 -1
- sqlmesh/core/engine_adapter/bigquery.py +146 -7
- sqlmesh/core/engine_adapter/clickhouse.py +17 -13
- sqlmesh/core/engine_adapter/databricks.py +33 -2
- sqlmesh/core/engine_adapter/fabric.py +10 -29
- sqlmesh/core/engine_adapter/mixins.py +142 -48
- sqlmesh/core/engine_adapter/mssql.py +15 -4
- sqlmesh/core/engine_adapter/mysql.py +2 -2
- sqlmesh/core/engine_adapter/postgres.py +9 -3
- sqlmesh/core/engine_adapter/redshift.py +4 -0
- sqlmesh/core/engine_adapter/risingwave.py +1 -0
- sqlmesh/core/engine_adapter/shared.py +6 -0
- sqlmesh/core/engine_adapter/snowflake.py +82 -11
- sqlmesh/core/engine_adapter/spark.py +14 -10
- sqlmesh/core/engine_adapter/trino.py +4 -2
- sqlmesh/core/environment.py +2 -0
- sqlmesh/core/janitor.py +181 -0
- sqlmesh/core/lineage.py +1 -0
- sqlmesh/core/linter/definition.py +13 -13
- sqlmesh/core/linter/rules/builtin.py +29 -0
- sqlmesh/core/macros.py +35 -13
- sqlmesh/core/model/common.py +2 -0
- sqlmesh/core/model/definition.py +82 -28
- sqlmesh/core/model/kind.py +66 -2
- sqlmesh/core/model/meta.py +108 -4
- sqlmesh/core/node.py +101 -1
- sqlmesh/core/plan/builder.py +18 -10
- sqlmesh/core/plan/common.py +199 -2
- sqlmesh/core/plan/definition.py +25 -6
- sqlmesh/core/plan/evaluator.py +75 -113
- sqlmesh/core/plan/explainer.py +90 -8
- sqlmesh/core/plan/stages.py +42 -21
- sqlmesh/core/renderer.py +78 -32
- sqlmesh/core/scheduler.py +102 -22
- sqlmesh/core/selector.py +137 -9
- sqlmesh/core/signal.py +64 -1
- sqlmesh/core/snapshot/__init__.py +2 -0
- sqlmesh/core/snapshot/definition.py +146 -34
- sqlmesh/core/snapshot/evaluator.py +689 -124
- sqlmesh/core/state_sync/__init__.py +0 -1
- sqlmesh/core/state_sync/base.py +55 -33
- sqlmesh/core/state_sync/cache.py +12 -7
- sqlmesh/core/state_sync/common.py +216 -111
- sqlmesh/core/state_sync/db/environment.py +6 -4
- sqlmesh/core/state_sync/db/facade.py +42 -24
- sqlmesh/core/state_sync/db/interval.py +27 -7
- sqlmesh/core/state_sync/db/migrator.py +34 -16
- sqlmesh/core/state_sync/db/snapshot.py +177 -169
- sqlmesh/core/table_diff.py +2 -2
- sqlmesh/core/test/context.py +2 -0
- sqlmesh/core/test/definition.py +14 -9
- sqlmesh/dbt/adapter.py +22 -16
- sqlmesh/dbt/basemodel.py +75 -56
- sqlmesh/dbt/builtin.py +116 -12
- sqlmesh/dbt/column.py +17 -5
- sqlmesh/dbt/common.py +19 -5
- sqlmesh/dbt/context.py +14 -1
- sqlmesh/dbt/loader.py +61 -9
- sqlmesh/dbt/manifest.py +174 -16
- sqlmesh/dbt/model.py +183 -85
- sqlmesh/dbt/package.py +16 -1
- sqlmesh/dbt/profile.py +3 -3
- sqlmesh/dbt/project.py +12 -7
- sqlmesh/dbt/seed.py +6 -1
- sqlmesh/dbt/source.py +13 -1
- sqlmesh/dbt/target.py +25 -6
- sqlmesh/dbt/test.py +36 -5
- sqlmesh/migrations/v0000_baseline.py +95 -0
- sqlmesh/migrations/v0061_mysql_fix_blob_text_type.py +5 -7
- sqlmesh/migrations/v0062_add_model_gateway.py +5 -1
- sqlmesh/migrations/v0063_change_signals.py +5 -3
- sqlmesh/migrations/v0064_join_when_matched_strings.py +5 -3
- sqlmesh/migrations/v0065_add_model_optimize.py +5 -1
- sqlmesh/migrations/v0066_add_auto_restatements.py +8 -3
- sqlmesh/migrations/v0067_add_tsql_date_full_precision.py +5 -1
- sqlmesh/migrations/v0068_include_unrendered_query_in_metadata_hash.py +5 -1
- sqlmesh/migrations/v0069_update_dev_table_suffix.py +5 -3
- sqlmesh/migrations/v0070_include_grains_in_metadata_hash.py +5 -1
- sqlmesh/migrations/v0071_add_dev_version_to_intervals.py +9 -5
- sqlmesh/migrations/v0072_add_environment_statements.py +5 -3
- sqlmesh/migrations/v0073_remove_symbolic_disable_restatement.py +5 -3
- sqlmesh/migrations/v0074_add_partition_by_time_column_property.py +5 -1
- sqlmesh/migrations/v0075_remove_validate_query.py +5 -3
- sqlmesh/migrations/v0076_add_cron_tz.py +5 -1
- sqlmesh/migrations/v0077_fix_column_type_hash_calculation.py +5 -1
- sqlmesh/migrations/v0078_warn_if_non_migratable_python_env.py +5 -3
- sqlmesh/migrations/v0079_add_gateway_managed_property.py +10 -5
- sqlmesh/migrations/v0080_add_batch_size_to_scd_type_2_models.py +5 -1
- sqlmesh/migrations/v0081_update_partitioned_by.py +5 -3
- sqlmesh/migrations/v0082_warn_if_incorrectly_duplicated_statements.py +5 -3
- sqlmesh/migrations/v0083_use_sql_for_scd_time_data_type_data_hash.py +5 -1
- sqlmesh/migrations/v0084_normalize_quote_when_matched_and_merge_filter.py +5 -1
- sqlmesh/migrations/v0085_deterministic_repr.py +5 -3
- sqlmesh/migrations/v0086_check_deterministic_bug.py +5 -3
- sqlmesh/migrations/v0087_normalize_blueprint_variables.py +5 -3
- sqlmesh/migrations/v0088_warn_about_variable_python_env_diffs.py +5 -3
- sqlmesh/migrations/v0089_add_virtual_environment_mode.py +5 -1
- sqlmesh/migrations/v0090_add_forward_only_column.py +9 -5
- sqlmesh/migrations/v0091_on_additive_change.py +5 -1
- sqlmesh/migrations/v0092_warn_about_dbt_data_type_diff.py +5 -3
- sqlmesh/migrations/v0093_use_raw_sql_in_fingerprint.py +5 -1
- sqlmesh/migrations/v0094_add_dev_version_and_fingerprint_columns.py +123 -0
- sqlmesh/migrations/v0095_warn_about_dbt_raw_sql_diff.py +49 -0
- sqlmesh/migrations/v0096_remove_plan_dags_table.py +13 -0
- sqlmesh/migrations/v0097_add_dbt_name_in_node.py +9 -0
- sqlmesh/migrations/{v0060_move_audits_to_model.py → v0098_add_dbt_node_info_in_node.py} +33 -16
- sqlmesh/migrations/v0099_add_last_altered_to_intervals.py +25 -0
- sqlmesh/migrations/v0100_add_grants_and_grants_target_layer.py +9 -0
- sqlmesh/utils/__init__.py +8 -1
- sqlmesh/utils/cache.py +5 -1
- sqlmesh/utils/connection_pool.py +2 -1
- sqlmesh/utils/dag.py +65 -10
- sqlmesh/utils/date.py +8 -1
- sqlmesh/utils/errors.py +8 -0
- sqlmesh/utils/jinja.py +54 -4
- sqlmesh/utils/pydantic.py +6 -6
- sqlmesh/utils/windows.py +13 -3
- {sqlmesh-0.213.1.dev1.dist-info → sqlmesh-0.227.2.dev4.dist-info}/METADATA +7 -10
- sqlmesh-0.227.2.dev4.dist-info/RECORD +370 -0
- sqlmesh_dbt/cli.py +70 -7
- sqlmesh_dbt/console.py +14 -6
- sqlmesh_dbt/operations.py +103 -24
- sqlmesh_dbt/selectors.py +39 -1
- web/client/dist/assets/{Audits-Ucsx1GzF.js → Audits-CBiYyyx-.js} +1 -1
- web/client/dist/assets/{Banner-BWDzvavM.js → Banner-DSRbUlO5.js} +1 -1
- web/client/dist/assets/{ChevronDownIcon-D2VL13Ah.js → ChevronDownIcon-MK_nrjD_.js} +1 -1
- web/client/dist/assets/{ChevronRightIcon-DWGYbf1l.js → ChevronRightIcon-CLWtT22Q.js} +1 -1
- web/client/dist/assets/{Content-DdHDZM3I.js → Content-BNuGZN5l.js} +1 -1
- web/client/dist/assets/{Content-Bikfy8fh.js → Content-CSHJyW0n.js} +1 -1
- web/client/dist/assets/{Data-CzAJH7rW.js → Data-C1oRDbLx.js} +1 -1
- web/client/dist/assets/{DataCatalog-BJF11g8f.js → DataCatalog-HXyX2-_j.js} +1 -1
- web/client/dist/assets/{Editor-s0SBpV2y.js → Editor-BDyfpUuw.js} +1 -1
- web/client/dist/assets/{Editor-DgLhgKnm.js → Editor-D0jNItwC.js} +1 -1
- web/client/dist/assets/{Errors-D0m0O1d3.js → Errors-BfuFLcPi.js} +1 -1
- web/client/dist/assets/{FileExplorer-CEv0vXkt.js → FileExplorer-BR9IE3he.js} +1 -1
- web/client/dist/assets/{Footer-BwzXn8Ew.js → Footer-CgBEtiAh.js} +1 -1
- web/client/dist/assets/{Header-6heDkEqG.js → Header-DSqR6nSO.js} +1 -1
- web/client/dist/assets/{Input-obuJsD6k.js → Input-B-oZ6fGO.js} +1 -1
- web/client/dist/assets/Lineage-DYQVwDbD.js +1 -0
- web/client/dist/assets/{ListboxShow-HM9_qyrt.js → ListboxShow-BE5-xevs.js} +1 -1
- web/client/dist/assets/{ModelLineage-zWdKo0U2.js → ModelLineage-DkIFAYo4.js} +1 -1
- web/client/dist/assets/{Models-Bcu66SRz.js → Models-D5dWr8RB.js} +1 -1
- web/client/dist/assets/{Page-BWEEQfIt.js → Page-C-XfU5BR.js} +1 -1
- web/client/dist/assets/{Plan-C4gXCqlf.js → Plan-ZEuTINBq.js} +1 -1
- web/client/dist/assets/{PlusCircleIcon-CVDO651q.js → PlusCircleIcon-DVXAHG8_.js} +1 -1
- web/client/dist/assets/{ReportErrors-BT6xFwAr.js → ReportErrors-B7FEPzMB.js} +1 -1
- web/client/dist/assets/{Root-ryJoBK4h.js → Root-8aZyhPxF.js} +1 -1
- web/client/dist/assets/{SearchList-DB04sPb9.js → SearchList-W_iT2G82.js} +1 -1
- web/client/dist/assets/{SelectEnvironment-CUYcXUu6.js → SelectEnvironment-C65jALmO.js} +1 -1
- web/client/dist/assets/{SourceList-Doo_9ZGp.js → SourceList-DSLO6nVJ.js} +1 -1
- web/client/dist/assets/{SourceListItem-D5Mj7Dly.js → SourceListItem-BHt8d9-I.js} +1 -1
- web/client/dist/assets/{SplitPane-qHmkD1qy.js → SplitPane-CViaZmw6.js} +1 -1
- web/client/dist/assets/{Tests-DH1Z74ML.js → Tests-DhaVt5t1.js} +1 -1
- web/client/dist/assets/{Welcome-DqUJUNMF.js → Welcome-DvpjH-_4.js} +1 -1
- web/client/dist/assets/context-BctCsyGb.js +71 -0
- web/client/dist/assets/{context-Dr54UHLi.js → context-DFNeGsFF.js} +1 -1
- web/client/dist/assets/{editor-DYIP1yQ4.js → editor-CcO28cqd.js} +1 -1
- web/client/dist/assets/{file-DarlIDVi.js → file-CvJN3aZO.js} +1 -1
- web/client/dist/assets/{floating-ui.react-dom-BH3TFvkM.js → floating-ui.react-dom-CjE-JNW1.js} +1 -1
- web/client/dist/assets/{help-Bl8wqaQc.js → help-DuPhjipa.js} +1 -1
- web/client/dist/assets/{index-D1sR7wpN.js → index-C-dJH7yZ.js} +1 -1
- web/client/dist/assets/{index-O3mjYpnE.js → index-Dj0i1-CA.js} +2 -2
- web/client/dist/assets/{plan-CehRrJUG.js → plan-BTRSbjKn.js} +1 -1
- web/client/dist/assets/{popover-CqgMRE0G.js → popover-_Sf0yvOI.js} +1 -1
- web/client/dist/assets/{project-6gxepOhm.js → project-BvSOI8MY.js} +1 -1
- web/client/dist/index.html +1 -1
- sqlmesh/integrations/llm.py +0 -56
- sqlmesh/migrations/v0001_init.py +0 -60
- sqlmesh/migrations/v0002_remove_identify.py +0 -5
- sqlmesh/migrations/v0003_move_batch_size.py +0 -34
- sqlmesh/migrations/v0004_environmnent_add_finalized_at.py +0 -23
- sqlmesh/migrations/v0005_create_seed_table.py +0 -24
- sqlmesh/migrations/v0006_change_seed_hash.py +0 -5
- sqlmesh/migrations/v0007_env_table_info_to_kind.py +0 -99
- sqlmesh/migrations/v0008_create_intervals_table.py +0 -38
- sqlmesh/migrations/v0009_remove_pre_post_hooks.py +0 -62
- sqlmesh/migrations/v0010_seed_hash_batch_size.py +0 -5
- sqlmesh/migrations/v0011_add_model_kind_name.py +0 -63
- sqlmesh/migrations/v0012_update_jinja_expressions.py +0 -86
- sqlmesh/migrations/v0013_serde_using_model_dialects.py +0 -87
- sqlmesh/migrations/v0014_fix_dev_intervals.py +0 -14
- sqlmesh/migrations/v0015_environment_add_promoted_snapshot_ids.py +0 -26
- sqlmesh/migrations/v0016_fix_windows_path.py +0 -59
- sqlmesh/migrations/v0017_fix_windows_seed_path.py +0 -55
- sqlmesh/migrations/v0018_rename_snapshot_model_to_node.py +0 -53
- sqlmesh/migrations/v0019_add_env_suffix_target.py +0 -28
- sqlmesh/migrations/v0020_remove_redundant_attributes_from_dbt_models.py +0 -80
- sqlmesh/migrations/v0021_fix_table_properties.py +0 -62
- sqlmesh/migrations/v0022_move_project_to_model.py +0 -54
- sqlmesh/migrations/v0023_fix_added_models_with_forward_only_parents.py +0 -65
- sqlmesh/migrations/v0024_replace_model_kind_name_enum_with_value.py +0 -55
- sqlmesh/migrations/v0025_fix_intervals_and_missing_change_category.py +0 -117
- sqlmesh/migrations/v0026_remove_dialect_from_seed.py +0 -55
- sqlmesh/migrations/v0027_minute_interval_to_five.py +0 -57
- sqlmesh/migrations/v0028_add_plan_dags_table.py +0 -29
- sqlmesh/migrations/v0029_generate_schema_types_using_dialect.py +0 -69
- sqlmesh/migrations/v0030_update_unrestorable_snapshots.py +0 -65
- sqlmesh/migrations/v0031_remove_dbt_target_fields.py +0 -65
- sqlmesh/migrations/v0032_add_sqlmesh_version.py +0 -25
- sqlmesh/migrations/v0033_mysql_fix_blob_text_type.py +0 -45
- sqlmesh/migrations/v0034_add_default_catalog.py +0 -367
- sqlmesh/migrations/v0035_add_catalog_name_override.py +0 -22
- sqlmesh/migrations/v0036_delete_plan_dags_bug_fix.py +0 -14
- sqlmesh/migrations/v0037_remove_dbt_is_incremental_macro.py +0 -61
- sqlmesh/migrations/v0038_add_expiration_ts_to_snapshot.py +0 -73
- sqlmesh/migrations/v0039_include_environment_in_plan_dag_spec.py +0 -68
- sqlmesh/migrations/v0040_add_previous_finalized_snapshots.py +0 -26
- sqlmesh/migrations/v0041_remove_hash_raw_query_attribute.py +0 -59
- sqlmesh/migrations/v0042_trim_indirect_versions.py +0 -66
- sqlmesh/migrations/v0043_fix_remove_obsolete_attributes_in_plan_dags.py +0 -61
- sqlmesh/migrations/v0044_quote_identifiers_in_model_attributes.py +0 -5
- sqlmesh/migrations/v0045_move_gateway_variable.py +0 -70
- sqlmesh/migrations/v0046_add_batch_concurrency.py +0 -8
- sqlmesh/migrations/v0047_change_scd_string_to_column.py +0 -5
- sqlmesh/migrations/v0048_drop_indirect_versions.py +0 -59
- sqlmesh/migrations/v0049_replace_identifier_with_version_in_seeds_table.py +0 -57
- sqlmesh/migrations/v0050_drop_seeds_table.py +0 -11
- sqlmesh/migrations/v0051_rename_column_descriptions.py +0 -65
- sqlmesh/migrations/v0052_add_normalize_name_in_environment_naming_info.py +0 -28
- sqlmesh/migrations/v0053_custom_model_kind_extra_attributes.py +0 -5
- sqlmesh/migrations/v0054_fix_trailing_comments.py +0 -5
- sqlmesh/migrations/v0055_add_updated_ts_unpaused_ts_ttl_ms_unrestorable_to_snapshot.py +0 -132
- sqlmesh/migrations/v0056_restore_table_indexes.py +0 -118
- sqlmesh/migrations/v0057_add_table_format.py +0 -5
- sqlmesh/migrations/v0058_add_requirements.py +0 -26
- sqlmesh/migrations/v0059_add_physical_version.py +0 -5
- sqlmesh-0.213.1.dev1.dist-info/RECORD +0 -421
- web/client/dist/assets/Lineage-D0Hgdz2v.js +0 -1
- web/client/dist/assets/context-DgX0fp2E.js +0 -68
- {sqlmesh-0.213.1.dev1.dist-info → sqlmesh-0.227.2.dev4.dist-info}/WHEEL +0 -0
- {sqlmesh-0.213.1.dev1.dist-info → sqlmesh-0.227.2.dev4.dist-info}/entry_points.txt +0 -0
- {sqlmesh-0.213.1.dev1.dist-info → sqlmesh-0.227.2.dev4.dist-info}/licenses/LICENSE +0 -0
- {sqlmesh-0.213.1.dev1.dist-info → sqlmesh-0.227.2.dev4.dist-info}/top_level.txt +0 -0
|
@@ -1,15 +1,15 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
|
-
|
|
3
|
-
from sqlmesh.core.config.linter import LinterConfig
|
|
4
|
-
from sqlmesh.core.model import Model
|
|
5
|
-
from sqlmesh.utils.errors import raise_config_error
|
|
6
|
-
from sqlmesh.core.console import LinterConsole, get_console
|
|
2
|
+
|
|
7
3
|
import operator as op
|
|
4
|
+
import typing as t
|
|
8
5
|
from collections.abc import Iterator, Iterable, Set, Mapping, Callable
|
|
9
6
|
from functools import reduce
|
|
10
|
-
|
|
11
|
-
from sqlmesh.core.linter
|
|
7
|
+
|
|
8
|
+
from sqlmesh.core.config.linter import LinterConfig
|
|
12
9
|
from sqlmesh.core.console import LinterConsole, get_console
|
|
10
|
+
from sqlmesh.core.linter.rule import Rule, RuleViolation, Range, Fix
|
|
11
|
+
from sqlmesh.core.model import Model
|
|
12
|
+
from sqlmesh.utils.errors import raise_config_error
|
|
13
13
|
|
|
14
14
|
if t.TYPE_CHECKING:
|
|
15
15
|
from sqlmesh.core.context import GenericContext
|
|
@@ -38,6 +38,12 @@ class Linter:
|
|
|
38
38
|
self.rules = rules
|
|
39
39
|
self.warn_rules = warn_rules
|
|
40
40
|
|
|
41
|
+
if overlapping := rules.intersection(warn_rules):
|
|
42
|
+
overlapping_rules = ", ".join(rule for rule in overlapping)
|
|
43
|
+
raise_config_error(
|
|
44
|
+
f"Rules cannot simultaneously warn and raise an error: [{overlapping_rules}]"
|
|
45
|
+
)
|
|
46
|
+
|
|
41
47
|
@classmethod
|
|
42
48
|
def from_rules(cls, all_rules: RuleSet, config: LinterConfig) -> Linter:
|
|
43
49
|
ignored_rules = select_rules(all_rules, config.ignored_rules)
|
|
@@ -46,12 +52,6 @@ class Linter:
|
|
|
46
52
|
rules = select_rules(included_rules, config.rules)
|
|
47
53
|
warn_rules = select_rules(included_rules, config.warn_rules)
|
|
48
54
|
|
|
49
|
-
if overlapping := rules.intersection(warn_rules):
|
|
50
|
-
overlapping_rules = ", ".join(rule for rule in overlapping)
|
|
51
|
-
raise_config_error(
|
|
52
|
-
f"Rules cannot simultaneously warn and raise an error: [{overlapping_rules}]"
|
|
53
|
-
)
|
|
54
|
-
|
|
55
55
|
return Linter(config.enabled, all_rules, rules, warn_rules)
|
|
56
56
|
|
|
57
57
|
def lint_model(
|
|
@@ -274,4 +274,33 @@ class NoMissingExternalModels(Rule):
|
|
|
274
274
|
)
|
|
275
275
|
|
|
276
276
|
|
|
277
|
+
class NoAmbiguousProjections(Rule):
|
|
278
|
+
"""All projections in a model must have unique & inferrable names or explicit aliases."""
|
|
279
|
+
|
|
280
|
+
def check_model(self, model: Model) -> t.Optional[RuleViolation]:
|
|
281
|
+
query = model.render_query()
|
|
282
|
+
if query is None:
|
|
283
|
+
return None
|
|
284
|
+
|
|
285
|
+
name_counts: t.Dict[str, int] = {}
|
|
286
|
+
projection_list = query.selects
|
|
287
|
+
for expression in projection_list:
|
|
288
|
+
alias = expression.output_name
|
|
289
|
+
if alias == "*":
|
|
290
|
+
continue
|
|
291
|
+
|
|
292
|
+
if not alias:
|
|
293
|
+
return self.violation(
|
|
294
|
+
f"Outer projection '{expression.sql(dialect=model.dialect)}' must have inferrable names or explicit aliases."
|
|
295
|
+
)
|
|
296
|
+
|
|
297
|
+
name_counts[alias] = name_counts.get(alias, 0) + 1
|
|
298
|
+
|
|
299
|
+
for name, count in name_counts.items():
|
|
300
|
+
if count > 1:
|
|
301
|
+
return self.violation(f"Found duplicate outer select name '{name}'")
|
|
302
|
+
|
|
303
|
+
return None
|
|
304
|
+
|
|
305
|
+
|
|
277
306
|
BUILTIN_RULES = RuleSet(subclasses(__name__, Rule, (Rule,)))
|
sqlmesh/core/macros.py
CHANGED
|
@@ -128,6 +128,17 @@ def _macro_str_replace(text: str) -> str:
|
|
|
128
128
|
return f"self.template({text}, locals())"
|
|
129
129
|
|
|
130
130
|
|
|
131
|
+
class CaseInsensitiveMapping(t.Dict[str, t.Any]):
|
|
132
|
+
def __init__(self, data: t.Dict[str, t.Any]) -> None:
|
|
133
|
+
super().__init__(data)
|
|
134
|
+
|
|
135
|
+
def __getitem__(self, key: str) -> t.Any:
|
|
136
|
+
return super().__getitem__(key.lower())
|
|
137
|
+
|
|
138
|
+
def get(self, key: str, default: t.Any = None, /) -> t.Any:
|
|
139
|
+
return super().get(key.lower(), default)
|
|
140
|
+
|
|
141
|
+
|
|
131
142
|
class MacroDialect(Python):
|
|
132
143
|
class Generator(Python.Generator):
|
|
133
144
|
TRANSFORMS = {
|
|
@@ -256,14 +267,18 @@ class MacroEvaluator:
|
|
|
256
267
|
changed = True
|
|
257
268
|
variables = self.variables
|
|
258
269
|
|
|
259
|
-
|
|
270
|
+
# This makes all variables case-insensitive, e.g. @X is the same as @x. We do this
|
|
271
|
+
# for consistency, since `variables` and `blueprint_variables` are normalized.
|
|
272
|
+
var_name = node.name.lower()
|
|
273
|
+
|
|
274
|
+
if var_name not in self.locals and var_name not in variables:
|
|
260
275
|
if not isinstance(node.parent, StagedFilePath):
|
|
261
276
|
raise SQLMeshError(f"Macro variable '{node.name}' is undefined.")
|
|
262
277
|
|
|
263
278
|
return node
|
|
264
279
|
|
|
265
280
|
# Precedence order is locals (e.g. @DEF) > blueprint variables > config variables
|
|
266
|
-
value = self.locals.get(
|
|
281
|
+
value = self.locals.get(var_name, variables.get(var_name))
|
|
267
282
|
if isinstance(value, list):
|
|
268
283
|
return exp.convert(
|
|
269
284
|
tuple(
|
|
@@ -313,11 +328,16 @@ class MacroEvaluator:
|
|
|
313
328
|
"""
|
|
314
329
|
# We try to convert all variables into sqlglot expressions because they're going to be converted
|
|
315
330
|
# into strings; in sql we don't convert strings because that would result in adding quotes
|
|
316
|
-
|
|
317
|
-
k: convert_sql(v, self.dialect)
|
|
331
|
+
base_mapping = {
|
|
332
|
+
k.lower(): convert_sql(v, self.dialect)
|
|
318
333
|
for k, v in chain(self.variables.items(), self.locals.items(), local_variables.items())
|
|
334
|
+
if k.lower()
|
|
335
|
+
not in (
|
|
336
|
+
"engine_adapter",
|
|
337
|
+
"snapshot",
|
|
338
|
+
)
|
|
319
339
|
}
|
|
320
|
-
return MacroStrTemplate(str(text)).safe_substitute(
|
|
340
|
+
return MacroStrTemplate(str(text)).safe_substitute(CaseInsensitiveMapping(base_mapping))
|
|
321
341
|
|
|
322
342
|
def evaluate(self, node: MacroFunc) -> exp.Expression | t.List[exp.Expression] | None:
|
|
323
343
|
if isinstance(node, MacroDef):
|
|
@@ -327,7 +347,9 @@ class MacroEvaluator:
|
|
|
327
347
|
args[0] if len(args) == 1 else exp.Tuple(expressions=list(args))
|
|
328
348
|
)
|
|
329
349
|
else:
|
|
330
|
-
|
|
350
|
+
# Make variables defined through `@DEF` case-insensitive
|
|
351
|
+
self.locals[node.name.lower()] = self.transform(node.expression)
|
|
352
|
+
|
|
331
353
|
return node
|
|
332
354
|
|
|
333
355
|
if isinstance(node, (MacroSQL, MacroStrReplace)):
|
|
@@ -630,7 +652,7 @@ def _norm_var_arg_lambda(
|
|
|
630
652
|
) -> exp.Expression | t.List[exp.Expression] | None:
|
|
631
653
|
if isinstance(node, (exp.Identifier, exp.Var)):
|
|
632
654
|
if not isinstance(node.parent, exp.Column):
|
|
633
|
-
name = node.name
|
|
655
|
+
name = node.name.lower()
|
|
634
656
|
if name in args:
|
|
635
657
|
return args[name].copy()
|
|
636
658
|
if name in evaluator.locals:
|
|
@@ -663,7 +685,7 @@ def _norm_var_arg_lambda(
|
|
|
663
685
|
return expressions, lambda args: func.this.transform(
|
|
664
686
|
substitute,
|
|
665
687
|
{
|
|
666
|
-
expression.name: arg
|
|
688
|
+
expression.name.lower(): arg
|
|
667
689
|
for expression, arg in zip(
|
|
668
690
|
func.expressions, args.expressions if isinstance(args, exp.Tuple) else [args]
|
|
669
691
|
)
|
|
@@ -1128,7 +1150,7 @@ def haversine_distance(
|
|
|
1128
1150
|
def pivot(
|
|
1129
1151
|
evaluator: MacroEvaluator,
|
|
1130
1152
|
column: SQL,
|
|
1131
|
-
values: t.List[
|
|
1153
|
+
values: t.List[exp.Expression],
|
|
1132
1154
|
alias: bool = True,
|
|
1133
1155
|
agg: exp.Expression = exp.Literal.string("SUM"),
|
|
1134
1156
|
cmp: exp.Expression = exp.Literal.string("="),
|
|
@@ -1146,10 +1168,10 @@ def pivot(
|
|
|
1146
1168
|
>>> from sqlmesh.core.macros import MacroEvaluator
|
|
1147
1169
|
>>> sql = "SELECT date_day, @PIVOT(status, ['cancelled', 'completed']) FROM rides GROUP BY 1"
|
|
1148
1170
|
>>> MacroEvaluator().transform(parse_one(sql)).sql()
|
|
1149
|
-
'SELECT date_day, SUM(CASE WHEN status = \\'cancelled\\' THEN 1 ELSE 0 END) AS "
|
|
1171
|
+
'SELECT date_day, SUM(CASE WHEN status = \\'cancelled\\' THEN 1 ELSE 0 END) AS "cancelled", SUM(CASE WHEN status = \\'completed\\' THEN 1 ELSE 0 END) AS "completed" FROM rides GROUP BY 1'
|
|
1150
1172
|
>>> sql = "SELECT @PIVOT(a, ['v'], then_value := tv, suffix := '_sfx', quote := FALSE)"
|
|
1151
1173
|
>>> MacroEvaluator(dialect="bigquery").transform(parse_one(sql)).sql("bigquery")
|
|
1152
|
-
"SELECT SUM(CASE WHEN a = 'v' THEN tv ELSE 0 END) AS
|
|
1174
|
+
"SELECT SUM(CASE WHEN a = 'v' THEN tv ELSE 0 END) AS v_sfx"
|
|
1153
1175
|
"""
|
|
1154
1176
|
aggregates: t.List[exp.Expression] = []
|
|
1155
1177
|
for value in values:
|
|
@@ -1157,12 +1179,12 @@ def pivot(
|
|
|
1157
1179
|
if distinct:
|
|
1158
1180
|
proj += "DISTINCT "
|
|
1159
1181
|
|
|
1160
|
-
proj += f"CASE WHEN {column} {cmp.name} {value} THEN {then_value} ELSE {else_value} END) "
|
|
1182
|
+
proj += f"CASE WHEN {column} {cmp.name} {value.sql(evaluator.dialect)} THEN {then_value} ELSE {else_value} END) "
|
|
1161
1183
|
node = evaluator.parse_one(proj)
|
|
1162
1184
|
|
|
1163
1185
|
if alias:
|
|
1164
1186
|
node = node.as_(
|
|
1165
|
-
f"{prefix.name}{value}{suffix.name}",
|
|
1187
|
+
f"{prefix.name}{value.name}{suffix.name}",
|
|
1166
1188
|
quoted=quote,
|
|
1167
1189
|
copy=False,
|
|
1168
1190
|
dialect=evaluator.dialect,
|
sqlmesh/core/model/common.py
CHANGED
|
@@ -641,6 +641,7 @@ properties_validator: t.Callable = field_validator(
|
|
|
641
641
|
"physical_properties_",
|
|
642
642
|
"virtual_properties_",
|
|
643
643
|
"materialization_properties_",
|
|
644
|
+
"grants_",
|
|
644
645
|
mode="before",
|
|
645
646
|
check_fields=False,
|
|
646
647
|
)(parse_properties)
|
|
@@ -662,6 +663,7 @@ depends_on_validator: t.Callable = field_validator(
|
|
|
662
663
|
|
|
663
664
|
class ParsableSql(PydanticModel):
|
|
664
665
|
sql: str
|
|
666
|
+
transaction: t.Optional[bool] = None
|
|
665
667
|
|
|
666
668
|
_parsed: t.Optional[exp.Expression] = None
|
|
667
669
|
_parsed_dialect: t.Optional[str] = None
|
sqlmesh/core/model/definition.py
CHANGED
|
@@ -67,6 +67,7 @@ if t.TYPE_CHECKING:
|
|
|
67
67
|
from sqlmesh.core.context import ExecutionContext
|
|
68
68
|
from sqlmesh.core.engine_adapter import EngineAdapter
|
|
69
69
|
from sqlmesh.core.engine_adapter._typing import QueryOrDF
|
|
70
|
+
from sqlmesh.core.engine_adapter.shared import DataObjectType
|
|
70
71
|
from sqlmesh.core.linter.rule import Rule
|
|
71
72
|
from sqlmesh.core.snapshot import DeployabilityIndex, Node, Snapshot
|
|
72
73
|
from sqlmesh.utils.jinja import MacroReference
|
|
@@ -362,6 +363,7 @@ class _Model(ModelMeta, frozen=True):
|
|
|
362
363
|
expand: t.Iterable[str] = tuple(),
|
|
363
364
|
deployability_index: t.Optional[DeployabilityIndex] = None,
|
|
364
365
|
engine_adapter: t.Optional[EngineAdapter] = None,
|
|
366
|
+
inside_transaction: t.Optional[bool] = True,
|
|
365
367
|
**kwargs: t.Any,
|
|
366
368
|
) -> t.List[exp.Expression]:
|
|
367
369
|
"""Renders pre-statements for a model.
|
|
@@ -383,7 +385,11 @@ class _Model(ModelMeta, frozen=True):
|
|
|
383
385
|
The list of rendered expressions.
|
|
384
386
|
"""
|
|
385
387
|
return self._render_statements(
|
|
386
|
-
|
|
388
|
+
[
|
|
389
|
+
stmt
|
|
390
|
+
for stmt in self.pre_statements
|
|
391
|
+
if stmt.args.get("transaction", True) == inside_transaction
|
|
392
|
+
],
|
|
387
393
|
start=start,
|
|
388
394
|
end=end,
|
|
389
395
|
execution_time=execution_time,
|
|
@@ -404,6 +410,7 @@ class _Model(ModelMeta, frozen=True):
|
|
|
404
410
|
expand: t.Iterable[str] = tuple(),
|
|
405
411
|
deployability_index: t.Optional[DeployabilityIndex] = None,
|
|
406
412
|
engine_adapter: t.Optional[EngineAdapter] = None,
|
|
413
|
+
inside_transaction: t.Optional[bool] = True,
|
|
407
414
|
**kwargs: t.Any,
|
|
408
415
|
) -> t.List[exp.Expression]:
|
|
409
416
|
"""Renders post-statements for a model.
|
|
@@ -419,13 +426,18 @@ class _Model(ModelMeta, frozen=True):
|
|
|
419
426
|
that depend on materialized tables. Model definitions are inlined and can thus be run end to
|
|
420
427
|
end on the fly.
|
|
421
428
|
deployability_index: Determines snapshots that are deployable in the context of this render.
|
|
429
|
+
inside_transaction: Whether to render hooks with transaction=True (inside) or transaction=False (outside).
|
|
422
430
|
kwargs: Additional kwargs to pass to the renderer.
|
|
423
431
|
|
|
424
432
|
Returns:
|
|
425
433
|
The list of rendered expressions.
|
|
426
434
|
"""
|
|
427
435
|
return self._render_statements(
|
|
428
|
-
|
|
436
|
+
[
|
|
437
|
+
stmt
|
|
438
|
+
for stmt in self.post_statements
|
|
439
|
+
if stmt.args.get("transaction", True) == inside_transaction
|
|
440
|
+
],
|
|
429
441
|
start=start,
|
|
430
442
|
end=end,
|
|
431
443
|
execution_time=execution_time,
|
|
@@ -566,6 +578,8 @@ class _Model(ModelMeta, frozen=True):
|
|
|
566
578
|
result = []
|
|
567
579
|
for v in value:
|
|
568
580
|
parsed = v.parse(self.dialect)
|
|
581
|
+
if getattr(v, "transaction", None) is not None:
|
|
582
|
+
parsed.set("transaction", v.transaction)
|
|
569
583
|
if not isinstance(parsed, exp.Semicolon):
|
|
570
584
|
result.append(parsed)
|
|
571
585
|
return result
|
|
@@ -594,7 +608,7 @@ class _Model(ModelMeta, frozen=True):
|
|
|
594
608
|
python_env=self.python_env,
|
|
595
609
|
only_execution_time=False,
|
|
596
610
|
default_catalog=self.default_catalog,
|
|
597
|
-
|
|
611
|
+
model=self,
|
|
598
612
|
)
|
|
599
613
|
return self._statement_renderer_cache[expression_key]
|
|
600
614
|
|
|
@@ -1186,6 +1200,8 @@ class _Model(ModelMeta, frozen=True):
|
|
|
1186
1200
|
gen(self.session_properties_) if self.session_properties_ else None,
|
|
1187
1201
|
*[gen(g) for g in self.grains],
|
|
1188
1202
|
*self._audit_metadata_hash_values(),
|
|
1203
|
+
json.dumps(self.grants, sort_keys=True) if self.grants else None,
|
|
1204
|
+
self.grants_target_layer,
|
|
1189
1205
|
]
|
|
1190
1206
|
|
|
1191
1207
|
for key, value in (self.virtual_properties or {}).items():
|
|
@@ -1197,6 +1213,9 @@ class _Model(ModelMeta, frozen=True):
|
|
|
1197
1213
|
for k, v in sorted(args.items()):
|
|
1198
1214
|
metadata.append(f"{k}:{gen(v)}")
|
|
1199
1215
|
|
|
1216
|
+
if self.dbt_node_info:
|
|
1217
|
+
metadata.append(self.dbt_node_info.json(sort_keys=True))
|
|
1218
|
+
|
|
1200
1219
|
metadata.extend(self._additional_metadata)
|
|
1201
1220
|
|
|
1202
1221
|
self._metadata_hash = hash_data(metadata)
|
|
@@ -1207,6 +1226,24 @@ class _Model(ModelMeta, frozen=True):
|
|
|
1207
1226
|
"""Return True if this is a model node"""
|
|
1208
1227
|
return True
|
|
1209
1228
|
|
|
1229
|
+
@property
|
|
1230
|
+
def grants_table_type(self) -> DataObjectType:
|
|
1231
|
+
"""Get the table type for grants application (TABLE, VIEW, MATERIALIZED_VIEW).
|
|
1232
|
+
|
|
1233
|
+
Returns:
|
|
1234
|
+
The DataObjectType that should be used when applying grants to this model.
|
|
1235
|
+
"""
|
|
1236
|
+
from sqlmesh.core.engine_adapter.shared import DataObjectType
|
|
1237
|
+
|
|
1238
|
+
if self.kind.is_view:
|
|
1239
|
+
if hasattr(self.kind, "materialized") and getattr(self.kind, "materialized", False):
|
|
1240
|
+
return DataObjectType.MATERIALIZED_VIEW
|
|
1241
|
+
return DataObjectType.VIEW
|
|
1242
|
+
if self.kind.is_managed:
|
|
1243
|
+
return DataObjectType.MANAGED_TABLE
|
|
1244
|
+
# All other materialized models are tables
|
|
1245
|
+
return DataObjectType.TABLE
|
|
1246
|
+
|
|
1210
1247
|
@property
|
|
1211
1248
|
def _additional_metadata(self) -> t.List[str]:
|
|
1212
1249
|
additional_metadata = []
|
|
@@ -1417,12 +1454,20 @@ class SqlModel(_Model):
|
|
|
1417
1454
|
|
|
1418
1455
|
unknown = exp.DataType.build("unknown")
|
|
1419
1456
|
|
|
1420
|
-
|
|
1457
|
+
columns_to_types = {}
|
|
1458
|
+
for select in query.selects:
|
|
1459
|
+
output_name = select.output_name
|
|
1460
|
+
|
|
1461
|
+
# If model validation is disabled, we cannot assume that projections
|
|
1462
|
+
# will have inferrable output names or even that they will be unique
|
|
1463
|
+
if not output_name or output_name in columns_to_types:
|
|
1464
|
+
return None
|
|
1465
|
+
|
|
1421
1466
|
# copy data type because it is used in the engine to build CTAS and other queries
|
|
1422
1467
|
# this can change the parent which will mess up the diffing algo
|
|
1423
|
-
|
|
1424
|
-
|
|
1425
|
-
|
|
1468
|
+
columns_to_types[output_name] = (select.type or unknown).copy()
|
|
1469
|
+
|
|
1470
|
+
self._columns_to_types = columns_to_types
|
|
1426
1471
|
|
|
1427
1472
|
if "*" in self._columns_to_types:
|
|
1428
1473
|
return None
|
|
@@ -1473,22 +1518,6 @@ class SqlModel(_Model):
|
|
|
1473
1518
|
if not projection_list:
|
|
1474
1519
|
raise_config_error("Query missing select statements", self._path)
|
|
1475
1520
|
|
|
1476
|
-
name_counts: t.Dict[str, int] = {}
|
|
1477
|
-
for expression in projection_list:
|
|
1478
|
-
alias = expression.output_name
|
|
1479
|
-
if alias == "*":
|
|
1480
|
-
continue
|
|
1481
|
-
if not alias:
|
|
1482
|
-
raise_config_error(
|
|
1483
|
-
f"Outer projection '{expression.sql(dialect=self.dialect)}' must have inferrable names or explicit aliases.",
|
|
1484
|
-
self._path,
|
|
1485
|
-
)
|
|
1486
|
-
name_counts[alias] = name_counts.get(alias, 0) + 1
|
|
1487
|
-
|
|
1488
|
-
for name, count in name_counts.items():
|
|
1489
|
-
if count > 1:
|
|
1490
|
-
raise_config_error(f"Found duplicate outer select name '{name}'", self._path)
|
|
1491
|
-
|
|
1492
1521
|
if self.depends_on_self and not self.annotated:
|
|
1493
1522
|
raise_config_error(
|
|
1494
1523
|
"Self-referencing models require inferrable column types. There are three options available to mitigate this issue: add explicit types to all projections in the outermost SELECT statement, leverage external models (https://sqlmesh.readthedocs.io/en/stable/concepts/models/external_models/), or use the `columns` model attribute (https://sqlmesh.readthedocs.io/en/stable/concepts/models/overview/#columns).",
|
|
@@ -1573,7 +1602,6 @@ class SqlModel(_Model):
|
|
|
1573
1602
|
self.dialect,
|
|
1574
1603
|
self.macro_definitions,
|
|
1575
1604
|
schema=self.mapping_schema,
|
|
1576
|
-
model_fqn=self.fqn,
|
|
1577
1605
|
path=self._path,
|
|
1578
1606
|
jinja_macro_registry=self.jinja_macros,
|
|
1579
1607
|
python_env=self.python_env,
|
|
@@ -1581,6 +1609,7 @@ class SqlModel(_Model):
|
|
|
1581
1609
|
default_catalog=self.default_catalog,
|
|
1582
1610
|
quote_identifiers=not no_quote_identifiers,
|
|
1583
1611
|
optimize_query=self.optimize_query,
|
|
1612
|
+
model=self,
|
|
1584
1613
|
)
|
|
1585
1614
|
|
|
1586
1615
|
@property
|
|
@@ -1828,6 +1857,12 @@ class SeedModel(_Model):
|
|
|
1828
1857
|
for column_name, column_hash in self.column_hashes.items():
|
|
1829
1858
|
data.append(column_name)
|
|
1830
1859
|
data.append(column_hash)
|
|
1860
|
+
|
|
1861
|
+
# Include grants in data hash for seed models to force recreation on grant changes
|
|
1862
|
+
# since seed models don't support migration
|
|
1863
|
+
data.append(json.dumps(self.grants, sort_keys=True) if self.grants else "")
|
|
1864
|
+
data.append(self.grants_target_layer)
|
|
1865
|
+
|
|
1831
1866
|
return data
|
|
1832
1867
|
|
|
1833
1868
|
|
|
@@ -1846,8 +1881,9 @@ class PythonModel(_Model):
|
|
|
1846
1881
|
super().validate_definition()
|
|
1847
1882
|
|
|
1848
1883
|
if self.kind and not self.kind.supports_python_models:
|
|
1849
|
-
|
|
1850
|
-
f"Cannot create Python model '{self.name}' as the '{self.kind.name}' kind doesn't support Python models"
|
|
1884
|
+
raise_config_error(
|
|
1885
|
+
f"Cannot create Python model '{self.name}' as the '{self.kind.name}' kind doesn't support Python models",
|
|
1886
|
+
self._path,
|
|
1851
1887
|
)
|
|
1852
1888
|
|
|
1853
1889
|
def render(
|
|
@@ -2569,9 +2605,17 @@ def _create_model(
|
|
|
2569
2605
|
if statement_field in kwargs:
|
|
2570
2606
|
# Macros extracted from these statements need to be treated as metadata only
|
|
2571
2607
|
is_metadata = statement_field == "on_virtual_update"
|
|
2572
|
-
|
|
2608
|
+
for stmt in kwargs[statement_field]:
|
|
2609
|
+
# Extract the expression if it's ParsableSql already
|
|
2610
|
+
expr = stmt.parse(dialect) if isinstance(stmt, ParsableSql) else stmt
|
|
2611
|
+
statements.append((expr, is_metadata))
|
|
2573
2612
|
kwargs[statement_field] = [
|
|
2574
|
-
|
|
2613
|
+
# this to retain the transaction information
|
|
2614
|
+
stmt
|
|
2615
|
+
if isinstance(stmt, ParsableSql)
|
|
2616
|
+
else ParsableSql.from_parsed_expression(
|
|
2617
|
+
stmt, dialect, use_meta_sql=use_original_sql
|
|
2618
|
+
)
|
|
2575
2619
|
for stmt in kwargs[statement_field]
|
|
2576
2620
|
]
|
|
2577
2621
|
|
|
@@ -2873,6 +2917,13 @@ def render_meta_fields(
|
|
|
2873
2917
|
for key, value in field_value.items():
|
|
2874
2918
|
if key in RUNTIME_RENDERED_MODEL_FIELDS:
|
|
2875
2919
|
rendered_dict[key] = parse_strings_with_macro_refs(value, dialect)
|
|
2920
|
+
elif (
|
|
2921
|
+
# don't parse kind auto_restatement_cron="@..." kwargs (e.g. @daily) into MacroVar
|
|
2922
|
+
key == "auto_restatement_cron"
|
|
2923
|
+
and isinstance(value, str)
|
|
2924
|
+
and value.lower() in CRON_SHORTCUTS
|
|
2925
|
+
):
|
|
2926
|
+
rendered_dict[key] = value
|
|
2876
2927
|
elif (rendered := render_field_value(value)) is not None:
|
|
2877
2928
|
rendered_dict[key] = rendered
|
|
2878
2929
|
|
|
@@ -3019,6 +3070,9 @@ META_FIELD_CONVERTER: t.Dict[str, t.Callable] = {
|
|
|
3019
3070
|
"formatting": str,
|
|
3020
3071
|
"optimize_query": str,
|
|
3021
3072
|
"virtual_environment_mode": lambda value: exp.Literal.string(value.value),
|
|
3073
|
+
"dbt_node_info_": lambda value: value.to_expression(),
|
|
3074
|
+
"grants_": lambda value: value,
|
|
3075
|
+
"grants_target_layer": lambda value: exp.Literal.string(value.value),
|
|
3022
3076
|
}
|
|
3023
3077
|
|
|
3024
3078
|
|
sqlmesh/core/model/kind.py
CHANGED
|
@@ -23,7 +23,7 @@ from sqlmesh.utils.pydantic import (
|
|
|
23
23
|
PydanticModel,
|
|
24
24
|
SQLGlotBool,
|
|
25
25
|
SQLGlotColumn,
|
|
26
|
-
|
|
26
|
+
SQLGlotListOfFieldsOrStar,
|
|
27
27
|
SQLGlotListOfFields,
|
|
28
28
|
SQLGlotPositiveInt,
|
|
29
29
|
SQLGlotString,
|
|
@@ -119,6 +119,10 @@ class ModelKindMixin:
|
|
|
119
119
|
def is_managed(self) -> bool:
|
|
120
120
|
return self.model_kind_name == ModelKindName.MANAGED
|
|
121
121
|
|
|
122
|
+
@property
|
|
123
|
+
def is_dbt_custom(self) -> bool:
|
|
124
|
+
return self.model_kind_name == ModelKindName.DBT_CUSTOM
|
|
125
|
+
|
|
122
126
|
@property
|
|
123
127
|
def is_symbolic(self) -> bool:
|
|
124
128
|
"""A symbolic model is one that doesn't execute at all."""
|
|
@@ -150,6 +154,11 @@ class ModelKindMixin:
|
|
|
150
154
|
def supports_python_models(self) -> bool:
|
|
151
155
|
return True
|
|
152
156
|
|
|
157
|
+
@property
|
|
158
|
+
def supports_grants(self) -> bool:
|
|
159
|
+
"""Whether this model kind supports grants configuration."""
|
|
160
|
+
return self.is_materialized or self.is_view
|
|
161
|
+
|
|
153
162
|
|
|
154
163
|
class ModelKindName(str, ModelKindMixin, Enum):
|
|
155
164
|
"""The kind of model, determining how this data is computed and stored in the warehouse."""
|
|
@@ -170,6 +179,7 @@ class ModelKindName(str, ModelKindMixin, Enum):
|
|
|
170
179
|
EXTERNAL = "EXTERNAL"
|
|
171
180
|
CUSTOM = "CUSTOM"
|
|
172
181
|
MANAGED = "MANAGED"
|
|
182
|
+
DBT_CUSTOM = "DBT_CUSTOM"
|
|
173
183
|
|
|
174
184
|
@property
|
|
175
185
|
def model_kind_name(self) -> t.Optional[ModelKindName]:
|
|
@@ -842,7 +852,7 @@ class SCDType2ByTimeKind(_SCDType2Kind):
|
|
|
842
852
|
|
|
843
853
|
class SCDType2ByColumnKind(_SCDType2Kind):
|
|
844
854
|
name: t.Literal[ModelKindName.SCD_TYPE_2_BY_COLUMN] = ModelKindName.SCD_TYPE_2_BY_COLUMN
|
|
845
|
-
columns:
|
|
855
|
+
columns: SQLGlotListOfFieldsOrStar
|
|
846
856
|
execution_time_as_valid_from: SQLGlotBool = False
|
|
847
857
|
updated_at_name: t.Optional[SQLGlotColumn] = None
|
|
848
858
|
|
|
@@ -887,6 +897,46 @@ class ManagedKind(_ModelKind):
|
|
|
887
897
|
return False
|
|
888
898
|
|
|
889
899
|
|
|
900
|
+
class DbtCustomKind(_ModelKind):
|
|
901
|
+
name: t.Literal[ModelKindName.DBT_CUSTOM] = ModelKindName.DBT_CUSTOM
|
|
902
|
+
materialization: str
|
|
903
|
+
adapter: str = "default"
|
|
904
|
+
definition: str
|
|
905
|
+
dialect: t.Optional[str] = Field(None, validate_default=True)
|
|
906
|
+
|
|
907
|
+
_dialect_validator = kind_dialect_validator
|
|
908
|
+
|
|
909
|
+
@field_validator("materialization", "adapter", "definition", mode="before")
|
|
910
|
+
@classmethod
|
|
911
|
+
def _validate_fields(cls, v: t.Any) -> str:
|
|
912
|
+
return validate_string(v)
|
|
913
|
+
|
|
914
|
+
@property
|
|
915
|
+
def data_hash_values(self) -> t.List[t.Optional[str]]:
|
|
916
|
+
return [
|
|
917
|
+
*super().data_hash_values,
|
|
918
|
+
self.materialization,
|
|
919
|
+
self.definition,
|
|
920
|
+
self.adapter,
|
|
921
|
+
self.dialect,
|
|
922
|
+
]
|
|
923
|
+
|
|
924
|
+
def to_expression(
|
|
925
|
+
self, expressions: t.Optional[t.List[exp.Expression]] = None, **kwargs: t.Any
|
|
926
|
+
) -> d.ModelKind:
|
|
927
|
+
return super().to_expression(
|
|
928
|
+
expressions=[
|
|
929
|
+
*(expressions or []),
|
|
930
|
+
*_properties(
|
|
931
|
+
{
|
|
932
|
+
"materialization": exp.Literal.string(self.materialization),
|
|
933
|
+
"adapter": exp.Literal.string(self.adapter),
|
|
934
|
+
}
|
|
935
|
+
),
|
|
936
|
+
],
|
|
937
|
+
)
|
|
938
|
+
|
|
939
|
+
|
|
890
940
|
class EmbeddedKind(_ModelKind):
|
|
891
941
|
name: t.Literal[ModelKindName.EMBEDDED] = ModelKindName.EMBEDDED
|
|
892
942
|
|
|
@@ -992,6 +1042,7 @@ ModelKind = t.Annotated[
|
|
|
992
1042
|
SCDType2ByColumnKind,
|
|
993
1043
|
CustomKind,
|
|
994
1044
|
ManagedKind,
|
|
1045
|
+
DbtCustomKind,
|
|
995
1046
|
],
|
|
996
1047
|
Field(discriminator="name"),
|
|
997
1048
|
]
|
|
@@ -1011,6 +1062,7 @@ MODEL_KIND_NAME_TO_TYPE: t.Dict[str, t.Type[ModelKind]] = {
|
|
|
1011
1062
|
ModelKindName.SCD_TYPE_2_BY_COLUMN: SCDType2ByColumnKind,
|
|
1012
1063
|
ModelKindName.CUSTOM: CustomKind,
|
|
1013
1064
|
ModelKindName.MANAGED: ManagedKind,
|
|
1065
|
+
ModelKindName.DBT_CUSTOM: DbtCustomKind,
|
|
1014
1066
|
}
|
|
1015
1067
|
|
|
1016
1068
|
|
|
@@ -1053,6 +1105,18 @@ def create_model_kind(v: t.Any, dialect: str, defaults: t.Dict[str, t.Any]) -> M
|
|
|
1053
1105
|
):
|
|
1054
1106
|
props[on_change_property] = defaults.get(on_change_property)
|
|
1055
1107
|
|
|
1108
|
+
# only pass the batch_concurrency user default to models inheriting from _IncrementalBy
|
|
1109
|
+
# that don't explicitly set it in the model definition, but ignore subclasses of _IncrementalBy
|
|
1110
|
+
# that hardcode a specific batch_concurrency
|
|
1111
|
+
if issubclass(kind_type, _IncrementalBy):
|
|
1112
|
+
BATCH_CONCURRENCY: t.Final = "batch_concurrency"
|
|
1113
|
+
if (
|
|
1114
|
+
props.get(BATCH_CONCURRENCY) is None
|
|
1115
|
+
and defaults.get(BATCH_CONCURRENCY) is not None
|
|
1116
|
+
and kind_type.all_field_infos()[BATCH_CONCURRENCY].default is None
|
|
1117
|
+
):
|
|
1118
|
+
props[BATCH_CONCURRENCY] = defaults.get(BATCH_CONCURRENCY)
|
|
1119
|
+
|
|
1056
1120
|
if kind_type == CustomKind:
|
|
1057
1121
|
# load the custom materialization class and check if it uses a custom kind type
|
|
1058
1122
|
from sqlmesh.core.snapshot.evaluator import get_custom_materialization_type
|