sqlmesh 0.213.1.dev1__py3-none-any.whl → 0.227.2.dev4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sqlmesh/__init__.py +12 -2
- sqlmesh/_version.py +2 -2
- sqlmesh/cli/main.py +0 -44
- sqlmesh/cli/project_init.py +11 -2
- sqlmesh/core/_typing.py +1 -0
- sqlmesh/core/audit/definition.py +8 -2
- sqlmesh/core/config/__init__.py +1 -1
- sqlmesh/core/config/connection.py +17 -5
- sqlmesh/core/config/dbt.py +13 -0
- sqlmesh/core/config/janitor.py +12 -0
- sqlmesh/core/config/loader.py +7 -0
- sqlmesh/core/config/model.py +2 -0
- sqlmesh/core/config/root.py +3 -0
- sqlmesh/core/console.py +81 -3
- sqlmesh/core/constants.py +1 -1
- sqlmesh/core/context.py +69 -26
- sqlmesh/core/dialect.py +3 -0
- sqlmesh/core/engine_adapter/_typing.py +2 -0
- sqlmesh/core/engine_adapter/base.py +322 -22
- sqlmesh/core/engine_adapter/base_postgres.py +17 -1
- sqlmesh/core/engine_adapter/bigquery.py +146 -7
- sqlmesh/core/engine_adapter/clickhouse.py +17 -13
- sqlmesh/core/engine_adapter/databricks.py +33 -2
- sqlmesh/core/engine_adapter/fabric.py +10 -29
- sqlmesh/core/engine_adapter/mixins.py +142 -48
- sqlmesh/core/engine_adapter/mssql.py +15 -4
- sqlmesh/core/engine_adapter/mysql.py +2 -2
- sqlmesh/core/engine_adapter/postgres.py +9 -3
- sqlmesh/core/engine_adapter/redshift.py +4 -0
- sqlmesh/core/engine_adapter/risingwave.py +1 -0
- sqlmesh/core/engine_adapter/shared.py +6 -0
- sqlmesh/core/engine_adapter/snowflake.py +82 -11
- sqlmesh/core/engine_adapter/spark.py +14 -10
- sqlmesh/core/engine_adapter/trino.py +4 -2
- sqlmesh/core/environment.py +2 -0
- sqlmesh/core/janitor.py +181 -0
- sqlmesh/core/lineage.py +1 -0
- sqlmesh/core/linter/definition.py +13 -13
- sqlmesh/core/linter/rules/builtin.py +29 -0
- sqlmesh/core/macros.py +35 -13
- sqlmesh/core/model/common.py +2 -0
- sqlmesh/core/model/definition.py +82 -28
- sqlmesh/core/model/kind.py +66 -2
- sqlmesh/core/model/meta.py +108 -4
- sqlmesh/core/node.py +101 -1
- sqlmesh/core/plan/builder.py +18 -10
- sqlmesh/core/plan/common.py +199 -2
- sqlmesh/core/plan/definition.py +25 -6
- sqlmesh/core/plan/evaluator.py +75 -113
- sqlmesh/core/plan/explainer.py +90 -8
- sqlmesh/core/plan/stages.py +42 -21
- sqlmesh/core/renderer.py +78 -32
- sqlmesh/core/scheduler.py +102 -22
- sqlmesh/core/selector.py +137 -9
- sqlmesh/core/signal.py +64 -1
- sqlmesh/core/snapshot/__init__.py +2 -0
- sqlmesh/core/snapshot/definition.py +146 -34
- sqlmesh/core/snapshot/evaluator.py +689 -124
- sqlmesh/core/state_sync/__init__.py +0 -1
- sqlmesh/core/state_sync/base.py +55 -33
- sqlmesh/core/state_sync/cache.py +12 -7
- sqlmesh/core/state_sync/common.py +216 -111
- sqlmesh/core/state_sync/db/environment.py +6 -4
- sqlmesh/core/state_sync/db/facade.py +42 -24
- sqlmesh/core/state_sync/db/interval.py +27 -7
- sqlmesh/core/state_sync/db/migrator.py +34 -16
- sqlmesh/core/state_sync/db/snapshot.py +177 -169
- sqlmesh/core/table_diff.py +2 -2
- sqlmesh/core/test/context.py +2 -0
- sqlmesh/core/test/definition.py +14 -9
- sqlmesh/dbt/adapter.py +22 -16
- sqlmesh/dbt/basemodel.py +75 -56
- sqlmesh/dbt/builtin.py +116 -12
- sqlmesh/dbt/column.py +17 -5
- sqlmesh/dbt/common.py +19 -5
- sqlmesh/dbt/context.py +14 -1
- sqlmesh/dbt/loader.py +61 -9
- sqlmesh/dbt/manifest.py +174 -16
- sqlmesh/dbt/model.py +183 -85
- sqlmesh/dbt/package.py +16 -1
- sqlmesh/dbt/profile.py +3 -3
- sqlmesh/dbt/project.py +12 -7
- sqlmesh/dbt/seed.py +6 -1
- sqlmesh/dbt/source.py +13 -1
- sqlmesh/dbt/target.py +25 -6
- sqlmesh/dbt/test.py +36 -5
- sqlmesh/migrations/v0000_baseline.py +95 -0
- sqlmesh/migrations/v0061_mysql_fix_blob_text_type.py +5 -7
- sqlmesh/migrations/v0062_add_model_gateway.py +5 -1
- sqlmesh/migrations/v0063_change_signals.py +5 -3
- sqlmesh/migrations/v0064_join_when_matched_strings.py +5 -3
- sqlmesh/migrations/v0065_add_model_optimize.py +5 -1
- sqlmesh/migrations/v0066_add_auto_restatements.py +8 -3
- sqlmesh/migrations/v0067_add_tsql_date_full_precision.py +5 -1
- sqlmesh/migrations/v0068_include_unrendered_query_in_metadata_hash.py +5 -1
- sqlmesh/migrations/v0069_update_dev_table_suffix.py +5 -3
- sqlmesh/migrations/v0070_include_grains_in_metadata_hash.py +5 -1
- sqlmesh/migrations/v0071_add_dev_version_to_intervals.py +9 -5
- sqlmesh/migrations/v0072_add_environment_statements.py +5 -3
- sqlmesh/migrations/v0073_remove_symbolic_disable_restatement.py +5 -3
- sqlmesh/migrations/v0074_add_partition_by_time_column_property.py +5 -1
- sqlmesh/migrations/v0075_remove_validate_query.py +5 -3
- sqlmesh/migrations/v0076_add_cron_tz.py +5 -1
- sqlmesh/migrations/v0077_fix_column_type_hash_calculation.py +5 -1
- sqlmesh/migrations/v0078_warn_if_non_migratable_python_env.py +5 -3
- sqlmesh/migrations/v0079_add_gateway_managed_property.py +10 -5
- sqlmesh/migrations/v0080_add_batch_size_to_scd_type_2_models.py +5 -1
- sqlmesh/migrations/v0081_update_partitioned_by.py +5 -3
- sqlmesh/migrations/v0082_warn_if_incorrectly_duplicated_statements.py +5 -3
- sqlmesh/migrations/v0083_use_sql_for_scd_time_data_type_data_hash.py +5 -1
- sqlmesh/migrations/v0084_normalize_quote_when_matched_and_merge_filter.py +5 -1
- sqlmesh/migrations/v0085_deterministic_repr.py +5 -3
- sqlmesh/migrations/v0086_check_deterministic_bug.py +5 -3
- sqlmesh/migrations/v0087_normalize_blueprint_variables.py +5 -3
- sqlmesh/migrations/v0088_warn_about_variable_python_env_diffs.py +5 -3
- sqlmesh/migrations/v0089_add_virtual_environment_mode.py +5 -1
- sqlmesh/migrations/v0090_add_forward_only_column.py +9 -5
- sqlmesh/migrations/v0091_on_additive_change.py +5 -1
- sqlmesh/migrations/v0092_warn_about_dbt_data_type_diff.py +5 -3
- sqlmesh/migrations/v0093_use_raw_sql_in_fingerprint.py +5 -1
- sqlmesh/migrations/v0094_add_dev_version_and_fingerprint_columns.py +123 -0
- sqlmesh/migrations/v0095_warn_about_dbt_raw_sql_diff.py +49 -0
- sqlmesh/migrations/v0096_remove_plan_dags_table.py +13 -0
- sqlmesh/migrations/v0097_add_dbt_name_in_node.py +9 -0
- sqlmesh/migrations/{v0060_move_audits_to_model.py → v0098_add_dbt_node_info_in_node.py} +33 -16
- sqlmesh/migrations/v0099_add_last_altered_to_intervals.py +25 -0
- sqlmesh/migrations/v0100_add_grants_and_grants_target_layer.py +9 -0
- sqlmesh/utils/__init__.py +8 -1
- sqlmesh/utils/cache.py +5 -1
- sqlmesh/utils/connection_pool.py +2 -1
- sqlmesh/utils/dag.py +65 -10
- sqlmesh/utils/date.py +8 -1
- sqlmesh/utils/errors.py +8 -0
- sqlmesh/utils/jinja.py +54 -4
- sqlmesh/utils/pydantic.py +6 -6
- sqlmesh/utils/windows.py +13 -3
- {sqlmesh-0.213.1.dev1.dist-info → sqlmesh-0.227.2.dev4.dist-info}/METADATA +7 -10
- sqlmesh-0.227.2.dev4.dist-info/RECORD +370 -0
- sqlmesh_dbt/cli.py +70 -7
- sqlmesh_dbt/console.py +14 -6
- sqlmesh_dbt/operations.py +103 -24
- sqlmesh_dbt/selectors.py +39 -1
- web/client/dist/assets/{Audits-Ucsx1GzF.js → Audits-CBiYyyx-.js} +1 -1
- web/client/dist/assets/{Banner-BWDzvavM.js → Banner-DSRbUlO5.js} +1 -1
- web/client/dist/assets/{ChevronDownIcon-D2VL13Ah.js → ChevronDownIcon-MK_nrjD_.js} +1 -1
- web/client/dist/assets/{ChevronRightIcon-DWGYbf1l.js → ChevronRightIcon-CLWtT22Q.js} +1 -1
- web/client/dist/assets/{Content-DdHDZM3I.js → Content-BNuGZN5l.js} +1 -1
- web/client/dist/assets/{Content-Bikfy8fh.js → Content-CSHJyW0n.js} +1 -1
- web/client/dist/assets/{Data-CzAJH7rW.js → Data-C1oRDbLx.js} +1 -1
- web/client/dist/assets/{DataCatalog-BJF11g8f.js → DataCatalog-HXyX2-_j.js} +1 -1
- web/client/dist/assets/{Editor-s0SBpV2y.js → Editor-BDyfpUuw.js} +1 -1
- web/client/dist/assets/{Editor-DgLhgKnm.js → Editor-D0jNItwC.js} +1 -1
- web/client/dist/assets/{Errors-D0m0O1d3.js → Errors-BfuFLcPi.js} +1 -1
- web/client/dist/assets/{FileExplorer-CEv0vXkt.js → FileExplorer-BR9IE3he.js} +1 -1
- web/client/dist/assets/{Footer-BwzXn8Ew.js → Footer-CgBEtiAh.js} +1 -1
- web/client/dist/assets/{Header-6heDkEqG.js → Header-DSqR6nSO.js} +1 -1
- web/client/dist/assets/{Input-obuJsD6k.js → Input-B-oZ6fGO.js} +1 -1
- web/client/dist/assets/Lineage-DYQVwDbD.js +1 -0
- web/client/dist/assets/{ListboxShow-HM9_qyrt.js → ListboxShow-BE5-xevs.js} +1 -1
- web/client/dist/assets/{ModelLineage-zWdKo0U2.js → ModelLineage-DkIFAYo4.js} +1 -1
- web/client/dist/assets/{Models-Bcu66SRz.js → Models-D5dWr8RB.js} +1 -1
- web/client/dist/assets/{Page-BWEEQfIt.js → Page-C-XfU5BR.js} +1 -1
- web/client/dist/assets/{Plan-C4gXCqlf.js → Plan-ZEuTINBq.js} +1 -1
- web/client/dist/assets/{PlusCircleIcon-CVDO651q.js → PlusCircleIcon-DVXAHG8_.js} +1 -1
- web/client/dist/assets/{ReportErrors-BT6xFwAr.js → ReportErrors-B7FEPzMB.js} +1 -1
- web/client/dist/assets/{Root-ryJoBK4h.js → Root-8aZyhPxF.js} +1 -1
- web/client/dist/assets/{SearchList-DB04sPb9.js → SearchList-W_iT2G82.js} +1 -1
- web/client/dist/assets/{SelectEnvironment-CUYcXUu6.js → SelectEnvironment-C65jALmO.js} +1 -1
- web/client/dist/assets/{SourceList-Doo_9ZGp.js → SourceList-DSLO6nVJ.js} +1 -1
- web/client/dist/assets/{SourceListItem-D5Mj7Dly.js → SourceListItem-BHt8d9-I.js} +1 -1
- web/client/dist/assets/{SplitPane-qHmkD1qy.js → SplitPane-CViaZmw6.js} +1 -1
- web/client/dist/assets/{Tests-DH1Z74ML.js → Tests-DhaVt5t1.js} +1 -1
- web/client/dist/assets/{Welcome-DqUJUNMF.js → Welcome-DvpjH-_4.js} +1 -1
- web/client/dist/assets/context-BctCsyGb.js +71 -0
- web/client/dist/assets/{context-Dr54UHLi.js → context-DFNeGsFF.js} +1 -1
- web/client/dist/assets/{editor-DYIP1yQ4.js → editor-CcO28cqd.js} +1 -1
- web/client/dist/assets/{file-DarlIDVi.js → file-CvJN3aZO.js} +1 -1
- web/client/dist/assets/{floating-ui.react-dom-BH3TFvkM.js → floating-ui.react-dom-CjE-JNW1.js} +1 -1
- web/client/dist/assets/{help-Bl8wqaQc.js → help-DuPhjipa.js} +1 -1
- web/client/dist/assets/{index-D1sR7wpN.js → index-C-dJH7yZ.js} +1 -1
- web/client/dist/assets/{index-O3mjYpnE.js → index-Dj0i1-CA.js} +2 -2
- web/client/dist/assets/{plan-CehRrJUG.js → plan-BTRSbjKn.js} +1 -1
- web/client/dist/assets/{popover-CqgMRE0G.js → popover-_Sf0yvOI.js} +1 -1
- web/client/dist/assets/{project-6gxepOhm.js → project-BvSOI8MY.js} +1 -1
- web/client/dist/index.html +1 -1
- sqlmesh/integrations/llm.py +0 -56
- sqlmesh/migrations/v0001_init.py +0 -60
- sqlmesh/migrations/v0002_remove_identify.py +0 -5
- sqlmesh/migrations/v0003_move_batch_size.py +0 -34
- sqlmesh/migrations/v0004_environmnent_add_finalized_at.py +0 -23
- sqlmesh/migrations/v0005_create_seed_table.py +0 -24
- sqlmesh/migrations/v0006_change_seed_hash.py +0 -5
- sqlmesh/migrations/v0007_env_table_info_to_kind.py +0 -99
- sqlmesh/migrations/v0008_create_intervals_table.py +0 -38
- sqlmesh/migrations/v0009_remove_pre_post_hooks.py +0 -62
- sqlmesh/migrations/v0010_seed_hash_batch_size.py +0 -5
- sqlmesh/migrations/v0011_add_model_kind_name.py +0 -63
- sqlmesh/migrations/v0012_update_jinja_expressions.py +0 -86
- sqlmesh/migrations/v0013_serde_using_model_dialects.py +0 -87
- sqlmesh/migrations/v0014_fix_dev_intervals.py +0 -14
- sqlmesh/migrations/v0015_environment_add_promoted_snapshot_ids.py +0 -26
- sqlmesh/migrations/v0016_fix_windows_path.py +0 -59
- sqlmesh/migrations/v0017_fix_windows_seed_path.py +0 -55
- sqlmesh/migrations/v0018_rename_snapshot_model_to_node.py +0 -53
- sqlmesh/migrations/v0019_add_env_suffix_target.py +0 -28
- sqlmesh/migrations/v0020_remove_redundant_attributes_from_dbt_models.py +0 -80
- sqlmesh/migrations/v0021_fix_table_properties.py +0 -62
- sqlmesh/migrations/v0022_move_project_to_model.py +0 -54
- sqlmesh/migrations/v0023_fix_added_models_with_forward_only_parents.py +0 -65
- sqlmesh/migrations/v0024_replace_model_kind_name_enum_with_value.py +0 -55
- sqlmesh/migrations/v0025_fix_intervals_and_missing_change_category.py +0 -117
- sqlmesh/migrations/v0026_remove_dialect_from_seed.py +0 -55
- sqlmesh/migrations/v0027_minute_interval_to_five.py +0 -57
- sqlmesh/migrations/v0028_add_plan_dags_table.py +0 -29
- sqlmesh/migrations/v0029_generate_schema_types_using_dialect.py +0 -69
- sqlmesh/migrations/v0030_update_unrestorable_snapshots.py +0 -65
- sqlmesh/migrations/v0031_remove_dbt_target_fields.py +0 -65
- sqlmesh/migrations/v0032_add_sqlmesh_version.py +0 -25
- sqlmesh/migrations/v0033_mysql_fix_blob_text_type.py +0 -45
- sqlmesh/migrations/v0034_add_default_catalog.py +0 -367
- sqlmesh/migrations/v0035_add_catalog_name_override.py +0 -22
- sqlmesh/migrations/v0036_delete_plan_dags_bug_fix.py +0 -14
- sqlmesh/migrations/v0037_remove_dbt_is_incremental_macro.py +0 -61
- sqlmesh/migrations/v0038_add_expiration_ts_to_snapshot.py +0 -73
- sqlmesh/migrations/v0039_include_environment_in_plan_dag_spec.py +0 -68
- sqlmesh/migrations/v0040_add_previous_finalized_snapshots.py +0 -26
- sqlmesh/migrations/v0041_remove_hash_raw_query_attribute.py +0 -59
- sqlmesh/migrations/v0042_trim_indirect_versions.py +0 -66
- sqlmesh/migrations/v0043_fix_remove_obsolete_attributes_in_plan_dags.py +0 -61
- sqlmesh/migrations/v0044_quote_identifiers_in_model_attributes.py +0 -5
- sqlmesh/migrations/v0045_move_gateway_variable.py +0 -70
- sqlmesh/migrations/v0046_add_batch_concurrency.py +0 -8
- sqlmesh/migrations/v0047_change_scd_string_to_column.py +0 -5
- sqlmesh/migrations/v0048_drop_indirect_versions.py +0 -59
- sqlmesh/migrations/v0049_replace_identifier_with_version_in_seeds_table.py +0 -57
- sqlmesh/migrations/v0050_drop_seeds_table.py +0 -11
- sqlmesh/migrations/v0051_rename_column_descriptions.py +0 -65
- sqlmesh/migrations/v0052_add_normalize_name_in_environment_naming_info.py +0 -28
- sqlmesh/migrations/v0053_custom_model_kind_extra_attributes.py +0 -5
- sqlmesh/migrations/v0054_fix_trailing_comments.py +0 -5
- sqlmesh/migrations/v0055_add_updated_ts_unpaused_ts_ttl_ms_unrestorable_to_snapshot.py +0 -132
- sqlmesh/migrations/v0056_restore_table_indexes.py +0 -118
- sqlmesh/migrations/v0057_add_table_format.py +0 -5
- sqlmesh/migrations/v0058_add_requirements.py +0 -26
- sqlmesh/migrations/v0059_add_physical_version.py +0 -5
- sqlmesh-0.213.1.dev1.dist-info/RECORD +0 -421
- web/client/dist/assets/Lineage-D0Hgdz2v.js +0 -1
- web/client/dist/assets/context-DgX0fp2E.js +0 -68
- {sqlmesh-0.213.1.dev1.dist-info → sqlmesh-0.227.2.dev4.dist-info}/WHEEL +0 -0
- {sqlmesh-0.213.1.dev1.dist-info → sqlmesh-0.227.2.dev4.dist-info}/entry_points.txt +0 -0
- {sqlmesh-0.213.1.dev1.dist-info → sqlmesh-0.227.2.dev4.dist-info}/licenses/LICENSE +0 -0
- {sqlmesh-0.213.1.dev1.dist-info → sqlmesh-0.227.2.dev4.dist-info}/top_level.txt +0 -0
sqlmesh/core/renderer.py
CHANGED
|
@@ -6,7 +6,7 @@ from contextlib import contextmanager
|
|
|
6
6
|
from functools import partial
|
|
7
7
|
from pathlib import Path
|
|
8
8
|
|
|
9
|
-
from sqlglot import exp,
|
|
9
|
+
from sqlglot import exp, Dialect
|
|
10
10
|
from sqlglot.errors import SqlglotError
|
|
11
11
|
from sqlglot.helper import ensure_list
|
|
12
12
|
from sqlglot.optimizer.annotate_types import annotate_types
|
|
@@ -16,14 +16,21 @@ from sqlglot.optimizer.simplify import simplify
|
|
|
16
16
|
from sqlmesh.core import constants as c
|
|
17
17
|
from sqlmesh.core import dialect as d
|
|
18
18
|
from sqlmesh.core.macros import MacroEvaluator, RuntimeStage
|
|
19
|
-
from sqlmesh.utils.date import
|
|
19
|
+
from sqlmesh.utils.date import (
|
|
20
|
+
TimeLike,
|
|
21
|
+
date_dict,
|
|
22
|
+
make_inclusive,
|
|
23
|
+
to_datetime,
|
|
24
|
+
make_ts_exclusive,
|
|
25
|
+
to_tstz,
|
|
26
|
+
)
|
|
20
27
|
from sqlmesh.utils.errors import (
|
|
21
28
|
ConfigError,
|
|
22
29
|
ParsetimeAdapterCallError,
|
|
23
30
|
SQLMeshError,
|
|
24
31
|
raise_config_error,
|
|
25
32
|
)
|
|
26
|
-
from sqlmesh.utils.jinja import JinjaMacroRegistry
|
|
33
|
+
from sqlmesh.utils.jinja import JinjaMacroRegistry, extract_error_details
|
|
27
34
|
from sqlmesh.utils.metaprogramming import Executable, prepare_env
|
|
28
35
|
|
|
29
36
|
if t.TYPE_CHECKING:
|
|
@@ -31,6 +38,7 @@ if t.TYPE_CHECKING:
|
|
|
31
38
|
from sqlglot.dialects.dialect import DialectType
|
|
32
39
|
|
|
33
40
|
from sqlmesh.core.linter.rule import Rule
|
|
41
|
+
from sqlmesh.core.model.definition import _Model
|
|
34
42
|
from sqlmesh.core.snapshot import DeployabilityIndex, Snapshot
|
|
35
43
|
|
|
36
44
|
|
|
@@ -50,9 +58,9 @@ class BaseExpressionRenderer:
|
|
|
50
58
|
schema: t.Optional[t.Dict[str, t.Any]] = None,
|
|
51
59
|
default_catalog: t.Optional[str] = None,
|
|
52
60
|
quote_identifiers: bool = True,
|
|
53
|
-
model_fqn: t.Optional[str] = None,
|
|
54
61
|
normalize_identifiers: bool = True,
|
|
55
62
|
optimize_query: t.Optional[bool] = True,
|
|
63
|
+
model: t.Optional[_Model] = None,
|
|
56
64
|
):
|
|
57
65
|
self._expression = expression
|
|
58
66
|
self._dialect = dialect
|
|
@@ -66,8 +74,9 @@ class BaseExpressionRenderer:
|
|
|
66
74
|
self._quote_identifiers = quote_identifiers
|
|
67
75
|
self.update_schema({} if schema is None else schema)
|
|
68
76
|
self._cache: t.List[t.Optional[exp.Expression]] = []
|
|
69
|
-
self._model_fqn =
|
|
77
|
+
self._model_fqn = model.fqn if model else None
|
|
70
78
|
self._optimize_query_flag = optimize_query is not False
|
|
79
|
+
self._model = model
|
|
71
80
|
|
|
72
81
|
def update_schema(self, schema: t.Dict[str, t.Any]) -> None:
|
|
73
82
|
self.schema = d.normalize_mapping_schema(schema, dialect=self._dialect)
|
|
@@ -187,52 +196,84 @@ class BaseExpressionRenderer:
|
|
|
187
196
|
**kwargs,
|
|
188
197
|
}
|
|
189
198
|
|
|
190
|
-
variables = kwargs.pop("variables", {})
|
|
191
|
-
jinja_env_kwargs = {
|
|
192
|
-
**{
|
|
193
|
-
**render_kwargs,
|
|
194
|
-
**_prepare_python_env_for_jinja(macro_evaluator, self._python_env),
|
|
195
|
-
**variables,
|
|
196
|
-
},
|
|
197
|
-
"snapshots": snapshots or {},
|
|
198
|
-
"table_mapping": table_mapping,
|
|
199
|
-
"deployability_index": deployability_index,
|
|
200
|
-
"default_catalog": self._default_catalog,
|
|
201
|
-
"runtime_stage": runtime_stage.value,
|
|
202
|
-
"resolve_table": _resolve_table,
|
|
203
|
-
}
|
|
204
199
|
if this_model:
|
|
205
200
|
render_kwargs["this_model"] = this_model
|
|
206
|
-
jinja_env_kwargs["this_model"] = this_model.sql(
|
|
207
|
-
dialect=self._dialect, identify=True, comments=False
|
|
208
|
-
)
|
|
209
201
|
|
|
210
|
-
|
|
202
|
+
macro_evaluator.locals.update(render_kwargs)
|
|
203
|
+
|
|
204
|
+
variables = kwargs.pop("variables", {})
|
|
205
|
+
if variables:
|
|
206
|
+
macro_evaluator.locals.setdefault(c.SQLMESH_VARS, {}).update(variables)
|
|
211
207
|
|
|
212
208
|
expressions = [self._expression]
|
|
213
209
|
if isinstance(self._expression, d.Jinja):
|
|
214
210
|
try:
|
|
211
|
+
jinja_env_kwargs = {
|
|
212
|
+
**{
|
|
213
|
+
**render_kwargs,
|
|
214
|
+
**_prepare_python_env_for_jinja(macro_evaluator, self._python_env),
|
|
215
|
+
**variables,
|
|
216
|
+
},
|
|
217
|
+
"snapshots": snapshots or {},
|
|
218
|
+
"table_mapping": table_mapping,
|
|
219
|
+
"deployability_index": deployability_index,
|
|
220
|
+
"default_catalog": self._default_catalog,
|
|
221
|
+
"runtime_stage": runtime_stage.value,
|
|
222
|
+
"resolve_table": _resolve_table,
|
|
223
|
+
"model_instance": self._model,
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
if this_model:
|
|
227
|
+
jinja_env_kwargs["this_model"] = this_model.sql(
|
|
228
|
+
dialect=self._dialect, identify=True, comments=False
|
|
229
|
+
)
|
|
230
|
+
|
|
231
|
+
if self._model and self._model.kind.is_incremental_by_time_range:
|
|
232
|
+
all_refs = list(
|
|
233
|
+
self._jinja_macro_registry.global_objs.get("sources", {}).values() # type: ignore
|
|
234
|
+
) + list(
|
|
235
|
+
self._jinja_macro_registry.global_objs.get("refs", {}).values() # type: ignore
|
|
236
|
+
)
|
|
237
|
+
for ref in all_refs:
|
|
238
|
+
if ref.event_time_filter:
|
|
239
|
+
ref.event_time_filter["start"] = render_kwargs["start_tstz"]
|
|
240
|
+
ref.event_time_filter["end"] = to_tstz(
|
|
241
|
+
make_ts_exclusive(render_kwargs["end_tstz"], dialect=self._dialect)
|
|
242
|
+
)
|
|
243
|
+
|
|
244
|
+
jinja_env = self._jinja_macro_registry.build_environment(**jinja_env_kwargs)
|
|
245
|
+
|
|
215
246
|
expressions = []
|
|
216
247
|
rendered_expression = jinja_env.from_string(self._expression.name).render()
|
|
217
248
|
logger.debug(
|
|
218
249
|
f"Rendered Jinja expression for model '{self._model_fqn}' at '{self._path}': '{rendered_expression}'"
|
|
219
250
|
)
|
|
220
|
-
if rendered_expression.strip():
|
|
221
|
-
expressions = [e for e in parse(rendered_expression, read=self._dialect) if e]
|
|
222
|
-
|
|
223
|
-
if not expressions:
|
|
224
|
-
raise ConfigError(f"Failed to parse an expression:\n{self._expression}")
|
|
225
251
|
except ParsetimeAdapterCallError:
|
|
226
252
|
raise
|
|
227
253
|
except Exception as ex:
|
|
228
254
|
raise ConfigError(
|
|
229
|
-
f"Could not render
|
|
255
|
+
f"Could not render jinja for '{self._path}'.\n" + extract_error_details(ex)
|
|
230
256
|
) from ex
|
|
231
257
|
|
|
232
|
-
|
|
258
|
+
if rendered_expression.strip():
|
|
259
|
+
# ensure there is actual SQL and not just comments and non-SQL jinja
|
|
260
|
+
dialect = Dialect.get_or_raise(self._dialect)
|
|
261
|
+
tokens = dialect.tokenize(rendered_expression)
|
|
233
262
|
|
|
234
|
-
|
|
235
|
-
|
|
263
|
+
if tokens:
|
|
264
|
+
try:
|
|
265
|
+
expressions = [
|
|
266
|
+
e for e in dialect.parser().parse(tokens, rendered_expression) if e
|
|
267
|
+
]
|
|
268
|
+
|
|
269
|
+
if not expressions:
|
|
270
|
+
raise ConfigError(
|
|
271
|
+
f"Failed to parse an expression:\n{rendered_expression}"
|
|
272
|
+
)
|
|
273
|
+
except Exception as ex:
|
|
274
|
+
raise ConfigError(
|
|
275
|
+
f"Could not parse the rendered jinja at '{self._path}'.\n{ex}"
|
|
276
|
+
) from ex
|
|
236
277
|
|
|
237
278
|
for definition in self._macro_definitions:
|
|
238
279
|
try:
|
|
@@ -542,6 +583,11 @@ class QueryRenderer(BaseExpressionRenderer):
|
|
|
542
583
|
expressions = [e for e in expressions if not isinstance(e, exp.Semicolon)]
|
|
543
584
|
|
|
544
585
|
if not expressions:
|
|
586
|
+
# We assume that if there are no expressions, then the model contains dynamic Jinja SQL
|
|
587
|
+
# and we thus treat it similar to models with adapter calls to match dbt's behavior.
|
|
588
|
+
if isinstance(self._expression, d.JinjaQuery):
|
|
589
|
+
return None
|
|
590
|
+
|
|
545
591
|
raise ConfigError(f"Failed to render query at '{self._path}':\n{self._expression}")
|
|
546
592
|
|
|
547
593
|
if len(expressions) > 1:
|
sqlmesh/core/scheduler.py
CHANGED
|
@@ -251,7 +251,9 @@ class Scheduler:
|
|
|
251
251
|
**kwargs,
|
|
252
252
|
)
|
|
253
253
|
|
|
254
|
-
self.state_sync.add_interval(
|
|
254
|
+
self.state_sync.add_interval(
|
|
255
|
+
snapshot, start, end, is_dev=not is_deployable, last_altered_ts=now_timestamp()
|
|
256
|
+
)
|
|
255
257
|
return audit_results
|
|
256
258
|
|
|
257
259
|
def run(
|
|
@@ -335,6 +337,7 @@ class Scheduler:
|
|
|
335
337
|
deployability_index: t.Optional[DeployabilityIndex],
|
|
336
338
|
environment_naming_info: EnvironmentNamingInfo,
|
|
337
339
|
dag: t.Optional[DAG[SnapshotId]] = None,
|
|
340
|
+
is_restatement: bool = False,
|
|
338
341
|
) -> t.Dict[Snapshot, Intervals]:
|
|
339
342
|
dag = dag or snapshots_to_dag(merged_intervals)
|
|
340
343
|
|
|
@@ -349,7 +352,7 @@ class Scheduler:
|
|
|
349
352
|
)
|
|
350
353
|
for snapshot, intervals in merged_intervals.items()
|
|
351
354
|
}
|
|
352
|
-
snapshot_batches = {}
|
|
355
|
+
snapshot_batches: t.Dict[Snapshot, Intervals] = {}
|
|
353
356
|
all_unready_intervals: t.Dict[str, set[Interval]] = {}
|
|
354
357
|
for snapshot_id in dag:
|
|
355
358
|
if snapshot_id not in snapshot_intervals:
|
|
@@ -361,12 +364,22 @@ class Scheduler:
|
|
|
361
364
|
|
|
362
365
|
adapter = self.snapshot_evaluator.get_adapter(snapshot.model_gateway)
|
|
363
366
|
|
|
367
|
+
parent_intervals: Intervals = []
|
|
368
|
+
for parent_id in snapshot.parents:
|
|
369
|
+
parent_snapshot, _ = snapshot_intervals.get(parent_id, (None, []))
|
|
370
|
+
if not parent_snapshot or parent_snapshot.is_external:
|
|
371
|
+
continue
|
|
372
|
+
|
|
373
|
+
parent_intervals.extend(snapshot_batches[parent_snapshot])
|
|
374
|
+
|
|
364
375
|
context = ExecutionContext(
|
|
365
376
|
adapter,
|
|
366
377
|
self.snapshots_by_name,
|
|
367
378
|
deployability_index,
|
|
368
379
|
default_dialect=adapter.dialect,
|
|
369
380
|
default_catalog=self.default_catalog,
|
|
381
|
+
is_restatement=is_restatement,
|
|
382
|
+
parent_intervals=parent_intervals,
|
|
370
383
|
)
|
|
371
384
|
|
|
372
385
|
intervals = self._check_ready_intervals(
|
|
@@ -416,11 +429,13 @@ class Scheduler:
|
|
|
416
429
|
start: t.Optional[TimeLike] = None,
|
|
417
430
|
end: t.Optional[TimeLike] = None,
|
|
418
431
|
allow_destructive_snapshots: t.Optional[t.Set[str]] = None,
|
|
432
|
+
selected_models: t.Optional[t.Set[str]] = None,
|
|
419
433
|
allow_additive_snapshots: t.Optional[t.Set[str]] = None,
|
|
420
434
|
selected_snapshot_ids: t.Optional[t.Set[SnapshotId]] = None,
|
|
421
435
|
run_environment_statements: bool = False,
|
|
422
436
|
audit_only: bool = False,
|
|
423
437
|
auto_restatement_triggers: t.Dict[SnapshotId, t.List[SnapshotId]] = {},
|
|
438
|
+
is_restatement: bool = False,
|
|
424
439
|
) -> t.Tuple[t.List[NodeExecutionFailedError[SchedulingUnit]], t.List[SchedulingUnit]]:
|
|
425
440
|
"""Runs precomputed batches of missing intervals.
|
|
426
441
|
|
|
@@ -445,12 +460,21 @@ class Scheduler:
|
|
|
445
460
|
if not selected_snapshots:
|
|
446
461
|
selected_snapshots = list(merged_intervals)
|
|
447
462
|
|
|
448
|
-
|
|
463
|
+
# Build the full DAG from all snapshots to preserve transitive dependencies
|
|
464
|
+
full_dag = snapshots_to_dag(self.snapshots.values())
|
|
465
|
+
|
|
466
|
+
# Create a subdag that includes the selected snapshots and all their upstream dependencies
|
|
467
|
+
# This ensures that transitive dependencies are preserved even when intermediate nodes are not selected
|
|
468
|
+
selected_snapshot_ids_set = {s.snapshot_id for s in selected_snapshots}
|
|
469
|
+
snapshot_dag = full_dag.subdag(*selected_snapshot_ids_set)
|
|
449
470
|
|
|
450
471
|
batched_intervals = self.batch_intervals(
|
|
451
|
-
merged_intervals,
|
|
472
|
+
merged_intervals,
|
|
473
|
+
deployability_index,
|
|
474
|
+
environment_naming_info,
|
|
475
|
+
dag=snapshot_dag,
|
|
476
|
+
is_restatement=is_restatement,
|
|
452
477
|
)
|
|
453
|
-
|
|
454
478
|
self.console.start_evaluation_progress(
|
|
455
479
|
batched_intervals,
|
|
456
480
|
environment_naming_info,
|
|
@@ -472,12 +496,20 @@ class Scheduler:
|
|
|
472
496
|
start=start,
|
|
473
497
|
end=end,
|
|
474
498
|
execution_time=execution_time,
|
|
499
|
+
selected_models=selected_models,
|
|
475
500
|
)
|
|
476
501
|
|
|
502
|
+
# We only need to create physical tables if the snapshot is not representative or if it
|
|
503
|
+
# needs backfill
|
|
504
|
+
snapshots_to_create_candidates = [
|
|
505
|
+
s
|
|
506
|
+
for s in selected_snapshots
|
|
507
|
+
if not deployability_index.is_representative(s) or s in batched_intervals
|
|
508
|
+
]
|
|
477
509
|
snapshots_to_create = {
|
|
478
510
|
s.snapshot_id
|
|
479
511
|
for s in self.snapshot_evaluator.get_snapshots_to_create(
|
|
480
|
-
|
|
512
|
+
snapshots_to_create_candidates, deployability_index
|
|
481
513
|
)
|
|
482
514
|
}
|
|
483
515
|
|
|
@@ -515,6 +547,10 @@ class Scheduler:
|
|
|
515
547
|
execution_time=execution_time,
|
|
516
548
|
)
|
|
517
549
|
else:
|
|
550
|
+
# If batch_index > 0, then the target table must exist since the first batch would have created it
|
|
551
|
+
target_table_exists = (
|
|
552
|
+
snapshot.snapshot_id not in snapshots_to_create or node.batch_index > 0
|
|
553
|
+
)
|
|
518
554
|
audit_results = self.evaluate(
|
|
519
555
|
snapshot=snapshot,
|
|
520
556
|
environment_naming_info=environment_naming_info,
|
|
@@ -525,7 +561,8 @@ class Scheduler:
|
|
|
525
561
|
batch_index=node.batch_index,
|
|
526
562
|
allow_destructive_snapshots=allow_destructive_snapshots,
|
|
527
563
|
allow_additive_snapshots=allow_additive_snapshots,
|
|
528
|
-
target_table_exists=
|
|
564
|
+
target_table_exists=target_table_exists,
|
|
565
|
+
selected_models=selected_models,
|
|
529
566
|
)
|
|
530
567
|
|
|
531
568
|
evaluation_duration_ms = now_timestamp() - execution_start_ts
|
|
@@ -595,6 +632,7 @@ class Scheduler:
|
|
|
595
632
|
start=start,
|
|
596
633
|
end=end,
|
|
597
634
|
execution_time=execution_time,
|
|
635
|
+
selected_models=selected_models,
|
|
598
636
|
)
|
|
599
637
|
|
|
600
638
|
self.state_sync.recycle()
|
|
@@ -621,6 +659,7 @@ class Scheduler:
|
|
|
621
659
|
}
|
|
622
660
|
snapshots_to_create = snapshots_to_create or set()
|
|
623
661
|
original_snapshots_to_create = snapshots_to_create.copy()
|
|
662
|
+
upstream_dependencies_cache: t.Dict[SnapshotId, t.Set[SchedulingUnit]] = {}
|
|
624
663
|
|
|
625
664
|
snapshot_dag = snapshot_dag or snapshots_to_dag(batches)
|
|
626
665
|
dag = DAG[SchedulingUnit]()
|
|
@@ -632,23 +671,17 @@ class Scheduler:
|
|
|
632
671
|
snapshot = self.snapshots_by_name[snapshot_id.name]
|
|
633
672
|
intervals = intervals_per_snapshot.get(snapshot.name, [])
|
|
634
673
|
|
|
635
|
-
upstream_dependencies: t.
|
|
674
|
+
upstream_dependencies: t.Set[SchedulingUnit] = set()
|
|
636
675
|
|
|
637
676
|
for p_sid in snapshot.parents:
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
for i, interval in enumerate(p_intervals):
|
|
647
|
-
upstream_dependencies.append(
|
|
648
|
-
EvaluateNode(
|
|
649
|
-
snapshot_name=p_sid.name, interval=interval, batch_index=i
|
|
650
|
-
)
|
|
651
|
-
)
|
|
677
|
+
upstream_dependencies.update(
|
|
678
|
+
self._find_upstream_dependencies(
|
|
679
|
+
p_sid,
|
|
680
|
+
intervals_per_snapshot,
|
|
681
|
+
original_snapshots_to_create,
|
|
682
|
+
upstream_dependencies_cache,
|
|
683
|
+
)
|
|
684
|
+
)
|
|
652
685
|
|
|
653
686
|
batch_concurrency = snapshot.node.batch_concurrency
|
|
654
687
|
batch_size = snapshot.node.batch_size
|
|
@@ -692,6 +725,49 @@ class Scheduler:
|
|
|
692
725
|
)
|
|
693
726
|
return dag
|
|
694
727
|
|
|
728
|
+
def _find_upstream_dependencies(
|
|
729
|
+
self,
|
|
730
|
+
parent_sid: SnapshotId,
|
|
731
|
+
intervals_per_snapshot: t.Dict[str, Intervals],
|
|
732
|
+
snapshots_to_create: t.Set[SnapshotId],
|
|
733
|
+
cache: t.Dict[SnapshotId, t.Set[SchedulingUnit]],
|
|
734
|
+
) -> t.Set[SchedulingUnit]:
|
|
735
|
+
if parent_sid not in self.snapshots:
|
|
736
|
+
return set()
|
|
737
|
+
if parent_sid in cache:
|
|
738
|
+
return cache[parent_sid]
|
|
739
|
+
|
|
740
|
+
p_intervals = intervals_per_snapshot.get(parent_sid.name, [])
|
|
741
|
+
|
|
742
|
+
parent_node: t.Optional[SchedulingUnit] = None
|
|
743
|
+
if p_intervals:
|
|
744
|
+
if len(p_intervals) > 1:
|
|
745
|
+
parent_node = DummyNode(snapshot_name=parent_sid.name)
|
|
746
|
+
else:
|
|
747
|
+
interval = p_intervals[0]
|
|
748
|
+
parent_node = EvaluateNode(
|
|
749
|
+
snapshot_name=parent_sid.name, interval=interval, batch_index=0
|
|
750
|
+
)
|
|
751
|
+
elif parent_sid in snapshots_to_create:
|
|
752
|
+
parent_node = CreateNode(snapshot_name=parent_sid.name)
|
|
753
|
+
|
|
754
|
+
if parent_node is not None:
|
|
755
|
+
cache[parent_sid] = {parent_node}
|
|
756
|
+
return {parent_node}
|
|
757
|
+
|
|
758
|
+
# This snapshot has no intervals and doesn't need creation which means
|
|
759
|
+
# that it can be a transitive dependency
|
|
760
|
+
transitive_deps: t.Set[SchedulingUnit] = set()
|
|
761
|
+
parent_snapshot = self.snapshots[parent_sid]
|
|
762
|
+
for grandparent_sid in parent_snapshot.parents:
|
|
763
|
+
transitive_deps.update(
|
|
764
|
+
self._find_upstream_dependencies(
|
|
765
|
+
grandparent_sid, intervals_per_snapshot, snapshots_to_create, cache
|
|
766
|
+
)
|
|
767
|
+
)
|
|
768
|
+
cache[parent_sid] = transitive_deps
|
|
769
|
+
return transitive_deps
|
|
770
|
+
|
|
695
771
|
def _run_or_audit(
|
|
696
772
|
self,
|
|
697
773
|
environment: str | EnvironmentNamingInfo,
|
|
@@ -801,6 +877,9 @@ class Scheduler:
|
|
|
801
877
|
run_environment_statements=run_environment_statements,
|
|
802
878
|
audit_only=audit_only,
|
|
803
879
|
auto_restatement_triggers=auto_restatement_triggers,
|
|
880
|
+
selected_models={
|
|
881
|
+
s.node.dbt_unique_id for s in merged_intervals if s.node.dbt_unique_id
|
|
882
|
+
},
|
|
804
883
|
)
|
|
805
884
|
|
|
806
885
|
return CompletionStatus.FAILURE if errors else CompletionStatus.SUCCESS
|
|
@@ -915,6 +994,7 @@ class Scheduler:
|
|
|
915
994
|
python_env=signals.python_env,
|
|
916
995
|
dialect=snapshot.model.dialect,
|
|
917
996
|
path=snapshot.model._path,
|
|
997
|
+
snapshot=snapshot,
|
|
918
998
|
kwargs=kwargs,
|
|
919
999
|
)
|
|
920
1000
|
except SQLMeshError as e:
|
sqlmesh/core/selector.py
CHANGED
|
@@ -3,6 +3,8 @@ from __future__ import annotations
|
|
|
3
3
|
import fnmatch
|
|
4
4
|
import typing as t
|
|
5
5
|
from pathlib import Path
|
|
6
|
+
from itertools import zip_longest
|
|
7
|
+
import abc
|
|
6
8
|
|
|
7
9
|
from sqlglot import exp
|
|
8
10
|
from sqlglot.errors import ParseError
|
|
@@ -14,6 +16,7 @@ from sqlmesh.core import constants as c
|
|
|
14
16
|
from sqlmesh.core.dialect import normalize_model_name
|
|
15
17
|
from sqlmesh.core.environment import Environment
|
|
16
18
|
from sqlmesh.core.model import update_model_schemas
|
|
19
|
+
from sqlmesh.core.audit import StandaloneAudit
|
|
17
20
|
from sqlmesh.utils import UniqueKeyDict
|
|
18
21
|
from sqlmesh.utils.dag import DAG
|
|
19
22
|
from sqlmesh.utils.git import GitClient
|
|
@@ -23,10 +26,11 @@ from sqlmesh.utils.errors import SQLMeshError
|
|
|
23
26
|
if t.TYPE_CHECKING:
|
|
24
27
|
from typing_extensions import Literal as Lit # noqa
|
|
25
28
|
from sqlmesh.core.model import Model
|
|
29
|
+
from sqlmesh.core.node import Node
|
|
26
30
|
from sqlmesh.core.state_sync import StateReader
|
|
27
31
|
|
|
28
32
|
|
|
29
|
-
class Selector:
|
|
33
|
+
class Selector(abc.ABC):
|
|
30
34
|
def __init__(
|
|
31
35
|
self,
|
|
32
36
|
state_reader: StateReader,
|
|
@@ -165,20 +169,20 @@ class Selector:
|
|
|
165
169
|
return models
|
|
166
170
|
|
|
167
171
|
def expand_model_selections(
|
|
168
|
-
self, model_selections: t.Iterable[str], models: t.Optional[t.Dict[str,
|
|
172
|
+
self, model_selections: t.Iterable[str], models: t.Optional[t.Dict[str, Node]] = None
|
|
169
173
|
) -> t.Set[str]:
|
|
170
|
-
"""Expands a set of model selections into a set of model
|
|
174
|
+
"""Expands a set of model selections into a set of model fqns that can be looked up in the Context.
|
|
171
175
|
|
|
172
176
|
Args:
|
|
173
177
|
model_selections: A set of model selections.
|
|
174
178
|
|
|
175
179
|
Returns:
|
|
176
|
-
A set of model
|
|
180
|
+
A set of model fqns.
|
|
177
181
|
"""
|
|
178
182
|
|
|
179
183
|
node = parse(" | ".join(f"({s})" for s in model_selections))
|
|
180
184
|
|
|
181
|
-
all_models = models or self._models
|
|
185
|
+
all_models: t.Dict[str, Node] = models or dict(self._models)
|
|
182
186
|
models_by_tags: t.Dict[str, t.Set[str]] = {}
|
|
183
187
|
|
|
184
188
|
for fqn, model in all_models.items():
|
|
@@ -194,10 +198,9 @@ class Selector:
|
|
|
194
198
|
return {
|
|
195
199
|
fqn
|
|
196
200
|
for fqn, model in all_models.items()
|
|
197
|
-
if fnmatch.fnmatchcase(model
|
|
201
|
+
if fnmatch.fnmatchcase(self._model_name(model), node.this)
|
|
198
202
|
}
|
|
199
|
-
|
|
200
|
-
return {fqn} if fqn in all_models else set()
|
|
203
|
+
return self._pattern_to_model_fqns(pattern, all_models)
|
|
201
204
|
if isinstance(node, exp.And):
|
|
202
205
|
return evaluate(node.left) & evaluate(node.right)
|
|
203
206
|
if isinstance(node, exp.Or):
|
|
@@ -225,6 +228,13 @@ class Selector:
|
|
|
225
228
|
if fnmatch.fnmatchcase(tag, pattern)
|
|
226
229
|
}
|
|
227
230
|
return models_by_tags.get(pattern, set())
|
|
231
|
+
if isinstance(node, ResourceType):
|
|
232
|
+
resource_type = node.name.lower()
|
|
233
|
+
return {
|
|
234
|
+
fqn
|
|
235
|
+
for fqn, model in all_models.items()
|
|
236
|
+
if self._matches_resource_type(resource_type, model)
|
|
237
|
+
}
|
|
228
238
|
if isinstance(node, Direction):
|
|
229
239
|
selected = set()
|
|
230
240
|
|
|
@@ -241,6 +251,117 @@ class Selector:
|
|
|
241
251
|
|
|
242
252
|
return evaluate(node)
|
|
243
253
|
|
|
254
|
+
@abc.abstractmethod
|
|
255
|
+
def _model_name(self, model: Node) -> str:
|
|
256
|
+
"""Given a model, return the name that a selector pattern contining wildcards should be fnmatch'd on"""
|
|
257
|
+
pass
|
|
258
|
+
|
|
259
|
+
@abc.abstractmethod
|
|
260
|
+
def _pattern_to_model_fqns(self, pattern: str, all_models: t.Dict[str, Node]) -> t.Set[str]:
|
|
261
|
+
"""Given a pattern, return the keys of the matching models from :all_models"""
|
|
262
|
+
pass
|
|
263
|
+
|
|
264
|
+
@abc.abstractmethod
|
|
265
|
+
def _matches_resource_type(self, resource_type: str, model: Node) -> bool:
|
|
266
|
+
"""Indicate whether or not the supplied model matches the supplied resource type"""
|
|
267
|
+
pass
|
|
268
|
+
|
|
269
|
+
|
|
270
|
+
class NativeSelector(Selector):
|
|
271
|
+
"""Implementation of selectors that matches objects based on SQLMesh native names"""
|
|
272
|
+
|
|
273
|
+
def _model_name(self, model: Node) -> str:
|
|
274
|
+
return model.name
|
|
275
|
+
|
|
276
|
+
def _pattern_to_model_fqns(self, pattern: str, all_models: t.Dict[str, Node]) -> t.Set[str]:
|
|
277
|
+
fqn = normalize_model_name(pattern, self._default_catalog, self._dialect)
|
|
278
|
+
return {fqn} if fqn in all_models else set()
|
|
279
|
+
|
|
280
|
+
def _matches_resource_type(self, resource_type: str, model: Node) -> bool:
|
|
281
|
+
if resource_type == "model":
|
|
282
|
+
return model.is_model
|
|
283
|
+
if resource_type == "audit":
|
|
284
|
+
return isinstance(model, StandaloneAudit)
|
|
285
|
+
|
|
286
|
+
raise SQLMeshError(f"Unsupported resource type: {resource_type}")
|
|
287
|
+
|
|
288
|
+
|
|
289
|
+
class DbtSelector(Selector):
|
|
290
|
+
"""Implementation of selectors that matches objects based on the DBT names instead of the SQLMesh native names"""
|
|
291
|
+
|
|
292
|
+
def _model_name(self, model: Node) -> str:
|
|
293
|
+
if dbt_fqn := model.dbt_fqn:
|
|
294
|
+
return dbt_fqn
|
|
295
|
+
raise SQLMeshError("dbt node information must be populated to use dbt selectors")
|
|
296
|
+
|
|
297
|
+
def _pattern_to_model_fqns(self, pattern: str, all_models: t.Dict[str, Node]) -> t.Set[str]:
|
|
298
|
+
# a pattern like "staging.customers" should match a model called "jaffle_shop.staging.customers"
|
|
299
|
+
# but not a model called "jaffle_shop.customers.staging"
|
|
300
|
+
# also a pattern like "aging" should not match "staging" so we need to consider components; not substrings
|
|
301
|
+
pattern_components = pattern.split(".")
|
|
302
|
+
first_pattern_component = pattern_components[0]
|
|
303
|
+
matches = set()
|
|
304
|
+
for fqn, model in all_models.items():
|
|
305
|
+
if not model.dbt_fqn:
|
|
306
|
+
continue
|
|
307
|
+
|
|
308
|
+
dbt_fqn_components = model.dbt_fqn.split(".")
|
|
309
|
+
try:
|
|
310
|
+
starting_idx = dbt_fqn_components.index(first_pattern_component)
|
|
311
|
+
except ValueError:
|
|
312
|
+
continue
|
|
313
|
+
for pattern_component, fqn_component in zip_longest(
|
|
314
|
+
pattern_components, dbt_fqn_components[starting_idx:]
|
|
315
|
+
):
|
|
316
|
+
if pattern_component and not fqn_component:
|
|
317
|
+
# the pattern still goes but we have run out of fqn components to match; no match
|
|
318
|
+
break
|
|
319
|
+
if fqn_component and not pattern_component:
|
|
320
|
+
# all elements of the pattern have matched elements of the fqn; match
|
|
321
|
+
matches.add(fqn)
|
|
322
|
+
break
|
|
323
|
+
if pattern_component != fqn_component:
|
|
324
|
+
# the pattern explicitly doesnt match a component; no match
|
|
325
|
+
break
|
|
326
|
+
else:
|
|
327
|
+
# called if no explicit break, indicating all components of the pattern matched all components of the fqn
|
|
328
|
+
matches.add(fqn)
|
|
329
|
+
return matches
|
|
330
|
+
|
|
331
|
+
def _matches_resource_type(self, resource_type: str, model: Node) -> bool:
|
|
332
|
+
"""
|
|
333
|
+
ref: https://docs.getdbt.com/reference/node-selection/methods#resource_type
|
|
334
|
+
|
|
335
|
+
# supported by SQLMesh
|
|
336
|
+
"model"
|
|
337
|
+
"seed"
|
|
338
|
+
"source" # external model
|
|
339
|
+
"test" # standalone audit
|
|
340
|
+
|
|
341
|
+
# not supported by SQLMesh yet, commented out to throw an error if someone tries to use them
|
|
342
|
+
"analysis"
|
|
343
|
+
"exposure"
|
|
344
|
+
"metric"
|
|
345
|
+
"saved_query"
|
|
346
|
+
"semantic_model"
|
|
347
|
+
"snapshot"
|
|
348
|
+
"unit_test"
|
|
349
|
+
"""
|
|
350
|
+
if resource_type not in ("model", "seed", "source", "test"):
|
|
351
|
+
raise SQLMeshError(f"Unsupported resource type: {resource_type}")
|
|
352
|
+
|
|
353
|
+
if isinstance(model, StandaloneAudit):
|
|
354
|
+
return resource_type == "test"
|
|
355
|
+
|
|
356
|
+
if resource_type == "model":
|
|
357
|
+
return model.is_model and not model.kind.is_external and not model.kind.is_seed
|
|
358
|
+
if resource_type == "source":
|
|
359
|
+
return model.kind.is_external
|
|
360
|
+
if resource_type == "seed":
|
|
361
|
+
return model.kind.is_seed
|
|
362
|
+
|
|
363
|
+
return False
|
|
364
|
+
|
|
244
365
|
|
|
245
366
|
class SelectorDialect(Dialect):
|
|
246
367
|
IDENTIFIERS_CAN_START_WITH_DIGIT = True
|
|
@@ -271,6 +392,10 @@ class Tag(exp.Expression):
|
|
|
271
392
|
pass
|
|
272
393
|
|
|
273
394
|
|
|
395
|
+
class ResourceType(exp.Expression):
|
|
396
|
+
pass
|
|
397
|
+
|
|
398
|
+
|
|
274
399
|
class Direction(exp.Expression):
|
|
275
400
|
pass
|
|
276
401
|
|
|
@@ -323,7 +448,8 @@ def parse(selector: str, dialect: DialectType = None) -> exp.Expression:
|
|
|
323
448
|
upstream = _match(TokenType.PLUS)
|
|
324
449
|
downstream = None
|
|
325
450
|
tag = _parse_kind("tag")
|
|
326
|
-
|
|
451
|
+
resource_type = False if tag else _parse_kind("resource_type")
|
|
452
|
+
git = False if resource_type else _parse_kind("git")
|
|
327
453
|
lstar = "*" if _match(TokenType.STAR) else ""
|
|
328
454
|
directions = {}
|
|
329
455
|
|
|
@@ -349,6 +475,8 @@ def parse(selector: str, dialect: DialectType = None) -> exp.Expression:
|
|
|
349
475
|
|
|
350
476
|
if tag:
|
|
351
477
|
this = Tag(this=this)
|
|
478
|
+
if resource_type:
|
|
479
|
+
this = ResourceType(this=this)
|
|
352
480
|
if git:
|
|
353
481
|
this = Git(this=this)
|
|
354
482
|
if directions:
|