sqlmesh 0.199.1.dev29__py3-none-any.whl → 0.199.1.dev31__py3-none-any.whl

sqlmesh/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.199.1.dev29'
-__version_tuple__ = version_tuple = (0, 199, 1, 'dev29')
+__version__ = version = '0.199.1.dev31'
+__version_tuple__ = version_tuple = (0, 199, 1, 'dev31')

sqlmesh/core/context.py

@@ -453,7 +453,7 @@ class GenericContext(BaseContext, t.Generic[C]):
         if not self._snapshot_evaluator:
             self._snapshot_evaluator = SnapshotEvaluator(
                 {
-                    gateway: adapter.with_settings(log_level=logging.INFO)
+                    gateway: adapter.with_settings(execute_log_level=logging.INFO)
                     for gateway, adapter in self.engine_adapters.items()
                 },
                 ddl_concurrent_tasks=self.concurrent_tasks,
@@ -520,7 +520,11 @@ class GenericContext(BaseContext, t.Generic[C]):
 
         return model
 
-    def scheduler(self, environment: t.Optional[str] = None) -> Scheduler:
+    def scheduler(
+        self,
+        environment: t.Optional[str] = None,
+        snapshot_evaluator: t.Optional[SnapshotEvaluator] = None,
+    ) -> Scheduler:
         """Returns the built-in scheduler.
 
         Args:
@@ -542,9 +546,11 @@ class GenericContext(BaseContext, t.Generic[C]):
         if not snapshots:
             raise ConfigError("No models were found")
 
-        return self.create_scheduler(snapshots)
+        return self.create_scheduler(snapshots, snapshot_evaluator or self.snapshot_evaluator)
 
-    def create_scheduler(self, snapshots: t.Iterable[Snapshot]) -> Scheduler:
+    def create_scheduler(
+        self, snapshots: t.Iterable[Snapshot], snapshot_evaluator: SnapshotEvaluator
+    ) -> Scheduler:
         """Creates the built-in scheduler.
 
         Args:
@@ -555,7 +561,7 @@ class GenericContext(BaseContext, t.Generic[C]):
         """
         return Scheduler(
             snapshots,
-            self.snapshot_evaluator,
+            snapshot_evaluator,
             self.state_sync,
             default_catalog=self.default_catalog,
             max_workers=self.concurrent_tasks,
@@ -1931,7 +1937,7 @@ class GenericContext(BaseContext, t.Generic[C]):
             )
 
         return TableDiff(
-            adapter=adapter.with_settings(logger.getEffectiveLevel()),
+            adapter=adapter.with_settings(execute_log_level=logger.getEffectiveLevel()),
             source=source,
             target=target,
             on=on,

sqlmesh/core/engine_adapter/base.py

@@ -147,19 +147,23 @@ class EngineAdapter:
         self._multithreaded = multithreaded
         self.correlation_id = correlation_id
 
-    def with_settings(self, log_level: int, **kwargs: t.Any) -> EngineAdapter:
+    def with_settings(self, **kwargs: t.Any) -> EngineAdapter:
+        extra_kwargs = {
+            "null_connection": True,
+            "execute_log_level": kwargs.pop("execute_log_level", self._execute_log_level),
+            **self._extra_config,
+            **kwargs,
+        }
+
         adapter = self.__class__(
             self._connection_pool,
             dialect=self.dialect,
             sql_gen_kwargs=self._sql_gen_kwargs,
             default_catalog=self._default_catalog,
-            execute_log_level=log_level,
             register_comments=self._register_comments,
-            null_connection=True,
             multithreaded=self._multithreaded,
             pretty_sql=self._pretty_sql,
-            **self._extra_config,
-            **kwargs,
+            **extra_kwargs,
         )
 
         return adapter

sqlmesh/core/model/common.py

@@ -440,6 +440,19 @@ def sorted_python_env_payloads(python_env: t.Dict[str, Executable]) -> t.List[st
     return [_executable_to_str(k, v) for k, v in sort_python_env(python_env)]
 
 
+def parse_strings_with_macro_refs(value: t.Any, dialect: DialectType) -> t.Any:
+    if isinstance(value, str) and "@" in value:
+        return exp.maybe_parse(value, dialect=dialect)
+
+    if isinstance(value, dict):
+        for k, v in dict(value).items():
+            value[k] = parse_strings_with_macro_refs(v, dialect)
+    elif isinstance(value, list):
+        value = [parse_strings_with_macro_refs(v, dialect) for v in value]
+
+    return value
+
+
 expression_validator: t.Callable = field_validator(
     "query",
     "expressions_",

sqlmesh/core/model/definition.py

@@ -27,6 +27,7 @@ from sqlmesh.core.model.common import (
     expression_validator,
     make_python_env,
     parse_dependencies,
+    parse_strings_with_macro_refs,
     single_value_or_tuple,
     sorted_python_env_payloads,
     validate_extra_and_required_fields,
@@ -72,13 +73,14 @@ if t.TYPE_CHECKING:
 
 logger = logging.getLogger(__name__)
 
+
+UNRENDERABLE_MODEL_FIELDS = {"cron", "description"}
+
 PROPERTIES = {"physical_properties", "session_properties", "virtual_properties"}
 
 RUNTIME_RENDERED_MODEL_FIELDS = {
     "audits",
     "signals",
-    "description",
-    "cron",
     "merge_filter",
 } | PROPERTIES
 
@@ -2469,6 +2471,9 @@ def _create_model(
         if isinstance(property_values, exp.Tuple):
             statements.extend(property_values.expressions)
 
+    if isinstance(getattr(kwargs.get("kind"), "merge_filter", None), exp.Expression):
+        statements.append(kwargs["kind"].merge_filter)
+
     jinja_macro_references, used_variables = extract_macro_references_and_variables(
         *(gen(e if isinstance(e, exp.Expression) else e[0]) for e in statements)
     )
@@ -2751,21 +2756,24 @@ def render_meta_fields(
 
     for field_name, field_info in ModelMeta.all_field_infos().items():
         field = field_info.alias or field_name
+        field_value = fields.get(field)
 
-        if field in RUNTIME_RENDERED_MODEL_FIELDS:
+        # We don't want to parse python model cron="@..." kwargs (e.g. @daily) into MacroVar
+        if field == "cron" or field_value is None:
             continue
 
-        field_value = fields.get(field)
-        if field_value is None:
+        if field in RUNTIME_RENDERED_MODEL_FIELDS:
+            fields[field] = parse_strings_with_macro_refs(field_value, dialect)
             continue
 
         if isinstance(field_value, dict):
             rendered_dict = {}
             for key, value in field_value.items():
                 if key in RUNTIME_RENDERED_MODEL_FIELDS:
-                    rendered_dict[key] = value
+                    rendered_dict[key] = parse_strings_with_macro_refs(value, dialect)
                 elif (rendered := render_field_value(value)) is not None:
                     rendered_dict[key] = rendered
+
             if rendered_dict:
                 fields[field] = rendered_dict
             else:

sqlmesh/core/plan/evaluator.py

@@ -38,6 +38,7 @@ from sqlmesh.core.snapshot import (
 )
 from sqlmesh.utils import to_snake_case
 from sqlmesh.core.state_sync import StateSync
+from sqlmesh.utils import CorrelationId
 from sqlmesh.utils.concurrency import NodeExecutionFailedError
 from sqlmesh.utils.errors import PlanError, SQLMeshError
 from sqlmesh.utils.dag import DAG
@@ -71,7 +72,7 @@ class BuiltInPlanEvaluator(PlanEvaluator):
         self,
         state_sync: StateSync,
         snapshot_evaluator: SnapshotEvaluator,
-        create_scheduler: t.Callable[[t.Iterable[Snapshot]], Scheduler],
+        create_scheduler: t.Callable[[t.Iterable[Snapshot], SnapshotEvaluator], Scheduler],
         default_catalog: t.Optional[str],
         console: t.Optional[Console] = None,
     ):
@@ -88,6 +89,9 @@ class BuiltInPlanEvaluator(PlanEvaluator):
         circuit_breaker: t.Optional[t.Callable[[], bool]] = None,
     ) -> None:
         self._circuit_breaker = circuit_breaker
+        self.snapshot_evaluator = self.snapshot_evaluator.set_correlation_id(
+            CorrelationId.from_plan_id(plan.plan_id)
+        )
 
         self.console.start_plan_evaluation(plan)
         analytics.collector.on_plan_apply_start(
@@ -106,6 +110,7 @@ class BuiltInPlanEvaluator(PlanEvaluator):
         else:
             analytics.collector.on_plan_apply_end(plan_id=plan.plan_id)
         finally:
+            self.snapshot_evaluator.recycle()
             self.console.stop_plan_evaluation()
 
     def _evaluate_stages(
@@ -228,7 +233,7 @@ class BuiltInPlanEvaluator(PlanEvaluator):
             self.console.log_success("SKIP: No model batches to execute")
             return
 
-        scheduler = self.create_scheduler(stage.all_snapshots.values())
+        scheduler = self.create_scheduler(stage.all_snapshots.values(), self.snapshot_evaluator)
         errors, _ = scheduler.run_merged_intervals(
             merged_intervals=stage.snapshot_to_intervals,
             deployability_index=stage.deployability_index,
@@ -249,7 +254,7 @@ class BuiltInPlanEvaluator(PlanEvaluator):
             return
 
         # If there are any snapshots to be audited, we'll reuse the scheduler's internals to audit them
-        scheduler = self.create_scheduler(audit_snapshots)
+        scheduler = self.create_scheduler(audit_snapshots, self.snapshot_evaluator)
         completion_status = scheduler.audit(
             plan.environment,
             plan.start,

sqlmesh/core/snapshot/evaluator.py

@@ -61,7 +61,7 @@ from sqlmesh.core.snapshot import (
     SnapshotTableCleanupTask,
 )
 from sqlmesh.core.snapshot.definition import parent_snapshots_by_name
-from sqlmesh.utils import random_id
+from sqlmesh.utils import random_id, CorrelationId
 from sqlmesh.utils.concurrency import (
     concurrent_apply_to_snapshots,
     concurrent_apply_to_values,
@@ -127,6 +127,7 @@ class SnapshotEvaluator:
             if not selected_gateway
             else self.adapters[selected_gateway]
         )
+        self.selected_gateway = selected_gateway
         self.ddl_concurrent_tasks = ddl_concurrent_tasks
 
     def evaluate(
@@ -1186,6 +1187,16 @@ class SnapshotEvaluator:
         )
         adapter.execute(snapshot.model.render_post_statements(**create_render_kwargs))
 
+    def set_correlation_id(self, correlation_id: CorrelationId) -> SnapshotEvaluator:
+        return SnapshotEvaluator(
+            {
+                gateway: adapter.with_settings(correlation_id=correlation_id)
+                for gateway, adapter in self.adapters.items()
+            },
+            self.ddl_concurrent_tasks,
+            self.selected_gateway,
+        )
+
 
 def _evaluation_strategy(snapshot: SnapshotInfoLike, adapter: EngineAdapter) -> EvaluationStrategy:
     klass: t.Type

{sqlmesh-0.199.1.dev29.dist-info → sqlmesh-0.199.1.dev31.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sqlmesh
-Version: 0.199.1.dev29
+Version: 0.199.1.dev31
 Summary: Next-generation data transformation framework
 Author-email: "TobikoData Inc." <engineering@tobikodata.com>
 License:                                  Apache License

{sqlmesh-0.199.1.dev29.dist-info → sqlmesh-0.199.1.dev31.dist-info}/RECORD

@@ -1,5 +1,5 @@
 sqlmesh/__init__.py,sha256=QjwaamJIdW7voXoS880yAx7tHm-Im4_mqGA-XJgafos,7631
-sqlmesh/_version.py,sha256=WRMy6e067qv6ndk_Iz3NNkkEe2ksj6RJCrSTGYZNbI8,530
+sqlmesh/_version.py,sha256=CgHjS2NBWQdSFZUQVN24C3Ezoj0vTyfhAquX6nAtixs,530
 sqlmesh/magics.py,sha256=R9U0Mp9ZQhDYe751nxAQDLaMv3zLAR4HcTdIflKTmEE,41709
 sqlmesh/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sqlmesh/cicd/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -13,7 +13,7 @@ sqlmesh/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sqlmesh/core/_typing.py,sha256=e1f7xO2aZCy38M_3VoDy-XI3shyd99-IFctgIFSgXWM,497
 sqlmesh/core/console.py,sha256=ObpdArJOXDOihWPXsWIUuwiQ64cMlroNWJHqQPkVqC0,160753
 sqlmesh/core/constants.py,sha256=gUABXWYgH_XOSbaZAJsR5o1dmA61JDCvBDyxqqXX3LM,2630
-sqlmesh/core/context.py,sha256=VO2Sp0u3CRmUoQI0z2Qwfzp4bhqKdtxlG0az1tND2_w,124223
+sqlmesh/core/context.py,sha256=AuwB0PiV1jW7WZDaMOo_RkfKKTvwq6IDjByCHLA1GKw,124433
 sqlmesh/core/context_diff.py,sha256=CeUmMBJzN__98YohrtAiIYPIZI9mrGBk6bVADWBLUw8,21692
 sqlmesh/core/dialect.py,sha256=1ykYwb9WZn_nCaX08PcQ0BJh2Pihv9SPvoMNjpMeCfs,50158
 sqlmesh/core/environment.py,sha256=7DbJX0r5nZRU3LhcgIxbSV-JosQYDjAt3V5Wc2Fpns8,13005
@@ -59,7 +59,7 @@ sqlmesh/core/config/ui.py,sha256=jsO-S6_d9NkLZGG5pT4mgKgxMF34KzkDociZAMvCX3U,278
 sqlmesh/core/engine_adapter/__init__.py,sha256=two_WTs4UnSNp-fscJh8vOunJq0Xi37EuDuMjKsKuNE,2281
 sqlmesh/core/engine_adapter/_typing.py,sha256=_63pYCThfC39UtPVHY3N0H6qJ1YBT32OqwwR3swolGM,1037
 sqlmesh/core/engine_adapter/athena.py,sha256=DMC94WfCH8m37cVPio43Q11J8RQE8tuf1nl0RtXZ_CM,26198
-sqlmesh/core/engine_adapter/base.py,sha256=7JbSAzz1U2umlmcUKYd-nIf__yRFArfwgcPkXewqoHU,104081
+sqlmesh/core/engine_adapter/base.py,sha256=9rRzY01cvPdrWGCGfT4qqsPbklt6vUWTQmK_C9DEiAo,104182
 sqlmesh/core/engine_adapter/base_postgres.py,sha256=DOsOdCJyQAVfA9S0GznuH0J1bonMqYVP_yJLDYcxFYM,6772
 sqlmesh/core/engine_adapter/bigquery.py,sha256=qJE8vHLKKAeWMvltmYaUvWS95jM_UEZ6UzYLlr58sGo,50424
 sqlmesh/core/engine_adapter/clickhouse.py,sha256=LoPdm_s_s-ZxS1XJPA3Et0iOHeJ1LgqTAFbwBdiqE7A,34912
@@ -86,9 +86,9 @@ sqlmesh/core/metric/definition.py,sha256=Yd5aVgsZCDPJ43aGP7WqtzZOuuSUtB8uJGVA6Jw
 sqlmesh/core/metric/rewriter.py,sha256=GiSTHfn2kinqCfNPYgZPRk93JFLzVaaejHtHDQ0yXZI,7326
 sqlmesh/core/model/__init__.py,sha256=C8GRZ53xuXEA9hQv3BQS9pNNyd9rZ06R_B96UYGhDu4,1644
 sqlmesh/core/model/cache.py,sha256=csun0RJguHzKX6-qITcOs4fVP4f8_Ts8qiUVV4sHY6Q,7869
-sqlmesh/core/model/common.py,sha256=PcQS3rHFNIXV_Z1dGwLQ7gLPbYVlrbWGGKAyc58-9Xk,17379
+sqlmesh/core/model/common.py,sha256=dPyqXLaDSnQgDVdbiOZ7YwomlaPIvGfyL-mHhc7bR1o,17830
 sqlmesh/core/model/decorator.py,sha256=wUUg5t0K559tV9Pqt6FE2EWXt5TPGb4wF2Yg4B4hX4w,9251
-sqlmesh/core/model/definition.py,sha256=mG6UPwxiZj0xP_cdHSVELEcXusAcltpgHBX-wiUSuPM,111963
+sqlmesh/core/model/definition.py,sha256=mAcbq5chgFgDQi0f_p-lByvbFK9kIQgQfROqvP5LnB8,112396
 sqlmesh/core/model/kind.py,sha256=8Dp8yiTiXltboZWrB2FJ-LQ3zx4Fv_5Tj0KdUIPwCHI,36358
 sqlmesh/core/model/meta.py,sha256=D9InI9M2-IXeb34vkXNQSLALL8xVXjZ_LXwI-Tqxm7I,21197
 sqlmesh/core/model/schema.py,sha256=_HMYfzK9wWXh7_CQDIIGnuQUD4aiX3o5D2cRp2sERzc,3387
@@ -96,14 +96,14 @@ sqlmesh/core/model/seed.py,sha256=a0M-1zY1gOkN5ph2GQyataEdBSCtq50YjeFk2LyvInI,50
 sqlmesh/core/plan/__init__.py,sha256=NKSvM7ZBVjw9ho3J65M1wFvG3KURB8PJ0FHHLmtSF44,443
 sqlmesh/core/plan/builder.py,sha256=TQVSGiGbQi9tro0tBlcL2f5BEvyOYNiLuB3yz8PRUzA,41859
 sqlmesh/core/plan/definition.py,sha256=yrX_uZMXi29ltkuowWhNUOi1kd0Uq0TQoM574EI6JHo,14019
-sqlmesh/core/plan/evaluator.py,sha256=FNA4oQ29lZsabKZU36T2wu7QQymwAfCu6zhirdPKuCg,20627
+sqlmesh/core/plan/evaluator.py,sha256=z5Rweyc-JgWxQck8suZSlb-s5DfN0FvC6Aq1eFPwetI,20923
 sqlmesh/core/plan/explainer.py,sha256=aBkvtM9eog_2NIglazSDdG_i9TTCdFSVOkCw5N8O-1k,12028
 sqlmesh/core/plan/stages.py,sha256=7iiVZLSRoI1kSEMyOuuW6gvUA-eyALUfdW5vDh_N4jU,22040
 sqlmesh/core/snapshot/__init__.py,sha256=Z5HTetNCZGoKF1WAKXk4LWL9dc8EC8lwS2O4TtoBmt4,1317
 sqlmesh/core/snapshot/cache.py,sha256=bgqCR2hyf6r2A_8QP1EnXFK25gDX37-Zg0YeMuETWxg,3934
 sqlmesh/core/snapshot/categorizer.py,sha256=qGG5870mu1jR2whKisSBH_GFbsyh96RnzJDgkN7DV9g,2482
 sqlmesh/core/snapshot/definition.py,sha256=JBrvHx9ZPVWxY649IOXOYs4RKPzgLYvfdxXsdCewad0,85738
-sqlmesh/core/snapshot/evaluator.py,sha256=JORjYIvfu1KAkhCazdybNTq7z7jGVE0kEFq3FFmAFJI,90065
+sqlmesh/core/snapshot/evaluator.py,sha256=yNVxlTIV8_mKmUsIJtQtjT8RZHDHA64pUPds9VZfOq8,90503
 sqlmesh/core/state_sync/__init__.py,sha256=B60xj-_Xk-4jas3UphLvwkuf04G-9AyGckcblXE0aRg,869
 sqlmesh/core/state_sync/base.py,sha256=AdqWmbhnvRFqVcFK6L01uCvpBVAYGuMrm4P0D3u_NaU,18334
 sqlmesh/core/state_sync/cache.py,sha256=0AsYdQyBqbLdJvkBQ-hDAipsvnn_jSps-nya-Q4XeL0,5337
@@ -284,7 +284,7 @@ sqlmesh/utils/pydantic.py,sha256=o_NsXbIpDqNpUA1Uc5xF0ZzoXQYYB0DfHwdRxBwXPNk,120
 sqlmesh/utils/rich.py,sha256=BpB47G0-JcU41LkGkeGKroexj4NOV5fox4-GOIXPgIc,3545
 sqlmesh/utils/windows.py,sha256=r7Yin4WEwbvH7sKA3hOJtRsclmscNuBeCklCfVPvtQs,611
 sqlmesh/utils/yaml.py,sha256=hvo5lmybYLcIx4p6CQwufS8X4MbwS60lEBnCmYJCXmU,2336
-sqlmesh-0.199.1.dev29.dist-info/licenses/LICENSE,sha256=OlMefUjgWJdULtf84BLW0AZZcY8DwdgQqb_1j2862j8,11346
+sqlmesh-0.199.1.dev31.dist-info/licenses/LICENSE,sha256=OlMefUjgWJdULtf84BLW0AZZcY8DwdgQqb_1j2862j8,11346
 web/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 web/client/dist/index.html,sha256=DAv6ABGogw4xRLb0T_92WP_S7XWyqVhwBgGKWtuQwU4,5472
 web/client/dist/assets/Audits-BuHAmFO8.js,sha256=zbYUF6WdUGMbnMm01GMNPb7xAJwj-o5wMdL56vPo2qE,1041
@@ -402,8 +402,8 @@ web/server/api/endpoints/models.py,sha256=ud-lRwriyI_OYg9tZchrBgeiXELiRTLNEodraf
 web/server/api/endpoints/modules.py,sha256=8hqqgonGay_mJmpCw0IdbjsPhWlQH2VLdKAqha-myac,468
 web/server/api/endpoints/plan.py,sha256=bbbY50W_2MsZSTxOHWMKz0tbIm75nsRSlPy8GI2fg9Q,9306
 web/server/api/endpoints/table_diff.py,sha256=SCy3-pr5x6bEEiOaZGCDbiHQhpgxwmnz3Acqa2Ie6jI,2090
-sqlmesh-0.199.1.dev29.dist-info/METADATA,sha256=vElYNZY_cexKSeT6e6NUHjytbluS_YGoXmcMI60U3nA,26659
-sqlmesh-0.199.1.dev29.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-sqlmesh-0.199.1.dev29.dist-info/entry_points.txt,sha256=pIYKtg-u-uO7BK_Y0qrdmfyj0bYrcMc3Xiqr5WE3VFk,121
-sqlmesh-0.199.1.dev29.dist-info/top_level.txt,sha256=bspjo5vjELQkuN0qKuTFIbsrUt3Zeo8D2mlKeHG1LdE,12
-sqlmesh-0.199.1.dev29.dist-info/RECORD,,
+sqlmesh-0.199.1.dev31.dist-info/METADATA,sha256=b2S5AOFx4aEUG1T59ljG0g7winpSYkR_WMjLrl8uUgw,26659
+sqlmesh-0.199.1.dev31.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+sqlmesh-0.199.1.dev31.dist-info/entry_points.txt,sha256=pIYKtg-u-uO7BK_Y0qrdmfyj0bYrcMc3Xiqr5WE3VFk,121
+sqlmesh-0.199.1.dev31.dist-info/top_level.txt,sha256=bspjo5vjELQkuN0qKuTFIbsrUt3Zeo8D2mlKeHG1LdE,12
+sqlmesh-0.199.1.dev31.dist-info/RECORD,,