sqlmesh 0.198.1.dev12__py3-none-any.whl → 0.198.1.dev13__py3-none-any.whl

sqlmesh/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.198.1.dev12'
-__version_tuple__ = version_tuple = (0, 198, 1, 'dev12')
+__version__ = version = '0.198.1.dev13'
+__version_tuple__ = version_tuple = (0, 198, 1, 'dev13')

sqlmesh/core/model/definition.py

@@ -627,8 +627,10 @@ class _Model(ModelMeta, frozen=True):
             {k: _render(v) for k, v in signal.items()} for name, signal in self.signals if not name
         ]
 
-    def render_signal_calls(self) -> t.Dict[str, t.Dict[str, t.Optional[exp.Expression]]]:
-        return {
+    def render_signal_calls(self) -> EvaluatableSignals:
+        python_env = self.python_env
+        env = prepare_env(python_env)
+        signals_to_kwargs = {
             name: {
                 k: seq_get(self._create_renderer(v).render() or [], 0) for k, v in kwargs.items()
             }
@@ -636,6 +638,12 @@ class _Model(ModelMeta, frozen=True):
             if name
         }
 
+        return EvaluatableSignals(
+            signals_to_kwargs=signals_to_kwargs,
+            python_env=python_env,
+            prepared_python_env=env,
+        )
+
     def render_merge_filter(
         self,
         *,
@@ -1857,6 +1865,15 @@ class AuditResult(PydanticModel):
     blocking: bool = True
 
 
+class EvaluatableSignals(PydanticModel):
+    signals_to_kwargs: t.Dict[str, t.Dict[str, t.Optional[exp.Expression]]]
+    """A mapping of signal names to the kwargs passed to the signal."""
+    python_env: t.Dict[str, Executable]
+    """The Python environment that should be used to evaluated the rendered signal calls."""
+    prepared_python_env: t.Dict[str, t.Any]
+    """The prepared Python environment that should be used to evaluated the rendered signal calls."""
+
+
 def _extract_blueprints(blueprints: t.Any, path: Path) -> t.List[t.Any]:
     if not blueprints:
         return [None]

sqlmesh/core/scheduler.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 import logging
 import typing as t
+import time
 from sqlglot import exp
 from sqlmesh.core import constants as c
 from sqlmesh.core.console import Console, get_console
@@ -24,6 +25,7 @@ from sqlmesh.core.snapshot import (
     snapshots_to_dag,
     Intervals,
 )
+from sqlmesh.core.snapshot.definition import check_ready_intervals
 from sqlmesh.core.snapshot.definition import (
     Interval,
     expand_range,
@@ -39,7 +41,16 @@ from sqlmesh.utils.date import (
     to_timestamp,
     validate_date_range,
 )
-from sqlmesh.utils.errors import AuditError, NodeAuditsErrors, CircuitBreakerError, SQLMeshError
+from sqlmesh.utils.errors import (
+    AuditError,
+    NodeAuditsErrors,
+    CircuitBreakerError,
+    SQLMeshError,
+    SignalEvalError,
+)
+
+if t.TYPE_CHECKING:
+    from sqlmesh.core.context import ExecutionContext
 
 logger = logging.getLogger(__name__)
 SnapshotToIntervals = t.Dict[Snapshot, Intervals]
@@ -304,12 +315,11 @@ class Scheduler:
                 default_catalog=self.default_catalog,
             )
 
-            intervals = snapshot.check_ready_intervals(
+            intervals = self._check_ready_intervals(
+                snapshot,
                 intervals,
                 context,
-                console=self.console,
-                default_catalog=self.default_catalog,
-                environment_naming_info=environment_naming_info,
+                environment_naming_info,
             )
             unready -= set(intervals)
 
@@ -709,6 +719,76 @@ class Scheduler:
 
         return audit_results
 
+    def _check_ready_intervals(
+        self,
+        snapshot: Snapshot,
+        intervals: Intervals,
+        context: ExecutionContext,
+        environment_naming_info: EnvironmentNamingInfo,
+    ) -> Intervals:
+        """Checks if the intervals are ready for evaluation for the given snapshot.
+
+        This implementation also includes the signal progress tracking.
+        Note that this will handle gaps in the provided intervals. The returned intervals
+        may introduce new gaps.
+
+        Args:
+            snapshot: The snapshot to check.
+            intervals: The intervals to check.
+            context: The context to use.
+            environment_naming_info: The environment naming info to use.
+
+        Returns:
+            The intervals that are ready for evaluation.
+        """
+        signals = snapshot.is_model and snapshot.model.render_signal_calls()
+
+        if not signals:
+            return intervals
+
+        self.console.start_signal_progress(
+            snapshot,
+            self.default_catalog,
+            environment_naming_info or EnvironmentNamingInfo(),
+        )
+
+        for signal_idx, (signal_name, kwargs) in enumerate(signals.signals_to_kwargs.items()):
+            # Capture intervals before signal check for display
+            intervals_to_check = merge_intervals(intervals)
+
+            signal_start_ts = time.perf_counter()
+
+            try:
+                intervals = check_ready_intervals(
+                    signals.prepared_python_env[signal_name],
+                    intervals,
+                    context,
+                    python_env=signals.python_env,
+                    dialect=snapshot.model.dialect,
+                    path=snapshot.model._path,
+                    kwargs=kwargs,
+                )
+            except SQLMeshError as e:
+                raise SignalEvalError(
+                    f"{e} '{signal_name}' for '{snapshot.model.name}' at {snapshot.model._path}"
+                )
+
+            duration = time.perf_counter() - signal_start_ts
+
+            self.console.update_signal_progress(
+                snapshot=snapshot,
+                signal_name=signal_name,
+                signal_idx=signal_idx,
+                total_signals=len(signals.signals_to_kwargs),
+                ready_intervals=merge_intervals(intervals),
+                check_intervals=intervals_to_check,
+                duration=duration,
+            )
+
+        self.console.stop_signal_progress()
+
+        return intervals
+
 
 def merged_missing_intervals(
     snapshots: t.Collection[Snapshot],

sqlmesh/core/snapshot/definition.py

@@ -1,7 +1,6 @@
 from __future__ import annotations
 
 import sys
-import time
 import typing as t
 from collections import defaultdict
 from datetime import datetime, timedelta
@@ -42,8 +41,6 @@ from sqlmesh.utils.date import (
 )
 from sqlmesh.utils.errors import SQLMeshError, SignalEvalError
 from sqlmesh.utils.metaprogramming import (
-    prepare_env,
-    print_exception,
     format_evaluated_code_exception,
     Executable,
 )
@@ -51,7 +48,6 @@ from sqlmesh.utils.hashing import hash_data
 from sqlmesh.utils.pydantic import PydanticModel, field_validator
 
 if t.TYPE_CHECKING:
-    from sqlmesh.core.console import Console
     from sqlglot.dialects.dialect import DialectType
     from sqlmesh.core.environment import EnvironmentNamingInfo
     from sqlmesh.core.context import ExecutionContext
@@ -971,9 +967,6 @@ class Snapshot(PydanticModel, SnapshotInfoMixin):
         self,
         intervals: Intervals,
         context: ExecutionContext,
-        console: t.Optional[Console] = None,
-        default_catalog: t.Optional[str] = None,
-        environment_naming_info: t.Optional[EnvironmentNamingInfo] = None,
     ) -> Intervals:
         """Returns a list of intervals that are considered ready by the provided signal.
 
@@ -981,59 +974,24 @@ class Snapshot(PydanticModel, SnapshotInfoMixin):
         may introduce new gaps.
         """
         signals = self.is_model and self.model.render_signal_calls()
-
         if not signals:
             return intervals
 
-        python_env = self.model.python_env
-        env = prepare_env(python_env)
-
-        if console:
-            console.start_signal_progress(
-                self,
-                default_catalog,
-                environment_naming_info or EnvironmentNamingInfo(),
-            )
-
-        for signal_idx, (signal_name, kwargs) in enumerate(signals.items()):
-            # Capture intervals before signal check for display
-            intervals_to_check = merge_intervals(intervals)
-
-            signal_start_ts = time.perf_counter()
-
+        for signal_name, kwargs in signals.signals_to_kwargs.items():
             try:
-                intervals = _check_ready_intervals(
-                    env[signal_name],
+                intervals = check_ready_intervals(
+                    signals.prepared_python_env[signal_name],
                     intervals,
                     context,
-                    python_env=python_env,
+                    python_env=signals.python_env,
                     dialect=self.model.dialect,
                     path=self.model._path,
                     kwargs=kwargs,
                 )
             except SQLMeshError as e:
-                print_exception(e, python_env)
-                raise SQLMeshError(
+                raise SignalEvalError(
                     f"{e} '{signal_name}' for '{self.model.name}' at {self.model._path}"
                 )
-
-            duration = time.perf_counter() - signal_start_ts
-
-            if console:
-                console.update_signal_progress(
-                    snapshot=self,
-                    signal_name=signal_name,
-                    signal_idx=signal_idx,
-                    total_signals=len(signals),
-                    ready_intervals=merge_intervals(intervals),
-                    check_intervals=intervals_to_check,
-                    duration=duration,
-                )
-
-        # Stop signal progress tracking
-        if console:
-            console.stop_signal_progress()
-
         return intervals
 
     def categorize_as(self, category: SnapshotChangeCategory) -> None:
@@ -2229,7 +2187,7 @@ def _contiguous_intervals(intervals: Intervals) -> t.List[Intervals]:
     return contiguous_intervals
 
 
-def _check_ready_intervals(
+def check_ready_intervals(
     check: t.Callable,
     intervals: Intervals,
     context: ExecutionContext,

{sqlmesh-0.198.1.dev12.dist-info → sqlmesh-0.198.1.dev13.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sqlmesh
-Version: 0.198.1.dev12
+Version: 0.198.1.dev13
 Summary: Next-generation data transformation framework
 Author-email: "TobikoData Inc." <engineering@tobikodata.com>
 License:                                  Apache License

{sqlmesh-0.198.1.dev12.dist-info → sqlmesh-0.198.1.dev13.dist-info}/RECORD

@@ -1,5 +1,5 @@
 sqlmesh/__init__.py,sha256=QjwaamJIdW7voXoS880yAx7tHm-Im4_mqGA-XJgafos,7631
-sqlmesh/_version.py,sha256=hnPFLgViYAuq9OQ8HMNnl3oJTWYxsMayOWfylgVZPU0,530
+sqlmesh/_version.py,sha256=Ot41ppAYTuEK_dNN-7AEmc6ave1-1Q5pztj9mQwkDoo,530
 sqlmesh/magics.py,sha256=R9U0Mp9ZQhDYe751nxAQDLaMv3zLAR4HcTdIflKTmEE,41709
 sqlmesh/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sqlmesh/cicd/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -24,7 +24,7 @@ sqlmesh/core/node.py,sha256=r4_1MRTPcU0MAlwZ5KzrykLT58B7dkKvHzPCsJmEKD8,14497
 sqlmesh/core/notification_target.py,sha256=PPGoDrgbRKxr27vJEu03XqNTQLYTw0ZF_b0yAapxGeI,16158
 sqlmesh/core/reference.py,sha256=k7OSkLqTjPR8WJjNeFj0xAJ297nZUMgb_iTVwKRRKjc,4875
 sqlmesh/core/renderer.py,sha256=ksdD2x2Q5Ns5fOLIGfWIznx_T3fXbP3CfwUhD5kVSOQ,26674
-sqlmesh/core/scheduler.py,sha256=TRuxqu3vIyOMZTnViKbalWBDkpnog2lCiMeScxvNvq4,36728
+sqlmesh/core/scheduler.py,sha256=lA3zhpz52HV-QBGoPirtqd2KakivnTuqiqZp09pP8js,39318
 sqlmesh/core/schema_diff.py,sha256=Sp2MoqAngp-yTYevkLaf9jb26SuAdGXoZ54g5MjWowU,30516
 sqlmesh/core/schema_loader.py,sha256=lvrI9k7hr1P4X4E6t3U-KrcuvdQuw0UdNlN90P2s52w,3563
 sqlmesh/core/selector.py,sha256=_hcktUQiKXCPE-mOqXClHUVWBfEZKMPVVNDtpSSRhbI,12660
@@ -88,7 +88,7 @@ sqlmesh/core/model/__init__.py,sha256=C8GRZ53xuXEA9hQv3BQS9pNNyd9rZ06R_B96UYGhDu
 sqlmesh/core/model/cache.py,sha256=csun0RJguHzKX6-qITcOs4fVP4f8_Ts8qiUVV4sHY6Q,7869
 sqlmesh/core/model/common.py,sha256=PcQS3rHFNIXV_Z1dGwLQ7gLPbYVlrbWGGKAyc58-9Xk,17379
 sqlmesh/core/model/decorator.py,sha256=wUUg5t0K559tV9Pqt6FE2EWXt5TPGb4wF2Yg4B4hX4w,9251
-sqlmesh/core/model/definition.py,sha256=bVvdPhQxtRwqWe-fCBuslOPanWPPxK4vcRrbI-Dl2rk,111272
+sqlmesh/core/model/definition.py,sha256=mG6UPwxiZj0xP_cdHSVELEcXusAcltpgHBX-wiUSuPM,111963
 sqlmesh/core/model/kind.py,sha256=8Dp8yiTiXltboZWrB2FJ-LQ3zx4Fv_5Tj0KdUIPwCHI,36358
 sqlmesh/core/model/meta.py,sha256=D9InI9M2-IXeb34vkXNQSLALL8xVXjZ_LXwI-Tqxm7I,21197
 sqlmesh/core/model/schema.py,sha256=OClbAeA8LpW-IiSiLT_TlDWkEqmIIvSk0GXq8SWI-GQ,3443
@@ -102,7 +102,7 @@ sqlmesh/core/plan/stages.py,sha256=5cvEbrQsw_cx1gayxwLRwkIehQPixrbUT9B4BbEOj3A,2
 sqlmesh/core/snapshot/__init__.py,sha256=Z5HTetNCZGoKF1WAKXk4LWL9dc8EC8lwS2O4TtoBmt4,1317
 sqlmesh/core/snapshot/cache.py,sha256=bgqCR2hyf6r2A_8QP1EnXFK25gDX37-Zg0YeMuETWxg,3934
 sqlmesh/core/snapshot/categorizer.py,sha256=qGG5870mu1jR2whKisSBH_GFbsyh96RnzJDgkN7DV9g,2482
-sqlmesh/core/snapshot/definition.py,sha256=3unI15dMHzBLbfXatMSeVqH4y11TtP2PDt-FeiQfP5k,86696
+sqlmesh/core/snapshot/definition.py,sha256=L7aNCiUUmsEdkvQOfroB9aZDW6ZitHRph1eYYq1rTLE,85363
 sqlmesh/core/snapshot/evaluator.py,sha256=3mjrSp9ggd9uKhuZURrY0HtuCc_WAPsH4PYvQlqmn9o,90443
 sqlmesh/core/state_sync/__init__.py,sha256=B60xj-_Xk-4jas3UphLvwkuf04G-9AyGckcblXE0aRg,869
 sqlmesh/core/state_sync/base.py,sha256=Drq4MDWWxO_J-sJIxsMG1-YSgj2L42n5WM2vh1sr6bc,18334
@@ -284,7 +284,7 @@ sqlmesh/utils/pydantic.py,sha256=o_NsXbIpDqNpUA1Uc5xF0ZzoXQYYB0DfHwdRxBwXPNk,120
 sqlmesh/utils/rich.py,sha256=BpB47G0-JcU41LkGkeGKroexj4NOV5fox4-GOIXPgIc,3545
 sqlmesh/utils/windows.py,sha256=r7Yin4WEwbvH7sKA3hOJtRsclmscNuBeCklCfVPvtQs,611
 sqlmesh/utils/yaml.py,sha256=hvo5lmybYLcIx4p6CQwufS8X4MbwS60lEBnCmYJCXmU,2336
-sqlmesh-0.198.1.dev12.dist-info/licenses/LICENSE,sha256=OlMefUjgWJdULtf84BLW0AZZcY8DwdgQqb_1j2862j8,11346
+sqlmesh-0.198.1.dev13.dist-info/licenses/LICENSE,sha256=OlMefUjgWJdULtf84BLW0AZZcY8DwdgQqb_1j2862j8,11346
 web/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 web/client/dist/index.html,sha256=kNpFjWxt34Pf-P-cROtPWi3GNSVk6zqA3eoN9tlZXI0,5472
 web/client/dist/assets/Audits-DKgXravk.js,sha256=SaVQhNlislD_yQc_KivO7SPi1JH0ts1GnDPD5Zssutw,1041
@@ -402,8 +402,8 @@ web/server/api/endpoints/models.py,sha256=ud-lRwriyI_OYg9tZchrBgeiXELiRTLNEodraf
 web/server/api/endpoints/modules.py,sha256=8hqqgonGay_mJmpCw0IdbjsPhWlQH2VLdKAqha-myac,468
 web/server/api/endpoints/plan.py,sha256=bbbY50W_2MsZSTxOHWMKz0tbIm75nsRSlPy8GI2fg9Q,9306
 web/server/api/endpoints/table_diff.py,sha256=SCy3-pr5x6bEEiOaZGCDbiHQhpgxwmnz3Acqa2Ie6jI,2090
-sqlmesh-0.198.1.dev12.dist-info/METADATA,sha256=qnZqaFCgJs7F_RfwlVT0HnHKk4nO1q2InfazHHOLH1c,26659
-sqlmesh-0.198.1.dev12.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-sqlmesh-0.198.1.dev12.dist-info/entry_points.txt,sha256=pIYKtg-u-uO7BK_Y0qrdmfyj0bYrcMc3Xiqr5WE3VFk,121
-sqlmesh-0.198.1.dev12.dist-info/top_level.txt,sha256=bspjo5vjELQkuN0qKuTFIbsrUt3Zeo8D2mlKeHG1LdE,12
-sqlmesh-0.198.1.dev12.dist-info/RECORD,,
+sqlmesh-0.198.1.dev13.dist-info/METADATA,sha256=tO04527OWfbw2b1VCQB8tQMxbS4NGkvObRntCNs2Npo,26659
+sqlmesh-0.198.1.dev13.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+sqlmesh-0.198.1.dev13.dist-info/entry_points.txt,sha256=pIYKtg-u-uO7BK_Y0qrdmfyj0bYrcMc3Xiqr5WE3VFk,121
+sqlmesh-0.198.1.dev13.dist-info/top_level.txt,sha256=bspjo5vjELQkuN0qKuTFIbsrUt3Zeo8D2mlKeHG1LdE,12
+sqlmesh-0.198.1.dev13.dist-info/RECORD,,