sqlmesh 0.197.1.dev11__py3-none-any.whl → 0.198.1.dev1__py3-none-any.whl

sqlmesh/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.197.1.dev11'
-__version_tuple__ = version_tuple = (0, 197, 1, 'dev11')
+__version__ = version = '0.198.1.dev1'
+__version_tuple__ = version_tuple = (0, 198, 1, 'dev1')

sqlmesh/core/console.py

@@ -330,7 +330,36 @@ class UnitTestConsole(abc.ABC):
         """
 
 
+class SignalConsole(abc.ABC):
+    @abc.abstractmethod
+    def start_signal_progress(
+        self,
+        snapshot: Snapshot,
+        default_catalog: t.Optional[str],
+        environment_naming_info: EnvironmentNamingInfo,
+    ) -> None:
+        """Indicates that signal checking has begun for a snapshot."""
+
+    @abc.abstractmethod
+    def update_signal_progress(
+        self,
+        snapshot: Snapshot,
+        signal_name: str,
+        signal_idx: int,
+        total_signals: int,
+        ready_intervals: Intervals,
+        check_intervals: Intervals,
+        duration: float,
+    ) -> None:
+        """Updates the signal checking progress."""
+
+    @abc.abstractmethod
+    def stop_signal_progress(self) -> None:
+        """Indicates that signal checking has completed for a snapshot."""
+
+
 class Console(
+    SignalConsole,
     PlanBuilderConsole,
     LinterConsole,
     StateExporterConsole,
@@ -536,6 +565,29 @@ class NoopConsole(Console):
     def stop_evaluation_progress(self, success: bool = True) -> None:
         pass
 
+    def start_signal_progress(
+        self,
+        snapshot: Snapshot,
+        default_catalog: t.Optional[str],
+        environment_naming_info: EnvironmentNamingInfo,
+    ) -> None:
+        pass
+
+    def update_signal_progress(
+        self,
+        snapshot: Snapshot,
+        signal_name: str,
+        signal_idx: int,
+        total_signals: int,
+        ready_intervals: Intervals,
+        check_intervals: Intervals,
+        duration: float,
+    ) -> None:
+        pass
+
+    def stop_signal_progress(self) -> None:
+        pass
+
     def start_creation_progress(
         self,
         snapshots: t.List[Snapshot],
@@ -860,6 +912,8 @@ class TerminalConsole(Console):
         self.table_diff_model_tasks: t.Dict[str, TaskID] = {}
         self.table_diff_progress_live: t.Optional[Live] = None
 
+        self.signal_status_tree: t.Optional[Tree] = None
+
         self.verbosity = verbosity
         self.dialect = dialect
         self.ignore_warnings = ignore_warnings
@@ -901,6 +955,9 @@ class TerminalConsole(Console):
         audit_only: bool = False,
     ) -> None:
         """Indicates that a new snapshot evaluation/auditing progress has begun."""
+        # Add a newline to separate signal checking from evaluation
+        self._print("")
+
         if not self.evaluation_progress_live:
             self.evaluation_total_progress = make_progress_bar(
                 "Executing model batches" if not audit_only else "Auditing models", self.console
@@ -1050,6 +1107,88 @@ class TerminalConsole(Console):
         self.environment_naming_info = EnvironmentNamingInfo()
         self.default_catalog = None
 
+    def start_signal_progress(
+        self,
+        snapshot: Snapshot,
+        default_catalog: t.Optional[str],
+        environment_naming_info: EnvironmentNamingInfo,
+    ) -> None:
+        """Indicates that signal checking has begun for a snapshot."""
+        display_name = snapshot.display_name(
+            environment_naming_info,
+            default_catalog if self.verbosity < Verbosity.VERY_VERBOSE else None,
+            dialect=self.dialect,
+        )
+        self.signal_status_tree = Tree(f"Checking signals for {display_name}")
+
+    def update_signal_progress(
+        self,
+        snapshot: Snapshot,
+        signal_name: str,
+        signal_idx: int,
+        total_signals: int,
+        ready_intervals: Intervals,
+        check_intervals: Intervals,
+        duration: float,
+    ) -> None:
+        """Updates the signal checking progress."""
+        tree = Tree(f"[{signal_idx + 1}/{total_signals}] {signal_name} {duration:.2f}s")
+
+        formatted_check_intervals = [_format_signal_interval(snapshot, i) for i in check_intervals]
+        formatted_ready_intervals = [_format_signal_interval(snapshot, i) for i in ready_intervals]
+
+        if not formatted_check_intervals:
+            formatted_check_intervals = ["no intervals"]
+        if not formatted_ready_intervals:
+            formatted_ready_intervals = ["no intervals"]
+
+        # Color coding to help detect partial interval ranges quickly
+        if ready_intervals == check_intervals:
+            msg = "All ready"
+            color = "green"
+        elif ready_intervals:
+            msg = "Some ready"
+            color = "yellow"
+        else:
+            msg = "None ready"
+            color = "red"
+
+        if self.verbosity < Verbosity.VERY_VERBOSE:
+            num_check_intervals = len(formatted_check_intervals)
+            if num_check_intervals > 3:
+                formatted_check_intervals = formatted_check_intervals[:3]
+                formatted_check_intervals.append(f"... and {num_check_intervals - 3} more")
+
+            num_ready_intervals = len(formatted_ready_intervals)
+            if num_ready_intervals > 3:
+                formatted_ready_intervals = formatted_ready_intervals[:3]
+                formatted_ready_intervals.append(f"... and {num_ready_intervals - 3} more")
+
+            check = ", ".join(formatted_check_intervals)
+            tree.add(f"Check: {check}")
+
+            ready = ", ".join(formatted_ready_intervals)
+            tree.add(f"[{color}]{msg}: {ready}[/{color}]")
+        else:
+            check_tree = Tree("Check")
+            tree.add(check_tree)
+            for interval in formatted_check_intervals:
+                check_tree.add(interval)
+
+            ready_tree = Tree(f"[{color}]{msg}[/{color}]")
+            tree.add(ready_tree)
+            for interval in formatted_ready_intervals:
+                ready_tree.add(f"[{color}]{interval}[/{color}]")
+
+        if self.signal_status_tree is not None:
+            self.signal_status_tree.add(tree)
+
+    def stop_signal_progress(self) -> None:
+        """Indicates that signal checking has completed for a snapshot."""
+        if self.signal_status_tree is not None:
+            self._print(self.signal_status_tree)
+            self.signal_status_tree = None
+
     def start_creation_progress(
         self,
         snapshots: t.List[Snapshot],
@@ -3810,19 +3949,34 @@ def _format_audits_errors(error: NodeAuditsErrors) -> str:
     return "  " + "\n".join(error_messages)
 
 
+def _format_interval(snapshot: Snapshot, interval: Interval) -> str:
+    """Format an interval with an optional prefix."""
+    inclusive_interval = make_inclusive(interval[0], interval[1])
+    if snapshot.model.interval_unit.is_date_granularity:
+        return f"{to_ds(inclusive_interval[0])} - {to_ds(inclusive_interval[1])}"
+
+    if inclusive_interval[0].date() == inclusive_interval[1].date():
+        # omit end date if interval start/end on same day
+        return f"{to_ds(inclusive_interval[0])} {inclusive_interval[0].strftime('%H:%M:%S')}-{inclusive_interval[1].strftime('%H:%M:%S')}"
+
+    return f"{inclusive_interval[0].strftime('%Y-%m-%d %H:%M:%S')} - {inclusive_interval[1].strftime('%Y-%m-%d %H:%M:%S')}"
+
+
+def _format_signal_interval(snapshot: Snapshot, interval: Interval) -> str:
+    """Format an interval for signal output (without 'insert' prefix)."""
+    return _format_interval(snapshot, interval)
+
+
 def _format_evaluation_model_interval(snapshot: Snapshot, interval: Interval) -> str:
+    """Format an interval for evaluation output (with 'insert' prefix)."""
     if snapshot.is_model and (
         snapshot.model.kind.is_incremental
         or snapshot.model.kind.is_managed
         or snapshot.model.kind.is_custom
     ):
-        inclusive_interval = make_inclusive(interval[0], interval[1])
-        if snapshot.model.interval_unit.is_date_granularity:
-            return f"insert {to_ds(inclusive_interval[0])} - {to_ds(inclusive_interval[1])}"
-        # omit end date if interval start/end on same day
-        if inclusive_interval[0].date() == inclusive_interval[1].date():
-            return f"insert {to_ds(inclusive_interval[0])} {inclusive_interval[0].strftime('%H:%M:%S')}-{inclusive_interval[1].strftime('%H:%M:%S')}"
-        return f"insert {inclusive_interval[0].strftime('%Y-%m-%d %H:%M:%S')} - {inclusive_interval[1].strftime('%Y-%m-%d %H:%M:%S')}"
+        formatted_interval = _format_interval(snapshot, interval)
+        return f"insert {formatted_interval}"
+
     return ""
 
 

sqlmesh/core/scheduler.py

@@ -269,6 +269,7 @@ class Scheduler:
         self,
         merged_intervals: SnapshotToIntervals,
         deployability_index: t.Optional[DeployabilityIndex],
+        environment_naming_info: EnvironmentNamingInfo,
     ) -> t.Dict[Snapshot, Intervals]:
         dag = snapshots_to_dag(merged_intervals)
 
@@ -303,7 +304,13 @@ class Scheduler:
                 default_catalog=self.default_catalog,
             )
 
-            intervals = snapshot.check_ready_intervals(intervals, context)
+            intervals = snapshot.check_ready_intervals(
+                intervals,
+                context,
+                console=self.console,
+                default_catalog=self.default_catalog,
+                environment_naming_info=environment_naming_info,
+            )
             unready -= set(intervals)
 
             for parent in snapshot.parents:
@@ -324,10 +331,14 @@ class Scheduler:
                 ):
                     batches.append((next_batch[0][0], next_batch[-1][-1]))
                     next_batch = []
+
                 next_batch.append(interval)
+
             if next_batch:
                 batches.append((next_batch[0][0], next_batch[-1][-1]))
+
             snapshot_batches[snapshot] = batches
+
         return snapshot_batches
 
     def run_merged_intervals(
@@ -359,7 +370,9 @@ class Scheduler:
         """
         execution_time = execution_time or now_timestamp()
 
-        batched_intervals = self.batch_intervals(merged_intervals, deployability_index)
+        batched_intervals = self.batch_intervals(
+            merged_intervals, deployability_index, environment_naming_info
+        )
 
         self.console.start_evaluation_progress(
             batched_intervals,

sqlmesh/core/snapshot/definition.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import sys
+import time
 import typing as t
 from collections import defaultdict
 from datetime import datetime, timedelta
@@ -50,6 +51,7 @@ from sqlmesh.utils.hashing import hash_data
 from sqlmesh.utils.pydantic import PydanticModel, field_validator
 
 if t.TYPE_CHECKING:
+    from sqlmesh.core.console import Console
     from sqlglot.dialects.dialect import DialectType
     from sqlmesh.core.environment import EnvironmentNamingInfo
     from sqlmesh.core.context import ExecutionContext
@@ -965,7 +967,14 @@ class Snapshot(PydanticModel, SnapshotInfoMixin):
             model_end_ts,
         )
 
-    def check_ready_intervals(self, intervals: Intervals, context: ExecutionContext) -> Intervals:
+    def check_ready_intervals(
+        self,
+        intervals: Intervals,
+        context: ExecutionContext,
+        console: t.Optional[Console] = None,
+        default_catalog: t.Optional[str] = None,
+        environment_naming_info: t.Optional[EnvironmentNamingInfo] = None,
+    ) -> Intervals:
         """Returns a list of intervals that are considered ready by the provided signal.
 
         Note that this will handle gaps in the provided intervals. The returned intervals
@@ -979,7 +988,19 @@ class Snapshot(PydanticModel, SnapshotInfoMixin):
         python_env = self.model.python_env
         env = prepare_env(python_env)
 
-        for signal_name, kwargs in signals.items():
+        if console:
+            console.start_signal_progress(
+                self,
+                default_catalog,
+                environment_naming_info or EnvironmentNamingInfo(),
+            )
+
+        for signal_idx, (signal_name, kwargs) in enumerate(signals.items()):
+            # Capture intervals before signal check for display
+            intervals_to_check = merge_intervals(intervals)
+
+            signal_start_ts = time.perf_counter()
+
             try:
                 intervals = _check_ready_intervals(
                     env[signal_name],
@@ -996,6 +1017,23 @@ class Snapshot(PydanticModel, SnapshotInfoMixin):
                     f"{e} '{signal_name}' for '{self.model.name}' at {self.model._path}"
                 )
 
+            duration = time.perf_counter() - signal_start_ts
+
+            if console:
+                console.update_signal_progress(
+                    snapshot=self,
+                    signal_name=signal_name,
+                    signal_idx=signal_idx,
+                    total_signals=len(signals),
+                    ready_intervals=merge_intervals(intervals),
+                    check_intervals=intervals_to_check,
+                    duration=duration,
+                )
+
+        # Stop signal progress tracking
+        if console:
+            console.stop_signal_progress()
+
         return intervals
 
     def categorize_as(self, category: SnapshotChangeCategory) -> None:

{sqlmesh-0.197.1.dev11.dist-info → sqlmesh-0.198.1.dev1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sqlmesh
-Version: 0.197.1.dev11
+Version: 0.198.1.dev1
 Summary: Next-generation data transformation framework
 Author-email: "TobikoData Inc." <engineering@tobikodata.com>
 License:                                  Apache License

{sqlmesh-0.197.1.dev11.dist-info → sqlmesh-0.198.1.dev1.dist-info}/RECORD

@@ -1,5 +1,5 @@
 sqlmesh/__init__.py,sha256=QjwaamJIdW7voXoS880yAx7tHm-Im4_mqGA-XJgafos,7631
-sqlmesh/_version.py,sha256=Gn3T2eA62VLpoFoj_Xuw8Id_w-JyLIM4P69sOv8yQEo,530
+sqlmesh/_version.py,sha256=0SX9VtCi6Q-w0R4nGDmoF9JyIVUHld097Vt4ne0i-Fo,528
 sqlmesh/magics.py,sha256=R9U0Mp9ZQhDYe751nxAQDLaMv3zLAR4HcTdIflKTmEE,41709
 sqlmesh/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sqlmesh/cicd/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -11,7 +11,7 @@ sqlmesh/cli/options.py,sha256=-4ZclMIeIg8tghLowtmQNmpEFlm0QtOVEh40NqRdFHA,2499
 sqlmesh/cli/project_init.py,sha256=5JmDmUK0WJs39hM-p76uzHYdo1Hnx6fKuJg_pR3Hn5A,16419
 sqlmesh/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sqlmesh/core/_typing.py,sha256=e1f7xO2aZCy38M_3VoDy-XI3shyd99-IFctgIFSgXWM,497
-sqlmesh/core/console.py,sha256=_B9rmOn0WbgP8DsST8Lg9ODGu9u0WA2D2jRrJRX0VMM,153805
+sqlmesh/core/console.py,sha256=HInr6MOcxgn8Z-B9fAoghoDXbBEZqkB5ClyZF-fRP58,159032
 sqlmesh/core/constants.py,sha256=gUABXWYgH_XOSbaZAJsR5o1dmA61JDCvBDyxqqXX3LM,2630
 sqlmesh/core/context.py,sha256=XQFLCzpiO8yYgIcgP0J1fG9mYBzrveaqGB8ZINR6W5A,119362
 sqlmesh/core/context_diff.py,sha256=TCzM8egBE35DskpouOB-GQM2Qf5iUHWbRgiB5yZ6tzg,21552
@@ -24,7 +24,7 @@ sqlmesh/core/node.py,sha256=r4_1MRTPcU0MAlwZ5KzrykLT58B7dkKvHzPCsJmEKD8,14497
 sqlmesh/core/notification_target.py,sha256=PPGoDrgbRKxr27vJEu03XqNTQLYTw0ZF_b0yAapxGeI,16158
 sqlmesh/core/reference.py,sha256=k7OSkLqTjPR8WJjNeFj0xAJ297nZUMgb_iTVwKRRKjc,4875
 sqlmesh/core/renderer.py,sha256=ksdD2x2Q5Ns5fOLIGfWIznx_T3fXbP3CfwUhD5kVSOQ,26674
-sqlmesh/core/scheduler.py,sha256=9ADqU9YNGABJjx4GOraPC_JbzOv0GpAig8v1w6Hg1ko,36417
+sqlmesh/core/scheduler.py,sha256=TRuxqu3vIyOMZTnViKbalWBDkpnog2lCiMeScxvNvq4,36728
 sqlmesh/core/schema_diff.py,sha256=Sp2MoqAngp-yTYevkLaf9jb26SuAdGXoZ54g5MjWowU,30516
 sqlmesh/core/schema_loader.py,sha256=lvrI9k7hr1P4X4E6t3U-KrcuvdQuw0UdNlN90P2s52w,3563
 sqlmesh/core/selector.py,sha256=_hcktUQiKXCPE-mOqXClHUVWBfEZKMPVVNDtpSSRhbI,12660
@@ -102,7 +102,7 @@ sqlmesh/core/plan/stages.py,sha256=5cvEbrQsw_cx1gayxwLRwkIehQPixrbUT9B4BbEOj3A,2
 sqlmesh/core/snapshot/__init__.py,sha256=Z5HTetNCZGoKF1WAKXk4LWL9dc8EC8lwS2O4TtoBmt4,1317
 sqlmesh/core/snapshot/cache.py,sha256=bgqCR2hyf6r2A_8QP1EnXFK25gDX37-Zg0YeMuETWxg,3934
 sqlmesh/core/snapshot/categorizer.py,sha256=qGG5870mu1jR2whKisSBH_GFbsyh96RnzJDgkN7DV9g,2482
-sqlmesh/core/snapshot/definition.py,sha256=zbNqxmIi136n4Ee-25vvEVK0pICQM2KVsgm2qGR12l0,85447
+sqlmesh/core/snapshot/definition.py,sha256=vLGhbV7BDCisq8ezJhI266aeI0fiK_12OkHGYOu7-po,86693
 sqlmesh/core/snapshot/evaluator.py,sha256=i5T5EZ4lEw7X7ZGapIhau6IxJ52nJcdvSixBu_utrhE,90251
 sqlmesh/core/state_sync/__init__.py,sha256=B60xj-_Xk-4jas3UphLvwkuf04G-9AyGckcblXE0aRg,869
 sqlmesh/core/state_sync/base.py,sha256=Drq4MDWWxO_J-sJIxsMG1-YSgj2L42n5WM2vh1sr6bc,18334
@@ -283,7 +283,7 @@ sqlmesh/utils/pydantic.py,sha256=I898jAFqa3qU6lG2uPpRebVFncOUo8ZrLWM_R3K3DHU,116
 sqlmesh/utils/rich.py,sha256=BpB47G0-JcU41LkGkeGKroexj4NOV5fox4-GOIXPgIc,3545
 sqlmesh/utils/windows.py,sha256=r7Yin4WEwbvH7sKA3hOJtRsclmscNuBeCklCfVPvtQs,611
 sqlmesh/utils/yaml.py,sha256=hvo5lmybYLcIx4p6CQwufS8X4MbwS60lEBnCmYJCXmU,2336
-sqlmesh-0.197.1.dev11.dist-info/licenses/LICENSE,sha256=OlMefUjgWJdULtf84BLW0AZZcY8DwdgQqb_1j2862j8,11346
+sqlmesh-0.198.1.dev1.dist-info/licenses/LICENSE,sha256=OlMefUjgWJdULtf84BLW0AZZcY8DwdgQqb_1j2862j8,11346
 web/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 web/client/dist/index.html,sha256=kNpFjWxt34Pf-P-cROtPWi3GNSVk6zqA3eoN9tlZXI0,5472
 web/client/dist/assets/Audits-DKgXravk.js,sha256=SaVQhNlislD_yQc_KivO7SPi1JH0ts1GnDPD5Zssutw,1041
@@ -399,10 +399,10 @@ web/server/api/endpoints/lineage.py,sha256=XU99RI-6Q9SidItGM5WLQMje8NhM5nZWHbtWx
 web/server/api/endpoints/meta.py,sha256=QKWWrHewGh5JS8mCSWXTKCIQwXw-TQfaT9tWt_SsNY4,1032
 web/server/api/endpoints/models.py,sha256=ud-lRwriyI_OYg9tZchrBgeiXELiRTLNEodrafB7Ndk,4697
 web/server/api/endpoints/modules.py,sha256=8hqqgonGay_mJmpCw0IdbjsPhWlQH2VLdKAqha-myac,468
-web/server/api/endpoints/plan.py,sha256=OYuQ_4dgbhG9ibTcX1a8mvYgH3ANxw9Fhbb4wg6QWqc,9222
+web/server/api/endpoints/plan.py,sha256=bbbY50W_2MsZSTxOHWMKz0tbIm75nsRSlPy8GI2fg9Q,9306
 web/server/api/endpoints/table_diff.py,sha256=SCy3-pr5x6bEEiOaZGCDbiHQhpgxwmnz3Acqa2Ie6jI,2090
-sqlmesh-0.197.1.dev11.dist-info/METADATA,sha256=GxMBbI3F4-uteCE6qxpwy8K7KehHzifXMAeO1K7BxtE,26659
-sqlmesh-0.197.1.dev11.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-sqlmesh-0.197.1.dev11.dist-info/entry_points.txt,sha256=pIYKtg-u-uO7BK_Y0qrdmfyj0bYrcMc3Xiqr5WE3VFk,121
-sqlmesh-0.197.1.dev11.dist-info/top_level.txt,sha256=bspjo5vjELQkuN0qKuTFIbsrUt3Zeo8D2mlKeHG1LdE,12
-sqlmesh-0.197.1.dev11.dist-info/RECORD,,
+sqlmesh-0.198.1.dev1.dist-info/METADATA,sha256=SQTcPVXm2LsE_jJwcSRRh_FYDEmXMf4-4pVY7x4HyIk,26658
+sqlmesh-0.198.1.dev1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+sqlmesh-0.198.1.dev1.dist-info/entry_points.txt,sha256=pIYKtg-u-uO7BK_Y0qrdmfyj0bYrcMc3Xiqr5WE3VFk,121
+sqlmesh-0.198.1.dev1.dist-info/top_level.txt,sha256=bspjo5vjELQkuN0qKuTFIbsrUt3Zeo8D2mlKeHG1LdE,12
+sqlmesh-0.198.1.dev1.dist-info/RECORD,,

web/server/api/endpoints/plan.py

@@ -7,6 +7,7 @@ from fastapi import APIRouter, Body, Depends, Request, Response
 from starlette.status import HTTP_204_NO_CONTENT
 
 from sqlmesh.core.context import Context
+from sqlmesh.core.environment import EnvironmentNamingInfo
 from sqlmesh.core.plan import Plan, PlanBuilder
 from sqlmesh.core.snapshot.definition import SnapshotChangeCategory
 from sqlmesh.utils.date import make_inclusive, to_ds
@@ -132,7 +133,7 @@ def _get_plan_changes(context: Context, plan: Plan) -> models.PlanChanges:
 def _get_plan_backfills(context: Context, plan: Plan) -> t.Dict[str, t.Any]:
     """Get plan backfills"""
     merged_intervals = context.scheduler().merged_missing_intervals()
-    batches = context.scheduler().batch_intervals(merged_intervals, None)
+    batches = context.scheduler().batch_intervals(merged_intervals, None, EnvironmentNamingInfo())
     tasks = {snapshot.name: len(intervals) for snapshot, intervals in batches.items()}
     snapshots = plan.context_diff.snapshots
     default_catalog = context.default_catalog