sqlmesh 0.198.1.dev11__py3-none-any.whl → 0.198.1.dev13__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sqlmesh/_version.py +2 -2
- sqlmesh/core/config/scheduler.py +1 -0
- sqlmesh/core/context.py +19 -36
- sqlmesh/core/model/definition.py +19 -2
- sqlmesh/core/plan/evaluator.py +2 -4
- sqlmesh/core/plan/explainer.py +0 -2
- sqlmesh/core/scheduler.py +85 -5
- sqlmesh/core/snapshot/definition.py +6 -48
- {sqlmesh-0.198.1.dev11.dist-info → sqlmesh-0.198.1.dev13.dist-info}/METADATA +1 -1
- {sqlmesh-0.198.1.dev11.dist-info → sqlmesh-0.198.1.dev13.dist-info}/RECORD +14 -14
- {sqlmesh-0.198.1.dev11.dist-info → sqlmesh-0.198.1.dev13.dist-info}/WHEEL +0 -0
- {sqlmesh-0.198.1.dev11.dist-info → sqlmesh-0.198.1.dev13.dist-info}/entry_points.txt +0 -0
- {sqlmesh-0.198.1.dev11.dist-info → sqlmesh-0.198.1.dev13.dist-info}/licenses/LICENSE +0 -0
- {sqlmesh-0.198.1.dev11.dist-info → sqlmesh-0.198.1.dev13.dist-info}/top_level.txt +0 -0
sqlmesh/_version.py
CHANGED
|
@@ -17,5 +17,5 @@ __version__: str
|
|
|
17
17
|
__version_tuple__: VERSION_TUPLE
|
|
18
18
|
version_tuple: VERSION_TUPLE
|
|
19
19
|
|
|
20
|
-
__version__ = version = '0.198.1.
|
|
21
|
-
__version_tuple__ = version_tuple = (0, 198, 1, '
|
|
20
|
+
__version__ = version = '0.198.1.dev13'
|
|
21
|
+
__version_tuple__ = version_tuple = (0, 198, 1, 'dev13')
|
sqlmesh/core/config/scheduler.py
CHANGED
|
@@ -130,6 +130,7 @@ class BuiltInSchedulerConfig(_EngineAdapterStateSyncSchedulerConfig, BaseConfig)
|
|
|
130
130
|
def create_plan_evaluator(self, context: GenericContext) -> PlanEvaluator:
|
|
131
131
|
return BuiltInPlanEvaluator(
|
|
132
132
|
state_sync=context.state_sync,
|
|
133
|
+
snapshot_evaluator=context.snapshot_evaluator,
|
|
133
134
|
create_scheduler=context.create_scheduler,
|
|
134
135
|
default_catalog=context.default_catalog,
|
|
135
136
|
console=context.console,
|
sqlmesh/core/context.py
CHANGED
|
@@ -116,7 +116,7 @@ from sqlmesh.core.test import (
|
|
|
116
116
|
run_tests,
|
|
117
117
|
)
|
|
118
118
|
from sqlmesh.core.user import User
|
|
119
|
-
from sqlmesh.utils import UniqueKeyDict, Verbosity
|
|
119
|
+
from sqlmesh.utils import UniqueKeyDict, Verbosity
|
|
120
120
|
from sqlmesh.utils.concurrency import concurrent_apply_to_values
|
|
121
121
|
from sqlmesh.utils.dag import DAG
|
|
122
122
|
from sqlmesh.utils.date import (
|
|
@@ -417,7 +417,7 @@ class GenericContext(BaseContext, t.Generic[C]):
|
|
|
417
417
|
self.config.get_state_connection(self.gateway) or self.connection_config
|
|
418
418
|
)
|
|
419
419
|
|
|
420
|
-
self.
|
|
420
|
+
self._snapshot_evaluator: t.Optional[SnapshotEvaluator] = None
|
|
421
421
|
|
|
422
422
|
self.console = get_console()
|
|
423
423
|
setattr(self.console, "dialect", self.config.dialect)
|
|
@@ -445,22 +445,18 @@ class GenericContext(BaseContext, t.Generic[C]):
|
|
|
445
445
|
self._engine_adapter = self.connection_config.create_engine_adapter()
|
|
446
446
|
return self._engine_adapter
|
|
447
447
|
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
if correlation_id not in self._snapshot_evaluators:
|
|
453
|
-
self._snapshot_evaluators[correlation_id] = SnapshotEvaluator(
|
|
448
|
+
@property
|
|
449
|
+
def snapshot_evaluator(self) -> SnapshotEvaluator:
|
|
450
|
+
if not self._snapshot_evaluator:
|
|
451
|
+
self._snapshot_evaluator = SnapshotEvaluator(
|
|
454
452
|
{
|
|
455
|
-
gateway: adapter.with_settings(
|
|
456
|
-
log_level=logging.INFO, correlation_id=correlation_id
|
|
457
|
-
)
|
|
453
|
+
gateway: adapter.with_settings(log_level=logging.INFO)
|
|
458
454
|
for gateway, adapter in self.engine_adapters.items()
|
|
459
455
|
},
|
|
460
456
|
ddl_concurrent_tasks=self.concurrent_tasks,
|
|
461
457
|
selected_gateway=self.selected_gateway,
|
|
462
458
|
)
|
|
463
|
-
return self.
|
|
459
|
+
return self._snapshot_evaluator
|
|
464
460
|
|
|
465
461
|
def execution_context(
|
|
466
462
|
self,
|
|
@@ -541,9 +537,7 @@ class GenericContext(BaseContext, t.Generic[C]):
|
|
|
541
537
|
|
|
542
538
|
return self.create_scheduler(snapshots)
|
|
543
539
|
|
|
544
|
-
def create_scheduler(
|
|
545
|
-
self, snapshots: t.Iterable[Snapshot], correlation_id: t.Optional[CorrelationId] = None
|
|
546
|
-
) -> Scheduler:
|
|
540
|
+
def create_scheduler(self, snapshots: t.Iterable[Snapshot]) -> Scheduler:
|
|
547
541
|
"""Creates the built-in scheduler.
|
|
548
542
|
|
|
549
543
|
Args:
|
|
@@ -554,7 +548,7 @@ class GenericContext(BaseContext, t.Generic[C]):
|
|
|
554
548
|
"""
|
|
555
549
|
return Scheduler(
|
|
556
550
|
snapshots,
|
|
557
|
-
self.snapshot_evaluator
|
|
551
|
+
self.snapshot_evaluator,
|
|
558
552
|
self.state_sync,
|
|
559
553
|
default_catalog=self.default_catalog,
|
|
560
554
|
max_workers=self.concurrent_tasks,
|
|
@@ -719,7 +713,7 @@ class GenericContext(BaseContext, t.Generic[C]):
|
|
|
719
713
|
NotificationEvent.RUN_START, environment=environment
|
|
720
714
|
)
|
|
721
715
|
analytics_run_id = analytics.collector.on_run_start(
|
|
722
|
-
engine_type=self.snapshot_evaluator
|
|
716
|
+
engine_type=self.snapshot_evaluator.adapter.dialect,
|
|
723
717
|
state_sync_type=self.state_sync.state_type(),
|
|
724
718
|
)
|
|
725
719
|
self._load_materializations()
|
|
@@ -1081,7 +1075,7 @@ class GenericContext(BaseContext, t.Generic[C]):
|
|
|
1081
1075
|
and not parent_snapshot.categorized
|
|
1082
1076
|
]
|
|
1083
1077
|
|
|
1084
|
-
df = self.snapshot_evaluator
|
|
1078
|
+
df = self.snapshot_evaluator.evaluate_and_fetch(
|
|
1085
1079
|
snapshot,
|
|
1086
1080
|
start=start,
|
|
1087
1081
|
end=end,
|
|
@@ -1593,12 +1587,7 @@ class GenericContext(BaseContext, t.Generic[C]):
|
|
|
1593
1587
|
default_catalog=self.default_catalog,
|
|
1594
1588
|
console=self.console,
|
|
1595
1589
|
)
|
|
1596
|
-
explainer.evaluate(
|
|
1597
|
-
plan.to_evaluatable(),
|
|
1598
|
-
snapshot_evaluator=self.snapshot_evaluator(
|
|
1599
|
-
correlation_id=CorrelationId.from_plan_id(plan.plan_id)
|
|
1600
|
-
),
|
|
1601
|
-
)
|
|
1590
|
+
explainer.evaluate(plan.to_evaluatable())
|
|
1602
1591
|
return
|
|
1603
1592
|
|
|
1604
1593
|
self.notification_target_manager.notify(
|
|
@@ -2121,7 +2110,7 @@ class GenericContext(BaseContext, t.Generic[C]):
|
|
|
2121
2110
|
errors = []
|
|
2122
2111
|
skipped_count = 0
|
|
2123
2112
|
for snapshot in snapshots:
|
|
2124
|
-
for audit_result in self.snapshot_evaluator
|
|
2113
|
+
for audit_result in self.snapshot_evaluator.audit(
|
|
2125
2114
|
snapshot=snapshot,
|
|
2126
2115
|
start=start,
|
|
2127
2116
|
end=end,
|
|
@@ -2153,7 +2142,7 @@ class GenericContext(BaseContext, t.Generic[C]):
|
|
|
2153
2142
|
self.console.log_status_update(f"Got {error.count} results, expected 0.")
|
|
2154
2143
|
if error.query:
|
|
2155
2144
|
self.console.show_sql(
|
|
2156
|
-
f"{error.query.sql(dialect=self.snapshot_evaluator
|
|
2145
|
+
f"{error.query.sql(dialect=self.snapshot_evaluator.adapter.dialect)}"
|
|
2157
2146
|
)
|
|
2158
2147
|
|
|
2159
2148
|
self.console.log_status_update("Done.")
|
|
@@ -2345,14 +2334,12 @@ class GenericContext(BaseContext, t.Generic[C]):
|
|
|
2345
2334
|
|
|
2346
2335
|
def close(self) -> None:
|
|
2347
2336
|
"""Releases all resources allocated by this context."""
|
|
2348
|
-
|
|
2349
|
-
|
|
2337
|
+
if self._snapshot_evaluator:
|
|
2338
|
+
self._snapshot_evaluator.close()
|
|
2350
2339
|
|
|
2351
2340
|
if self._state_sync:
|
|
2352
2341
|
self._state_sync.close()
|
|
2353
2342
|
|
|
2354
|
-
self._snapshot_evaluators.clear()
|
|
2355
|
-
|
|
2356
2343
|
def _run(
|
|
2357
2344
|
self,
|
|
2358
2345
|
environment: str,
|
|
@@ -2403,11 +2390,7 @@ class GenericContext(BaseContext, t.Generic[C]):
|
|
|
2403
2390
|
|
|
2404
2391
|
def _apply(self, plan: Plan, circuit_breaker: t.Optional[t.Callable[[], bool]]) -> None:
|
|
2405
2392
|
self._scheduler.create_plan_evaluator(self).evaluate(
|
|
2406
|
-
plan.to_evaluatable(),
|
|
2407
|
-
snapshot_evaluator=self.snapshot_evaluator(
|
|
2408
|
-
correlation_id=CorrelationId.from_plan_id(plan.plan_id)
|
|
2409
|
-
),
|
|
2410
|
-
circuit_breaker=circuit_breaker,
|
|
2393
|
+
plan.to_evaluatable(), circuit_breaker=circuit_breaker
|
|
2411
2394
|
)
|
|
2412
2395
|
|
|
2413
2396
|
@python_api_analytics
|
|
@@ -2700,7 +2683,7 @@ class GenericContext(BaseContext, t.Generic[C]):
|
|
|
2700
2683
|
)
|
|
2701
2684
|
|
|
2702
2685
|
# Remove the expired snapshots tables
|
|
2703
|
-
self.snapshot_evaluator
|
|
2686
|
+
self.snapshot_evaluator.cleanup(
|
|
2704
2687
|
target_snapshots=cleanup_targets,
|
|
2705
2688
|
on_complete=self.console.update_cleanup_progress,
|
|
2706
2689
|
)
|
sqlmesh/core/model/definition.py
CHANGED
|
@@ -627,8 +627,10 @@ class _Model(ModelMeta, frozen=True):
|
|
|
627
627
|
{k: _render(v) for k, v in signal.items()} for name, signal in self.signals if not name
|
|
628
628
|
]
|
|
629
629
|
|
|
630
|
-
def render_signal_calls(self) ->
|
|
631
|
-
|
|
630
|
+
def render_signal_calls(self) -> EvaluatableSignals:
|
|
631
|
+
python_env = self.python_env
|
|
632
|
+
env = prepare_env(python_env)
|
|
633
|
+
signals_to_kwargs = {
|
|
632
634
|
name: {
|
|
633
635
|
k: seq_get(self._create_renderer(v).render() or [], 0) for k, v in kwargs.items()
|
|
634
636
|
}
|
|
@@ -636,6 +638,12 @@ class _Model(ModelMeta, frozen=True):
|
|
|
636
638
|
if name
|
|
637
639
|
}
|
|
638
640
|
|
|
641
|
+
return EvaluatableSignals(
|
|
642
|
+
signals_to_kwargs=signals_to_kwargs,
|
|
643
|
+
python_env=python_env,
|
|
644
|
+
prepared_python_env=env,
|
|
645
|
+
)
|
|
646
|
+
|
|
639
647
|
def render_merge_filter(
|
|
640
648
|
self,
|
|
641
649
|
*,
|
|
@@ -1857,6 +1865,15 @@ class AuditResult(PydanticModel):
|
|
|
1857
1865
|
blocking: bool = True
|
|
1858
1866
|
|
|
1859
1867
|
|
|
1868
|
+
class EvaluatableSignals(PydanticModel):
|
|
1869
|
+
signals_to_kwargs: t.Dict[str, t.Dict[str, t.Optional[exp.Expression]]]
|
|
1870
|
+
"""A mapping of signal names to the kwargs passed to the signal."""
|
|
1871
|
+
python_env: t.Dict[str, Executable]
|
|
1872
|
+
"""The Python environment that should be used to evaluated the rendered signal calls."""
|
|
1873
|
+
prepared_python_env: t.Dict[str, t.Any]
|
|
1874
|
+
"""The prepared Python environment that should be used to evaluated the rendered signal calls."""
|
|
1875
|
+
|
|
1876
|
+
|
|
1860
1877
|
def _extract_blueprints(blueprints: t.Any, path: Path) -> t.List[t.Any]:
|
|
1861
1878
|
if not blueprints:
|
|
1862
1879
|
return [None]
|
sqlmesh/core/plan/evaluator.py
CHANGED
|
@@ -51,7 +51,6 @@ class PlanEvaluator(abc.ABC):
|
|
|
51
51
|
def evaluate(
|
|
52
52
|
self,
|
|
53
53
|
plan: EvaluatablePlan,
|
|
54
|
-
snapshot_evaluator: SnapshotEvaluator,
|
|
55
54
|
circuit_breaker: t.Optional[t.Callable[[], bool]] = None,
|
|
56
55
|
) -> None:
|
|
57
56
|
"""Evaluates a plan by pushing snapshots and backfilling data.
|
|
@@ -63,7 +62,6 @@ class PlanEvaluator(abc.ABC):
|
|
|
63
62
|
|
|
64
63
|
Args:
|
|
65
64
|
plan: The plan to evaluate.
|
|
66
|
-
snapshot_evaluator: The snapshot evaluator to use.
|
|
67
65
|
circuit_breaker: The circuit breaker to use.
|
|
68
66
|
"""
|
|
69
67
|
|
|
@@ -72,11 +70,13 @@ class BuiltInPlanEvaluator(PlanEvaluator):
|
|
|
72
70
|
def __init__(
|
|
73
71
|
self,
|
|
74
72
|
state_sync: StateSync,
|
|
73
|
+
snapshot_evaluator: SnapshotEvaluator,
|
|
75
74
|
create_scheduler: t.Callable[[t.Iterable[Snapshot]], Scheduler],
|
|
76
75
|
default_catalog: t.Optional[str],
|
|
77
76
|
console: t.Optional[Console] = None,
|
|
78
77
|
):
|
|
79
78
|
self.state_sync = state_sync
|
|
79
|
+
self.snapshot_evaluator = snapshot_evaluator
|
|
80
80
|
self.create_scheduler = create_scheduler
|
|
81
81
|
self.default_catalog = default_catalog
|
|
82
82
|
self.console = console or get_console()
|
|
@@ -85,11 +85,9 @@ class BuiltInPlanEvaluator(PlanEvaluator):
|
|
|
85
85
|
def evaluate(
|
|
86
86
|
self,
|
|
87
87
|
plan: EvaluatablePlan,
|
|
88
|
-
snapshot_evaluator: SnapshotEvaluator,
|
|
89
88
|
circuit_breaker: t.Optional[t.Callable[[], bool]] = None,
|
|
90
89
|
) -> None:
|
|
91
90
|
self._circuit_breaker = circuit_breaker
|
|
92
|
-
self.snapshot_evaluator = snapshot_evaluator
|
|
93
91
|
|
|
94
92
|
self.console.start_plan_evaluation(plan)
|
|
95
93
|
analytics.collector.on_plan_apply_start(
|
sqlmesh/core/plan/explainer.py
CHANGED
|
@@ -20,7 +20,6 @@ from sqlmesh.core.snapshot.definition import (
|
|
|
20
20
|
from sqlmesh.utils import Verbosity, rich as srich, to_snake_case
|
|
21
21
|
from sqlmesh.utils.date import to_ts
|
|
22
22
|
from sqlmesh.utils.errors import SQLMeshError
|
|
23
|
-
from sqlmesh.core.snapshot.evaluator import SnapshotEvaluator
|
|
24
23
|
|
|
25
24
|
|
|
26
25
|
logger = logging.getLogger(__name__)
|
|
@@ -40,7 +39,6 @@ class PlanExplainer(PlanEvaluator):
|
|
|
40
39
|
def evaluate(
|
|
41
40
|
self,
|
|
42
41
|
plan: EvaluatablePlan,
|
|
43
|
-
snapshot_evaluator: SnapshotEvaluator,
|
|
44
42
|
circuit_breaker: t.Optional[t.Callable[[], bool]] = None,
|
|
45
43
|
) -> None:
|
|
46
44
|
plan_stages = stages.build_plan_stages(plan, self.state_reader, self.default_catalog)
|
sqlmesh/core/scheduler.py
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
import logging
|
|
3
3
|
import typing as t
|
|
4
|
+
import time
|
|
4
5
|
from sqlglot import exp
|
|
5
6
|
from sqlmesh.core import constants as c
|
|
6
7
|
from sqlmesh.core.console import Console, get_console
|
|
@@ -24,6 +25,7 @@ from sqlmesh.core.snapshot import (
|
|
|
24
25
|
snapshots_to_dag,
|
|
25
26
|
Intervals,
|
|
26
27
|
)
|
|
28
|
+
from sqlmesh.core.snapshot.definition import check_ready_intervals
|
|
27
29
|
from sqlmesh.core.snapshot.definition import (
|
|
28
30
|
Interval,
|
|
29
31
|
expand_range,
|
|
@@ -39,7 +41,16 @@ from sqlmesh.utils.date import (
|
|
|
39
41
|
to_timestamp,
|
|
40
42
|
validate_date_range,
|
|
41
43
|
)
|
|
42
|
-
from sqlmesh.utils.errors import
|
|
44
|
+
from sqlmesh.utils.errors import (
|
|
45
|
+
AuditError,
|
|
46
|
+
NodeAuditsErrors,
|
|
47
|
+
CircuitBreakerError,
|
|
48
|
+
SQLMeshError,
|
|
49
|
+
SignalEvalError,
|
|
50
|
+
)
|
|
51
|
+
|
|
52
|
+
if t.TYPE_CHECKING:
|
|
53
|
+
from sqlmesh.core.context import ExecutionContext
|
|
43
54
|
|
|
44
55
|
logger = logging.getLogger(__name__)
|
|
45
56
|
SnapshotToIntervals = t.Dict[Snapshot, Intervals]
|
|
@@ -304,12 +315,11 @@ class Scheduler:
|
|
|
304
315
|
default_catalog=self.default_catalog,
|
|
305
316
|
)
|
|
306
317
|
|
|
307
|
-
intervals =
|
|
318
|
+
intervals = self._check_ready_intervals(
|
|
319
|
+
snapshot,
|
|
308
320
|
intervals,
|
|
309
321
|
context,
|
|
310
|
-
|
|
311
|
-
default_catalog=self.default_catalog,
|
|
312
|
-
environment_naming_info=environment_naming_info,
|
|
322
|
+
environment_naming_info,
|
|
313
323
|
)
|
|
314
324
|
unready -= set(intervals)
|
|
315
325
|
|
|
@@ -709,6 +719,76 @@ class Scheduler:
|
|
|
709
719
|
|
|
710
720
|
return audit_results
|
|
711
721
|
|
|
722
|
+
def _check_ready_intervals(
|
|
723
|
+
self,
|
|
724
|
+
snapshot: Snapshot,
|
|
725
|
+
intervals: Intervals,
|
|
726
|
+
context: ExecutionContext,
|
|
727
|
+
environment_naming_info: EnvironmentNamingInfo,
|
|
728
|
+
) -> Intervals:
|
|
729
|
+
"""Checks if the intervals are ready for evaluation for the given snapshot.
|
|
730
|
+
|
|
731
|
+
This implementation also includes the signal progress tracking.
|
|
732
|
+
Note that this will handle gaps in the provided intervals. The returned intervals
|
|
733
|
+
may introduce new gaps.
|
|
734
|
+
|
|
735
|
+
Args:
|
|
736
|
+
snapshot: The snapshot to check.
|
|
737
|
+
intervals: The intervals to check.
|
|
738
|
+
context: The context to use.
|
|
739
|
+
environment_naming_info: The environment naming info to use.
|
|
740
|
+
|
|
741
|
+
Returns:
|
|
742
|
+
The intervals that are ready for evaluation.
|
|
743
|
+
"""
|
|
744
|
+
signals = snapshot.is_model and snapshot.model.render_signal_calls()
|
|
745
|
+
|
|
746
|
+
if not signals:
|
|
747
|
+
return intervals
|
|
748
|
+
|
|
749
|
+
self.console.start_signal_progress(
|
|
750
|
+
snapshot,
|
|
751
|
+
self.default_catalog,
|
|
752
|
+
environment_naming_info or EnvironmentNamingInfo(),
|
|
753
|
+
)
|
|
754
|
+
|
|
755
|
+
for signal_idx, (signal_name, kwargs) in enumerate(signals.signals_to_kwargs.items()):
|
|
756
|
+
# Capture intervals before signal check for display
|
|
757
|
+
intervals_to_check = merge_intervals(intervals)
|
|
758
|
+
|
|
759
|
+
signal_start_ts = time.perf_counter()
|
|
760
|
+
|
|
761
|
+
try:
|
|
762
|
+
intervals = check_ready_intervals(
|
|
763
|
+
signals.prepared_python_env[signal_name],
|
|
764
|
+
intervals,
|
|
765
|
+
context,
|
|
766
|
+
python_env=signals.python_env,
|
|
767
|
+
dialect=snapshot.model.dialect,
|
|
768
|
+
path=snapshot.model._path,
|
|
769
|
+
kwargs=kwargs,
|
|
770
|
+
)
|
|
771
|
+
except SQLMeshError as e:
|
|
772
|
+
raise SignalEvalError(
|
|
773
|
+
f"{e} '{signal_name}' for '{snapshot.model.name}' at {snapshot.model._path}"
|
|
774
|
+
)
|
|
775
|
+
|
|
776
|
+
duration = time.perf_counter() - signal_start_ts
|
|
777
|
+
|
|
778
|
+
self.console.update_signal_progress(
|
|
779
|
+
snapshot=snapshot,
|
|
780
|
+
signal_name=signal_name,
|
|
781
|
+
signal_idx=signal_idx,
|
|
782
|
+
total_signals=len(signals.signals_to_kwargs),
|
|
783
|
+
ready_intervals=merge_intervals(intervals),
|
|
784
|
+
check_intervals=intervals_to_check,
|
|
785
|
+
duration=duration,
|
|
786
|
+
)
|
|
787
|
+
|
|
788
|
+
self.console.stop_signal_progress()
|
|
789
|
+
|
|
790
|
+
return intervals
|
|
791
|
+
|
|
712
792
|
|
|
713
793
|
def merged_missing_intervals(
|
|
714
794
|
snapshots: t.Collection[Snapshot],
|
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
3
|
import sys
|
|
4
|
-
import time
|
|
5
4
|
import typing as t
|
|
6
5
|
from collections import defaultdict
|
|
7
6
|
from datetime import datetime, timedelta
|
|
@@ -42,8 +41,6 @@ from sqlmesh.utils.date import (
|
|
|
42
41
|
)
|
|
43
42
|
from sqlmesh.utils.errors import SQLMeshError, SignalEvalError
|
|
44
43
|
from sqlmesh.utils.metaprogramming import (
|
|
45
|
-
prepare_env,
|
|
46
|
-
print_exception,
|
|
47
44
|
format_evaluated_code_exception,
|
|
48
45
|
Executable,
|
|
49
46
|
)
|
|
@@ -51,7 +48,6 @@ from sqlmesh.utils.hashing import hash_data
|
|
|
51
48
|
from sqlmesh.utils.pydantic import PydanticModel, field_validator
|
|
52
49
|
|
|
53
50
|
if t.TYPE_CHECKING:
|
|
54
|
-
from sqlmesh.core.console import Console
|
|
55
51
|
from sqlglot.dialects.dialect import DialectType
|
|
56
52
|
from sqlmesh.core.environment import EnvironmentNamingInfo
|
|
57
53
|
from sqlmesh.core.context import ExecutionContext
|
|
@@ -971,9 +967,6 @@ class Snapshot(PydanticModel, SnapshotInfoMixin):
|
|
|
971
967
|
self,
|
|
972
968
|
intervals: Intervals,
|
|
973
969
|
context: ExecutionContext,
|
|
974
|
-
console: t.Optional[Console] = None,
|
|
975
|
-
default_catalog: t.Optional[str] = None,
|
|
976
|
-
environment_naming_info: t.Optional[EnvironmentNamingInfo] = None,
|
|
977
970
|
) -> Intervals:
|
|
978
971
|
"""Returns a list of intervals that are considered ready by the provided signal.
|
|
979
972
|
|
|
@@ -981,59 +974,24 @@ class Snapshot(PydanticModel, SnapshotInfoMixin):
|
|
|
981
974
|
may introduce new gaps.
|
|
982
975
|
"""
|
|
983
976
|
signals = self.is_model and self.model.render_signal_calls()
|
|
984
|
-
|
|
985
977
|
if not signals:
|
|
986
978
|
return intervals
|
|
987
979
|
|
|
988
|
-
|
|
989
|
-
env = prepare_env(python_env)
|
|
990
|
-
|
|
991
|
-
if console:
|
|
992
|
-
console.start_signal_progress(
|
|
993
|
-
self,
|
|
994
|
-
default_catalog,
|
|
995
|
-
environment_naming_info or EnvironmentNamingInfo(),
|
|
996
|
-
)
|
|
997
|
-
|
|
998
|
-
for signal_idx, (signal_name, kwargs) in enumerate(signals.items()):
|
|
999
|
-
# Capture intervals before signal check for display
|
|
1000
|
-
intervals_to_check = merge_intervals(intervals)
|
|
1001
|
-
|
|
1002
|
-
signal_start_ts = time.perf_counter()
|
|
1003
|
-
|
|
980
|
+
for signal_name, kwargs in signals.signals_to_kwargs.items():
|
|
1004
981
|
try:
|
|
1005
|
-
intervals =
|
|
1006
|
-
|
|
982
|
+
intervals = check_ready_intervals(
|
|
983
|
+
signals.prepared_python_env[signal_name],
|
|
1007
984
|
intervals,
|
|
1008
985
|
context,
|
|
1009
|
-
python_env=python_env,
|
|
986
|
+
python_env=signals.python_env,
|
|
1010
987
|
dialect=self.model.dialect,
|
|
1011
988
|
path=self.model._path,
|
|
1012
989
|
kwargs=kwargs,
|
|
1013
990
|
)
|
|
1014
991
|
except SQLMeshError as e:
|
|
1015
|
-
|
|
1016
|
-
raise SQLMeshError(
|
|
992
|
+
raise SignalEvalError(
|
|
1017
993
|
f"{e} '{signal_name}' for '{self.model.name}' at {self.model._path}"
|
|
1018
994
|
)
|
|
1019
|
-
|
|
1020
|
-
duration = time.perf_counter() - signal_start_ts
|
|
1021
|
-
|
|
1022
|
-
if console:
|
|
1023
|
-
console.update_signal_progress(
|
|
1024
|
-
snapshot=self,
|
|
1025
|
-
signal_name=signal_name,
|
|
1026
|
-
signal_idx=signal_idx,
|
|
1027
|
-
total_signals=len(signals),
|
|
1028
|
-
ready_intervals=merge_intervals(intervals),
|
|
1029
|
-
check_intervals=intervals_to_check,
|
|
1030
|
-
duration=duration,
|
|
1031
|
-
)
|
|
1032
|
-
|
|
1033
|
-
# Stop signal progress tracking
|
|
1034
|
-
if console:
|
|
1035
|
-
console.stop_signal_progress()
|
|
1036
|
-
|
|
1037
995
|
return intervals
|
|
1038
996
|
|
|
1039
997
|
def categorize_as(self, category: SnapshotChangeCategory) -> None:
|
|
@@ -2229,7 +2187,7 @@ def _contiguous_intervals(intervals: Intervals) -> t.List[Intervals]:
|
|
|
2229
2187
|
return contiguous_intervals
|
|
2230
2188
|
|
|
2231
2189
|
|
|
2232
|
-
def
|
|
2190
|
+
def check_ready_intervals(
|
|
2233
2191
|
check: t.Callable,
|
|
2234
2192
|
intervals: Intervals,
|
|
2235
2193
|
context: ExecutionContext,
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
sqlmesh/__init__.py,sha256=QjwaamJIdW7voXoS880yAx7tHm-Im4_mqGA-XJgafos,7631
|
|
2
|
-
sqlmesh/_version.py,sha256=
|
|
2
|
+
sqlmesh/_version.py,sha256=Ot41ppAYTuEK_dNN-7AEmc6ave1-1Q5pztj9mQwkDoo,530
|
|
3
3
|
sqlmesh/magics.py,sha256=R9U0Mp9ZQhDYe751nxAQDLaMv3zLAR4HcTdIflKTmEE,41709
|
|
4
4
|
sqlmesh/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
5
5
|
sqlmesh/cicd/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
@@ -13,7 +13,7 @@ sqlmesh/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
|
13
13
|
sqlmesh/core/_typing.py,sha256=e1f7xO2aZCy38M_3VoDy-XI3shyd99-IFctgIFSgXWM,497
|
|
14
14
|
sqlmesh/core/console.py,sha256=HInr6MOcxgn8Z-B9fAoghoDXbBEZqkB5ClyZF-fRP58,159032
|
|
15
15
|
sqlmesh/core/constants.py,sha256=gUABXWYgH_XOSbaZAJsR5o1dmA61JDCvBDyxqqXX3LM,2630
|
|
16
|
-
sqlmesh/core/context.py,sha256=
|
|
16
|
+
sqlmesh/core/context.py,sha256=WyaHN2Fh3FMMHN5MQNnfq5fm89wkfJujq-dkhccRId0,118534
|
|
17
17
|
sqlmesh/core/context_diff.py,sha256=TCzM8egBE35DskpouOB-GQM2Qf5iUHWbRgiB5yZ6tzg,21552
|
|
18
18
|
sqlmesh/core/dialect.py,sha256=1ykYwb9WZn_nCaX08PcQ0BJh2Pihv9SPvoMNjpMeCfs,50158
|
|
19
19
|
sqlmesh/core/environment.py,sha256=7DbJX0r5nZRU3LhcgIxbSV-JosQYDjAt3V5Wc2Fpns8,13005
|
|
@@ -24,7 +24,7 @@ sqlmesh/core/node.py,sha256=r4_1MRTPcU0MAlwZ5KzrykLT58B7dkKvHzPCsJmEKD8,14497
|
|
|
24
24
|
sqlmesh/core/notification_target.py,sha256=PPGoDrgbRKxr27vJEu03XqNTQLYTw0ZF_b0yAapxGeI,16158
|
|
25
25
|
sqlmesh/core/reference.py,sha256=k7OSkLqTjPR8WJjNeFj0xAJ297nZUMgb_iTVwKRRKjc,4875
|
|
26
26
|
sqlmesh/core/renderer.py,sha256=ksdD2x2Q5Ns5fOLIGfWIznx_T3fXbP3CfwUhD5kVSOQ,26674
|
|
27
|
-
sqlmesh/core/scheduler.py,sha256=
|
|
27
|
+
sqlmesh/core/scheduler.py,sha256=lA3zhpz52HV-QBGoPirtqd2KakivnTuqiqZp09pP8js,39318
|
|
28
28
|
sqlmesh/core/schema_diff.py,sha256=Sp2MoqAngp-yTYevkLaf9jb26SuAdGXoZ54g5MjWowU,30516
|
|
29
29
|
sqlmesh/core/schema_loader.py,sha256=lvrI9k7hr1P4X4E6t3U-KrcuvdQuw0UdNlN90P2s52w,3563
|
|
30
30
|
sqlmesh/core/selector.py,sha256=_hcktUQiKXCPE-mOqXClHUVWBfEZKMPVVNDtpSSRhbI,12660
|
|
@@ -54,7 +54,7 @@ sqlmesh/core/config/naming.py,sha256=KVGaxUgswoQajXR_cHFJvgkmEFcU1V182GkOH1SJxuE
|
|
|
54
54
|
sqlmesh/core/config/plan.py,sha256=wJEDfocMcKSMN0-5gVojjIcmJZ0376LJU3tkGMTm-Zc,1558
|
|
55
55
|
sqlmesh/core/config/root.py,sha256=tZxArC4kjPqrt43R9uErH_awwwNQVaNIaTAb-ot9WiU,15529
|
|
56
56
|
sqlmesh/core/config/run.py,sha256=rIK0QnN_vArrwps2TXGVEbvao2AnGZgzXo7EliJa42E,852
|
|
57
|
-
sqlmesh/core/config/scheduler.py,sha256=
|
|
57
|
+
sqlmesh/core/config/scheduler.py,sha256=k3BVMpy5ZrC05ZlyE3vec8ZGfTjUjWyfsKZtYgflbyA,7029
|
|
58
58
|
sqlmesh/core/config/ui.py,sha256=jsO-S6_d9NkLZGG5pT4mgKgxMF34KzkDociZAMvCX3U,278
|
|
59
59
|
sqlmesh/core/engine_adapter/__init__.py,sha256=two_WTs4UnSNp-fscJh8vOunJq0Xi37EuDuMjKsKuNE,2281
|
|
60
60
|
sqlmesh/core/engine_adapter/_typing.py,sha256=_63pYCThfC39UtPVHY3N0H6qJ1YBT32OqwwR3swolGM,1037
|
|
@@ -88,7 +88,7 @@ sqlmesh/core/model/__init__.py,sha256=C8GRZ53xuXEA9hQv3BQS9pNNyd9rZ06R_B96UYGhDu
|
|
|
88
88
|
sqlmesh/core/model/cache.py,sha256=csun0RJguHzKX6-qITcOs4fVP4f8_Ts8qiUVV4sHY6Q,7869
|
|
89
89
|
sqlmesh/core/model/common.py,sha256=PcQS3rHFNIXV_Z1dGwLQ7gLPbYVlrbWGGKAyc58-9Xk,17379
|
|
90
90
|
sqlmesh/core/model/decorator.py,sha256=wUUg5t0K559tV9Pqt6FE2EWXt5TPGb4wF2Yg4B4hX4w,9251
|
|
91
|
-
sqlmesh/core/model/definition.py,sha256=
|
|
91
|
+
sqlmesh/core/model/definition.py,sha256=mG6UPwxiZj0xP_cdHSVELEcXusAcltpgHBX-wiUSuPM,111963
|
|
92
92
|
sqlmesh/core/model/kind.py,sha256=8Dp8yiTiXltboZWrB2FJ-LQ3zx4Fv_5Tj0KdUIPwCHI,36358
|
|
93
93
|
sqlmesh/core/model/meta.py,sha256=D9InI9M2-IXeb34vkXNQSLALL8xVXjZ_LXwI-Tqxm7I,21197
|
|
94
94
|
sqlmesh/core/model/schema.py,sha256=OClbAeA8LpW-IiSiLT_TlDWkEqmIIvSk0GXq8SWI-GQ,3443
|
|
@@ -96,13 +96,13 @@ sqlmesh/core/model/seed.py,sha256=a0M-1zY1gOkN5ph2GQyataEdBSCtq50YjeFk2LyvInI,50
|
|
|
96
96
|
sqlmesh/core/plan/__init__.py,sha256=NKSvM7ZBVjw9ho3J65M1wFvG3KURB8PJ0FHHLmtSF44,443
|
|
97
97
|
sqlmesh/core/plan/builder.py,sha256=g07h-NOsOr_4spnOrlQ4KrFjh3RmbvHSydEuSOsShNc,41416
|
|
98
98
|
sqlmesh/core/plan/definition.py,sha256=OxgbvkMGNdlvYF2BjFJt_lINsBw38Fq1IX25P8_Z_Z0,13734
|
|
99
|
-
sqlmesh/core/plan/evaluator.py,sha256=
|
|
100
|
-
sqlmesh/core/plan/explainer.py,sha256=
|
|
99
|
+
sqlmesh/core/plan/evaluator.py,sha256=4Gt9svTUe-0YSDawlSbJNed5fkZASbTPsYyytQFXy14,20559
|
|
100
|
+
sqlmesh/core/plan/explainer.py,sha256=aBkvtM9eog_2NIglazSDdG_i9TTCdFSVOkCw5N8O-1k,12028
|
|
101
101
|
sqlmesh/core/plan/stages.py,sha256=5cvEbrQsw_cx1gayxwLRwkIehQPixrbUT9B4BbEOj3A,21972
|
|
102
102
|
sqlmesh/core/snapshot/__init__.py,sha256=Z5HTetNCZGoKF1WAKXk4LWL9dc8EC8lwS2O4TtoBmt4,1317
|
|
103
103
|
sqlmesh/core/snapshot/cache.py,sha256=bgqCR2hyf6r2A_8QP1EnXFK25gDX37-Zg0YeMuETWxg,3934
|
|
104
104
|
sqlmesh/core/snapshot/categorizer.py,sha256=qGG5870mu1jR2whKisSBH_GFbsyh96RnzJDgkN7DV9g,2482
|
|
105
|
-
sqlmesh/core/snapshot/definition.py,sha256=
|
|
105
|
+
sqlmesh/core/snapshot/definition.py,sha256=L7aNCiUUmsEdkvQOfroB9aZDW6ZitHRph1eYYq1rTLE,85363
|
|
106
106
|
sqlmesh/core/snapshot/evaluator.py,sha256=3mjrSp9ggd9uKhuZURrY0HtuCc_WAPsH4PYvQlqmn9o,90443
|
|
107
107
|
sqlmesh/core/state_sync/__init__.py,sha256=B60xj-_Xk-4jas3UphLvwkuf04G-9AyGckcblXE0aRg,869
|
|
108
108
|
sqlmesh/core/state_sync/base.py,sha256=Drq4MDWWxO_J-sJIxsMG1-YSgj2L42n5WM2vh1sr6bc,18334
|
|
@@ -284,7 +284,7 @@ sqlmesh/utils/pydantic.py,sha256=o_NsXbIpDqNpUA1Uc5xF0ZzoXQYYB0DfHwdRxBwXPNk,120
|
|
|
284
284
|
sqlmesh/utils/rich.py,sha256=BpB47G0-JcU41LkGkeGKroexj4NOV5fox4-GOIXPgIc,3545
|
|
285
285
|
sqlmesh/utils/windows.py,sha256=r7Yin4WEwbvH7sKA3hOJtRsclmscNuBeCklCfVPvtQs,611
|
|
286
286
|
sqlmesh/utils/yaml.py,sha256=hvo5lmybYLcIx4p6CQwufS8X4MbwS60lEBnCmYJCXmU,2336
|
|
287
|
-
sqlmesh-0.198.1.
|
|
287
|
+
sqlmesh-0.198.1.dev13.dist-info/licenses/LICENSE,sha256=OlMefUjgWJdULtf84BLW0AZZcY8DwdgQqb_1j2862j8,11346
|
|
288
288
|
web/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
289
289
|
web/client/dist/index.html,sha256=kNpFjWxt34Pf-P-cROtPWi3GNSVk6zqA3eoN9tlZXI0,5472
|
|
290
290
|
web/client/dist/assets/Audits-DKgXravk.js,sha256=SaVQhNlislD_yQc_KivO7SPi1JH0ts1GnDPD5Zssutw,1041
|
|
@@ -402,8 +402,8 @@ web/server/api/endpoints/models.py,sha256=ud-lRwriyI_OYg9tZchrBgeiXELiRTLNEodraf
|
|
|
402
402
|
web/server/api/endpoints/modules.py,sha256=8hqqgonGay_mJmpCw0IdbjsPhWlQH2VLdKAqha-myac,468
|
|
403
403
|
web/server/api/endpoints/plan.py,sha256=bbbY50W_2MsZSTxOHWMKz0tbIm75nsRSlPy8GI2fg9Q,9306
|
|
404
404
|
web/server/api/endpoints/table_diff.py,sha256=SCy3-pr5x6bEEiOaZGCDbiHQhpgxwmnz3Acqa2Ie6jI,2090
|
|
405
|
-
sqlmesh-0.198.1.
|
|
406
|
-
sqlmesh-0.198.1.
|
|
407
|
-
sqlmesh-0.198.1.
|
|
408
|
-
sqlmesh-0.198.1.
|
|
409
|
-
sqlmesh-0.198.1.
|
|
405
|
+
sqlmesh-0.198.1.dev13.dist-info/METADATA,sha256=tO04527OWfbw2b1VCQB8tQMxbS4NGkvObRntCNs2Npo,26659
|
|
406
|
+
sqlmesh-0.198.1.dev13.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
407
|
+
sqlmesh-0.198.1.dev13.dist-info/entry_points.txt,sha256=pIYKtg-u-uO7BK_Y0qrdmfyj0bYrcMc3Xiqr5WE3VFk,121
|
|
408
|
+
sqlmesh-0.198.1.dev13.dist-info/top_level.txt,sha256=bspjo5vjELQkuN0qKuTFIbsrUt3Zeo8D2mlKeHG1LdE,12
|
|
409
|
+
sqlmesh-0.198.1.dev13.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|