sqlmesh 0.209.1.dev32__py3-none-any.whl → 0.209.1.dev34__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sqlmesh/_version.py +2 -2
- sqlmesh/core/selector.py +7 -7
- sqlmesh/core/state_sync/db/facade.py +1 -1
- sqlmesh/core/state_sync/db/migrator.py +1 -1
- sqlmesh/core/state_sync/db/snapshot.py +60 -87
- sqlmesh/core/state_sync/db/utils.py +15 -0
- sqlmesh/migrations/v0090_add_forward_only_column.py +100 -0
- {sqlmesh-0.209.1.dev32.dist-info → sqlmesh-0.209.1.dev34.dist-info}/METADATA +1 -1
- {sqlmesh-0.209.1.dev32.dist-info → sqlmesh-0.209.1.dev34.dist-info}/RECORD +13 -12
- {sqlmesh-0.209.1.dev32.dist-info → sqlmesh-0.209.1.dev34.dist-info}/WHEEL +0 -0
- {sqlmesh-0.209.1.dev32.dist-info → sqlmesh-0.209.1.dev34.dist-info}/entry_points.txt +0 -0
- {sqlmesh-0.209.1.dev32.dist-info → sqlmesh-0.209.1.dev34.dist-info}/licenses/LICENSE +0 -0
- {sqlmesh-0.209.1.dev32.dist-info → sqlmesh-0.209.1.dev34.dist-info}/top_level.txt +0 -0
sqlmesh/_version.py
CHANGED
|
@@ -17,5 +17,5 @@ __version__: str
|
|
|
17
17
|
__version_tuple__: VERSION_TUPLE
|
|
18
18
|
version_tuple: VERSION_TUPLE
|
|
19
19
|
|
|
20
|
-
__version__ = version = '0.209.1.
|
|
21
|
-
__version_tuple__ = version_tuple = (0, 209, 1, '
|
|
20
|
+
__version__ = version = '0.209.1.dev34'
|
|
21
|
+
__version_tuple__ = version_tuple = (0, 209, 1, 'dev34')
|
sqlmesh/core/selector.py
CHANGED
|
@@ -178,10 +178,10 @@ class Selector:
|
|
|
178
178
|
|
|
179
179
|
node = parse(" | ".join(f"({s})" for s in model_selections))
|
|
180
180
|
|
|
181
|
-
|
|
181
|
+
all_models = models or self._models
|
|
182
182
|
models_by_tags: t.Dict[str, t.Set[str]] = {}
|
|
183
183
|
|
|
184
|
-
for fqn, model in
|
|
184
|
+
for fqn, model in all_models.items():
|
|
185
185
|
for tag in model.tags:
|
|
186
186
|
tag = tag.lower()
|
|
187
187
|
models_by_tags.setdefault(tag, set())
|
|
@@ -193,11 +193,11 @@ class Selector:
|
|
|
193
193
|
if "*" in pattern:
|
|
194
194
|
return {
|
|
195
195
|
fqn
|
|
196
|
-
for fqn, model in
|
|
196
|
+
for fqn, model in all_models.items()
|
|
197
197
|
if fnmatch.fnmatchcase(model.name, node.this)
|
|
198
198
|
}
|
|
199
199
|
fqn = normalize_model_name(pattern, self._default_catalog, self._dialect)
|
|
200
|
-
return {fqn} if fqn in
|
|
200
|
+
return {fqn} if fqn in all_models else set()
|
|
201
201
|
if isinstance(node, exp.And):
|
|
202
202
|
return evaluate(node.left) & evaluate(node.right)
|
|
203
203
|
if isinstance(node, exp.Or):
|
|
@@ -205,7 +205,7 @@ class Selector:
|
|
|
205
205
|
if isinstance(node, exp.Paren):
|
|
206
206
|
return evaluate(node.this)
|
|
207
207
|
if isinstance(node, exp.Not):
|
|
208
|
-
return set(
|
|
208
|
+
return set(all_models) - evaluate(node.this)
|
|
209
209
|
if isinstance(node, Git):
|
|
210
210
|
target_branch = node.name
|
|
211
211
|
git_modified_files = {
|
|
@@ -213,7 +213,7 @@ class Selector:
|
|
|
213
213
|
*self._git_client.list_uncommitted_changed_files(),
|
|
214
214
|
*self._git_client.list_committed_changed_files(target_branch=target_branch),
|
|
215
215
|
}
|
|
216
|
-
return {m.fqn for m in
|
|
216
|
+
return {m.fqn for m in all_models.values() if m._path in git_modified_files}
|
|
217
217
|
if isinstance(node, Tag):
|
|
218
218
|
pattern = node.name.lower()
|
|
219
219
|
|
|
@@ -232,7 +232,7 @@ class Selector:
|
|
|
232
232
|
selected.add(model_name)
|
|
233
233
|
if node.args.get("up"):
|
|
234
234
|
for u in self._dag.upstream(model_name):
|
|
235
|
-
if u in
|
|
235
|
+
if u in all_models:
|
|
236
236
|
selected.add(u)
|
|
237
237
|
if node.args.get("down"):
|
|
238
238
|
selected.update(self._dag.downstream(model_name))
|
|
@@ -256,7 +256,7 @@ class EngineAdapterStateSync(StateSync):
|
|
|
256
256
|
def unpause_snapshots(
|
|
257
257
|
self, snapshots: t.Collection[SnapshotInfoLike], unpaused_dt: TimeLike
|
|
258
258
|
) -> None:
|
|
259
|
-
self.snapshot_state.unpause_snapshots(snapshots, unpaused_dt
|
|
259
|
+
self.snapshot_state.unpause_snapshots(snapshots, unpaused_dt)
|
|
260
260
|
|
|
261
261
|
def invalidate_environment(self, name: str, protect_prod: bool = True) -> None:
|
|
262
262
|
self.environment_state.invalidate_environment(name, protect_prod)
|
|
@@ -396,7 +396,7 @@ class StateMigrator:
|
|
|
396
396
|
if updated_prod_environment:
|
|
397
397
|
try:
|
|
398
398
|
self.snapshot_state.unpause_snapshots(
|
|
399
|
-
updated_prod_environment.snapshots, now_timestamp()
|
|
399
|
+
updated_prod_environment.snapshots, now_timestamp()
|
|
400
400
|
)
|
|
401
401
|
except Exception:
|
|
402
402
|
logger.warning("Failed to unpause migrated snapshots", exc_info=True)
|
|
@@ -10,6 +10,7 @@ from pydantic import Field
|
|
|
10
10
|
|
|
11
11
|
from sqlmesh.core.engine_adapter import EngineAdapter
|
|
12
12
|
from sqlmesh.core.state_sync.db.utils import (
|
|
13
|
+
snapshot_name_filter,
|
|
13
14
|
snapshot_name_version_filter,
|
|
14
15
|
snapshot_id_filter,
|
|
15
16
|
fetchone,
|
|
@@ -32,15 +33,13 @@ from sqlmesh.core.snapshot import (
|
|
|
32
33
|
SnapshotChangeCategory,
|
|
33
34
|
)
|
|
34
35
|
from sqlmesh.utils.migration import index_text_type, blob_text_type
|
|
35
|
-
from sqlmesh.utils.date import now_timestamp, TimeLike,
|
|
36
|
+
from sqlmesh.utils.date import now_timestamp, TimeLike, to_timestamp
|
|
36
37
|
from sqlmesh.utils.pydantic import PydanticModel
|
|
37
38
|
from sqlmesh.utils import unique
|
|
38
39
|
|
|
39
40
|
if t.TYPE_CHECKING:
|
|
40
41
|
import pandas as pd
|
|
41
42
|
|
|
42
|
-
from sqlmesh.core.state_sync.db.interval import IntervalState
|
|
43
|
-
|
|
44
43
|
|
|
45
44
|
logger = logging.getLogger(__name__)
|
|
46
45
|
|
|
@@ -70,6 +69,7 @@ class SnapshotState:
|
|
|
70
69
|
"unpaused_ts": exp.DataType.build("bigint"),
|
|
71
70
|
"ttl_ms": exp.DataType.build("bigint"),
|
|
72
71
|
"unrestorable": exp.DataType.build("boolean"),
|
|
72
|
+
"forward_only": exp.DataType.build("boolean"),
|
|
73
73
|
}
|
|
74
74
|
|
|
75
75
|
self._auto_restatement_columns_to_types = {
|
|
@@ -112,84 +112,52 @@ class SnapshotState:
|
|
|
112
112
|
self,
|
|
113
113
|
snapshots: t.Collection[SnapshotInfoLike],
|
|
114
114
|
unpaused_dt: TimeLike,
|
|
115
|
-
interval_state: IntervalState,
|
|
116
115
|
) -> None:
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
Args:
|
|
120
|
-
snapshots: The snapshots to unpause.
|
|
121
|
-
unpaused_dt: The timestamp to unpause the snapshots at.
|
|
122
|
-
interval_state: The interval state to use to remove intervals when needed.
|
|
123
|
-
"""
|
|
124
|
-
current_ts = now()
|
|
125
|
-
|
|
126
|
-
target_snapshot_ids = {s.snapshot_id for s in snapshots}
|
|
127
|
-
same_version_snapshots = self._get_snapshots_with_same_version(
|
|
128
|
-
snapshots, lock_for_update=True
|
|
116
|
+
unrestorable_snapshots_by_forward_only: t.Dict[bool, t.List[SnapshotNameVersion]] = (
|
|
117
|
+
defaultdict(list)
|
|
129
118
|
)
|
|
130
|
-
target_snapshots_by_version = {
|
|
131
|
-
(s.name, s.version): s
|
|
132
|
-
for s in same_version_snapshots
|
|
133
|
-
if s.snapshot_id in target_snapshot_ids
|
|
134
|
-
}
|
|
135
|
-
|
|
136
|
-
unpaused_snapshots: t.Dict[int, t.List[SnapshotId]] = defaultdict(list)
|
|
137
|
-
paused_snapshots: t.List[SnapshotId] = []
|
|
138
|
-
unrestorable_snapshots: t.List[SnapshotId] = []
|
|
139
|
-
|
|
140
|
-
for snapshot in same_version_snapshots:
|
|
141
|
-
is_target_snapshot = snapshot.snapshot_id in target_snapshot_ids
|
|
142
|
-
if is_target_snapshot and not snapshot.unpaused_ts:
|
|
143
|
-
logger.info("Unpausing snapshot %s", snapshot.snapshot_id)
|
|
144
|
-
snapshot.set_unpaused_ts(unpaused_dt)
|
|
145
|
-
assert snapshot.unpaused_ts is not None
|
|
146
|
-
unpaused_snapshots[snapshot.unpaused_ts].append(snapshot.snapshot_id)
|
|
147
|
-
elif not is_target_snapshot:
|
|
148
|
-
target_snapshot = target_snapshots_by_version[(snapshot.name, snapshot.version)]
|
|
149
|
-
if (
|
|
150
|
-
target_snapshot.normalized_effective_from_ts
|
|
151
|
-
and not target_snapshot.disable_restatement
|
|
152
|
-
):
|
|
153
|
-
# Making sure that there are no overlapping intervals.
|
|
154
|
-
effective_from_ts = target_snapshot.normalized_effective_from_ts
|
|
155
|
-
logger.info(
|
|
156
|
-
"Removing all intervals after '%s' for snapshot %s, superseded by snapshot %s",
|
|
157
|
-
target_snapshot.effective_from,
|
|
158
|
-
snapshot.snapshot_id,
|
|
159
|
-
target_snapshot.snapshot_id,
|
|
160
|
-
)
|
|
161
|
-
full_snapshot = snapshot.full_snapshot
|
|
162
|
-
interval_state.remove_intervals(
|
|
163
|
-
[
|
|
164
|
-
(
|
|
165
|
-
full_snapshot,
|
|
166
|
-
full_snapshot.get_removal_interval(effective_from_ts, current_ts),
|
|
167
|
-
)
|
|
168
|
-
]
|
|
169
|
-
)
|
|
170
119
|
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
120
|
+
for snapshot in snapshots:
|
|
121
|
+
# We need to mark all other snapshots that have forward-only opposite to the target snapshot as unrestorable
|
|
122
|
+
unrestorable_snapshots_by_forward_only[not snapshot.is_forward_only].append(
|
|
123
|
+
snapshot.name_version
|
|
124
|
+
)
|
|
175
125
|
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
or (snapshot.is_forward_only and not target_snapshot.is_forward_only)
|
|
179
|
-
):
|
|
180
|
-
logger.info("Marking snapshot %s as unrestorable", snapshot.snapshot_id)
|
|
181
|
-
snapshot.unrestorable = True
|
|
182
|
-
unrestorable_snapshots.append(snapshot.snapshot_id)
|
|
126
|
+
updated_ts = now_timestamp()
|
|
127
|
+
unpaused_ts = to_timestamp(unpaused_dt)
|
|
183
128
|
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
129
|
+
# Pause all snapshots with target names first
|
|
130
|
+
for where in snapshot_name_filter(
|
|
131
|
+
[s.name for s in snapshots],
|
|
132
|
+
batch_size=self.SNAPSHOT_BATCH_SIZE,
|
|
133
|
+
):
|
|
134
|
+
self.engine_adapter.update_table(
|
|
135
|
+
self.snapshots_table,
|
|
136
|
+
{"unpaused_ts": None, "updated_ts": updated_ts},
|
|
137
|
+
where=where,
|
|
138
|
+
)
|
|
187
139
|
|
|
188
|
-
|
|
189
|
-
|
|
140
|
+
# Now unpause the target snapshots
|
|
141
|
+
self._update_snapshots(
|
|
142
|
+
[s.snapshot_id for s in snapshots],
|
|
143
|
+
unpaused_ts=unpaused_ts,
|
|
144
|
+
updated_ts=updated_ts,
|
|
145
|
+
)
|
|
190
146
|
|
|
191
|
-
|
|
192
|
-
|
|
147
|
+
# Mark unrestorable snapshots
|
|
148
|
+
for forward_only, snapshot_name_versions in unrestorable_snapshots_by_forward_only.items():
|
|
149
|
+
forward_only_exp = exp.column("forward_only").is_(exp.convert(forward_only))
|
|
150
|
+
for where in snapshot_name_version_filter(
|
|
151
|
+
self.engine_adapter,
|
|
152
|
+
snapshot_name_versions,
|
|
153
|
+
batch_size=self.SNAPSHOT_BATCH_SIZE,
|
|
154
|
+
alias=None,
|
|
155
|
+
):
|
|
156
|
+
self.engine_adapter.update_table(
|
|
157
|
+
self.snapshots_table,
|
|
158
|
+
{"unrestorable": True, "updated_ts": updated_ts},
|
|
159
|
+
where=forward_only_exp.and_(where),
|
|
160
|
+
)
|
|
193
161
|
|
|
194
162
|
def get_expired_snapshots(
|
|
195
163
|
self,
|
|
@@ -414,7 +382,8 @@ class SnapshotState:
|
|
|
414
382
|
**kwargs: t.Any,
|
|
415
383
|
) -> None:
|
|
416
384
|
properties = kwargs
|
|
417
|
-
|
|
385
|
+
if "updated_ts" not in properties:
|
|
386
|
+
properties["updated_ts"] = now_timestamp()
|
|
418
387
|
|
|
419
388
|
for where in snapshot_id_filter(
|
|
420
389
|
self.engine_adapter, snapshots, batch_size=self.SNAPSHOT_BATCH_SIZE
|
|
@@ -466,6 +435,7 @@ class SnapshotState:
|
|
|
466
435
|
updated_ts,
|
|
467
436
|
unpaused_ts,
|
|
468
437
|
unrestorable,
|
|
438
|
+
forward_only,
|
|
469
439
|
next_auto_restatement_ts,
|
|
470
440
|
) in fetchall(self.engine_adapter, query):
|
|
471
441
|
snapshot = parse_snapshot(
|
|
@@ -473,6 +443,7 @@ class SnapshotState:
|
|
|
473
443
|
updated_ts=updated_ts,
|
|
474
444
|
unpaused_ts=unpaused_ts,
|
|
475
445
|
unrestorable=unrestorable,
|
|
446
|
+
forward_only=forward_only,
|
|
476
447
|
next_auto_restatement_ts=next_auto_restatement_ts,
|
|
477
448
|
)
|
|
478
449
|
snapshot_id = snapshot.snapshot_id
|
|
@@ -502,6 +473,7 @@ class SnapshotState:
|
|
|
502
473
|
"updated_ts",
|
|
503
474
|
"unpaused_ts",
|
|
504
475
|
"unrestorable",
|
|
476
|
+
"forward_only",
|
|
505
477
|
"next_auto_restatement_ts",
|
|
506
478
|
)
|
|
507
479
|
.from_(exp.to_table(self.snapshots_table).as_("snapshots"))
|
|
@@ -528,6 +500,7 @@ class SnapshotState:
|
|
|
528
500
|
updated_ts,
|
|
529
501
|
unpaused_ts,
|
|
530
502
|
unrestorable,
|
|
503
|
+
forward_only,
|
|
531
504
|
next_auto_restatement_ts,
|
|
532
505
|
) in fetchall(self.engine_adapter, query):
|
|
533
506
|
snapshot_id = SnapshotId(name=name, identifier=identifier)
|
|
@@ -535,6 +508,7 @@ class SnapshotState:
|
|
|
535
508
|
snapshot.updated_ts = updated_ts
|
|
536
509
|
snapshot.unpaused_ts = unpaused_ts
|
|
537
510
|
snapshot.unrestorable = unrestorable
|
|
511
|
+
snapshot.forward_only = forward_only
|
|
538
512
|
snapshot.next_auto_restatement_ts = next_auto_restatement_ts
|
|
539
513
|
cached_snapshots_in_state.add(snapshot_id)
|
|
540
514
|
|
|
@@ -568,6 +542,7 @@ class SnapshotState:
|
|
|
568
542
|
"snapshots.updated_ts",
|
|
569
543
|
"snapshots.unpaused_ts",
|
|
570
544
|
"snapshots.unrestorable",
|
|
545
|
+
"snapshots.forward_only",
|
|
571
546
|
"auto_restatements.next_auto_restatement_ts",
|
|
572
547
|
)
|
|
573
548
|
.from_(exp.to_table(self.snapshots_table).as_("snapshots"))
|
|
@@ -623,6 +598,7 @@ class SnapshotState:
|
|
|
623
598
|
"updated_ts",
|
|
624
599
|
"unpaused_ts",
|
|
625
600
|
"unrestorable",
|
|
601
|
+
"forward_only",
|
|
626
602
|
)
|
|
627
603
|
.from_(exp.to_table(self.snapshots_table).as_("snapshots"))
|
|
628
604
|
.where(where)
|
|
@@ -640,9 +616,10 @@ class SnapshotState:
|
|
|
640
616
|
updated_ts=updated_ts,
|
|
641
617
|
unpaused_ts=unpaused_ts,
|
|
642
618
|
unrestorable=unrestorable,
|
|
619
|
+
forward_only=forward_only,
|
|
643
620
|
snapshot=snapshot,
|
|
644
621
|
)
|
|
645
|
-
for snapshot, name, identifier, version, updated_ts, unpaused_ts, unrestorable in snapshot_rows
|
|
622
|
+
for snapshot, name, identifier, version, updated_ts, unpaused_ts, unrestorable, forward_only in snapshot_rows
|
|
646
623
|
]
|
|
647
624
|
|
|
648
625
|
|
|
@@ -651,6 +628,7 @@ def parse_snapshot(
|
|
|
651
628
|
updated_ts: int,
|
|
652
629
|
unpaused_ts: t.Optional[int],
|
|
653
630
|
unrestorable: bool,
|
|
631
|
+
forward_only: bool,
|
|
654
632
|
next_auto_restatement_ts: t.Optional[int],
|
|
655
633
|
) -> Snapshot:
|
|
656
634
|
return Snapshot(
|
|
@@ -659,6 +637,7 @@ def parse_snapshot(
|
|
|
659
637
|
"updated_ts": updated_ts,
|
|
660
638
|
"unpaused_ts": unpaused_ts,
|
|
661
639
|
"unrestorable": unrestorable,
|
|
640
|
+
"forward_only": forward_only,
|
|
662
641
|
"next_auto_restatement_ts": next_auto_restatement_ts,
|
|
663
642
|
}
|
|
664
643
|
)
|
|
@@ -673,6 +652,7 @@ def _snapshot_to_json(snapshot: Snapshot) -> str:
|
|
|
673
652
|
"updated_ts",
|
|
674
653
|
"unpaused_ts",
|
|
675
654
|
"unrestorable",
|
|
655
|
+
"forward_only",
|
|
676
656
|
"next_auto_restatement_ts",
|
|
677
657
|
}
|
|
678
658
|
)
|
|
@@ -693,6 +673,7 @@ def _snapshots_to_df(snapshots: t.Iterable[Snapshot]) -> pd.DataFrame:
|
|
|
693
673
|
"unpaused_ts": snapshot.unpaused_ts,
|
|
694
674
|
"ttl_ms": snapshot.ttl_ms,
|
|
695
675
|
"unrestorable": snapshot.unrestorable,
|
|
676
|
+
"forward_only": snapshot.forward_only,
|
|
696
677
|
}
|
|
697
678
|
for snapshot in snapshots
|
|
698
679
|
]
|
|
@@ -762,19 +743,10 @@ class SharedVersionSnapshot(PydanticModel):
|
|
|
762
743
|
"updated_ts": self.updated_ts,
|
|
763
744
|
"unpaused_ts": self.unpaused_ts,
|
|
764
745
|
"unrestorable": self.unrestorable,
|
|
746
|
+
"forward_only": self.forward_only,
|
|
765
747
|
}
|
|
766
748
|
)
|
|
767
749
|
|
|
768
|
-
def set_unpaused_ts(self, unpaused_dt: t.Optional[TimeLike]) -> None:
|
|
769
|
-
"""Sets the timestamp for when this snapshot was unpaused.
|
|
770
|
-
|
|
771
|
-
Args:
|
|
772
|
-
unpaused_dt: The datetime object of when this snapshot was unpaused.
|
|
773
|
-
"""
|
|
774
|
-
self.unpaused_ts = (
|
|
775
|
-
to_timestamp(self.interval_unit.cron_floor(unpaused_dt)) if unpaused_dt else None
|
|
776
|
-
)
|
|
777
|
-
|
|
778
750
|
@classmethod
|
|
779
751
|
def from_snapshot_record(
|
|
780
752
|
cls,
|
|
@@ -785,6 +757,7 @@ class SharedVersionSnapshot(PydanticModel):
|
|
|
785
757
|
updated_ts: int,
|
|
786
758
|
unpaused_ts: t.Optional[int],
|
|
787
759
|
unrestorable: bool,
|
|
760
|
+
forward_only: bool,
|
|
788
761
|
snapshot: str,
|
|
789
762
|
) -> SharedVersionSnapshot:
|
|
790
763
|
raw_snapshot = json.loads(snapshot)
|
|
@@ -803,5 +776,5 @@ class SharedVersionSnapshot(PydanticModel):
|
|
|
803
776
|
disable_restatement=raw_node.get("kind", {}).get("disable_restatement", False),
|
|
804
777
|
effective_from=raw_snapshot.get("effective_from"),
|
|
805
778
|
raw_snapshot=raw_snapshot,
|
|
806
|
-
forward_only=
|
|
779
|
+
forward_only=forward_only,
|
|
807
780
|
)
|
|
@@ -22,6 +22,21 @@ except ImportError:
|
|
|
22
22
|
T = t.TypeVar("T")
|
|
23
23
|
|
|
24
24
|
|
|
25
|
+
def snapshot_name_filter(
|
|
26
|
+
snapshot_names: t.Iterable[str],
|
|
27
|
+
batch_size: int,
|
|
28
|
+
alias: t.Optional[str] = None,
|
|
29
|
+
) -> t.Iterator[exp.Condition]:
|
|
30
|
+
names = sorted(snapshot_names)
|
|
31
|
+
|
|
32
|
+
if not names:
|
|
33
|
+
yield exp.false()
|
|
34
|
+
else:
|
|
35
|
+
batches = create_batches(names, batch_size=batch_size)
|
|
36
|
+
for names in batches:
|
|
37
|
+
yield exp.column("name", table=alias).isin(*names)
|
|
38
|
+
|
|
39
|
+
|
|
25
40
|
def snapshot_id_filter(
|
|
26
41
|
engine_adapter: EngineAdapter,
|
|
27
42
|
snapshot_ids: t.Iterable[SnapshotIdLike],
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
"""Add forward_only column to the snapshots table."""
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
|
|
5
|
+
from sqlglot import exp
|
|
6
|
+
|
|
7
|
+
from sqlmesh.utils.migration import index_text_type, blob_text_type
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def migrate(state_sync, **kwargs): # type: ignore
|
|
11
|
+
import pandas as pd
|
|
12
|
+
|
|
13
|
+
engine_adapter = state_sync.engine_adapter
|
|
14
|
+
schema = state_sync.schema
|
|
15
|
+
snapshots_table = "_snapshots"
|
|
16
|
+
if schema:
|
|
17
|
+
snapshots_table = f"{schema}.{snapshots_table}"
|
|
18
|
+
|
|
19
|
+
alter_table_exp = exp.Alter(
|
|
20
|
+
this=exp.to_table(snapshots_table),
|
|
21
|
+
kind="TABLE",
|
|
22
|
+
actions=[
|
|
23
|
+
exp.ColumnDef(
|
|
24
|
+
this=exp.to_column("forward_only"),
|
|
25
|
+
kind=exp.DataType.build("boolean"),
|
|
26
|
+
)
|
|
27
|
+
],
|
|
28
|
+
)
|
|
29
|
+
engine_adapter.execute(alter_table_exp)
|
|
30
|
+
|
|
31
|
+
new_snapshots = []
|
|
32
|
+
|
|
33
|
+
for (
|
|
34
|
+
name,
|
|
35
|
+
identifier,
|
|
36
|
+
version,
|
|
37
|
+
snapshot,
|
|
38
|
+
kind_name,
|
|
39
|
+
updated_ts,
|
|
40
|
+
unpaused_ts,
|
|
41
|
+
ttl_ms,
|
|
42
|
+
unrestorable,
|
|
43
|
+
forward_only,
|
|
44
|
+
) in engine_adapter.fetchall(
|
|
45
|
+
exp.select(
|
|
46
|
+
"name",
|
|
47
|
+
"identifier",
|
|
48
|
+
"version",
|
|
49
|
+
"snapshot",
|
|
50
|
+
"kind_name",
|
|
51
|
+
"updated_ts",
|
|
52
|
+
"unpaused_ts",
|
|
53
|
+
"ttl_ms",
|
|
54
|
+
"unrestorable",
|
|
55
|
+
"forward_only",
|
|
56
|
+
).from_(snapshots_table),
|
|
57
|
+
quote_identifiers=True,
|
|
58
|
+
):
|
|
59
|
+
parsed_snapshot = json.loads(snapshot)
|
|
60
|
+
|
|
61
|
+
forward_only = parsed_snapshot.get("forward_only")
|
|
62
|
+
if forward_only is None:
|
|
63
|
+
forward_only = parsed_snapshot.get("change_category") == 3
|
|
64
|
+
|
|
65
|
+
new_snapshots.append(
|
|
66
|
+
{
|
|
67
|
+
"name": name,
|
|
68
|
+
"identifier": identifier,
|
|
69
|
+
"version": version,
|
|
70
|
+
"snapshot": json.dumps(parsed_snapshot),
|
|
71
|
+
"kind_name": kind_name,
|
|
72
|
+
"updated_ts": updated_ts,
|
|
73
|
+
"unpaused_ts": unpaused_ts,
|
|
74
|
+
"ttl_ms": ttl_ms,
|
|
75
|
+
"unrestorable": unrestorable,
|
|
76
|
+
"forward_only": forward_only,
|
|
77
|
+
}
|
|
78
|
+
)
|
|
79
|
+
|
|
80
|
+
if new_snapshots:
|
|
81
|
+
engine_adapter.delete_from(snapshots_table, "TRUE")
|
|
82
|
+
index_type = index_text_type(engine_adapter.dialect)
|
|
83
|
+
blob_type = blob_text_type(engine_adapter.dialect)
|
|
84
|
+
|
|
85
|
+
engine_adapter.insert_append(
|
|
86
|
+
snapshots_table,
|
|
87
|
+
pd.DataFrame(new_snapshots),
|
|
88
|
+
columns_to_types={
|
|
89
|
+
"name": exp.DataType.build(index_type),
|
|
90
|
+
"identifier": exp.DataType.build(index_type),
|
|
91
|
+
"version": exp.DataType.build(index_type),
|
|
92
|
+
"snapshot": exp.DataType.build(blob_type),
|
|
93
|
+
"kind_name": exp.DataType.build(index_type),
|
|
94
|
+
"updated_ts": exp.DataType.build("bigint"),
|
|
95
|
+
"unpaused_ts": exp.DataType.build("bigint"),
|
|
96
|
+
"ttl_ms": exp.DataType.build("bigint"),
|
|
97
|
+
"unrestorable": exp.DataType.build("boolean"),
|
|
98
|
+
"forward_only": exp.DataType.build("boolean"),
|
|
99
|
+
},
|
|
100
|
+
)
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
sqlmesh/__init__.py,sha256=QjwaamJIdW7voXoS880yAx7tHm-Im4_mqGA-XJgafos,7631
|
|
2
|
-
sqlmesh/_version.py,sha256
|
|
2
|
+
sqlmesh/_version.py,sha256=QpqyjAfKoC7gzkkp4JKBbDY4F-PE0FeoOq6tEL_HW8o,530
|
|
3
3
|
sqlmesh/magics.py,sha256=bEzYqXHcM0P3CjyUZ1O9arlmXAdLAhoglKbQ29fD9AE,41842
|
|
4
4
|
sqlmesh/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
5
5
|
sqlmesh/cicd/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
@@ -27,7 +27,7 @@ sqlmesh/core/renderer.py,sha256=PMUmmtwqk3FA5lN-16OWl6buCbrVa1M8NIgRXCYEB8k,2680
|
|
|
27
27
|
sqlmesh/core/scheduler.py,sha256=I30K5NyeM5HuZxyZ62Vgs1VoMWPNA0gev0nveWugmLM,40544
|
|
28
28
|
sqlmesh/core/schema_diff.py,sha256=Sp2MoqAngp-yTYevkLaf9jb26SuAdGXoZ54g5MjWowU,30516
|
|
29
29
|
sqlmesh/core/schema_loader.py,sha256=_Pq2RSw91uthqv1vNi_eHmLlzhtGz_APMJ0wAJZYuvk,3677
|
|
30
|
-
sqlmesh/core/selector.py,sha256=
|
|
30
|
+
sqlmesh/core/selector.py,sha256=9bqtBx7Onl8zZb-G4OsGVYVwkdm24j_qnp5HgXLFhR4,12837
|
|
31
31
|
sqlmesh/core/signal.py,sha256=yVLXewYP6eH0vt-EwaIaI2WwHz6hWsVr-Kb2m4kP1FU,1353
|
|
32
32
|
sqlmesh/core/table_diff.py,sha256=d1Cqlz2i6ktaFY_y7DoPe_FJqyx2FHD-6_s1sYFRFBU,28846
|
|
33
33
|
sqlmesh/core/user.py,sha256=EJ6R4R1iK67n80vBoCCsidF56IR7xEYqiCEO-nrVMso,1660
|
|
@@ -112,11 +112,11 @@ sqlmesh/core/state_sync/common.py,sha256=x8Nkox6VmkD-lYZkjRcNe3iVsChR1HBroTorf1k
|
|
|
112
112
|
sqlmesh/core/state_sync/export_import.py,sha256=WJ7sFX2b_RPAFE4yspJRi_4JQnaO23xpJgRxwXD0agA,8709
|
|
113
113
|
sqlmesh/core/state_sync/db/__init__.py,sha256=jGCqKYB7npq4eh6H2LyfBAJWdOP-IFu0DC4CMiQgYvw,107
|
|
114
114
|
sqlmesh/core/state_sync/db/environment.py,sha256=iTxrdNlKB36S54PUdlqEqlxiAZJtKNExHBtsIbaXT30,13675
|
|
115
|
-
sqlmesh/core/state_sync/db/facade.py,sha256=
|
|
115
|
+
sqlmesh/core/state_sync/db/facade.py,sha256=7OMZt_T3TvavxeC95uwBWe7uD_PtgRyf-s7oYrTkVPg,25245
|
|
116
116
|
sqlmesh/core/state_sync/db/interval.py,sha256=aMQSF42bVOUU05g7oizcrKR6TBVzR6TJppxzlxkmJFU,17985
|
|
117
|
-
sqlmesh/core/state_sync/db/migrator.py,sha256=
|
|
118
|
-
sqlmesh/core/state_sync/db/snapshot.py,sha256=
|
|
119
|
-
sqlmesh/core/state_sync/db/utils.py,sha256=
|
|
117
|
+
sqlmesh/core/state_sync/db/migrator.py,sha256=av5-dw_0T-u171L8Aepv6nDEg14GFVibdM_hYWq3Hz0,18118
|
|
118
|
+
sqlmesh/core/state_sync/db/snapshot.py,sha256=R8zhaXjIvWU3dQ_yZNohL0JKyi10gFrJAnkrsKuTEXk,28199
|
|
119
|
+
sqlmesh/core/state_sync/db/utils.py,sha256=8KjRmOjP5CLuSRkYBUE2k34V-UYB0iSyuO0rWBQ_5pY,4218
|
|
120
120
|
sqlmesh/core/state_sync/db/version.py,sha256=fl5g1tNMYu8O6WTN4J3bciwjGq0c5nZlRDlxxRKGU78,2210
|
|
121
121
|
sqlmesh/core/test/__init__.py,sha256=e83TJPwPRR_rAG29Y0OVbZb-5oWVBzz-_wrcd22Qk10,418
|
|
122
122
|
sqlmesh/core/test/context.py,sha256=9mmskKXpzsIkW_I31uN1n3Ioj8ngvyGAqyL_AZvVHpc,2253
|
|
@@ -267,6 +267,7 @@ sqlmesh/migrations/v0086_check_deterministic_bug.py,sha256=WiYcmhQD434GXWiiakQtc
|
|
|
267
267
|
sqlmesh/migrations/v0087_normalize_blueprint_variables.py,sha256=GI6KKA-6PtONUoOnyvLpRTUmPjbpa-CKhDa0iqJW01U,4421
|
|
268
268
|
sqlmesh/migrations/v0088_warn_about_variable_python_env_diffs.py,sha256=zCLT6Yqwny73SlfACWjO1bZfs8lZ7gJBov5Iz4akwiQ,3254
|
|
269
269
|
sqlmesh/migrations/v0089_add_virtual_environment_mode.py,sha256=rm6k7CRy2uHozo1Oy17D8WhiV1JldPNCpJ8PukoHSwM,122
|
|
270
|
+
sqlmesh/migrations/v0090_add_forward_only_column.py,sha256=0mzqPAmZS7KkBpb0tox-iLRAH3ZZVR8gWicZY27El_0,3012
|
|
270
271
|
sqlmesh/schedulers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
271
272
|
sqlmesh/utils/__init__.py,sha256=51VqA2etTZ6kX54250s1hN5ui7D89OeUTUQ9kgMze7s,11554
|
|
272
273
|
sqlmesh/utils/aws.py,sha256=krxyhrKtsMJDEjT567UbGZogjJ9JRHkvAV-Cg3FVndM,1062
|
|
@@ -291,7 +292,7 @@ sqlmesh/utils/pydantic.py,sha256=o_NsXbIpDqNpUA1Uc5xF0ZzoXQYYB0DfHwdRxBwXPNk,120
|
|
|
291
292
|
sqlmesh/utils/rich.py,sha256=cwQ5nJ6sgz64xHtoh6_ec7ReV5YpsOGhMtUJnwoRfEI,3549
|
|
292
293
|
sqlmesh/utils/windows.py,sha256=r7Yin4WEwbvH7sKA3hOJtRsclmscNuBeCklCfVPvtQs,611
|
|
293
294
|
sqlmesh/utils/yaml.py,sha256=hvo5lmybYLcIx4p6CQwufS8X4MbwS60lEBnCmYJCXmU,2336
|
|
294
|
-
sqlmesh-0.209.1.
|
|
295
|
+
sqlmesh-0.209.1.dev34.dist-info/licenses/LICENSE,sha256=OlMefUjgWJdULtf84BLW0AZZcY8DwdgQqb_1j2862j8,11346
|
|
295
296
|
sqlmesh_dbt/__init__.py,sha256=awYS5y5mz-1NUmx6i5h5NSTJ7tidRl9NC0FAnFWSF6U,350
|
|
296
297
|
sqlmesh_dbt/cli.py,sha256=Ekn77WLMmyyr31aoDoWrZp5DS0rfiddy7pNohBK8G0s,2115
|
|
297
298
|
sqlmesh_dbt/console.py,sha256=1B0B5_R1DlrEkyWVDl8G78-B6_K-ZtLPsQEhqPCfLEU,188
|
|
@@ -413,8 +414,8 @@ web/server/api/endpoints/models.py,sha256=kwj0s7uve3iZSMfmjkoPVMFMeY1sD0peTeyrWf
|
|
|
413
414
|
web/server/api/endpoints/modules.py,sha256=8hqqgonGay_mJmpCw0IdbjsPhWlQH2VLdKAqha-myac,468
|
|
414
415
|
web/server/api/endpoints/plan.py,sha256=bbbY50W_2MsZSTxOHWMKz0tbIm75nsRSlPy8GI2fg9Q,9306
|
|
415
416
|
web/server/api/endpoints/table_diff.py,sha256=SCy3-pr5x6bEEiOaZGCDbiHQhpgxwmnz3Acqa2Ie6jI,2090
|
|
416
|
-
sqlmesh-0.209.1.
|
|
417
|
-
sqlmesh-0.209.1.
|
|
418
|
-
sqlmesh-0.209.1.
|
|
419
|
-
sqlmesh-0.209.1.
|
|
420
|
-
sqlmesh-0.209.1.
|
|
417
|
+
sqlmesh-0.209.1.dev34.dist-info/METADATA,sha256=19_jNfeYb5fP6Ksx9DAdXAc7g7rFBmoJB6Jq7hVZYos,26686
|
|
418
|
+
sqlmesh-0.209.1.dev34.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
419
|
+
sqlmesh-0.209.1.dev34.dist-info/entry_points.txt,sha256=sHAf6tQczIM8xZoduN4qaUjV7QEPVUUW_LCT8EDUMv4,155
|
|
420
|
+
sqlmesh-0.209.1.dev34.dist-info/top_level.txt,sha256=RQ-33FPe2IgL0rgossAfJkCRtqslz9b7wFARqiWLC5Q,24
|
|
421
|
+
sqlmesh-0.209.1.dev34.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|