query-cascade 0.3.25__tar.gz → 0.3.26__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {query_cascade-0.3.25 → query_cascade-0.3.26}/PKG-INFO +1 -1
- {query_cascade-0.3.25 → query_cascade-0.3.26}/pyproject.toml +1 -1
- {query_cascade-0.3.25 → query_cascade-0.3.26}/src/cascade/_evaluator.py +78 -0
- {query_cascade-0.3.25 → query_cascade-0.3.26}/src/query_cascade.egg-info/PKG-INFO +1 -1
- {query_cascade-0.3.25 → query_cascade-0.3.26}/src/query_cascade.egg-info/SOURCES.txt +1 -0
- query_cascade-0.3.26/tests/test_iterative_verification.py +175 -0
- {query_cascade-0.3.25 → query_cascade-0.3.26}/README.md +0 -0
- {query_cascade-0.3.25 → query_cascade-0.3.26}/setup.cfg +0 -0
- {query_cascade-0.3.25 → query_cascade-0.3.26}/src/cascade/__init__.py +0 -0
- {query_cascade-0.3.25 → query_cascade-0.3.26}/src/cascade/_ast_rewrite.py +0 -0
- {query_cascade-0.3.25 → query_cascade-0.3.26}/src/cascade/_canonical.py +0 -0
- {query_cascade-0.3.25 → query_cascade-0.3.26}/src/cascade/_collections.py +0 -0
- {query_cascade-0.3.25 → query_cascade-0.3.26}/src/cascade/_disk_cache.py +0 -0
- {query_cascade-0.3.25 → query_cascade-0.3.26}/src/cascade/_errors.py +0 -0
- {query_cascade-0.3.25 → query_cascade-0.3.26}/src/cascade/_graph_export.py +0 -0
- {query_cascade-0.3.25 → query_cascade-0.3.26}/src/cascade/_incremental.py +0 -0
- {query_cascade-0.3.25 → query_cascade-0.3.26}/src/cascade/_persistence.py +0 -0
- {query_cascade-0.3.25 → query_cascade-0.3.26}/src/cascade/_runtime.py +0 -0
- {query_cascade-0.3.25 → query_cascade-0.3.26}/src/cascade/_scheduler.py +0 -0
- {query_cascade-0.3.25 → query_cascade-0.3.26}/src/cascade/_serde.py +0 -0
- {query_cascade-0.3.25 → query_cascade-0.3.26}/src/cascade/_state.py +0 -0
- {query_cascade-0.3.25 → query_cascade-0.3.26}/src/cascade/_store.py +0 -0
- {query_cascade-0.3.25 → query_cascade-0.3.26}/src/cascade/_synthetic_graph.py +0 -0
- {query_cascade-0.3.25 → query_cascade-0.3.26}/src/cascade/engine.py +0 -0
- {query_cascade-0.3.25 → query_cascade-0.3.26}/src/query_cascade.egg-info/dependency_links.txt +0 -0
- {query_cascade-0.3.25 → query_cascade-0.3.26}/src/query_cascade.egg-info/requires.txt +0 -0
- {query_cascade-0.3.25 → query_cascade-0.3.26}/src/query_cascade.egg-info/top_level.txt +0 -0
- {query_cascade-0.3.25 → query_cascade-0.3.26}/tests/test_ast_rewrite.py +0 -0
- {query_cascade-0.3.25 → query_cascade-0.3.26}/tests/test_async.py +0 -0
- {query_cascade-0.3.25 → query_cascade-0.3.26}/tests/test_async_coverage.py +0 -0
- {query_cascade-0.3.25 → query_cascade-0.3.26}/tests/test_code_versioning.py +0 -0
- {query_cascade-0.3.25 → query_cascade-0.3.26}/tests/test_datetime.py +0 -0
- {query_cascade-0.3.25 → query_cascade-0.3.26}/tests/test_dependency_parallelism.py +0 -0
- {query_cascade-0.3.25 → query_cascade-0.3.26}/tests/test_dirty_propagation.py +0 -0
- {query_cascade-0.3.25 → query_cascade-0.3.26}/tests/test_disk_cache.py +0 -0
- {query_cascade-0.3.25 → query_cascade-0.3.26}/tests/test_disk_format2.py +0 -0
- {query_cascade-0.3.25 → query_cascade-0.3.26}/tests/test_engine.py +0 -0
- {query_cascade-0.3.25 → query_cascade-0.3.26}/tests/test_engine_compute_many.py +0 -0
- {query_cascade-0.3.25 → query_cascade-0.3.26}/tests/test_engine_concurrency.py +0 -0
- {query_cascade-0.3.25 → query_cascade-0.3.26}/tests/test_engine_persistence_and_tracing.py +0 -0
- {query_cascade-0.3.25 → query_cascade-0.3.26}/tests/test_engine_private.py +0 -0
- {query_cascade-0.3.25 → query_cascade-0.3.26}/tests/test_engine_private2.py +0 -0
- {query_cascade-0.3.25 → query_cascade-0.3.26}/tests/test_engine_sweep.py +0 -0
- {query_cascade-0.3.25 → query_cascade-0.3.26}/tests/test_error_caching.py +0 -0
- {query_cascade-0.3.25 → query_cascade-0.3.26}/tests/test_error_caching_persistence.py +0 -0
- {query_cascade-0.3.25 → query_cascade-0.3.26}/tests/test_evaluator_and_synthetic_coverage.py +0 -0
- {query_cascade-0.3.25 → query_cascade-0.3.26}/tests/test_examples.py +0 -0
- {query_cascade-0.3.25 → query_cascade-0.3.26}/tests/test_fixed_point.py +0 -0
- {query_cascade-0.3.25 → query_cascade-0.3.26}/tests/test_incremental_collections.py +0 -0
- {query_cascade-0.3.25 → query_cascade-0.3.26}/tests/test_incremental_graph.py +0 -0
- {query_cascade-0.3.25 → query_cascade-0.3.26}/tests/test_incremental_persistence.py +0 -0
- {query_cascade-0.3.25 → query_cascade-0.3.26}/tests/test_incremental_pipeline.py +0 -0
- {query_cascade-0.3.25 → query_cascade-0.3.26}/tests/test_internal_invariants.py +0 -0
- {query_cascade-0.3.25 → query_cascade-0.3.26}/tests/test_memoize_false.py +0 -0
- {query_cascade-0.3.25 → query_cascade-0.3.26}/tests/test_overhead_optimizations.py +0 -0
- {query_cascade-0.3.25 → query_cascade-0.3.26}/tests/test_performance.py +0 -0
- {query_cascade-0.3.25 → query_cascade-0.3.26}/tests/test_persistence_permutations.py +0 -0
- {query_cascade-0.3.25 → query_cascade-0.3.26}/tests/test_repro_touch_memo_keyerror.py +0 -0
- {query_cascade-0.3.25 → query_cascade-0.3.26}/tests/test_scale_behavior.py +0 -0
- {query_cascade-0.3.25 → query_cascade-0.3.26}/tests/test_serde.py +0 -0
- {query_cascade-0.3.25 → query_cascade-0.3.26}/tests/test_stateful_engine_invariants.py +0 -0
- {query_cascade-0.3.25 → query_cascade-0.3.26}/tests/test_store_coverage.py +0 -0
- {query_cascade-0.3.25 → query_cascade-0.3.26}/tests/test_subgraph_stats_export.py +0 -0
- {query_cascade-0.3.25 → query_cascade-0.3.26}/tests/test_sweep_unaccessed.py +0 -0
- {query_cascade-0.3.25 → query_cascade-0.3.26}/tests/test_ttl.py +0 -0
|
@@ -566,6 +566,7 @@ class Evaluator:
|
|
|
566
566
|
self._store.trace_event("cache_red_ttl", key)
|
|
567
567
|
return False
|
|
568
568
|
|
|
569
|
+
self._settle_dependencies(key, entry, snapshot)
|
|
569
570
|
for dep in entry.deps:
|
|
570
571
|
dep_changed_at = self.dependency_changed_at(dep.key, snapshot)
|
|
571
572
|
if dep_changed_at != dep.observed_changed_at:
|
|
@@ -575,6 +576,83 @@ class Evaluator:
|
|
|
575
576
|
return False
|
|
576
577
|
return True
|
|
577
578
|
|
|
579
|
+
def _settle_dependencies(
|
|
580
|
+
self, root: QueryKey, root_entry: MemoEntry, snapshot: Snapshot
|
|
581
|
+
) -> None:
|
|
582
|
+
"""Verify ``root``'s unverified query dependencies deepest-first.
|
|
583
|
+
|
|
584
|
+
``dependency_changed_at`` verifies a stale dependency by recursing into
|
|
585
|
+
it, which costs several Python frames per level, so a long chain of
|
|
586
|
+
stale memos (pinned snapshots, TTL or cycle memos below, memos loaded by
|
|
587
|
+
``load()``) overflowed the recursion limit. This explicit-stack pass
|
|
588
|
+
settles each dependency only once its own dependencies are settled, so
|
|
589
|
+
the nested call it makes finds everything beneath already verified and
|
|
590
|
+
returns at constant depth.
|
|
591
|
+
|
|
592
|
+
It visits exactly what the recursive walk would, in the same order: a
|
|
593
|
+
memo's dependencies are examined in recorded order, and the scan stops
|
|
594
|
+
at the first one whose ``changed_at`` differs, because the memo will be
|
|
595
|
+
recomputed and its later dependencies may no longer be read. Anything
|
|
596
|
+
it cannot handle (missing memos, fixed-point cycles, frames already
|
|
597
|
+
executing) is left to the normal path in ``try_mark_green``.
|
|
598
|
+
"""
|
|
599
|
+
store = self._store
|
|
600
|
+
runtime = self._runtime_var.get()
|
|
601
|
+
executing = (
|
|
602
|
+
{frame.key for frame in runtime.stack} if runtime is not None else set()
|
|
603
|
+
)
|
|
604
|
+
revision = snapshot.revision
|
|
605
|
+
# Frames are [key, memo, index of the next dependency to examine].
|
|
606
|
+
stack: list[list[Any]] = [[root, root_entry, 0]]
|
|
607
|
+
visiting = {root}
|
|
608
|
+
settled: set[QueryKey] = set()
|
|
609
|
+
while stack:
|
|
610
|
+
frame = stack[-1]
|
|
611
|
+
deps = frame[1].deps
|
|
612
|
+
index = frame[2]
|
|
613
|
+
pushed = False
|
|
614
|
+
while index < len(deps):
|
|
615
|
+
dep = deps[index]
|
|
616
|
+
dep_key = dep.key
|
|
617
|
+
if dep_key[0] == "input":
|
|
618
|
+
version = store.input_version_at((dep_key[1], dep_key[2]), revision)
|
|
619
|
+
changed_at = -1 if version is None else version.changed_at
|
|
620
|
+
if changed_at != dep.observed_changed_at:
|
|
621
|
+
break
|
|
622
|
+
index += 1
|
|
623
|
+
continue
|
|
624
|
+
with store.lock:
|
|
625
|
+
dep_memo = store.memos.get(dep_key)
|
|
626
|
+
current = dep_memo is not None and store.is_current_locked(
|
|
627
|
+
dep_memo, revision
|
|
628
|
+
)
|
|
629
|
+
if current:
|
|
630
|
+
if dep_memo.changed_at != dep.observed_changed_at: # type: ignore[union-attr]
|
|
631
|
+
break
|
|
632
|
+
index += 1
|
|
633
|
+
continue
|
|
634
|
+
if dep_key in settled or dep_key in visiting or dep_key in executing:
|
|
635
|
+
break
|
|
636
|
+
if dep_memo is None:
|
|
637
|
+
# Nothing to walk; computing it is the only way forward.
|
|
638
|
+
self.dependency_changed_at(dep_key, snapshot)
|
|
639
|
+
settled.add(dep_key)
|
|
640
|
+
continue
|
|
641
|
+
frame[2] = index
|
|
642
|
+
stack.append([dep_key, dep_memo, 0])
|
|
643
|
+
visiting.add(dep_key)
|
|
644
|
+
pushed = True
|
|
645
|
+
break
|
|
646
|
+
if pushed:
|
|
647
|
+
continue
|
|
648
|
+
stack.pop()
|
|
649
|
+
visiting.discard(frame[0])
|
|
650
|
+
if stack:
|
|
651
|
+
# All of this memo's examined dependencies are now verified,
|
|
652
|
+
# so verifying the memo itself does not recurse further.
|
|
653
|
+
self.dependency_changed_at(frame[0], snapshot)
|
|
654
|
+
settled.add(frame[0])
|
|
655
|
+
|
|
578
656
|
async def try_mark_green_async(
|
|
579
657
|
self, key: QueryKey, entry: MemoEntry, snapshot: Snapshot
|
|
580
658
|
) -> bool: # pragma: no cover
|
|
@@ -48,6 +48,7 @@ tests/test_incremental_graph.py
|
|
|
48
48
|
tests/test_incremental_persistence.py
|
|
49
49
|
tests/test_incremental_pipeline.py
|
|
50
50
|
tests/test_internal_invariants.py
|
|
51
|
+
tests/test_iterative_verification.py
|
|
51
52
|
tests/test_memoize_false.py
|
|
52
53
|
tests/test_overhead_optimizations.py
|
|
53
54
|
tests/test_performance.py
|
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
"""The slow verification path must not recurse once per dependency level.
|
|
2
|
+
|
|
3
|
+
Chains are built bottom-up so the initial compute never recurses deeply; only
|
|
4
|
+
verification could. Each case would overflow Python's default recursion limit
|
|
5
|
+
(1000 frames) with the recursive walk.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import sys
|
|
11
|
+
from collections import Counter
|
|
12
|
+
from typing import Any
|
|
13
|
+
|
|
14
|
+
import pytest
|
|
15
|
+
|
|
16
|
+
from cascade import Engine
|
|
17
|
+
|
|
18
|
+
DEPTH = 3000
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def _chain(engine: Engine, *, ttl_bottom: bool, runs: Counter[str]) -> tuple[Any, ...]:
|
|
22
|
+
@engine.input
|
|
23
|
+
def leaf() -> int:
|
|
24
|
+
return 1
|
|
25
|
+
|
|
26
|
+
@engine.input
|
|
27
|
+
def noise() -> int:
|
|
28
|
+
return 0
|
|
29
|
+
|
|
30
|
+
def bottom_body() -> int:
|
|
31
|
+
runs["bottom"] += 1
|
|
32
|
+
return leaf()
|
|
33
|
+
|
|
34
|
+
bottom = (
|
|
35
|
+
engine.query(ttl=3600.0)(bottom_body)
|
|
36
|
+
if ttl_bottom
|
|
37
|
+
else engine.query(bottom_body)
|
|
38
|
+
)
|
|
39
|
+
|
|
40
|
+
@engine.query
|
|
41
|
+
def level(n: int) -> int:
|
|
42
|
+
runs["level"] += 1
|
|
43
|
+
return (bottom() if n == 0 else level(n - 1)) + 1
|
|
44
|
+
|
|
45
|
+
leaf.set(1)
|
|
46
|
+
noise.set(0)
|
|
47
|
+
for n in range(DEPTH):
|
|
48
|
+
level(n)
|
|
49
|
+
runs.clear()
|
|
50
|
+
return leaf, noise, level
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
@pytest.fixture(autouse=True)
|
|
54
|
+
def _default_recursion_limit() -> Any:
|
|
55
|
+
old = sys.getrecursionlimit()
|
|
56
|
+
sys.setrecursionlimit(1000)
|
|
57
|
+
yield
|
|
58
|
+
sys.setrecursionlimit(old)
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def test_deep_chain_above_ttl_memo_verifies_without_recursion() -> None:
|
|
62
|
+
runs: Counter[str] = Counter()
|
|
63
|
+
engine = Engine(max_entries=DEPTH * 2)
|
|
64
|
+
_, noise, level = _chain(engine, ttl_bottom=True, runs=runs)
|
|
65
|
+
noise.set(1) # unrelated bump; the TTL memo keeps everything above it dirty
|
|
66
|
+
assert level(DEPTH - 1) == DEPTH + 1
|
|
67
|
+
assert runs == {}
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def test_deep_chain_read_at_pinned_snapshot() -> None:
|
|
71
|
+
runs: Counter[str] = Counter()
|
|
72
|
+
engine = Engine(max_entries=DEPTH * 2)
|
|
73
|
+
leaf, _, level = _chain(engine, ttl_bottom=False, runs=runs)
|
|
74
|
+
old = engine.snapshot()
|
|
75
|
+
leaf.set(2)
|
|
76
|
+
assert level(DEPTH - 1) == DEPTH + 2
|
|
77
|
+
runs.clear()
|
|
78
|
+
# Every memo was verified at the newer revision, so the old snapshot read
|
|
79
|
+
# walks (and, since leaf differs there, recomputes) the whole chain.
|
|
80
|
+
assert level(DEPTH - 1, snapshot=old) == DEPTH + 1
|
|
81
|
+
assert runs == {"bottom": 1, "level": DEPTH}
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def test_deep_chain_changed_leaf_recomputes_every_level() -> None:
|
|
85
|
+
runs: Counter[str] = Counter()
|
|
86
|
+
engine = Engine(max_entries=DEPTH * 2)
|
|
87
|
+
leaf, _, level = _chain(engine, ttl_bottom=True, runs=runs)
|
|
88
|
+
leaf.set(5)
|
|
89
|
+
assert level(DEPTH - 1) == DEPTH + 5
|
|
90
|
+
assert runs == {"bottom": 1, "level": DEPTH}
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def test_deep_chain_with_evicted_middle_memo() -> None:
|
|
94
|
+
runs: Counter[str] = Counter()
|
|
95
|
+
engine = Engine(max_entries=DEPTH * 2)
|
|
96
|
+
_, noise, level = _chain(engine, ttl_bottom=True, runs=runs)
|
|
97
|
+
store = engine._internals._store # noqa: SLF001
|
|
98
|
+
with store.lock:
|
|
99
|
+
store.drop_memo_locked(("query", level.id, (DEPTH // 2,)))
|
|
100
|
+
noise.set(1)
|
|
101
|
+
assert level(DEPTH - 1) == DEPTH + 1
|
|
102
|
+
# The dropped level is recomputed. With no previous memo to compare
|
|
103
|
+
# against it cannot backdate, so the level above reruns once, finds the
|
|
104
|
+
# same value, backdates, and nothing further up reruns.
|
|
105
|
+
assert runs == {"level": 2}
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def test_verification_stops_at_first_changed_dependency() -> None:
|
|
109
|
+
"""A dependency read after one that changed must not be verified: the
|
|
110
|
+
recomputation may no longer read it, and verifying it could fail."""
|
|
111
|
+
engine = Engine()
|
|
112
|
+
runs: Counter[str] = Counter()
|
|
113
|
+
|
|
114
|
+
@engine.input
|
|
115
|
+
def flag() -> bool:
|
|
116
|
+
return True
|
|
117
|
+
|
|
118
|
+
@engine.input
|
|
119
|
+
def x() -> int:
|
|
120
|
+
return 1
|
|
121
|
+
|
|
122
|
+
@engine.query(ttl=3600.0)
|
|
123
|
+
def gate() -> bool:
|
|
124
|
+
return flag()
|
|
125
|
+
|
|
126
|
+
@engine.query
|
|
127
|
+
def fragile() -> int:
|
|
128
|
+
runs["fragile"] += 1
|
|
129
|
+
if x() < 0:
|
|
130
|
+
raise ValueError("fragile read with negative x")
|
|
131
|
+
return x()
|
|
132
|
+
|
|
133
|
+
@engine.query
|
|
134
|
+
def top() -> int:
|
|
135
|
+
runs["top"] += 1
|
|
136
|
+
return fragile() if gate() else 0
|
|
137
|
+
|
|
138
|
+
flag.set(True)
|
|
139
|
+
x.set(1)
|
|
140
|
+
assert top() == 1
|
|
141
|
+
runs.clear()
|
|
142
|
+
with engine.transaction():
|
|
143
|
+
flag.set(False)
|
|
144
|
+
x.set(-1)
|
|
145
|
+
assert top() == 0
|
|
146
|
+
assert runs == {"top": 1}
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def test_cycle_memos_fall_back_to_the_normal_path() -> None:
|
|
150
|
+
engine = Engine()
|
|
151
|
+
|
|
152
|
+
@engine.input
|
|
153
|
+
def limit() -> int:
|
|
154
|
+
return 3
|
|
155
|
+
|
|
156
|
+
@engine.input
|
|
157
|
+
def noise() -> int:
|
|
158
|
+
return 0
|
|
159
|
+
|
|
160
|
+
@engine.query(fixed_point=0)
|
|
161
|
+
def grow() -> int:
|
|
162
|
+
return min(grow() + 1, limit())
|
|
163
|
+
|
|
164
|
+
@engine.query
|
|
165
|
+
def wrap(n: int) -> int:
|
|
166
|
+
return grow() if n == 0 else wrap(n - 1)
|
|
167
|
+
|
|
168
|
+
limit.set(3)
|
|
169
|
+
noise.set(0)
|
|
170
|
+
for n in range(50):
|
|
171
|
+
wrap(n)
|
|
172
|
+
noise.set(1)
|
|
173
|
+
assert wrap(49) == 3
|
|
174
|
+
limit.set(6)
|
|
175
|
+
assert wrap(49) == 6
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{query_cascade-0.3.25 → query_cascade-0.3.26}/src/query_cascade.egg-info/dependency_links.txt
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{query_cascade-0.3.25 → query_cascade-0.3.26}/tests/test_evaluator_and_synthetic_coverage.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|