kahn-queue 1.0.1__tar.gz → 1.0.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {kahn_queue-1.0.1 → kahn_queue-1.0.2}/PKG-INFO +4 -2
- {kahn_queue-1.0.1 → kahn_queue-1.0.2}/README.md +3 -1
- {kahn_queue-1.0.1 → kahn_queue-1.0.2}/pyproject.toml +1 -1
- {kahn_queue-1.0.1 → kahn_queue-1.0.2}/src/kahnQueue/concurrent_kahn_queue.py +2 -2
- {kahn_queue-1.0.1 → kahn_queue-1.0.2}/src/kahnQueue/default_kahn_queue.py +1 -1
- {kahn_queue-1.0.1 → kahn_queue-1.0.2}/src/kahnQueue/kahn_queue.py +1 -1
- {kahn_queue-1.0.1 → kahn_queue-1.0.2}/src/kahn_queue.egg-info/PKG-INFO +4 -2
- {kahn_queue-1.0.1 → kahn_queue-1.0.2}/src/scheduler.py +1 -1
- {kahn_queue-1.0.1 → kahn_queue-1.0.2}/tests/test_concurrent_kahn_queue.py +27 -27
- {kahn_queue-1.0.1 → kahn_queue-1.0.2}/tests/test_default_kahn_queue.py +11 -11
- {kahn_queue-1.0.1 → kahn_queue-1.0.2}/setup.cfg +0 -0
- {kahn_queue-1.0.1 → kahn_queue-1.0.2}/src/dag.py +0 -0
- {kahn_queue-1.0.1 → kahn_queue-1.0.2}/src/exception.py +0 -0
- {kahn_queue-1.0.1 → kahn_queue-1.0.2}/src/kahnQueue/__init__.py +0 -0
- {kahn_queue-1.0.1 → kahn_queue-1.0.2}/src/kahnQueue/node_machine.py +0 -0
- {kahn_queue-1.0.1 → kahn_queue-1.0.2}/src/kahnQueue/node_state.py +0 -0
- {kahn_queue-1.0.1 → kahn_queue-1.0.2}/src/kahn_queue.egg-info/SOURCES.txt +0 -0
- {kahn_queue-1.0.1 → kahn_queue-1.0.2}/src/kahn_queue.egg-info/dependency_links.txt +0 -0
- {kahn_queue-1.0.1 → kahn_queue-1.0.2}/src/kahn_queue.egg-info/top_level.txt +0 -0
- {kahn_queue-1.0.1 → kahn_queue-1.0.2}/src/tracker.py +0 -0
- {kahn_queue-1.0.1 → kahn_queue-1.0.2}/src/utils/__init__.py +0 -0
- {kahn_queue-1.0.1 → kahn_queue-1.0.2}/src/utils/state_machine.py +0 -0
- {kahn_queue-1.0.1 → kahn_queue-1.0.2}/tests/test_dag.py +0 -0
- {kahn_queue-1.0.1 → kahn_queue-1.0.2}/tests/test_kahn_scheduler.py +0 -0
- {kahn_queue-1.0.1 → kahn_queue-1.0.2}/tests/test_node_machine.py +0 -0
- {kahn_queue-1.0.1 → kahn_queue-1.0.2}/tests/test_node_progress_tracker.py +0 -0
- {kahn_queue-1.0.1 → kahn_queue-1.0.2}/tests/test_state_machine.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: kahn-queue
|
|
3
|
-
Version: 1.0.
|
|
3
|
+
Version: 1.0.2
|
|
4
4
|
Summary: Lightweight Kahn-based ready queue for dependency-driven scheduling and workflows
|
|
5
5
|
License: MIT
|
|
6
6
|
Project-URL: Homepage, https://flashlock.github.io/kahn-queue/
|
|
@@ -11,6 +11,8 @@ Description-Content-Type: text/markdown
|
|
|
11
11
|
|
|
12
12
|
# kahn-queue (Python)
|
|
13
13
|
|
|
14
|
+
[](https://pypi.org/project/kahn-queue/)
|
|
15
|
+
|
|
14
16
|
## Getting started
|
|
15
17
|
|
|
16
18
|
- **Python:** **3.10+** recommended (venv + `requirements-dev.txt`; no version pinned in-repo).
|
|
@@ -100,7 +102,7 @@ dag = b.build()
|
|
|
100
102
|
|
|
101
103
|
q = DefaultKahnQueue(dag)
|
|
102
104
|
|
|
103
|
-
ready = list(q.
|
|
105
|
+
ready = list(q.peek())
|
|
104
106
|
|
|
105
107
|
while ready:
|
|
106
108
|
id_ = ready.pop(0)
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
# kahn-queue (Python)
|
|
2
2
|
|
|
3
|
+
[](https://pypi.org/project/kahn-queue/)
|
|
4
|
+
|
|
3
5
|
## Getting started
|
|
4
6
|
|
|
5
7
|
- **Python:** **3.10+** recommended (venv + `requirements-dev.txt`; no version pinned in-repo).
|
|
@@ -89,7 +91,7 @@ dag = b.build()
|
|
|
89
91
|
|
|
90
92
|
q = DefaultKahnQueue(dag)
|
|
91
93
|
|
|
92
|
-
ready = list(q.
|
|
94
|
+
ready = list(q.peek())
|
|
93
95
|
|
|
94
96
|
while ready:
|
|
95
97
|
id_ = ready.pop(0)
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "kahn-queue"
|
|
7
|
-
version = "1.0.
|
|
7
|
+
version = "1.0.2"
|
|
8
8
|
description = "Lightweight Kahn-based ready queue for dependency-driven scheduling and workflows"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.10"
|
|
@@ -10,7 +10,7 @@ from kahnQueue.kahn_queue import KahnQueue
|
|
|
10
10
|
|
|
11
11
|
|
|
12
12
|
class ConcurrentKahnQueue(KahnQueue):
|
|
13
|
-
"""``KahnQueue`` for concurrent ``pop`` and ``prune`` calls. ``
|
|
13
|
+
"""``KahnQueue`` for concurrent ``pop`` and ``prune`` calls. ``peek()`` may not
|
|
14
14
|
reflect a consistent snapshot if other threads update the queue at the same time; coordinate
|
|
15
15
|
externally if you need strict ordering or visibility. For single-threaded use, prefer
|
|
16
16
|
``DefaultKahnQueue``.
|
|
@@ -68,7 +68,7 @@ class ConcurrentKahnQueue(KahnQueue):
|
|
|
68
68
|
|
|
69
69
|
return sorted(affected)
|
|
70
70
|
|
|
71
|
-
def
|
|
71
|
+
def peek(self) -> List[int]:
|
|
72
72
|
# Deterministic ordering for sequential callers; may not reflect a consistent snapshot
|
|
73
73
|
# under concurrent updates.
|
|
74
74
|
return [m.id for m in self._node_machines if m.is_(NodeState.READY)]
|
|
@@ -55,6 +55,6 @@ class DefaultKahnQueue(KahnQueue):
|
|
|
55
55
|
|
|
56
56
|
return sorted(affected)
|
|
57
57
|
|
|
58
|
-
def
|
|
58
|
+
def peek(self) -> List[int]:
|
|
59
59
|
# Deterministic: ids are scanned in ascending order.
|
|
60
60
|
return [m.id for m in self._node_machines if m.is_(NodeState.READY)]
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: kahn-queue
|
|
3
|
-
Version: 1.0.
|
|
3
|
+
Version: 1.0.2
|
|
4
4
|
Summary: Lightweight Kahn-based ready queue for dependency-driven scheduling and workflows
|
|
5
5
|
License: MIT
|
|
6
6
|
Project-URL: Homepage, https://flashlock.github.io/kahn-queue/
|
|
@@ -11,6 +11,8 @@ Description-Content-Type: text/markdown
|
|
|
11
11
|
|
|
12
12
|
# kahn-queue (Python)
|
|
13
13
|
|
|
14
|
+
[](https://pypi.org/project/kahn-queue/)
|
|
15
|
+
|
|
14
16
|
## Getting started
|
|
15
17
|
|
|
16
18
|
- **Python:** **3.10+** recommended (venv + `requirements-dev.txt`; no version pinned in-repo).
|
|
@@ -100,7 +102,7 @@ dag = b.build()
|
|
|
100
102
|
|
|
101
103
|
q = DefaultKahnQueue(dag)
|
|
102
104
|
|
|
103
|
-
ready = list(q.
|
|
105
|
+
ready = list(q.peek())
|
|
104
106
|
|
|
105
107
|
while ready:
|
|
106
108
|
id_ = ready.pop(0)
|
|
@@ -43,7 +43,7 @@ class KahnScheduler(Generic[T]):
|
|
|
43
43
|
"""Invokes ``execute_node`` for each id the queue reports ready now."""
|
|
44
44
|
if self.is_finished:
|
|
45
45
|
return
|
|
46
|
-
for node_id in self._queue.
|
|
46
|
+
for node_id in self._queue.peek():
|
|
47
47
|
self._execute_node(node_id, self)
|
|
48
48
|
|
|
49
49
|
def signal_complete(self, node_id: int) -> None:
|
|
@@ -14,10 +14,10 @@ def _force_node_state(q: ConcurrentKahnQueue, id_: int, state: NodeState) -> Non
|
|
|
14
14
|
q._node_machines[id_]._state = state # mirrors Java tests using reflection
|
|
15
15
|
|
|
16
16
|
|
|
17
|
-
def
|
|
17
|
+
def test_basics_peek_and_pop_validation():
|
|
18
18
|
empty = Dag.builder().build()
|
|
19
19
|
q0 = ConcurrentKahnQueue(empty)
|
|
20
|
-
assert q0.
|
|
20
|
+
assert q0.peek() == []
|
|
21
21
|
with pytest.raises(IndexError):
|
|
22
22
|
q0.pop(0)
|
|
23
23
|
|
|
@@ -28,7 +28,7 @@ def test_basics_ready_ids_and_pop_validation():
|
|
|
28
28
|
chain.connect(root, mid).connect(mid, leaf)
|
|
29
29
|
dag_chain = chain.build()
|
|
30
30
|
qc = ConcurrentKahnQueue(dag_chain)
|
|
31
|
-
assert qc.
|
|
31
|
+
assert qc.peek() == [root]
|
|
32
32
|
|
|
33
33
|
join = Dag.builder()
|
|
34
34
|
a = join.add("a")
|
|
@@ -37,13 +37,13 @@ def test_basics_ready_ids_and_pop_validation():
|
|
|
37
37
|
join.connect(a, jn).connect(c, jn)
|
|
38
38
|
dag_join = join.build()
|
|
39
39
|
qj = ConcurrentKahnQueue(dag_join)
|
|
40
|
-
assert qj.
|
|
40
|
+
assert qj.peek() == [a, c]
|
|
41
41
|
|
|
42
42
|
one = Dag.builder()
|
|
43
43
|
only = one.add("x")
|
|
44
44
|
dag_one = one.build()
|
|
45
45
|
q1 = ConcurrentKahnQueue(dag_one)
|
|
46
|
-
assert q1.
|
|
46
|
+
assert q1.peek() == [only]
|
|
47
47
|
with pytest.raises(ValueError) as ex:
|
|
48
48
|
q1.pop(only)
|
|
49
49
|
assert "Pop failed. Node" in str(ex.value)
|
|
@@ -83,12 +83,12 @@ def test_prune_updates_ready_set():
|
|
|
83
83
|
b.connect(a, join).connect(c, join)
|
|
84
84
|
dag = b.build()
|
|
85
85
|
q = ConcurrentKahnQueue(dag)
|
|
86
|
-
assert q.
|
|
86
|
+
assert q.peek() == [a, c]
|
|
87
87
|
q.prune(a)
|
|
88
|
-
assert q.
|
|
88
|
+
assert q.peek() == [c]
|
|
89
89
|
|
|
90
90
|
|
|
91
|
-
def
|
|
91
|
+
def test_peek_stable_under_sequential_repeats():
|
|
92
92
|
b = Dag.builder()
|
|
93
93
|
a = b.add("a")
|
|
94
94
|
c = b.add("c")
|
|
@@ -96,10 +96,10 @@ def test_ready_ids_stable_under_sequential_repeats():
|
|
|
96
96
|
b.connect(a, join).connect(c, join)
|
|
97
97
|
dag = b.build()
|
|
98
98
|
q = ConcurrentKahnQueue(dag)
|
|
99
|
-
snapshot = q.
|
|
99
|
+
snapshot = q.peek()
|
|
100
100
|
for _ in range(10_000):
|
|
101
|
-
assert q.
|
|
102
|
-
for id_ in q.
|
|
101
|
+
assert q.peek() == snapshot
|
|
102
|
+
for id_ in q.peek():
|
|
103
103
|
Dag.validate_node(id_, dag.size())
|
|
104
104
|
|
|
105
105
|
|
|
@@ -109,10 +109,10 @@ def test_prune_second_call_throws():
|
|
|
109
109
|
dag = b.build()
|
|
110
110
|
q = ConcurrentKahnQueue(dag)
|
|
111
111
|
assert q.prune(r) == [r]
|
|
112
|
-
assert q.
|
|
112
|
+
assert q.peek() == []
|
|
113
113
|
# ConcurrentKahnQueue.prune is idempotent for already-pruned branches.
|
|
114
114
|
assert q.prune(r) == []
|
|
115
|
-
for id_ in q.
|
|
115
|
+
for id_ in q.peek():
|
|
116
116
|
Dag.validate_node(id_, dag.size())
|
|
117
117
|
|
|
118
118
|
|
|
@@ -128,12 +128,12 @@ def test_kahn_progression_pop_active_node_returns_promoted_dependents():
|
|
|
128
128
|
assert q.pop(root) == [mid]
|
|
129
129
|
assert q.pop(mid) == [leaf]
|
|
130
130
|
assert q.pop(leaf) == []
|
|
131
|
-
assert q.
|
|
132
|
-
for id_ in q.
|
|
131
|
+
assert q.peek() == []
|
|
132
|
+
for id_ in q.peek():
|
|
133
133
|
Dag.validate_node(id_, dag.size())
|
|
134
134
|
|
|
135
135
|
|
|
136
|
-
def
|
|
136
|
+
def test_concurrent_peek_stress_reads_match_snapshot():
|
|
137
137
|
for _round in range(25):
|
|
138
138
|
b = Dag.builder()
|
|
139
139
|
x = b.add("x")
|
|
@@ -142,18 +142,18 @@ def test_concurrent_ready_ids_stress_reads_match_snapshot():
|
|
|
142
142
|
dag = b.build()
|
|
143
143
|
q = ConcurrentKahnQueue(dag)
|
|
144
144
|
expected = [x]
|
|
145
|
-
assert q.
|
|
145
|
+
assert q.peek() == expected
|
|
146
146
|
|
|
147
147
|
def worker():
|
|
148
148
|
for _i in range(500):
|
|
149
|
-
if q.
|
|
150
|
-
raise AssertionError("
|
|
149
|
+
if q.peek() != expected:
|
|
150
|
+
raise AssertionError("peek drifted during concurrent read")
|
|
151
151
|
|
|
152
152
|
with concurrent.futures.ThreadPoolExecutor(max_workers=8) as pool:
|
|
153
153
|
futures = [pool.submit(worker) for _ in range(8)]
|
|
154
154
|
for f in futures:
|
|
155
155
|
f.result()
|
|
156
|
-
assert q.
|
|
156
|
+
assert q.peek() == expected
|
|
157
157
|
|
|
158
158
|
|
|
159
159
|
def test_concurrent_disjoint_prune():
|
|
@@ -168,7 +168,7 @@ def test_concurrent_disjoint_prune():
|
|
|
168
168
|
f2 = pool.submit(q.prune, right)
|
|
169
169
|
assert f1.result() == [left]
|
|
170
170
|
assert f2.result() == [right]
|
|
171
|
-
assert q.
|
|
171
|
+
assert q.peek() == []
|
|
172
172
|
|
|
173
173
|
|
|
174
174
|
def test_concurrent_pop_failures_do_not_mutate_ready():
|
|
@@ -177,7 +177,7 @@ def test_concurrent_pop_failures_do_not_mutate_ready():
|
|
|
177
177
|
only = b.add("x")
|
|
178
178
|
dag = b.build()
|
|
179
179
|
q = ConcurrentKahnQueue(dag)
|
|
180
|
-
before = q.
|
|
180
|
+
before = q.peek()
|
|
181
181
|
|
|
182
182
|
def worker():
|
|
183
183
|
try:
|
|
@@ -189,7 +189,7 @@ def test_concurrent_pop_failures_do_not_mutate_ready():
|
|
|
189
189
|
with concurrent.futures.ThreadPoolExecutor(max_workers=32) as pool:
|
|
190
190
|
results = list(pool.map(lambda _: worker(), range(32)))
|
|
191
191
|
assert all(results)
|
|
192
|
-
assert q.
|
|
192
|
+
assert q.peek() == before
|
|
193
193
|
|
|
194
194
|
|
|
195
195
|
def test_concurrent_same_id_prune_contention():
|
|
@@ -210,7 +210,7 @@ def test_concurrent_same_id_prune_contention():
|
|
|
210
210
|
results = [f.result(timeout=30) for f in futures]
|
|
211
211
|
assert results.count([root]) == 1
|
|
212
212
|
assert results.count([]) == 7
|
|
213
|
-
assert q.
|
|
213
|
+
assert q.peek() == []
|
|
214
214
|
|
|
215
215
|
|
|
216
216
|
def test_concurrent_overlapping_prune():
|
|
@@ -235,14 +235,14 @@ def test_concurrent_overlapping_prune():
|
|
|
235
235
|
s1 = f1.result(timeout=30)
|
|
236
236
|
s2 = f2.result(timeout=30)
|
|
237
237
|
|
|
238
|
-
for id_ in q.
|
|
238
|
+
for id_ in q.peek():
|
|
239
239
|
assert 0 <= id_ < dag.size()
|
|
240
240
|
|
|
241
241
|
assert set(s1) | set(s2) == {r, m, l}
|
|
242
242
|
|
|
243
243
|
any_full = (set(s1) == {r, m, l}) or (set(s2) == {r, m, l})
|
|
244
244
|
if any_full:
|
|
245
|
-
assert q.
|
|
245
|
+
assert q.peek() == []
|
|
246
246
|
|
|
247
247
|
|
|
248
248
|
def test_prune_mutation_visible_to_other_thread_after_future_get():
|
|
@@ -254,7 +254,7 @@ def test_prune_mutation_visible_to_other_thread_after_future_get():
|
|
|
254
254
|
with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool:
|
|
255
255
|
done = pool.submit(q.prune, left)
|
|
256
256
|
assert done.result(timeout=10) == [left]
|
|
257
|
-
assert q.
|
|
257
|
+
assert q.peek() == [right]
|
|
258
258
|
|
|
259
259
|
|
|
260
260
|
def _get_prune_result_or_illegal_state(fut: "concurrent.futures.Future[set[int]]"):
|
|
@@ -9,15 +9,15 @@ def _force_node_state(q: DefaultKahnQueue, id_: int, state: NodeState) -> None:
|
|
|
9
9
|
q._node_machines[id_]._state = state # mirrors Java tests using reflection
|
|
10
10
|
|
|
11
11
|
|
|
12
|
-
def
|
|
12
|
+
def test_empty_dag_peek_empty_and_pop_rejects_invalid_id():
|
|
13
13
|
dag = Dag.builder().build()
|
|
14
14
|
q = DefaultKahnQueue(dag)
|
|
15
|
-
assert q.
|
|
15
|
+
assert q.peek() == []
|
|
16
16
|
with pytest.raises(IndexError):
|
|
17
17
|
q.pop(0)
|
|
18
18
|
|
|
19
19
|
|
|
20
|
-
def
|
|
20
|
+
def test_peek_contains_only_zero_in_degree_nodes():
|
|
21
21
|
b = Dag.builder()
|
|
22
22
|
root = b.add("root")
|
|
23
23
|
mid = b.add("mid")
|
|
@@ -25,10 +25,10 @@ def test_ready_ids_contains_only_zero_in_degree_nodes():
|
|
|
25
25
|
b.connect(root, mid).connect(mid, leaf)
|
|
26
26
|
dag = b.build()
|
|
27
27
|
q = DefaultKahnQueue(dag)
|
|
28
|
-
assert q.
|
|
28
|
+
assert q.peek() == [root]
|
|
29
29
|
|
|
30
30
|
|
|
31
|
-
def
|
|
31
|
+
def test_peek_two_independent_roots():
|
|
32
32
|
b = Dag.builder()
|
|
33
33
|
a = b.add("a")
|
|
34
34
|
c = b.add("c")
|
|
@@ -36,7 +36,7 @@ def test_ready_ids_two_independent_roots():
|
|
|
36
36
|
b.connect(a, join).connect(c, join)
|
|
37
37
|
dag = b.build()
|
|
38
38
|
q = DefaultKahnQueue(dag)
|
|
39
|
-
assert q.
|
|
39
|
+
assert q.peek() == [a, c]
|
|
40
40
|
|
|
41
41
|
|
|
42
42
|
def test_pop_throws_when_node_is_ready_not_active():
|
|
@@ -44,7 +44,7 @@ def test_pop_throws_when_node_is_ready_not_active():
|
|
|
44
44
|
only = b.add("x")
|
|
45
45
|
dag = b.build()
|
|
46
46
|
q = DefaultKahnQueue(dag)
|
|
47
|
-
assert q.
|
|
47
|
+
assert q.peek() == [only]
|
|
48
48
|
with pytest.raises(ValueError) as ex:
|
|
49
49
|
q.pop(only)
|
|
50
50
|
assert "Pop failed. Node" in str(ex.value)
|
|
@@ -90,9 +90,9 @@ def test_prune_removes_ids_from_ready_set():
|
|
|
90
90
|
b.connect(a, join).connect(c, join)
|
|
91
91
|
dag = b.build()
|
|
92
92
|
q = DefaultKahnQueue(dag)
|
|
93
|
-
assert q.
|
|
93
|
+
assert q.peek() == [a, c]
|
|
94
94
|
q.prune(a)
|
|
95
|
-
assert q.
|
|
95
|
+
assert q.peek() == [c]
|
|
96
96
|
|
|
97
97
|
|
|
98
98
|
def test_prune_second_call_throws():
|
|
@@ -101,7 +101,7 @@ def test_prune_second_call_throws():
|
|
|
101
101
|
dag = b.build()
|
|
102
102
|
q = DefaultKahnQueue(dag)
|
|
103
103
|
assert q.prune(r) == [r]
|
|
104
|
-
assert q.
|
|
104
|
+
assert q.peek() == []
|
|
105
105
|
# DefaultKahnQueue.prune is idempotent for already-pruned branches.
|
|
106
106
|
assert q.prune(r) == []
|
|
107
107
|
|
|
@@ -119,5 +119,5 @@ def test_kahn_progression_pop_active_node_returns_promoted_dependents():
|
|
|
119
119
|
assert q.pop(root) == [mid]
|
|
120
120
|
assert q.pop(mid) == [leaf]
|
|
121
121
|
assert q.pop(leaf) == []
|
|
122
|
-
assert q.
|
|
122
|
+
assert q.peek() == []
|
|
123
123
|
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|