harness-sdk-python 0.18.0__tar.gz → 0.20.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {harness_sdk_python-0.18.0 → harness_sdk_python-0.20.0}/PKG-INFO +3 -3
- {harness_sdk_python-0.18.0 → harness_sdk_python-0.20.0}/pyproject.toml +2 -2
- {harness_sdk_python-0.18.0 → harness_sdk_python-0.20.0}/src/harness_sdk/run_manager.py +114 -39
- harness_sdk_python-0.20.0/tests/test_admit_steer.py +118 -0
- {harness_sdk_python-0.18.0 → harness_sdk_python-0.20.0}/tests/test_batches.py +2 -14
- {harness_sdk_python-0.18.0 → harness_sdk_python-0.20.0}/tests/test_durability.py +65 -5
- {harness_sdk_python-0.18.0 → harness_sdk_python-0.20.0}/tests/test_outcomes.py +32 -0
- {harness_sdk_python-0.18.0 → harness_sdk_python-0.20.0}/tests/test_restore.py +1 -0
- {harness_sdk_python-0.18.0 → harness_sdk_python-0.20.0}/tests/test_run_crash_sweep.py +39 -4
- {harness_sdk_python-0.18.0 → harness_sdk_python-0.20.0}/tests/test_stop_continue.py +52 -2
- {harness_sdk_python-0.18.0 → harness_sdk_python-0.20.0}/tests/test_stopping_window.py +36 -17
- {harness_sdk_python-0.18.0 → harness_sdk_python-0.20.0}/.gitignore +0 -0
- {harness_sdk_python-0.18.0 → harness_sdk_python-0.20.0}/README.md +0 -0
- {harness_sdk_python-0.18.0 → harness_sdk_python-0.20.0}/examples/__init__.py +0 -0
- {harness_sdk_python-0.18.0 → harness_sdk_python-0.20.0}/examples/runs_app.py +0 -0
- {harness_sdk_python-0.18.0 → harness_sdk_python-0.20.0}/src/harness_sdk/__init__.py +0 -0
- {harness_sdk_python-0.18.0 → harness_sdk_python-0.20.0}/src/harness_sdk/fenced_postgres.py +0 -0
- {harness_sdk_python-0.18.0 → harness_sdk_python-0.20.0}/src/harness_sdk/linear_thread.py +0 -0
- {harness_sdk_python-0.18.0 → harness_sdk_python-0.20.0}/tests/run_helpers.py +0 -0
- {harness_sdk_python-0.18.0 → harness_sdk_python-0.20.0}/tests/test_applied_visibility.py +0 -0
- {harness_sdk_python-0.18.0 → harness_sdk_python-0.20.0}/tests/test_branch_anchor.py +0 -0
- {harness_sdk_python-0.18.0 → harness_sdk_python-0.20.0}/tests/test_dispatching.py +0 -0
- {harness_sdk_python-0.18.0 → harness_sdk_python-0.20.0}/tests/test_edit_dispatched.py +0 -0
- {harness_sdk_python-0.18.0 → harness_sdk_python-0.20.0}/tests/test_edit_reload.py +0 -0
- {harness_sdk_python-0.18.0 → harness_sdk_python-0.20.0}/tests/test_enqueue.py +0 -0
- {harness_sdk_python-0.18.0 → harness_sdk_python-0.20.0}/tests/test_fenced_postgres.py +0 -0
- {harness_sdk_python-0.18.0 → harness_sdk_python-0.20.0}/tests/test_input_required.py +0 -0
- {harness_sdk_python-0.18.0 → harness_sdk_python-0.20.0}/tests/test_linear_thread.py +0 -0
- {harness_sdk_python-0.18.0 → harness_sdk_python-0.20.0}/tests/test_meta.py +0 -0
- {harness_sdk_python-0.18.0 → harness_sdk_python-0.20.0}/tests/test_placement.py +0 -0
- {harness_sdk_python-0.18.0 → harness_sdk_python-0.20.0}/tests/test_prepare_hooks.py +0 -0
- {harness_sdk_python-0.18.0 → harness_sdk_python-0.20.0}/tests/test_rewind_during_run.py +0 -0
- {harness_sdk_python-0.18.0 → harness_sdk_python-0.20.0}/tests/test_settle.py +0 -0
- {harness_sdk_python-0.18.0 → harness_sdk_python-0.20.0}/tests/test_steer.py +0 -0
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: harness-sdk-python
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.20.0
|
|
4
4
|
Summary: RunManager: the harness-sdk runs subsystem for Python Statewire hosts
|
|
5
5
|
Project-URL: Repository, https://github.com/assistant-ui/harness-sdk
|
|
6
6
|
License-Expression: MIT
|
|
7
7
|
Requires-Python: <4.0,>=3.12
|
|
8
|
-
Requires-Dist: pinned<0.
|
|
9
|
-
Requires-Dist: statewire<0.
|
|
8
|
+
Requires-Dist: pinned<0.14,>=0.13.0
|
|
9
|
+
Requires-Dist: statewire<0.11,>=0.10.0
|
|
10
10
|
Provides-Extra: deepagents
|
|
11
11
|
Requires-Dist: deepagents>=0.6.12; extra == 'deepagents'
|
|
12
12
|
Requires-Dist: langchain-core>=0.3; extra == 'deepagents'
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "harness-sdk-python"
|
|
3
|
-
version = "0.
|
|
3
|
+
version = "0.20.0"
|
|
4
4
|
description = "RunManager: the harness-sdk runs subsystem for Python Statewire hosts"
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
license = "MIT"
|
|
7
7
|
requires-python = ">=3.12,<4.0"
|
|
8
|
-
dependencies = ["statewire>=0.
|
|
8
|
+
dependencies = ["statewire>=0.10.0,<0.11", "pinned>=0.13.0,<0.14"]
|
|
9
9
|
|
|
10
10
|
[project.optional-dependencies]
|
|
11
11
|
postgres = ["langgraph-checkpoint-postgres>=2.0.0"]
|
|
@@ -6,13 +6,21 @@ tray's durable records are the machine's authoritative state, and the mounted
|
|
|
6
6
|
view rides ``state["runs"][0]`` (plus a top-level ``status`` mirror). A crash
|
|
7
7
|
at any point resumes the machine from its records: the in-flight dispatch
|
|
8
8
|
redelivers from the outbox (``ctx.attempts`` counts durable deliveries,
|
|
9
|
-
``ctx.state``/``set_state`` is the run's durable checkpoint), lanes,
|
|
10
|
-
|
|
11
|
-
|
|
9
|
+
``ctx.state``/``set_state`` is the run's durable checkpoint), and lanes, a
|
|
10
|
+
staged rewind (``nextDispatch``), and pending input requests all hydrate
|
|
11
|
+
back.
|
|
12
|
+
|
|
13
|
+
``status`` is derived, never stored: a recorded dispatch (the outbox) means
|
|
14
|
+
``running`` — ``stopping`` while a live stop intent (a staged stop or staged
|
|
15
|
+
rewind) is open — and with no recorded dispatch the durable facts decide:
|
|
16
|
+
``halted`` reads ``error``/``stopped``, pending ``inputRequests`` read
|
|
17
|
+
``input-required``, and bare facts read ``ready``. A crash drops live stop
|
|
18
|
+
intent; the pooled ``run/stop`` redelivers and re-stages it, and a staged
|
|
19
|
+
rewind hydrates back from ``nextDispatch``.
|
|
12
20
|
|
|
13
21
|
Commands stage entries into an intake list and an advance applies them in
|
|
14
|
-
order, then takes exactly one action based on status and net
|
|
15
|
-
(rewind > send-dispatch > continue > input-resume). Every initiator applies
|
|
22
|
+
order, then takes exactly one action based on the derived status and net
|
|
23
|
+
effect (rewind > send-dispatch > continue > input-resume). Every initiator applies
|
|
16
24
|
and settles accepted with its state-visible application, and every verb's
|
|
17
25
|
verdict follows the admission's durable flush. ``entry["dispatching"]`` is
|
|
18
26
|
the unconfirmed batch record; ``ctx.applied()`` confirms it durably,
|
|
@@ -29,8 +37,11 @@ own continuation state (``ctx.state`` or an external checkpointer).
|
|
|
29
37
|
``epoch`` counts the entry's dispatches (not redeliveries); a ``run/stop``
|
|
30
38
|
carrying an ``epoch`` other than the live one rejects. A mid-run stop or
|
|
31
39
|
rewind opens a stopping window (``stopping: {reason}`` plus, for rewinds, the
|
|
32
|
-
staged dispatch as ``nextDispatch``)
|
|
33
|
-
|
|
40
|
+
staged dispatch as ``nextDispatch``) and ``status`` reads ``stopping`` until
|
|
41
|
+
the run settles; the ``run/stop`` verdict follows that settle and its flush,
|
|
42
|
+
so a settled stop covers every durable effect the runner enlisted. On run end
|
|
43
|
+
one envelope promotes ``nextDispatch`` into ``dispatching`` and clears the
|
|
44
|
+
window.
|
|
34
45
|
"""
|
|
35
46
|
|
|
36
47
|
import asyncio
|
|
@@ -38,7 +49,7 @@ import logging
|
|
|
38
49
|
from dataclasses import dataclass, field
|
|
39
50
|
from typing import Any, AsyncIterator, Callable, Coroutine, Iterable, Protocol
|
|
40
51
|
|
|
41
|
-
from pinned.trays import Tray, TrayDescriptor, TrayCtx
|
|
52
|
+
from pinned.trays import Tray, TrayDescriptor, TrayCtx, inbox
|
|
42
53
|
from pinned.traystore import persistent, persistent_outbox, persistent_queue
|
|
43
54
|
from statepatch import observable
|
|
44
55
|
from statewire import StatewireReject
|
|
@@ -78,6 +89,15 @@ def _reject(reason: str, message: str) -> StatewireReject:
|
|
|
78
89
|
return StatewireReject(message, payload={"reason": reason})
|
|
79
90
|
|
|
80
91
|
|
|
92
|
+
def _public(message: dict[str, Any]) -> dict[str, Any]:
|
|
93
|
+
"""The message without its admission tag; the wire state is public."""
|
|
94
|
+
return {k: v for k, v in message.items() if k != "~admit"}
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def _public_record(record: dict[str, Any]) -> dict[str, Any]:
|
|
98
|
+
return {**record, "messages": [_public(m) for m in record["messages"]]}
|
|
99
|
+
|
|
100
|
+
|
|
81
101
|
def _run_id_of(params: Any) -> str:
|
|
82
102
|
run_id = params.get("runId") if isinstance(params, dict) else None
|
|
83
103
|
if not isinstance(run_id, str) or run_id == "":
|
|
@@ -180,8 +200,8 @@ class _Effects:
|
|
|
180
200
|
|
|
181
201
|
|
|
182
202
|
class RunManager(Tray):
|
|
183
|
-
"""One thread's run machine: durable lanes, dispatch outbox,
|
|
184
|
-
|
|
203
|
+
"""One thread's run machine: durable lanes, dispatch outbox, staged
|
|
204
|
+
rewind, and input requests, driven by the tray pump; status is derived."""
|
|
185
205
|
|
|
186
206
|
capabilities: frozenset[str] = frozenset()
|
|
187
207
|
max_queued: int = 50
|
|
@@ -195,15 +215,17 @@ class RunManager(Tray):
|
|
|
195
215
|
batchApplied = persistent(False)
|
|
196
216
|
confirmedTakes = persistent(0)
|
|
197
217
|
runId = persistent(None)
|
|
198
|
-
status = persistent("ready")
|
|
199
218
|
epoch = persistent(0)
|
|
200
|
-
|
|
219
|
+
halted = persistent(None) # off-wire settle fact: "error" | "stop" | None
|
|
201
220
|
nextDispatch = persistent(None)
|
|
202
221
|
inputRequests = persistent(None)
|
|
203
222
|
error = persistent(None)
|
|
204
223
|
stopReason = persistent(None)
|
|
205
224
|
dispatchedIds = persistent([])
|
|
225
|
+
status = observable("ready") # view: derived by _publish_status
|
|
226
|
+
stopping = observable(None) # view: the open stop window, derived
|
|
206
227
|
dispatching = observable(None) # view: the unconfirmed batch record
|
|
228
|
+
steerView = observable([]) # view: the steer lane without admission tags
|
|
207
229
|
|
|
208
230
|
state = observable(
|
|
209
231
|
{
|
|
@@ -211,12 +233,13 @@ class RunManager(Tray):
|
|
|
211
233
|
"status": status,
|
|
212
234
|
"epoch": epoch,
|
|
213
235
|
"queue": queue,
|
|
214
|
-
"steerQueue":
|
|
236
|
+
"steerQueue": steerView,
|
|
215
237
|
"dispatching": dispatching,
|
|
216
238
|
"stopping": stopping,
|
|
217
239
|
"nextDispatch": nextDispatch,
|
|
218
240
|
"inputRequests": inputRequests,
|
|
219
241
|
"error": error,
|
|
242
|
+
"stopReason": stopReason,
|
|
220
243
|
}
|
|
221
244
|
)
|
|
222
245
|
|
|
@@ -225,6 +248,7 @@ class RunManager(Tray):
|
|
|
225
248
|
self._intake: list[Any] = []
|
|
226
249
|
self._outcome: Any = None
|
|
227
250
|
self._stop_staged = False
|
|
251
|
+
self._stop_waiters: list["asyncio.Future[Any]"] = []
|
|
228
252
|
self._staged_rewind: _Rewind | None = None
|
|
229
253
|
self._batch: list[tuple[str, dict[str, Any]]] = []
|
|
230
254
|
self._pending_takes = 0
|
|
@@ -245,7 +269,10 @@ class RunManager(Tray):
|
|
|
245
269
|
self._thread = api
|
|
246
270
|
self._schedule = api.schedule
|
|
247
271
|
self._hydrate_machine()
|
|
272
|
+
self._mirror_steer()
|
|
273
|
+
unsubscribe = self.steerQueue.subscribe(lambda _op: self._mirror_steer())
|
|
248
274
|
yield
|
|
275
|
+
unsubscribe()
|
|
249
276
|
|
|
250
277
|
class Thread(Protocol):
|
|
251
278
|
"""The thread projection RunManager reads; the host owns the tree."""
|
|
@@ -277,14 +304,14 @@ class RunManager(Tray):
|
|
|
277
304
|
record = recorded["item"]["record"]
|
|
278
305
|
lanes = recorded["item"]["lanes"]
|
|
279
306
|
self._batch = list(zip(lanes, record["messages"]))
|
|
280
|
-
self.dispatching.value = record
|
|
307
|
+
self.dispatching.value = _public_record(record)
|
|
281
308
|
else:
|
|
282
309
|
self.dispatching.value = None
|
|
283
310
|
if recorded is not None:
|
|
284
311
|
# heal a dispatch torn after its outbox write landed
|
|
285
312
|
self._clear_input()
|
|
286
313
|
self.error.value = None
|
|
287
|
-
self.
|
|
314
|
+
self.halted.value = None
|
|
288
315
|
staged = self.nextDispatch.value
|
|
289
316
|
if (
|
|
290
317
|
staged is not None
|
|
@@ -317,6 +344,7 @@ class RunManager(Tray):
|
|
|
317
344
|
}
|
|
318
345
|
if not self._in_flight():
|
|
319
346
|
self._idle.set()
|
|
347
|
+
self._publish_status()
|
|
320
348
|
self._schedule(self._advance) # reconcile a torn advance segment
|
|
321
349
|
|
|
322
350
|
# ─── State access ───────────────────────────────────────
|
|
@@ -324,17 +352,36 @@ class RunManager(Tray):
|
|
|
324
352
|
def _in_flight(self) -> bool:
|
|
325
353
|
return self.outbox.value is not None
|
|
326
354
|
|
|
327
|
-
def
|
|
328
|
-
|
|
355
|
+
def _stop_intent(self) -> str | None:
|
|
356
|
+
if self._stop_staged:
|
|
357
|
+
return "stop"
|
|
358
|
+
if self._staged_rewind is not None:
|
|
359
|
+
return self._staged_rewind.type
|
|
360
|
+
return None
|
|
329
361
|
|
|
330
|
-
def
|
|
331
|
-
self.
|
|
362
|
+
def _status(self) -> str:
|
|
363
|
+
if self._in_flight():
|
|
364
|
+
return "running" if self._stop_intent() is None else "stopping"
|
|
365
|
+
if self.halted.value == "error":
|
|
366
|
+
return "error"
|
|
367
|
+
if self.halted.value == "stop":
|
|
368
|
+
return "stopped"
|
|
369
|
+
if self.inputRequests.value:
|
|
370
|
+
return "input-required"
|
|
371
|
+
return "ready"
|
|
372
|
+
|
|
373
|
+
def _publish_status(self) -> None:
|
|
374
|
+
status = self._status()
|
|
375
|
+
if self.status.value != status:
|
|
376
|
+
self.status.value = status
|
|
377
|
+
window = {"reason": self._stop_intent()} if status == "stopping" else None
|
|
378
|
+
if self.stopping.value != window:
|
|
379
|
+
self.stopping.value = window
|
|
332
380
|
|
|
333
381
|
def _to_ready(self) -> None:
|
|
334
382
|
self.runId.value = None
|
|
335
383
|
self.epoch.value = 0
|
|
336
384
|
self.dispatchedIds.value = []
|
|
337
|
-
self.status.value = "ready"
|
|
338
385
|
|
|
339
386
|
def _lane(self, lane: str) -> Any:
|
|
340
387
|
return self.queue if lane == "queue" else self.steerQueue
|
|
@@ -356,6 +403,9 @@ class RunManager(Tray):
|
|
|
356
403
|
def _is_dispatching(self, message_id: str) -> bool:
|
|
357
404
|
return any(item["id"] == message_id for _, item in self._batch)
|
|
358
405
|
|
|
406
|
+
def _mirror_steer(self) -> None:
|
|
407
|
+
self.steerView.value = [_public(item) for item in self.steerQueue.value]
|
|
408
|
+
|
|
359
409
|
def _sync_steering(self) -> None:
|
|
360
410
|
ctx = self._run_live
|
|
361
411
|
if ctx is None:
|
|
@@ -381,6 +431,12 @@ class RunManager(Tray):
|
|
|
381
431
|
self._apply(entry, fx)
|
|
382
432
|
return fx
|
|
383
433
|
|
|
434
|
+
def _resolve_stop_waiters(self) -> None:
|
|
435
|
+
waiters, self._stop_waiters = self._stop_waiters, []
|
|
436
|
+
for future in waiters:
|
|
437
|
+
if not future.done():
|
|
438
|
+
future.set_result(None)
|
|
439
|
+
|
|
384
440
|
def _settle_applied(self, fx: _Effects) -> None:
|
|
385
441
|
for future, applied, result in fx.staged_sends:
|
|
386
442
|
applied()
|
|
@@ -417,11 +473,13 @@ class RunManager(Tray):
|
|
|
417
473
|
self._idle_action(fx)
|
|
418
474
|
if self.outcome.value is not None:
|
|
419
475
|
self.outcome.value = None # the follow-on's writes are enlisted ahead
|
|
476
|
+
self._resolve_stop_waiters()
|
|
420
477
|
if self._in_flight() and (
|
|
421
478
|
self._stop_staged or self._staged_rewind is not None
|
|
422
479
|
):
|
|
423
480
|
if self._run_live is not None:
|
|
424
481
|
self._run_live.stop_requested.set()
|
|
482
|
+
self._publish_status()
|
|
425
483
|
self._settle_applied(fx)
|
|
426
484
|
if not self._in_flight():
|
|
427
485
|
self._idle.set()
|
|
@@ -433,8 +491,8 @@ class RunManager(Tray):
|
|
|
433
491
|
if self._staged_rewind is not None:
|
|
434
492
|
self._staged_rewind = None
|
|
435
493
|
self.nextDispatch.value = None
|
|
436
|
-
self.
|
|
437
|
-
|
|
494
|
+
self._stop_waiters.append(entry.future) # the verdict waits for the settle
|
|
495
|
+
elif not entry.future.done():
|
|
438
496
|
entry.future.set_result(None)
|
|
439
497
|
return
|
|
440
498
|
if isinstance(entry, _Continue):
|
|
@@ -494,14 +552,13 @@ class RunManager(Tray):
|
|
|
494
552
|
self._input_requests = [dict(r) for r in requests]
|
|
495
553
|
self._input_answers = {}
|
|
496
554
|
self.inputRequests.value = [dict(r) for r in requests]
|
|
497
|
-
self._set_status("input-required")
|
|
498
555
|
return
|
|
499
556
|
if outcome["dispatchQueue"] and self._pop_dispatchable():
|
|
500
557
|
return
|
|
501
558
|
error = outcome.get("error")
|
|
502
559
|
if error is not None:
|
|
503
560
|
self.error.value = error
|
|
504
|
-
self.
|
|
561
|
+
self.halted.value = "error" if kind == "error" else "stop"
|
|
505
562
|
|
|
506
563
|
# ─── Dispatch and settle ────────────────────────────────
|
|
507
564
|
|
|
@@ -564,10 +621,10 @@ class RunManager(Tray):
|
|
|
564
621
|
self._fresh = True
|
|
565
622
|
self._reclaim(item) # the record owns its sources: consume them behind it
|
|
566
623
|
self.nextDispatch.value = None
|
|
567
|
-
self.stopping.value = None
|
|
568
624
|
self._clear_input()
|
|
569
625
|
self.batchApplied.value = False
|
|
570
626
|
self.error.value = None
|
|
627
|
+
self.halted.value = None
|
|
571
628
|
stripped = [
|
|
572
629
|
{k: v for k, v in message.items() if k != "meta"} for message in messages
|
|
573
630
|
]
|
|
@@ -576,8 +633,7 @@ class RunManager(Tray):
|
|
|
576
633
|
added = [m["id"] for m in stripped if m["id"] not in known]
|
|
577
634
|
if added:
|
|
578
635
|
self.dispatchedIds.value = [*known, *added]
|
|
579
|
-
self.
|
|
580
|
-
self.dispatching.value = record
|
|
636
|
+
self.dispatching.value = _public_record(record)
|
|
581
637
|
self._idle.clear()
|
|
582
638
|
self.kick()
|
|
583
639
|
|
|
@@ -610,7 +666,7 @@ class RunManager(Tray):
|
|
|
610
666
|
def _make_run_ctx(self, item: Any) -> "RunManager.RunContext":
|
|
611
667
|
record = item["record"]
|
|
612
668
|
messages = [
|
|
613
|
-
{k: v for k, v in message.items() if k
|
|
669
|
+
{k: v for k, v in message.items() if k not in ("meta", "~admit")}
|
|
614
670
|
for message in record["messages"]
|
|
615
671
|
]
|
|
616
672
|
ctx = RunManager.RunContext(
|
|
@@ -628,7 +684,7 @@ class RunManager(Tray):
|
|
|
628
684
|
),
|
|
629
685
|
_tray_ctx=self._make_ctx(),
|
|
630
686
|
)
|
|
631
|
-
if self.
|
|
687
|
+
if self._stop_intent() is not None:
|
|
632
688
|
ctx.stop_requested.set()
|
|
633
689
|
return ctx
|
|
634
690
|
|
|
@@ -690,8 +746,10 @@ class RunManager(Tray):
|
|
|
690
746
|
)
|
|
691
747
|
rewind.applied()
|
|
692
748
|
rewind.applied_sent = True
|
|
693
|
-
self.
|
|
749
|
+
self.halted.value = "stop"
|
|
694
750
|
self.outcome.value = None
|
|
751
|
+
self._resolve_stop_waiters()
|
|
752
|
+
self._publish_status()
|
|
695
753
|
self._settle_applied(fx)
|
|
696
754
|
self._idle.set()
|
|
697
755
|
raise # staged intake settled; a cancelled run starts no dispatch
|
|
@@ -710,7 +768,7 @@ class RunManager(Tray):
|
|
|
710
768
|
"error": error,
|
|
711
769
|
}
|
|
712
770
|
self._settle(ctx)
|
|
713
|
-
self.
|
|
771
|
+
self.halted.value = "error"
|
|
714
772
|
self.error.value = error
|
|
715
773
|
self._advance()
|
|
716
774
|
return
|
|
@@ -766,7 +824,6 @@ class RunManager(Tray):
|
|
|
766
824
|
self.dispatchedIds.value = kept
|
|
767
825
|
|
|
768
826
|
def _finish_settle(self) -> None:
|
|
769
|
-
self.stopping.value = None
|
|
770
827
|
self.confirmedTakes.value = 0
|
|
771
828
|
self.batchApplied.value = False
|
|
772
829
|
self._clear_dispatch_state() # commits confirmed takes; drops the run's checkpoint
|
|
@@ -811,7 +868,8 @@ class RunManager(Tray):
|
|
|
811
868
|
self._lane(lane).insert(0, item)
|
|
812
869
|
|
|
813
870
|
def _dispatchable(self, lane: str) -> bool:
|
|
814
|
-
|
|
871
|
+
# an unconsumed outcome means the settle is mid-drain: its follow-on decides
|
|
872
|
+
if self._in_flight() or self._staged_rewind is not None or self.outcome.value is not None:
|
|
815
873
|
return False
|
|
816
874
|
if self._batch:
|
|
817
875
|
return True
|
|
@@ -849,7 +907,7 @@ class RunManager(Tray):
|
|
|
849
907
|
self._batch.append((e.lane, self._stamped(e.message, e.meta)))
|
|
850
908
|
self.dispatching.value = {
|
|
851
909
|
"trigger": self._send_trigger(),
|
|
852
|
-
"messages": [item for _, item in self._batch],
|
|
910
|
+
"messages": [_public(item) for _, item in self._batch],
|
|
853
911
|
}
|
|
854
912
|
|
|
855
913
|
def _dispatch_staged(
|
|
@@ -879,7 +937,6 @@ class RunManager(Tray):
|
|
|
879
937
|
def _stage_rewind(self, rewind: _Rewind) -> None:
|
|
880
938
|
self._staged_rewind = rewind
|
|
881
939
|
if self._in_flight():
|
|
882
|
-
self.stopping.value = {"reason": rewind.type}
|
|
883
940
|
self.nextDispatch.value = self._record_of(
|
|
884
941
|
rewind.type,
|
|
885
942
|
list(rewind.messages),
|
|
@@ -1321,6 +1378,15 @@ class RunManager(Tray):
|
|
|
1321
1378
|
) -> Any:
|
|
1322
1379
|
return await self._send("steerQueue", params, meta, applied)
|
|
1323
1380
|
|
|
1381
|
+
@inbox
|
|
1382
|
+
def admit_steer(self, message: dict[str, Any]) -> None:
|
|
1383
|
+
"""Durably admit one steer message; a ``ctx.dispatch`` target."""
|
|
1384
|
+
if not isinstance(message, dict) or not isinstance(message.get("id"), str):
|
|
1385
|
+
raise TypeError(f"a steer message needs a string id, got {message!r}")
|
|
1386
|
+
self.steerQueue.append(self._stamped(message, message.get("meta")))
|
|
1387
|
+
self._sync_steering()
|
|
1388
|
+
self._schedule(self._advance)
|
|
1389
|
+
|
|
1324
1390
|
async def dequeue(self, params: Any) -> None:
|
|
1325
1391
|
message_id = params.get("messageId") if isinstance(params, dict) else None
|
|
1326
1392
|
if not isinstance(message_id, str):
|
|
@@ -1328,7 +1394,7 @@ class RunManager(Tray):
|
|
|
1328
1394
|
return await self._stage(_Dequeue(message_id))
|
|
1329
1395
|
|
|
1330
1396
|
def _check_rewind_gate(self, command: str) -> None:
|
|
1331
|
-
if self._status()
|
|
1397
|
+
if self._status() in ("running", "stopping"):
|
|
1332
1398
|
if "rewind-during-run" not in self.capabilities:
|
|
1333
1399
|
raise _reject(
|
|
1334
1400
|
"capability-missing",
|
|
@@ -1402,7 +1468,11 @@ class RunManager(Tray):
|
|
|
1402
1468
|
if epoch is not _ABSENT and (isinstance(epoch, bool) or not isinstance(epoch, int)):
|
|
1403
1469
|
raise _reject("invalid-message", "epoch must be an integer")
|
|
1404
1470
|
status = self._status()
|
|
1405
|
-
|
|
1471
|
+
stop_held = self._stop_staged
|
|
1472
|
+
# a rewind-held window still admits a stop: it clears the staged rewind
|
|
1473
|
+
if status not in ("running", "stopping") or (
|
|
1474
|
+
status == "stopping" and stop_held
|
|
1475
|
+
):
|
|
1406
1476
|
raise _reject("wrong-state", f"run/stop is rejected in {status}")
|
|
1407
1477
|
run_id = _run_id_of(params)
|
|
1408
1478
|
if run_id != self.runId.value:
|
|
@@ -1551,13 +1621,17 @@ class RunManager(Tray):
|
|
|
1551
1621
|
added = [item["id"] for item in items if item["id"] not in known]
|
|
1552
1622
|
if added:
|
|
1553
1623
|
manager.dispatchedIds.value = [*known, *added]
|
|
1554
|
-
manager.dispatching.value = {
|
|
1624
|
+
manager.dispatching.value = {
|
|
1625
|
+
"trigger": "steer",
|
|
1626
|
+
"messages": [_public(item) for item in items],
|
|
1627
|
+
}
|
|
1555
1628
|
self._ctx._message_meta.update(
|
|
1556
1629
|
{item["id"]: item.get("meta") for item in items}
|
|
1557
1630
|
)
|
|
1558
1631
|
manager._sync_steering()
|
|
1559
1632
|
return tuple(
|
|
1560
|
-
{k: v for k, v in item.items() if k
|
|
1633
|
+
{k: v for k, v in item.items() if k not in ("meta", "~admit")}
|
|
1634
|
+
for item in items
|
|
1561
1635
|
)
|
|
1562
1636
|
|
|
1563
1637
|
@dataclass(frozen=True, eq=False)
|
|
@@ -1644,6 +1718,7 @@ class RunManager(Tray):
|
|
|
1644
1718
|
|
|
1645
1719
|
def enlist(self, effect: Any) -> None:
|
|
1646
1720
|
"""Enlist an opaque durable effect ordered with the machine's writes."""
|
|
1721
|
+
self._ensure_active()
|
|
1647
1722
|
self._tray_ctx.enlist(effect)
|
|
1648
1723
|
|
|
1649
1724
|
def _ensure_active(self) -> None:
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
"""Contract: ``RunManager.admit_steer`` is the ``ctx.dispatch`` target for
|
|
2
|
+
durable steering — a pooled command admits one steer message exactly once
|
|
3
|
+
across a crash redelivery, and the public seams (``steering.take()``,
|
|
4
|
+
``ctx.messages``, the replicated state view) never carry the admission tag
|
|
5
|
+
the durable queue record keeps for redelivery dedup."""
|
|
6
|
+
|
|
7
|
+
import asyncio
|
|
8
|
+
import copy
|
|
9
|
+
|
|
10
|
+
from pinned import PinnedWorker
|
|
11
|
+
from pinned.testing import MemoryStateStore
|
|
12
|
+
from run_helpers import Script, add, msg
|
|
13
|
+
from statewire_helpers import apply_ops, attach_of, post_frame, statewire_client
|
|
14
|
+
|
|
15
|
+
from harness_sdk import RunManager, run_manager
|
|
16
|
+
from statewire import Statewire, command
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def make_host(script: Script):
|
|
20
|
+
class Host(Statewire):
|
|
21
|
+
gate = asyncio.Event()
|
|
22
|
+
live = None
|
|
23
|
+
|
|
24
|
+
async def lifespan(self):
|
|
25
|
+
type(self).live = self
|
|
26
|
+
self.state = {}
|
|
27
|
+
yield
|
|
28
|
+
|
|
29
|
+
async def get_message_meta(self, message_id):
|
|
30
|
+
return await script.get_message_meta(message_id)
|
|
31
|
+
|
|
32
|
+
@run_manager()
|
|
33
|
+
async def runs(self, ctx):
|
|
34
|
+
return await script.run(ctx)
|
|
35
|
+
|
|
36
|
+
@command("run/steerDurable")
|
|
37
|
+
async def steer_durable(self, params, *, ctx):
|
|
38
|
+
runs = await self.runs.get()
|
|
39
|
+
outcome = await ctx.dispatch(runs.admit_steer, params["message"])
|
|
40
|
+
await type(self).gate.wait()
|
|
41
|
+
return {"outcome": outcome}
|
|
42
|
+
|
|
43
|
+
@command("run/enqueue")
|
|
44
|
+
async def run_enqueue(self, params, *, ctx):
|
|
45
|
+
runs = await self.runs.get()
|
|
46
|
+
return await runs.enqueue(params, applied=ctx.applied)
|
|
47
|
+
|
|
48
|
+
return Host
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def statement(seq, method, params):
|
|
52
|
+
return {"method": method, "params": [params], "seq": seq}
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
async def cmds_until(stream, entry, limit=10):
|
|
56
|
+
seen = []
|
|
57
|
+
for _ in range(limit):
|
|
58
|
+
event = await stream.next_event()
|
|
59
|
+
seen.extend(event.get("cmd", []))
|
|
60
|
+
if entry in seen:
|
|
61
|
+
return seen
|
|
62
|
+
raise AssertionError(f"{entry} never rode the stream: {seen}")
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
async def test_admit_steer_admits_exactly_once_and_the_run_sees_no_tag():
|
|
66
|
+
store = MemoryStateStore()
|
|
67
|
+
script = Script()
|
|
68
|
+
host = make_host(script)
|
|
69
|
+
async with statewire_client(host, worker=PinnedWorker(state_store=store)) as (
|
|
70
|
+
app,
|
|
71
|
+
client,
|
|
72
|
+
):
|
|
73
|
+
async with attach_of(app, {"headers": {}}) as stream:
|
|
74
|
+
await stream.next_event()
|
|
75
|
+
frame = {
|
|
76
|
+
"cmd": [statement(1, "run/steerDurable", {"message": msg("s1")})]
|
|
77
|
+
}
|
|
78
|
+
assert (await post_frame(client, frame)).status_code == 200
|
|
79
|
+
machine = await host.live.runs.get()
|
|
80
|
+
while not machine.steerQueue.value:
|
|
81
|
+
await asyncio.sleep(0)
|
|
82
|
+
await host.live._frontier.wait()
|
|
83
|
+
|
|
84
|
+
script2 = Script()
|
|
85
|
+
host2 = make_host(script2)
|
|
86
|
+
host2.gate.set()
|
|
87
|
+
async with statewire_client(host2, worker=PinnedWorker(state_store=store)) as (
|
|
88
|
+
app,
|
|
89
|
+
client,
|
|
90
|
+
):
|
|
91
|
+
async with attach_of(app, {"headers": {}}) as stream:
|
|
92
|
+
replica: dict = {}
|
|
93
|
+
apply_ops(replica, copy.deepcopy(await stream.next_event()))
|
|
94
|
+
# replay of the redelivered seq re-emits its recorded outcome
|
|
95
|
+
frame = {
|
|
96
|
+
"cmd": [statement(1, "run/steerDurable", {"message": msg("s1")})]
|
|
97
|
+
}
|
|
98
|
+
assert (await post_frame(client, frame)).status_code == 200
|
|
99
|
+
await cmds_until(
|
|
100
|
+
stream,
|
|
101
|
+
{"seq": 1, "type": "result", "payload": {"outcome": "admitted"}},
|
|
102
|
+
)
|
|
103
|
+
machine = await host2.live.runs.get()
|
|
104
|
+
# the durable record keeps the dedup tag; the state view never shows it
|
|
105
|
+
(entry,) = machine.steerQueue.value
|
|
106
|
+
assert entry["~admit"] == ["c1", 1]
|
|
107
|
+
assert replica["runs"][0]["steerQueue"] == [msg("s1")]
|
|
108
|
+
|
|
109
|
+
frame = {"cmd": [statement(2, "run/enqueue", add("m1"))]}
|
|
110
|
+
assert (await post_frame(client, frame)).status_code == 200
|
|
111
|
+
call = await script2.next_call()
|
|
112
|
+
call.applied()
|
|
113
|
+
(taken,) = call.ctx.steering.take()
|
|
114
|
+
assert taken == msg("s1")
|
|
115
|
+
call.applied()
|
|
116
|
+
call.finish(RunManager.Complete())
|
|
117
|
+
await cmds_until(stream, {"seq": 2, "dur": True})
|
|
118
|
+
assert machine.steerQueue.value == []
|
|
@@ -59,27 +59,15 @@ async def test_steer_and_stop_in_one_batch_nets_to_stop():
|
|
|
59
59
|
[("run/steer", add("s1")), ("run/stop", {"runId": await drv.run_id()})]
|
|
60
60
|
)
|
|
61
61
|
assert (await drv.res(first))["type"] == "accepted"
|
|
62
|
-
assert (await drv.res(first + 1))["type"] == "
|
|
62
|
+
assert (await drv.res(first + 1, terminal=False))["type"] == "pending"
|
|
63
63
|
await asyncio.wait_for(call.ctx.stop_requested.wait(), 5)
|
|
64
64
|
call.finish(RunManager.Stop(dispatch_queue=False))
|
|
65
|
+
assert (await drv.res(first + 1))["type"] == "accepted"
|
|
65
66
|
await drv.wait_status("stopped")
|
|
66
67
|
assert queue_ids(drv.replica, "steerQueue") == ["s1"]
|
|
67
68
|
script.no_call()
|
|
68
69
|
|
|
69
70
|
|
|
70
|
-
async def test_stop_settles_at_the_stopping_write():
|
|
71
|
-
script = Script()
|
|
72
|
-
async with run_host(script) as (drv, host):
|
|
73
|
-
drv.post("run/enqueue", add("m1"))
|
|
74
|
-
call = await script.next_call()
|
|
75
|
-
call.applied()
|
|
76
|
-
settled = await drv.stop()
|
|
77
|
-
assert settled["type"] == "accepted"
|
|
78
|
-
assert drv.replica["status"] == "running"
|
|
79
|
-
call.finish(RunManager.Stop(dispatch_queue=False))
|
|
80
|
-
await drv.wait_status("stopped")
|
|
81
|
-
|
|
82
|
-
|
|
83
71
|
class TailScript(Script):
|
|
84
72
|
"""get_message_meta suspends, pushing staging past the handler prefix."""
|
|
85
73
|
|
|
@@ -1,8 +1,10 @@
|
|
|
1
1
|
"""Contract: the durable seams of the run machine. A verdict never outruns the
|
|
2
2
|
admission's flush; a crash redelivers the in-flight dispatch with its checkpoint
|
|
3
|
-
and its epoch; and the rich mid-flight state —
|
|
4
|
-
|
|
5
|
-
half-answered input requests — all hydrate back on the restarted machine.
|
|
3
|
+
and its epoch; and the rich mid-flight state — a staged rewind with its
|
|
4
|
+
derived stopping window, the applied batch, an unconfirmed steering take, and
|
|
5
|
+
half-answered input requests — all hydrate back on the restarted machine. A
|
|
6
|
+
bare stop window is live intent: it drops at the crash and a re-issued stop
|
|
7
|
+
re-opens it on the redelivered run.
|
|
6
8
|
Torn storage segments are swept exhaustively in test_run_crash_sweep.py.
|
|
7
9
|
"""
|
|
8
10
|
|
|
@@ -110,9 +112,11 @@ async def test_a_stale_epoch_stop_rejects_across_a_redelivery():
|
|
|
110
112
|
assert redelivered.ctx.attempts == 2
|
|
111
113
|
await drv2.wait(lambda s: run_of(s)["epoch"] == 1) # deliveries never bump it
|
|
112
114
|
assert_rejected(await drv2.stop({"epoch": 2}, terminal=False), "wrong-state")
|
|
113
|
-
|
|
114
|
-
assert
|
|
115
|
+
pending = await drv2.stop({"epoch": 1}, terminal=False)
|
|
116
|
+
assert pending["type"] == "pending"
|
|
117
|
+
await asyncio.wait_for(redelivered.ctx.stop_requested.wait(), 5)
|
|
115
118
|
redelivered.finish(RunManager.Stop(dispatch_queue=False))
|
|
119
|
+
assert (await drv2.res(pending["seq"]))["type"] == "accepted"
|
|
116
120
|
await drv2.wait_status("stopped")
|
|
117
121
|
|
|
118
122
|
|
|
@@ -159,6 +163,7 @@ async def test_a_staged_rewind_survives_a_crash_in_the_stopping_window():
|
|
|
159
163
|
async with run_host(script2, store=crashed, capabilities=REWIND) as (drv2, host2):
|
|
160
164
|
redelivered = await script2.next_call()
|
|
161
165
|
assert redelivered.ctx.stop_requested.is_set()
|
|
166
|
+
assert drv2.replica["status"] == "stopping" # the window hydrates with its status
|
|
162
167
|
redelivered.finish(RunManager.Stop(dispatch_queue=False))
|
|
163
168
|
rewind = await script2.next_call()
|
|
164
169
|
assert rewind.ctx.trigger == "message-edit"
|
|
@@ -169,6 +174,34 @@ async def test_a_staged_rewind_survives_a_crash_in_the_stopping_window():
|
|
|
169
174
|
await drv2.wait_status("ready")
|
|
170
175
|
|
|
171
176
|
|
|
177
|
+
async def test_a_crash_in_the_stop_window_redelivers_the_run_without_the_window():
|
|
178
|
+
store = MemoryStateStore()
|
|
179
|
+
script = Script()
|
|
180
|
+
async with run_host(script, store=store) as (drv, host):
|
|
181
|
+
drv.post("run/enqueue", add("m1"))
|
|
182
|
+
call = await script.next_call()
|
|
183
|
+
call.applied()
|
|
184
|
+
await drv.stop(terminal=False)
|
|
185
|
+
await asyncio.wait_for(call.ctx.stop_requested.wait(), 5)
|
|
186
|
+
await drv.wait_status("stopping")
|
|
187
|
+
await flush(host)
|
|
188
|
+
crashed = survivors(store)
|
|
189
|
+
|
|
190
|
+
script2 = Script()
|
|
191
|
+
async with run_host(script2, store=crashed) as (drv2, host2):
|
|
192
|
+
assert drv2.replica["status"] == "running"
|
|
193
|
+
assert run_of(drv2.replica)["stopping"] is None
|
|
194
|
+
redelivered = await script2.next_call()
|
|
195
|
+
assert not redelivered.ctx.stop_requested.is_set()
|
|
196
|
+
pending = await drv2.stop(terminal=False) # a re-issued stop re-opens it
|
|
197
|
+
await asyncio.wait_for(redelivered.ctx.stop_requested.wait(), 5)
|
|
198
|
+
await drv2.wait_status("stopping")
|
|
199
|
+
redelivered.finish(RunManager.Stop(dispatch_queue=False))
|
|
200
|
+
assert (await drv2.res(pending["seq"]))["type"] == "accepted"
|
|
201
|
+
await drv2.wait_status("stopped")
|
|
202
|
+
assert run_of(drv2.replica)["stopping"] is None
|
|
203
|
+
|
|
204
|
+
|
|
172
205
|
async def test_an_applied_batch_redelivers_as_a_resume():
|
|
173
206
|
store = MemoryStateStore()
|
|
174
207
|
script = Script()
|
|
@@ -269,3 +302,30 @@ async def test_input_answers_survive_a_restart_mid_answering():
|
|
|
269
302
|
resumed.applied()
|
|
270
303
|
resumed.finish(RunManager.Complete())
|
|
271
304
|
await drv2.wait_status("ready")
|
|
305
|
+
|
|
306
|
+
|
|
307
|
+
async def test_the_stop_reason_survives_a_restart_until_the_next_dispatch():
|
|
308
|
+
store = MemoryStateStore()
|
|
309
|
+
script = Script()
|
|
310
|
+
async with run_host(script, store=store) as (drv, host):
|
|
311
|
+
drv.post("run/enqueue", add("m1"))
|
|
312
|
+
call = await script.next_call()
|
|
313
|
+
call.applied()
|
|
314
|
+
await drv.stop({"reason": "user-stop"}, terminal=False)
|
|
315
|
+
call.finish(RunManager.Stop(dispatch_queue=False))
|
|
316
|
+
await drv.wait_status("stopped")
|
|
317
|
+
assert drv.run["stopReason"] == "user-stop"
|
|
318
|
+
await flush(host)
|
|
319
|
+
crashed = survivors(store)
|
|
320
|
+
|
|
321
|
+
script2 = Script()
|
|
322
|
+
async with run_host(script2, store=crashed) as (drv2, host2):
|
|
323
|
+
assert drv2.replica["status"] == "stopped"
|
|
324
|
+
assert drv2.run["stopReason"] == "user-stop"
|
|
325
|
+
drv2.post("run/enqueue", add("m2"))
|
|
326
|
+
drained = await script2.next_call()
|
|
327
|
+
assert drained.ctx.trigger == "stop-continue"
|
|
328
|
+
await drv2.wait(lambda s: run_of(s)["stopReason"] is None)
|
|
329
|
+
drained.applied()
|
|
330
|
+
drained.finish(RunManager.Complete())
|
|
331
|
+
await drv2.wait_status("ready")
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import asyncio
|
|
2
|
+
|
|
1
3
|
import pytest
|
|
2
4
|
from run_helpers import Script, add, machine, queue_ids, run_host, run_of
|
|
3
5
|
|
|
@@ -21,6 +23,7 @@ async def test_activation_mounts_the_ready_entry():
|
|
|
21
23
|
"nextDispatch": None,
|
|
22
24
|
"inputRequests": None,
|
|
23
25
|
"error": None,
|
|
26
|
+
"stopReason": None,
|
|
24
27
|
}
|
|
25
28
|
],
|
|
26
29
|
}
|
|
@@ -136,6 +139,35 @@ async def test_unknown_capability_rejects_at_decoration():
|
|
|
136
139
|
run_manager(capabilities=("time-travel",))
|
|
137
140
|
|
|
138
141
|
|
|
142
|
+
async def test_a_send_landing_in_the_settle_drain_parks_for_the_follow_on():
|
|
143
|
+
script = Script()
|
|
144
|
+
async with run_host(script) as (drv, host):
|
|
145
|
+
runs = await machine(host.live)
|
|
146
|
+
drv.post("run/enqueue", add("m1"))
|
|
147
|
+
call = await script.next_call()
|
|
148
|
+
call.applied()
|
|
149
|
+
held: list = []
|
|
150
|
+
real, runs._schedule = runs._schedule, held.append
|
|
151
|
+
try:
|
|
152
|
+
task = asyncio.ensure_future(
|
|
153
|
+
runs.enqueue(add("m2", anchor="m1"), applied=lambda: None)
|
|
154
|
+
)
|
|
155
|
+
for _ in range(100):
|
|
156
|
+
if runs._intake:
|
|
157
|
+
break
|
|
158
|
+
await asyncio.sleep(0)
|
|
159
|
+
assert runs._intake, "the enqueue never staged"
|
|
160
|
+
finally:
|
|
161
|
+
runs._schedule = real
|
|
162
|
+
call.finish(RunManager.Complete()) # the enqueue lands in the settle advance
|
|
163
|
+
follow = await script.next_call()
|
|
164
|
+
assert [m["id"] for m in follow.ctx.messages] == ["m2"]
|
|
165
|
+
await asyncio.wait_for(task, 5)
|
|
166
|
+
follow.applied()
|
|
167
|
+
follow.finish(RunManager.Complete())
|
|
168
|
+
await drv.wait_status("ready")
|
|
169
|
+
|
|
170
|
+
|
|
139
171
|
async def test_idle_spans_the_drain_chain():
|
|
140
172
|
script = Script()
|
|
141
173
|
async with run_host(script) as (drv, host):
|
|
@@ -4,9 +4,11 @@ A SinglesStore lands writes one key at a time and dies for good after its
|
|
|
4
4
|
limit; each scenario is swept over every crash point. Whatever prefix
|
|
5
5
|
survived, the restarted machine must satisfy: an accepted message or edit is
|
|
6
6
|
either consumed by a durably settled dispatch or delivered exactly once after
|
|
7
|
-
the restart; an accepted input answer reaches a run's input_outcomes;
|
|
8
|
-
|
|
9
|
-
|
|
7
|
+
the restart; an accepted input answer reaches a run's input_outcomes; the
|
|
8
|
+
published status matches the derivation facts (a recorded dispatch reads
|
|
9
|
+
running/stopping, "stopped"/"error" carry the halted fact, "input-required"
|
|
10
|
+
carries inputRequests, the stop window rides status "stopping"); and a fresh
|
|
11
|
+
send still reaches a run.
|
|
10
12
|
|
|
11
13
|
The redelivery lands on the correct arm of the run callback: a batch not yet
|
|
12
14
|
durably applied re-presents in dispatch mode with the full record, a durably
|
|
@@ -264,8 +266,20 @@ async def flush(runs) -> None:
|
|
|
264
266
|
|
|
265
267
|
|
|
266
268
|
def check_wire(runs) -> None:
|
|
267
|
-
|
|
269
|
+
status = runs.status.value
|
|
270
|
+
in_flight = runs.outbox.value is not None
|
|
271
|
+
assert (status in ("running", "stopping")) == in_flight, (
|
|
272
|
+
f"status {status!r} contradicts the dispatch record"
|
|
273
|
+
)
|
|
274
|
+
if status == "input-required":
|
|
268
275
|
assert runs.inputRequests.value, "input-required without inputRequests"
|
|
276
|
+
if status in ("stopped", "error"):
|
|
277
|
+
expected = "stop" if status == "stopped" else "error"
|
|
278
|
+
assert runs.halted.value == expected, f"{status} without its halted fact"
|
|
279
|
+
if status == "stopping":
|
|
280
|
+
assert runs.stopping.value is not None, "stopping without an open stop window"
|
|
281
|
+
else:
|
|
282
|
+
assert runs.stopping.value is None, "an open stop window without its status"
|
|
269
283
|
|
|
270
284
|
|
|
271
285
|
async def settle_call(runs, call, seen: set[str]) -> None:
|
|
@@ -564,6 +578,25 @@ async def verify_rewind(runs, script, trace, seen, calls):
|
|
|
564
578
|
await probe(runs, script, seen, "a1")
|
|
565
579
|
|
|
566
580
|
|
|
581
|
+
async def drive_stop(runs, script, trace, spawn):
|
|
582
|
+
send = spawn(runs.enqueue(add("m1"), applied=lambda: None))
|
|
583
|
+
call = await script.next_call(2)
|
|
584
|
+
await step(send)
|
|
585
|
+
trace.accepted.add("m1")
|
|
586
|
+
apply_batch(call, "batch")
|
|
587
|
+
stop = spawn(runs.stop({"reason": "user-stop"}, applied=lambda: None))
|
|
588
|
+
await step(call.ctx.stop_requested.wait())
|
|
589
|
+
call.finish(RunManager.Stop(dispatch_queue=False))
|
|
590
|
+
await step(stop)
|
|
591
|
+
await flush(runs)
|
|
592
|
+
assert runs.status.value == "stopped"
|
|
593
|
+
assert runs.stopReason.value == "user-stop"
|
|
594
|
+
|
|
595
|
+
|
|
596
|
+
async def verify_stop(runs, script, trace, seen, calls):
|
|
597
|
+
await probe(runs, script, seen, None)
|
|
598
|
+
|
|
599
|
+
|
|
567
600
|
async def drive_cancel(runs, script, trace, spawn):
|
|
568
601
|
send = spawn(runs.enqueue(add("m1"), applied=lambda: None))
|
|
569
602
|
call = await script.next_call(2)
|
|
@@ -614,6 +647,8 @@ test_rewind_promotion_survives_every_crash_point = sweep(
|
|
|
614
647
|
drive_rewind, verify_rewind, capabilities=REWIND, thread=True
|
|
615
648
|
)
|
|
616
649
|
|
|
650
|
+
test_stop_settle_survives_every_crash_point = sweep(drive_stop, verify_stop)
|
|
651
|
+
|
|
617
652
|
test_cancel_with_in_drain_rewind_survives_every_crash_point = sweep(
|
|
618
653
|
drive_cancel, verify_cancel, capabilities=REWIND, thread=True
|
|
619
654
|
)
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import asyncio
|
|
2
2
|
|
|
3
3
|
import pytest
|
|
4
|
-
from run_helpers import Script, add, assert_rejected, run_host, run_of
|
|
4
|
+
from run_helpers import Script, add, assert_rejected, machine, run_host, run_of
|
|
5
5
|
|
|
6
6
|
from harness_sdk import RunManager
|
|
7
7
|
|
|
@@ -15,7 +15,7 @@ async def test_stop_is_not_optimistic():
|
|
|
15
15
|
pending = await drv.stop(terminal=False)
|
|
16
16
|
assert pending["type"] == "pending"
|
|
17
17
|
await asyncio.wait_for(call.ctx.stop_requested.wait(), 5)
|
|
18
|
-
|
|
18
|
+
await drv.wait_status("stopping")
|
|
19
19
|
call.finish(RunManager.Stop(dispatch_queue=False))
|
|
20
20
|
settled = await drv.res(pending["seq"])
|
|
21
21
|
assert settled["type"] == "accepted"
|
|
@@ -45,12 +45,15 @@ async def test_stop_reason_is_visible_on_the_start_context():
|
|
|
45
45
|
call = await script.next_call()
|
|
46
46
|
call.applied()
|
|
47
47
|
assert call.ctx.stop_reason is None
|
|
48
|
+
await drv.run_id()
|
|
49
|
+
assert drv.run["stopReason"] is None
|
|
48
50
|
pending = await drv.stop({"reason": "user-stop"}, terminal=False)
|
|
49
51
|
await asyncio.wait_for(call.ctx.stop_requested.wait(), 5)
|
|
50
52
|
assert call.ctx.stop_reason == "user-stop"
|
|
51
53
|
call.finish(RunManager.Stop(dispatch_queue=False))
|
|
52
54
|
assert (await drv.res(pending["seq"]))["type"] == "accepted"
|
|
53
55
|
await drv.wait_status("stopped")
|
|
56
|
+
assert drv.run["stopReason"] == "user-stop"
|
|
54
57
|
|
|
55
58
|
|
|
56
59
|
async def test_stop_reason_resets_for_the_next_run():
|
|
@@ -64,6 +67,7 @@ async def test_stop_reason_resets_for_the_next_run():
|
|
|
64
67
|
call.finish(RunManager.Stop(dispatch_queue=True))
|
|
65
68
|
drain = await script.next_call()
|
|
66
69
|
assert drain.ctx.stop_reason is None
|
|
70
|
+
await drv.wait(lambda s: run_of(s)["stopReason"] is None)
|
|
67
71
|
|
|
68
72
|
|
|
69
73
|
async def test_stop_rejects_malformed_params():
|
|
@@ -195,6 +199,52 @@ async def test_stop_settles_even_when_the_run_completes_normally():
|
|
|
195
199
|
await drv.wait_status("ready")
|
|
196
200
|
|
|
197
201
|
|
|
202
|
+
async def test_stop_verdict_awaits_the_settle_and_the_checkpoint_drain():
|
|
203
|
+
script = Script()
|
|
204
|
+
async with run_host(script) as (drv, host):
|
|
205
|
+
drv.post("run/enqueue", add("m1"))
|
|
206
|
+
call = await script.next_call()
|
|
207
|
+
call.applied()
|
|
208
|
+
release = asyncio.Event()
|
|
209
|
+
durable = asyncio.Event()
|
|
210
|
+
|
|
211
|
+
async def checkpoint():
|
|
212
|
+
await release.wait()
|
|
213
|
+
durable.set()
|
|
214
|
+
|
|
215
|
+
call.ctx.enlist(checkpoint())
|
|
216
|
+
runs = await machine(host.live)
|
|
217
|
+
verdict = asyncio.ensure_future(
|
|
218
|
+
runs.stop({"runId": await drv.run_id()}, applied=lambda: None)
|
|
219
|
+
)
|
|
220
|
+
await asyncio.wait_for(call.ctx.stop_requested.wait(), 5)
|
|
221
|
+
await asyncio.sleep(0.02)
|
|
222
|
+
assert not verdict.done() # the run is still winding down
|
|
223
|
+
call.finish(RunManager.Stop(dispatch_queue=False))
|
|
224
|
+
await asyncio.wait_for(runs.idle.wait(), 5)
|
|
225
|
+
assert runs.status.value == "stopped"
|
|
226
|
+
await asyncio.sleep(0.02)
|
|
227
|
+
assert not verdict.done() # the checkpoint drain is still open
|
|
228
|
+
release.set()
|
|
229
|
+
assert await asyncio.wait_for(verdict, 5) is None
|
|
230
|
+
assert durable.is_set() # every enlisted checkpoint durable before the verdict
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
async def test_a_late_enlist_after_the_run_settled_raises():
|
|
234
|
+
script = Script()
|
|
235
|
+
async with run_host(script) as (drv, host):
|
|
236
|
+
drv.post("run/enqueue", add("m1"))
|
|
237
|
+
call = await script.next_call()
|
|
238
|
+
await drv.run_id()
|
|
239
|
+
call.applied()
|
|
240
|
+
call.finish(RunManager.Complete())
|
|
241
|
+
await drv.wait(lambda s: run_of(s)["runId"] is None)
|
|
242
|
+
straggler = asyncio.sleep(0)
|
|
243
|
+
with pytest.raises(RuntimeError, match="settled"):
|
|
244
|
+
call.ctx.enlist(straggler)
|
|
245
|
+
straggler.close()
|
|
246
|
+
|
|
247
|
+
|
|
198
248
|
@pytest.mark.parametrize(
|
|
199
249
|
"end,continue_type",
|
|
200
250
|
[("error", "error-continue"), ("stopped", "stop-continue")],
|
|
@@ -1,10 +1,12 @@
|
|
|
1
1
|
"""Contract: the stopping window. Three openers — run/stop, mid-run
|
|
2
|
-
edit/reload, and the enqueue/steer edit-of-dispatched form —
|
|
3
|
-
``stopping: {reason}`` (plus the staged rewind as ``nextDispatch``) and
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
2
|
+
edit/reload, and the enqueue/steer edit-of-dispatched form — publish
|
|
3
|
+
``stopping: {reason}`` (plus the staged rewind as ``nextDispatch``) and flip
|
|
4
|
+
``status`` to "stopping" in the same envelope. Rewind commands settle
|
|
5
|
+
accepted with that write; the run/stop verdict follows the run's settle. On
|
|
6
|
+
run end one envelope promotes ``nextDispatch`` into ``dispatching`` and
|
|
7
|
+
clears the window; a later stop/rewind replaces the staged dispatch, whose
|
|
8
|
+
command already settled at its admission write. A second run/stop inside the
|
|
9
|
+
window rejects wrong-state."""
|
|
8
10
|
|
|
9
11
|
import asyncio
|
|
10
12
|
import copy
|
|
@@ -13,6 +15,7 @@ import pytest
|
|
|
13
15
|
from run_helpers import (
|
|
14
16
|
Script,
|
|
15
17
|
add,
|
|
18
|
+
assert_rejected,
|
|
16
19
|
edit_params,
|
|
17
20
|
msg,
|
|
18
21
|
queue_ids,
|
|
@@ -57,26 +60,40 @@ async def start_run(drv, script, *, parent="p0"):
|
|
|
57
60
|
return call
|
|
58
61
|
|
|
59
62
|
|
|
60
|
-
async def
|
|
63
|
+
async def test_stop_opens_the_window_and_flips_status_to_stopping():
|
|
61
64
|
script = Script()
|
|
62
65
|
async with run_host(script) as (drv, host):
|
|
63
66
|
call = await start_run(drv, script)
|
|
64
|
-
|
|
65
|
-
assert
|
|
66
|
-
|
|
67
|
+
pending = await drv.stop(terminal=False)
|
|
68
|
+
assert pending["type"] == "pending"
|
|
69
|
+
await drv.wait_status("stopping")
|
|
67
70
|
assert drv.run["stopping"] == {"reason": "stop"}
|
|
68
71
|
assert drv.run["nextDispatch"] is None
|
|
69
72
|
opened = next(
|
|
70
|
-
state
|
|
71
|
-
for env, state in states(drv)
|
|
72
|
-
if any(r["seq"] == settled["seq"] for r in env.get("res", []))
|
|
73
|
+
state for _, state in states(drv) if state.get("status") == "stopping"
|
|
73
74
|
)
|
|
74
75
|
assert window_of(opened) == ({"reason": "stop"}, None)
|
|
75
76
|
call.finish(RunManager.Stop(dispatch_queue=False))
|
|
77
|
+
assert (await drv.res(pending["seq"]))["type"] == "accepted"
|
|
76
78
|
await drv.wait_status("stopped")
|
|
77
79
|
assert drv.run["stopping"] is None
|
|
78
80
|
|
|
79
81
|
|
|
82
|
+
async def test_a_second_stop_inside_the_window_rejects_wrong_state():
|
|
83
|
+
script = Script()
|
|
84
|
+
async with run_host(script) as (drv, host):
|
|
85
|
+
call = await start_run(drv, script)
|
|
86
|
+
run_id = await drv.run_id()
|
|
87
|
+
pending = await drv.stop(terminal=False)
|
|
88
|
+
await drv.wait_status("stopping")
|
|
89
|
+
assert_rejected(
|
|
90
|
+
await drv.command("run/stop", {"runId": run_id}), "wrong-state"
|
|
91
|
+
)
|
|
92
|
+
call.finish(RunManager.Stop(dispatch_queue=False))
|
|
93
|
+
assert (await drv.res(pending["seq"]))["type"] == "accepted"
|
|
94
|
+
await drv.wait_status("stopped")
|
|
95
|
+
|
|
96
|
+
|
|
80
97
|
async def test_mid_run_edit_stages_the_window_and_promotes_in_one_envelope():
|
|
81
98
|
script = Script()
|
|
82
99
|
async with run_host(script, capabilities=("rewind", "rewind-during-run")) as (
|
|
@@ -87,7 +104,7 @@ async def test_mid_run_edit_stages_the_window_and_promotes_in_one_envelope():
|
|
|
87
104
|
seq = drv.post("run/edit", edit_params("m1", msg("m2"), anchor="p0"))
|
|
88
105
|
await asyncio.wait_for(call.ctx.stop_requested.wait(), 5)
|
|
89
106
|
await drv.wait(lambda s: run_of(s)["stopping"] is not None)
|
|
90
|
-
assert drv.replica["status"] == "
|
|
107
|
+
assert drv.replica["status"] == "stopping"
|
|
91
108
|
assert drv.run["stopping"] == {"reason": "message-edit"}
|
|
92
109
|
staged = drv.run["nextDispatch"]
|
|
93
110
|
assert staged == {
|
|
@@ -161,7 +178,7 @@ async def test_edit_of_dispatched_message_opens_the_window(command):
|
|
|
161
178
|
call = await start_run(drv, script)
|
|
162
179
|
seq = drv.post(command, target(message=msg("m1", "edited")))
|
|
163
180
|
await drv.wait(lambda s: run_of(s)["stopping"] is not None)
|
|
164
|
-
assert drv.replica["status"] == "
|
|
181
|
+
assert drv.replica["status"] == "stopping"
|
|
165
182
|
assert drv.run["stopping"] == {"reason": "message-edit"}
|
|
166
183
|
assert drv.run["nextDispatch"] == {
|
|
167
184
|
"trigger": "message-edit",
|
|
@@ -210,11 +227,12 @@ async def test_stop_clears_a_staged_rewind():
|
|
|
210
227
|
seq = drv.post("run/edit", edit_params("m1", msg("m2"), anchor="p0"))
|
|
211
228
|
await drv.wait(lambda s: run_of(s)["nextDispatch"] is not None)
|
|
212
229
|
assert (await drv.res(seq))["type"] == "accepted"
|
|
213
|
-
|
|
214
|
-
assert
|
|
230
|
+
pending = await drv.stop(terminal=False)
|
|
231
|
+
assert pending["type"] == "pending"
|
|
215
232
|
await drv.wait(lambda s: run_of(s)["stopping"] == {"reason": "stop"})
|
|
216
233
|
assert drv.run["nextDispatch"] is None
|
|
217
234
|
call.finish(RunManager.Stop(dispatch_queue=False))
|
|
235
|
+
assert (await drv.res(pending["seq"]))["type"] == "accepted"
|
|
218
236
|
await drv.wait_status("stopped")
|
|
219
237
|
script.no_call()
|
|
220
238
|
|
|
@@ -253,6 +271,7 @@ ENTRY_KEYS = {
|
|
|
253
271
|
"nextDispatch",
|
|
254
272
|
"inputRequests",
|
|
255
273
|
"error",
|
|
274
|
+
"stopReason",
|
|
256
275
|
}
|
|
257
276
|
|
|
258
277
|
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|