harness-sdk-python 0.18.0__tar.gz → 0.19.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (34) hide show
  1. {harness_sdk_python-0.18.0 → harness_sdk_python-0.19.0}/PKG-INFO +3 -3
  2. {harness_sdk_python-0.18.0 → harness_sdk_python-0.19.0}/pyproject.toml +2 -2
  3. {harness_sdk_python-0.18.0 → harness_sdk_python-0.19.0}/src/harness_sdk/run_manager.py +66 -14
  4. harness_sdk_python-0.19.0/tests/test_admit_steer.py +118 -0
  5. {harness_sdk_python-0.18.0 → harness_sdk_python-0.19.0}/tests/test_batches.py +2 -14
  6. {harness_sdk_python-0.18.0 → harness_sdk_python-0.19.0}/tests/test_durability.py +56 -2
  7. {harness_sdk_python-0.18.0 → harness_sdk_python-0.19.0}/tests/test_outcomes.py +1 -0
  8. {harness_sdk_python-0.18.0 → harness_sdk_python-0.19.0}/tests/test_restore.py +1 -0
  9. {harness_sdk_python-0.18.0 → harness_sdk_python-0.19.0}/tests/test_run_crash_sweep.py +6 -2
  10. {harness_sdk_python-0.18.0 → harness_sdk_python-0.19.0}/tests/test_stop_continue.py +52 -2
  11. {harness_sdk_python-0.18.0 → harness_sdk_python-0.19.0}/tests/test_stopping_window.py +35 -16
  12. {harness_sdk_python-0.18.0 → harness_sdk_python-0.19.0}/.gitignore +0 -0
  13. {harness_sdk_python-0.18.0 → harness_sdk_python-0.19.0}/README.md +0 -0
  14. {harness_sdk_python-0.18.0 → harness_sdk_python-0.19.0}/examples/__init__.py +0 -0
  15. {harness_sdk_python-0.18.0 → harness_sdk_python-0.19.0}/examples/runs_app.py +0 -0
  16. {harness_sdk_python-0.18.0 → harness_sdk_python-0.19.0}/src/harness_sdk/__init__.py +0 -0
  17. {harness_sdk_python-0.18.0 → harness_sdk_python-0.19.0}/src/harness_sdk/fenced_postgres.py +0 -0
  18. {harness_sdk_python-0.18.0 → harness_sdk_python-0.19.0}/src/harness_sdk/linear_thread.py +0 -0
  19. {harness_sdk_python-0.18.0 → harness_sdk_python-0.19.0}/tests/run_helpers.py +0 -0
  20. {harness_sdk_python-0.18.0 → harness_sdk_python-0.19.0}/tests/test_applied_visibility.py +0 -0
  21. {harness_sdk_python-0.18.0 → harness_sdk_python-0.19.0}/tests/test_branch_anchor.py +0 -0
  22. {harness_sdk_python-0.18.0 → harness_sdk_python-0.19.0}/tests/test_dispatching.py +0 -0
  23. {harness_sdk_python-0.18.0 → harness_sdk_python-0.19.0}/tests/test_edit_dispatched.py +0 -0
  24. {harness_sdk_python-0.18.0 → harness_sdk_python-0.19.0}/tests/test_edit_reload.py +0 -0
  25. {harness_sdk_python-0.18.0 → harness_sdk_python-0.19.0}/tests/test_enqueue.py +0 -0
  26. {harness_sdk_python-0.18.0 → harness_sdk_python-0.19.0}/tests/test_fenced_postgres.py +0 -0
  27. {harness_sdk_python-0.18.0 → harness_sdk_python-0.19.0}/tests/test_input_required.py +0 -0
  28. {harness_sdk_python-0.18.0 → harness_sdk_python-0.19.0}/tests/test_linear_thread.py +0 -0
  29. {harness_sdk_python-0.18.0 → harness_sdk_python-0.19.0}/tests/test_meta.py +0 -0
  30. {harness_sdk_python-0.18.0 → harness_sdk_python-0.19.0}/tests/test_placement.py +0 -0
  31. {harness_sdk_python-0.18.0 → harness_sdk_python-0.19.0}/tests/test_prepare_hooks.py +0 -0
  32. {harness_sdk_python-0.18.0 → harness_sdk_python-0.19.0}/tests/test_rewind_during_run.py +0 -0
  33. {harness_sdk_python-0.18.0 → harness_sdk_python-0.19.0}/tests/test_settle.py +0 -0
  34. {harness_sdk_python-0.18.0 → harness_sdk_python-0.19.0}/tests/test_steer.py +0 -0
@@ -1,12 +1,12 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: harness-sdk-python
3
- Version: 0.18.0
3
+ Version: 0.19.0
4
4
  Summary: RunManager: the harness-sdk runs subsystem for Python Statewire hosts
5
5
  Project-URL: Repository, https://github.com/assistant-ui/harness-sdk
6
6
  License-Expression: MIT
7
7
  Requires-Python: <4.0,>=3.12
8
- Requires-Dist: pinned<0.13,>=0.12.0
9
- Requires-Dist: statewire<0.10,>=0.9.2
8
+ Requires-Dist: pinned<0.14,>=0.13.0
9
+ Requires-Dist: statewire<0.11,>=0.10.0
10
10
  Provides-Extra: deepagents
11
11
  Requires-Dist: deepagents>=0.6.12; extra == 'deepagents'
12
12
  Requires-Dist: langchain-core>=0.3; extra == 'deepagents'
@@ -1,11 +1,11 @@
1
1
  [project]
2
2
  name = "harness-sdk-python"
3
- version = "0.18.0"
3
+ version = "0.19.0"
4
4
  description = "RunManager: the harness-sdk runs subsystem for Python Statewire hosts"
5
5
  readme = "README.md"
6
6
  license = "MIT"
7
7
  requires-python = ">=3.12,<4.0"
8
- dependencies = ["statewire>=0.9.2,<0.10", "pinned>=0.12.0,<0.13"]
8
+ dependencies = ["statewire>=0.10.0,<0.11", "pinned>=0.13.0,<0.14"]
9
9
 
10
10
  [project.optional-dependencies]
11
11
  postgres = ["langgraph-checkpoint-postgres>=2.0.0"]
@@ -29,8 +29,11 @@ own continuation state (``ctx.state`` or an external checkpointer).
29
29
  ``epoch`` counts the entry's dispatches (not redeliveries); a ``run/stop``
30
30
  carrying an ``epoch`` other than the live one rejects. A mid-run stop or
31
31
  rewind opens a stopping window (``stopping: {reason}`` plus, for rewinds, the
32
- staged dispatch as ``nextDispatch``); on run end one envelope promotes
33
- ``nextDispatch`` into ``dispatching`` and clears the window.
32
+ staged dispatch as ``nextDispatch``) and ``status`` reads ``stopping`` until
33
+ the run settles; the ``run/stop`` verdict follows that settle and its flush,
34
+ so a settled stop covers every durable effect the runner enlisted. On run end
35
+ one envelope promotes ``nextDispatch`` into ``dispatching`` and clears the
36
+ window.
34
37
  """
35
38
 
36
39
  import asyncio
@@ -38,7 +41,7 @@ import logging
38
41
  from dataclasses import dataclass, field
39
42
  from typing import Any, AsyncIterator, Callable, Coroutine, Iterable, Protocol
40
43
 
41
- from pinned.trays import Tray, TrayDescriptor, TrayCtx
44
+ from pinned.trays import Tray, TrayDescriptor, TrayCtx, inbox
42
45
  from pinned.traystore import persistent, persistent_outbox, persistent_queue
43
46
  from statepatch import observable
44
47
  from statewire import StatewireReject
@@ -78,6 +81,15 @@ def _reject(reason: str, message: str) -> StatewireReject:
78
81
  return StatewireReject(message, payload={"reason": reason})
79
82
 
80
83
 
84
+ def _public(message: dict[str, Any]) -> dict[str, Any]:
85
+ """The message without its admission tag; the wire state is public."""
86
+ return {k: v for k, v in message.items() if k != "~admit"}
87
+
88
+
89
+ def _public_record(record: dict[str, Any]) -> dict[str, Any]:
90
+ return {**record, "messages": [_public(m) for m in record["messages"]]}
91
+
92
+
81
93
  def _run_id_of(params: Any) -> str:
82
94
  run_id = params.get("runId") if isinstance(params, dict) else None
83
95
  if not isinstance(run_id, str) or run_id == "":
@@ -204,6 +216,7 @@ class RunManager(Tray):
204
216
  stopReason = persistent(None)
205
217
  dispatchedIds = persistent([])
206
218
  dispatching = observable(None) # view: the unconfirmed batch record
219
+ steerView = observable([]) # view: the steer lane without admission tags
207
220
 
208
221
  state = observable(
209
222
  {
@@ -211,12 +224,13 @@ class RunManager(Tray):
211
224
  "status": status,
212
225
  "epoch": epoch,
213
226
  "queue": queue,
214
- "steerQueue": steerQueue,
227
+ "steerQueue": steerView,
215
228
  "dispatching": dispatching,
216
229
  "stopping": stopping,
217
230
  "nextDispatch": nextDispatch,
218
231
  "inputRequests": inputRequests,
219
232
  "error": error,
233
+ "stopReason": stopReason,
220
234
  }
221
235
  )
222
236
 
@@ -225,6 +239,7 @@ class RunManager(Tray):
225
239
  self._intake: list[Any] = []
226
240
  self._outcome: Any = None
227
241
  self._stop_staged = False
242
+ self._stop_waiters: list["asyncio.Future[Any]"] = []
228
243
  self._staged_rewind: _Rewind | None = None
229
244
  self._batch: list[tuple[str, dict[str, Any]]] = []
230
245
  self._pending_takes = 0
@@ -245,7 +260,10 @@ class RunManager(Tray):
245
260
  self._thread = api
246
261
  self._schedule = api.schedule
247
262
  self._hydrate_machine()
263
+ self._mirror_steer()
264
+ unsubscribe = self.steerQueue.subscribe(lambda _op: self._mirror_steer())
248
265
  yield
266
+ unsubscribe()
249
267
 
250
268
  class Thread(Protocol):
251
269
  """The thread projection RunManager reads; the host owns the tree."""
@@ -277,14 +295,16 @@ class RunManager(Tray):
277
295
  record = recorded["item"]["record"]
278
296
  lanes = recorded["item"]["lanes"]
279
297
  self._batch = list(zip(lanes, record["messages"]))
280
- self.dispatching.value = record
298
+ self.dispatching.value = _public_record(record)
281
299
  else:
282
300
  self.dispatching.value = None
283
301
  if recorded is not None:
284
302
  # heal a dispatch torn after its outbox write landed
285
303
  self._clear_input()
286
304
  self.error.value = None
287
- self._set_status("running")
305
+ self._set_status(
306
+ "stopping" if self.stopping.value is not None else "running"
307
+ )
288
308
  staged = self.nextDispatch.value
289
309
  if (
290
310
  staged is not None
@@ -356,6 +376,9 @@ class RunManager(Tray):
356
376
  def _is_dispatching(self, message_id: str) -> bool:
357
377
  return any(item["id"] == message_id for _, item in self._batch)
358
378
 
379
+ def _mirror_steer(self) -> None:
380
+ self.steerView.value = [_public(item) for item in self.steerQueue.value]
381
+
359
382
  def _sync_steering(self) -> None:
360
383
  ctx = self._run_live
361
384
  if ctx is None:
@@ -381,6 +404,12 @@ class RunManager(Tray):
381
404
  self._apply(entry, fx)
382
405
  return fx
383
406
 
407
+ def _resolve_stop_waiters(self) -> None:
408
+ waiters, self._stop_waiters = self._stop_waiters, []
409
+ for future in waiters:
410
+ if not future.done():
411
+ future.set_result(None)
412
+
384
413
  def _settle_applied(self, fx: _Effects) -> None:
385
414
  for future, applied, result in fx.staged_sends:
386
415
  applied()
@@ -417,6 +446,7 @@ class RunManager(Tray):
417
446
  self._idle_action(fx)
418
447
  if self.outcome.value is not None:
419
448
  self.outcome.value = None # the follow-on's writes are enlisted ahead
449
+ self._resolve_stop_waiters()
420
450
  if self._in_flight() and (
421
451
  self._stop_staged or self._staged_rewind is not None
422
452
  ):
@@ -434,7 +464,9 @@ class RunManager(Tray):
434
464
  self._staged_rewind = None
435
465
  self.nextDispatch.value = None
436
466
  self.stopping.value = {"reason": "stop"}
437
- if not entry.future.done():
467
+ self._set_status("stopping")
468
+ self._stop_waiters.append(entry.future) # the verdict waits for the settle
469
+ elif not entry.future.done():
438
470
  entry.future.set_result(None)
439
471
  return
440
472
  if isinstance(entry, _Continue):
@@ -577,7 +609,7 @@ class RunManager(Tray):
577
609
  if added:
578
610
  self.dispatchedIds.value = [*known, *added]
579
611
  self._set_status("running")
580
- self.dispatching.value = record
612
+ self.dispatching.value = _public_record(record)
581
613
  self._idle.clear()
582
614
  self.kick()
583
615
 
@@ -610,7 +642,7 @@ class RunManager(Tray):
610
642
  def _make_run_ctx(self, item: Any) -> "RunManager.RunContext":
611
643
  record = item["record"]
612
644
  messages = [
613
- {k: v for k, v in message.items() if k != "meta"}
645
+ {k: v for k, v in message.items() if k not in ("meta", "~admit")}
614
646
  for message in record["messages"]
615
647
  ]
616
648
  ctx = RunManager.RunContext(
@@ -692,6 +724,7 @@ class RunManager(Tray):
692
724
  rewind.applied_sent = True
693
725
  self._set_status("stopped")
694
726
  self.outcome.value = None
727
+ self._resolve_stop_waiters()
695
728
  self._settle_applied(fx)
696
729
  self._idle.set()
697
730
  raise # staged intake settled; a cancelled run starts no dispatch
@@ -849,7 +882,7 @@ class RunManager(Tray):
849
882
  self._batch.append((e.lane, self._stamped(e.message, e.meta)))
850
883
  self.dispatching.value = {
851
884
  "trigger": self._send_trigger(),
852
- "messages": [item for _, item in self._batch],
885
+ "messages": [_public(item) for _, item in self._batch],
853
886
  }
854
887
 
855
888
  def _dispatch_staged(
@@ -880,6 +913,7 @@ class RunManager(Tray):
880
913
  self._staged_rewind = rewind
881
914
  if self._in_flight():
882
915
  self.stopping.value = {"reason": rewind.type}
916
+ self._set_status("stopping")
883
917
  self.nextDispatch.value = self._record_of(
884
918
  rewind.type,
885
919
  list(rewind.messages),
@@ -1321,6 +1355,15 @@ class RunManager(Tray):
1321
1355
  ) -> Any:
1322
1356
  return await self._send("steerQueue", params, meta, applied)
1323
1357
 
1358
+ @inbox
1359
+ def admit_steer(self, message: dict[str, Any]) -> None:
1360
+ """Durably admit one steer message; a ``ctx.dispatch`` target."""
1361
+ if not isinstance(message, dict) or not isinstance(message.get("id"), str):
1362
+ raise TypeError(f"a steer message needs a string id, got {message!r}")
1363
+ self.steerQueue.append(self._stamped(message, message.get("meta")))
1364
+ self._sync_steering()
1365
+ self._schedule(self._advance)
1366
+
1324
1367
  async def dequeue(self, params: Any) -> None:
1325
1368
  message_id = params.get("messageId") if isinstance(params, dict) else None
1326
1369
  if not isinstance(message_id, str):
@@ -1328,7 +1371,7 @@ class RunManager(Tray):
1328
1371
  return await self._stage(_Dequeue(message_id))
1329
1372
 
1330
1373
  def _check_rewind_gate(self, command: str) -> None:
1331
- if self._status() == "running":
1374
+ if self._status() in ("running", "stopping"):
1332
1375
  if "rewind-during-run" not in self.capabilities:
1333
1376
  raise _reject(
1334
1377
  "capability-missing",
@@ -1402,7 +1445,11 @@ class RunManager(Tray):
1402
1445
  if epoch is not _ABSENT and (isinstance(epoch, bool) or not isinstance(epoch, int)):
1403
1446
  raise _reject("invalid-message", "epoch must be an integer")
1404
1447
  status = self._status()
1405
- if status != "running":
1448
+ stop_held = (self.stopping.value or {}).get("reason") == "stop"
1449
+ # a rewind-held window still admits a stop: it clears the staged rewind
1450
+ if status not in ("running", "stopping") or (
1451
+ status == "stopping" and stop_held
1452
+ ):
1406
1453
  raise _reject("wrong-state", f"run/stop is rejected in {status}")
1407
1454
  run_id = _run_id_of(params)
1408
1455
  if run_id != self.runId.value:
@@ -1551,13 +1598,17 @@ class RunManager(Tray):
1551
1598
  added = [item["id"] for item in items if item["id"] not in known]
1552
1599
  if added:
1553
1600
  manager.dispatchedIds.value = [*known, *added]
1554
- manager.dispatching.value = {"trigger": "steer", "messages": items}
1601
+ manager.dispatching.value = {
1602
+ "trigger": "steer",
1603
+ "messages": [_public(item) for item in items],
1604
+ }
1555
1605
  self._ctx._message_meta.update(
1556
1606
  {item["id"]: item.get("meta") for item in items}
1557
1607
  )
1558
1608
  manager._sync_steering()
1559
1609
  return tuple(
1560
- {k: v for k, v in item.items() if k != "meta"} for item in items
1610
+ {k: v for k, v in item.items() if k not in ("meta", "~admit")}
1611
+ for item in items
1561
1612
  )
1562
1613
 
1563
1614
  @dataclass(frozen=True, eq=False)
@@ -1644,6 +1695,7 @@ class RunManager(Tray):
1644
1695
 
1645
1696
  def enlist(self, effect: Any) -> None:
1646
1697
  """Enlist an opaque durable effect ordered with the machine's writes."""
1698
+ self._ensure_active()
1647
1699
  self._tray_ctx.enlist(effect)
1648
1700
 
1649
1701
  def _ensure_active(self) -> None:
@@ -0,0 +1,118 @@
1
+ """Contract: ``RunManager.admit_steer`` is the ``ctx.dispatch`` target for
2
+ durable steering — a pooled command admits one steer message exactly once
3
+ across a crash redelivery, and the public seams (``steering.take()``,
4
+ ``ctx.messages``, the replicated state view) never carry the admission tag
5
+ the durable queue record keeps for redelivery dedup."""
6
+
7
+ import asyncio
8
+ import copy
9
+
10
+ from pinned import PinnedWorker
11
+ from pinned.testing import MemoryStateStore
12
+ from run_helpers import Script, add, msg
13
+ from statewire_helpers import apply_ops, attach_of, post_frame, statewire_client
14
+
15
+ from harness_sdk import RunManager, run_manager
16
+ from statewire import Statewire, command
17
+
18
+
19
+ def make_host(script: Script):
20
+ class Host(Statewire):
21
+ gate = asyncio.Event()
22
+ live = None
23
+
24
+ async def lifespan(self):
25
+ type(self).live = self
26
+ self.state = {}
27
+ yield
28
+
29
+ async def get_message_meta(self, message_id):
30
+ return await script.get_message_meta(message_id)
31
+
32
+ @run_manager()
33
+ async def runs(self, ctx):
34
+ return await script.run(ctx)
35
+
36
+ @command("run/steerDurable")
37
+ async def steer_durable(self, params, *, ctx):
38
+ runs = await self.runs.get()
39
+ outcome = await ctx.dispatch(runs.admit_steer, params["message"])
40
+ await type(self).gate.wait()
41
+ return {"outcome": outcome}
42
+
43
+ @command("run/enqueue")
44
+ async def run_enqueue(self, params, *, ctx):
45
+ runs = await self.runs.get()
46
+ return await runs.enqueue(params, applied=ctx.applied)
47
+
48
+ return Host
49
+
50
+
51
+ def statement(seq, method, params):
52
+ return {"method": method, "params": [params], "seq": seq}
53
+
54
+
55
+ async def cmds_until(stream, entry, limit=10):
56
+ seen = []
57
+ for _ in range(limit):
58
+ event = await stream.next_event()
59
+ seen.extend(event.get("cmd", []))
60
+ if entry in seen:
61
+ return seen
62
+ raise AssertionError(f"{entry} never rode the stream: {seen}")
63
+
64
+
65
+ async def test_admit_steer_admits_exactly_once_and_the_run_sees_no_tag():
66
+ store = MemoryStateStore()
67
+ script = Script()
68
+ host = make_host(script)
69
+ async with statewire_client(host, worker=PinnedWorker(state_store=store)) as (
70
+ app,
71
+ client,
72
+ ):
73
+ async with attach_of(app, {"headers": {}}) as stream:
74
+ await stream.next_event()
75
+ frame = {
76
+ "cmd": [statement(1, "run/steerDurable", {"message": msg("s1")})]
77
+ }
78
+ assert (await post_frame(client, frame)).status_code == 200
79
+ machine = await host.live.runs.get()
80
+ while not machine.steerQueue.value:
81
+ await asyncio.sleep(0)
82
+ await host.live._frontier.wait()
83
+
84
+ script2 = Script()
85
+ host2 = make_host(script2)
86
+ host2.gate.set()
87
+ async with statewire_client(host2, worker=PinnedWorker(state_store=store)) as (
88
+ app,
89
+ client,
90
+ ):
91
+ async with attach_of(app, {"headers": {}}) as stream:
92
+ replica: dict = {}
93
+ apply_ops(replica, copy.deepcopy(await stream.next_event()))
94
+ # replay of the redelivered seq re-emits its recorded outcome
95
+ frame = {
96
+ "cmd": [statement(1, "run/steerDurable", {"message": msg("s1")})]
97
+ }
98
+ assert (await post_frame(client, frame)).status_code == 200
99
+ await cmds_until(
100
+ stream,
101
+ {"seq": 1, "type": "result", "payload": {"outcome": "admitted"}},
102
+ )
103
+ machine = await host2.live.runs.get()
104
+ # the durable record keeps the dedup tag; the state view never shows it
105
+ (entry,) = machine.steerQueue.value
106
+ assert entry["~admit"] == ["c1", 1]
107
+ assert replica["runs"][0]["steerQueue"] == [msg("s1")]
108
+
109
+ frame = {"cmd": [statement(2, "run/enqueue", add("m1"))]}
110
+ assert (await post_frame(client, frame)).status_code == 200
111
+ call = await script2.next_call()
112
+ call.applied()
113
+ (taken,) = call.ctx.steering.take()
114
+ assert taken == msg("s1")
115
+ call.applied()
116
+ call.finish(RunManager.Complete())
117
+ await cmds_until(stream, {"seq": 2, "dur": True})
118
+ assert machine.steerQueue.value == []
@@ -59,27 +59,15 @@ async def test_steer_and_stop_in_one_batch_nets_to_stop():
59
59
  [("run/steer", add("s1")), ("run/stop", {"runId": await drv.run_id()})]
60
60
  )
61
61
  assert (await drv.res(first))["type"] == "accepted"
62
- assert (await drv.res(first + 1))["type"] == "accepted"
62
+ assert (await drv.res(first + 1, terminal=False))["type"] == "pending"
63
63
  await asyncio.wait_for(call.ctx.stop_requested.wait(), 5)
64
64
  call.finish(RunManager.Stop(dispatch_queue=False))
65
+ assert (await drv.res(first + 1))["type"] == "accepted"
65
66
  await drv.wait_status("stopped")
66
67
  assert queue_ids(drv.replica, "steerQueue") == ["s1"]
67
68
  script.no_call()
68
69
 
69
70
 
70
- async def test_stop_settles_at_the_stopping_write():
71
- script = Script()
72
- async with run_host(script) as (drv, host):
73
- drv.post("run/enqueue", add("m1"))
74
- call = await script.next_call()
75
- call.applied()
76
- settled = await drv.stop()
77
- assert settled["type"] == "accepted"
78
- assert drv.replica["status"] == "running"
79
- call.finish(RunManager.Stop(dispatch_queue=False))
80
- await drv.wait_status("stopped")
81
-
82
-
83
71
  class TailScript(Script):
84
72
  """get_message_meta suspends, pushing staging past the handler prefix."""
85
73
 
@@ -110,9 +110,11 @@ async def test_a_stale_epoch_stop_rejects_across_a_redelivery():
110
110
  assert redelivered.ctx.attempts == 2
111
111
  await drv2.wait(lambda s: run_of(s)["epoch"] == 1) # deliveries never bump it
112
112
  assert_rejected(await drv2.stop({"epoch": 2}, terminal=False), "wrong-state")
113
- assert (await drv2.stop({"epoch": 1}))["type"] == "accepted"
114
- assert redelivered.ctx.stop_requested.is_set()
113
+ pending = await drv2.stop({"epoch": 1}, terminal=False)
114
+ assert pending["type"] == "pending"
115
+ await asyncio.wait_for(redelivered.ctx.stop_requested.wait(), 5)
115
116
  redelivered.finish(RunManager.Stop(dispatch_queue=False))
117
+ assert (await drv2.res(pending["seq"]))["type"] == "accepted"
116
118
  await drv2.wait_status("stopped")
117
119
 
118
120
 
@@ -159,6 +161,7 @@ async def test_a_staged_rewind_survives_a_crash_in_the_stopping_window():
159
161
  async with run_host(script2, store=crashed, capabilities=REWIND) as (drv2, host2):
160
162
  redelivered = await script2.next_call()
161
163
  assert redelivered.ctx.stop_requested.is_set()
164
+ assert drv2.replica["status"] == "stopping" # the window hydrates with its status
162
165
  redelivered.finish(RunManager.Stop(dispatch_queue=False))
163
166
  rewind = await script2.next_call()
164
167
  assert rewind.ctx.trigger == "message-edit"
@@ -169,6 +172,30 @@ async def test_a_staged_rewind_survives_a_crash_in_the_stopping_window():
169
172
  await drv2.wait_status("ready")
170
173
 
171
174
 
175
+ async def test_the_stopping_status_survives_a_crash_in_the_stop_window():
176
+ store = MemoryStateStore()
177
+ script = Script()
178
+ async with run_host(script, store=store) as (drv, host):
179
+ drv.post("run/enqueue", add("m1"))
180
+ call = await script.next_call()
181
+ call.applied()
182
+ await drv.stop(terminal=False)
183
+ await asyncio.wait_for(call.ctx.stop_requested.wait(), 5)
184
+ await drv.wait_status("stopping")
185
+ await flush(host)
186
+ crashed = survivors(store)
187
+
188
+ script2 = Script()
189
+ async with run_host(script2, store=crashed) as (drv2, host2):
190
+ assert drv2.replica["status"] == "stopping"
191
+ assert run_of(drv2.replica)["stopping"] == {"reason": "stop"}
192
+ redelivered = await script2.next_call()
193
+ assert redelivered.ctx.stop_requested.is_set()
194
+ redelivered.finish(RunManager.Stop(dispatch_queue=False))
195
+ await drv2.wait_status("stopped")
196
+ assert run_of(drv2.replica)["stopping"] is None
197
+
198
+
172
199
  async def test_an_applied_batch_redelivers_as_a_resume():
173
200
  store = MemoryStateStore()
174
201
  script = Script()
@@ -269,3 +296,30 @@ async def test_input_answers_survive_a_restart_mid_answering():
269
296
  resumed.applied()
270
297
  resumed.finish(RunManager.Complete())
271
298
  await drv2.wait_status("ready")
299
+
300
+
301
+ async def test_the_stop_reason_survives_a_restart_until_the_next_dispatch():
302
+ store = MemoryStateStore()
303
+ script = Script()
304
+ async with run_host(script, store=store) as (drv, host):
305
+ drv.post("run/enqueue", add("m1"))
306
+ call = await script.next_call()
307
+ call.applied()
308
+ await drv.stop({"reason": "user-stop"}, terminal=False)
309
+ call.finish(RunManager.Stop(dispatch_queue=False))
310
+ await drv.wait_status("stopped")
311
+ assert drv.run["stopReason"] == "user-stop"
312
+ await flush(host)
313
+ crashed = survivors(store)
314
+
315
+ script2 = Script()
316
+ async with run_host(script2, store=crashed) as (drv2, host2):
317
+ assert drv2.replica["status"] == "stopped"
318
+ assert drv2.run["stopReason"] == "user-stop"
319
+ drv2.post("run/enqueue", add("m2"))
320
+ drained = await script2.next_call()
321
+ assert drained.ctx.trigger == "stop-continue"
322
+ await drv2.wait(lambda s: run_of(s)["stopReason"] is None)
323
+ drained.applied()
324
+ drained.finish(RunManager.Complete())
325
+ await drv2.wait_status("ready")
@@ -21,6 +21,7 @@ async def test_activation_mounts_the_ready_entry():
21
21
  "nextDispatch": None,
22
22
  "inputRequests": None,
23
23
  "error": None,
24
+ "stopReason": None,
24
25
  }
25
26
  ],
26
27
  }
@@ -40,6 +40,7 @@ async def test_a_fresh_store_seeds_ready():
40
40
  "nextDispatch": None,
41
41
  "inputRequests": None,
42
42
  "error": None,
43
+ "stopReason": None,
43
44
  }
44
45
  script.no_call()
45
46
 
@@ -5,8 +5,8 @@ limit; each scenario is swept over every crash point. Whatever prefix
5
5
  survived, the restarted machine must satisfy: an accepted message or edit is
6
6
  either consumed by a durably settled dispatch or delivered exactly once after
7
7
  the restart; an accepted input answer reaches a run's input_outcomes; status
8
- "input-required" always carries inputRequests; and a fresh send still reaches
9
- a run.
8
+ "input-required" always carries inputRequests; an open stop window and status
9
+ "stopping" imply each other; and a fresh send still reaches a run.
10
10
 
11
11
  The redelivery lands on the correct arm of the run callback: a batch not yet
12
12
  durably applied re-presents in dispatch mode with the full record, a durably
@@ -266,6 +266,10 @@ async def flush(runs) -> None:
266
266
  def check_wire(runs) -> None:
267
267
  if runs.status.value == "input-required":
268
268
  assert runs.inputRequests.value, "input-required without inputRequests"
269
+ if runs.stopping.value is not None:
270
+ assert runs.status.value == "stopping", "an open stop window without its status"
271
+ elif runs.status.value == "stopping":
272
+ raise AssertionError("stopping status without an open stop window")
269
273
 
270
274
 
271
275
  async def settle_call(runs, call, seen: set[str]) -> None:
@@ -1,7 +1,7 @@
1
1
  import asyncio
2
2
 
3
3
  import pytest
4
- from run_helpers import Script, add, assert_rejected, run_host, run_of
4
+ from run_helpers import Script, add, assert_rejected, machine, run_host, run_of
5
5
 
6
6
  from harness_sdk import RunManager
7
7
 
@@ -15,7 +15,7 @@ async def test_stop_is_not_optimistic():
15
15
  pending = await drv.stop(terminal=False)
16
16
  assert pending["type"] == "pending"
17
17
  await asyncio.wait_for(call.ctx.stop_requested.wait(), 5)
18
- assert drv.replica["status"] == "running"
18
+ await drv.wait_status("stopping")
19
19
  call.finish(RunManager.Stop(dispatch_queue=False))
20
20
  settled = await drv.res(pending["seq"])
21
21
  assert settled["type"] == "accepted"
@@ -45,12 +45,15 @@ async def test_stop_reason_is_visible_on_the_start_context():
45
45
  call = await script.next_call()
46
46
  call.applied()
47
47
  assert call.ctx.stop_reason is None
48
+ await drv.run_id()
49
+ assert drv.run["stopReason"] is None
48
50
  pending = await drv.stop({"reason": "user-stop"}, terminal=False)
49
51
  await asyncio.wait_for(call.ctx.stop_requested.wait(), 5)
50
52
  assert call.ctx.stop_reason == "user-stop"
51
53
  call.finish(RunManager.Stop(dispatch_queue=False))
52
54
  assert (await drv.res(pending["seq"]))["type"] == "accepted"
53
55
  await drv.wait_status("stopped")
56
+ assert drv.run["stopReason"] == "user-stop"
54
57
 
55
58
 
56
59
  async def test_stop_reason_resets_for_the_next_run():
@@ -64,6 +67,7 @@ async def test_stop_reason_resets_for_the_next_run():
64
67
  call.finish(RunManager.Stop(dispatch_queue=True))
65
68
  drain = await script.next_call()
66
69
  assert drain.ctx.stop_reason is None
70
+ await drv.wait(lambda s: run_of(s)["stopReason"] is None)
67
71
 
68
72
 
69
73
  async def test_stop_rejects_malformed_params():
@@ -195,6 +199,52 @@ async def test_stop_settles_even_when_the_run_completes_normally():
195
199
  await drv.wait_status("ready")
196
200
 
197
201
 
202
+ async def test_stop_verdict_awaits_the_settle_and_the_checkpoint_drain():
203
+ script = Script()
204
+ async with run_host(script) as (drv, host):
205
+ drv.post("run/enqueue", add("m1"))
206
+ call = await script.next_call()
207
+ call.applied()
208
+ release = asyncio.Event()
209
+ durable = asyncio.Event()
210
+
211
+ async def checkpoint():
212
+ await release.wait()
213
+ durable.set()
214
+
215
+ call.ctx.enlist(checkpoint())
216
+ runs = await machine(host.live)
217
+ verdict = asyncio.ensure_future(
218
+ runs.stop({"runId": await drv.run_id()}, applied=lambda: None)
219
+ )
220
+ await asyncio.wait_for(call.ctx.stop_requested.wait(), 5)
221
+ await asyncio.sleep(0.02)
222
+ assert not verdict.done() # the run is still winding down
223
+ call.finish(RunManager.Stop(dispatch_queue=False))
224
+ await asyncio.wait_for(runs.idle.wait(), 5)
225
+ assert runs.status.value == "stopped"
226
+ await asyncio.sleep(0.02)
227
+ assert not verdict.done() # the checkpoint drain is still open
228
+ release.set()
229
+ assert await asyncio.wait_for(verdict, 5) is None
230
+ assert durable.is_set() # every enlisted checkpoint durable before the verdict
231
+
232
+
233
+ async def test_a_late_enlist_after_the_run_settled_raises():
234
+ script = Script()
235
+ async with run_host(script) as (drv, host):
236
+ drv.post("run/enqueue", add("m1"))
237
+ call = await script.next_call()
238
+ await drv.run_id()
239
+ call.applied()
240
+ call.finish(RunManager.Complete())
241
+ await drv.wait(lambda s: run_of(s)["runId"] is None)
242
+ straggler = asyncio.sleep(0)
243
+ with pytest.raises(RuntimeError, match="settled"):
244
+ call.ctx.enlist(straggler)
245
+ straggler.close()
246
+
247
+
198
248
  @pytest.mark.parametrize(
199
249
  "end,continue_type",
200
250
  [("error", "error-continue"), ("stopped", "stop-continue")],
@@ -1,10 +1,12 @@
1
1
  """Contract: the stopping window. Three openers — run/stop, mid-run
2
2
  edit/reload, and the enqueue/steer edit-of-dispatched form — write
3
- ``stopping: {reason}`` (plus the staged rewind as ``nextDispatch``) and
4
- settle accepted with that write, ``status`` staying "running". On run end
5
- one envelope promotes ``nextDispatch`` into ``dispatching`` and clears the
6
- window; a later stop/rewind replaces the staged dispatch, whose command
7
- already settled at its admission write."""
3
+ ``stopping: {reason}`` (plus the staged rewind as ``nextDispatch``) and flip
4
+ ``status`` to "stopping" in the same envelope. Rewind commands settle
5
+ accepted with that write; the run/stop verdict follows the run's settle. On
6
+ run end one envelope promotes ``nextDispatch`` into ``dispatching`` and
7
+ clears the window; a later stop/rewind replaces the staged dispatch, whose
8
+ command already settled at its admission write. A second run/stop inside the
9
+ window rejects wrong-state."""
8
10
 
9
11
  import asyncio
10
12
  import copy
@@ -13,6 +15,7 @@ import pytest
13
15
  from run_helpers import (
14
16
  Script,
15
17
  add,
18
+ assert_rejected,
16
19
  edit_params,
17
20
  msg,
18
21
  queue_ids,
@@ -57,26 +60,40 @@ async def start_run(drv, script, *, parent="p0"):
57
60
  return call
58
61
 
59
62
 
60
- async def test_stop_opens_the_window_and_settles_with_its_write():
63
+ async def test_stop_opens_the_window_and_flips_status_to_stopping():
61
64
  script = Script()
62
65
  async with run_host(script) as (drv, host):
63
66
  call = await start_run(drv, script)
64
- settled = await drv.stop()
65
- assert settled["type"] == "accepted"
66
- assert drv.replica["status"] == "running"
67
+ pending = await drv.stop(terminal=False)
68
+ assert pending["type"] == "pending"
69
+ await drv.wait_status("stopping")
67
70
  assert drv.run["stopping"] == {"reason": "stop"}
68
71
  assert drv.run["nextDispatch"] is None
69
72
  opened = next(
70
- state
71
- for env, state in states(drv)
72
- if any(r["seq"] == settled["seq"] for r in env.get("res", []))
73
+ state for _, state in states(drv) if state.get("status") == "stopping"
73
74
  )
74
75
  assert window_of(opened) == ({"reason": "stop"}, None)
75
76
  call.finish(RunManager.Stop(dispatch_queue=False))
77
+ assert (await drv.res(pending["seq"]))["type"] == "accepted"
76
78
  await drv.wait_status("stopped")
77
79
  assert drv.run["stopping"] is None
78
80
 
79
81
 
82
+ async def test_a_second_stop_inside_the_window_rejects_wrong_state():
83
+ script = Script()
84
+ async with run_host(script) as (drv, host):
85
+ call = await start_run(drv, script)
86
+ run_id = await drv.run_id()
87
+ pending = await drv.stop(terminal=False)
88
+ await drv.wait_status("stopping")
89
+ assert_rejected(
90
+ await drv.command("run/stop", {"runId": run_id}), "wrong-state"
91
+ )
92
+ call.finish(RunManager.Stop(dispatch_queue=False))
93
+ assert (await drv.res(pending["seq"]))["type"] == "accepted"
94
+ await drv.wait_status("stopped")
95
+
96
+
80
97
  async def test_mid_run_edit_stages_the_window_and_promotes_in_one_envelope():
81
98
  script = Script()
82
99
  async with run_host(script, capabilities=("rewind", "rewind-during-run")) as (
@@ -87,7 +104,7 @@ async def test_mid_run_edit_stages_the_window_and_promotes_in_one_envelope():
87
104
  seq = drv.post("run/edit", edit_params("m1", msg("m2"), anchor="p0"))
88
105
  await asyncio.wait_for(call.ctx.stop_requested.wait(), 5)
89
106
  await drv.wait(lambda s: run_of(s)["stopping"] is not None)
90
- assert drv.replica["status"] == "running"
107
+ assert drv.replica["status"] == "stopping"
91
108
  assert drv.run["stopping"] == {"reason": "message-edit"}
92
109
  staged = drv.run["nextDispatch"]
93
110
  assert staged == {
@@ -161,7 +178,7 @@ async def test_edit_of_dispatched_message_opens_the_window(command):
161
178
  call = await start_run(drv, script)
162
179
  seq = drv.post(command, target(message=msg("m1", "edited")))
163
180
  await drv.wait(lambda s: run_of(s)["stopping"] is not None)
164
- assert drv.replica["status"] == "running"
181
+ assert drv.replica["status"] == "stopping"
165
182
  assert drv.run["stopping"] == {"reason": "message-edit"}
166
183
  assert drv.run["nextDispatch"] == {
167
184
  "trigger": "message-edit",
@@ -210,11 +227,12 @@ async def test_stop_clears_a_staged_rewind():
210
227
  seq = drv.post("run/edit", edit_params("m1", msg("m2"), anchor="p0"))
211
228
  await drv.wait(lambda s: run_of(s)["nextDispatch"] is not None)
212
229
  assert (await drv.res(seq))["type"] == "accepted"
213
- settled = await drv.stop()
214
- assert settled["type"] == "accepted"
230
+ pending = await drv.stop(terminal=False)
231
+ assert pending["type"] == "pending"
215
232
  await drv.wait(lambda s: run_of(s)["stopping"] == {"reason": "stop"})
216
233
  assert drv.run["nextDispatch"] is None
217
234
  call.finish(RunManager.Stop(dispatch_queue=False))
235
+ assert (await drv.res(pending["seq"]))["type"] == "accepted"
218
236
  await drv.wait_status("stopped")
219
237
  script.no_call()
220
238
 
@@ -253,6 +271,7 @@ ENTRY_KEYS = {
253
271
  "nextDispatch",
254
272
  "inputRequests",
255
273
  "error",
274
+ "stopReason",
256
275
  }
257
276
 
258
277