harness-sdk-python 0.17.0__tar.gz → 0.19.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (34) hide show
  1. {harness_sdk_python-0.17.0 → harness_sdk_python-0.19.0}/PKG-INFO +3 -3
  2. {harness_sdk_python-0.17.0 → harness_sdk_python-0.19.0}/pyproject.toml +2 -2
  3. {harness_sdk_python-0.17.0 → harness_sdk_python-0.19.0}/src/harness_sdk/run_manager.py +245 -65
  4. harness_sdk_python-0.19.0/tests/test_admit_steer.py +118 -0
  5. {harness_sdk_python-0.17.0 → harness_sdk_python-0.19.0}/tests/test_batches.py +2 -14
  6. {harness_sdk_python-0.17.0 → harness_sdk_python-0.19.0}/tests/test_durability.py +82 -63
  7. {harness_sdk_python-0.17.0 → harness_sdk_python-0.19.0}/tests/test_edit_dispatched.py +2 -3
  8. {harness_sdk_python-0.17.0 → harness_sdk_python-0.19.0}/tests/test_edit_reload.py +6 -1
  9. {harness_sdk_python-0.17.0 → harness_sdk_python-0.19.0}/tests/test_enqueue.py +7 -2
  10. {harness_sdk_python-0.17.0 → harness_sdk_python-0.19.0}/tests/test_fenced_postgres.py +8 -1
  11. {harness_sdk_python-0.17.0 → harness_sdk_python-0.19.0}/tests/test_input_required.py +1 -12
  12. {harness_sdk_python-0.17.0 → harness_sdk_python-0.19.0}/tests/test_outcomes.py +1 -0
  13. {harness_sdk_python-0.17.0 → harness_sdk_python-0.19.0}/tests/test_restore.py +1 -0
  14. {harness_sdk_python-0.17.0 → harness_sdk_python-0.19.0}/tests/test_rewind_during_run.py +2 -3
  15. harness_sdk_python-0.19.0/tests/test_run_crash_sweep.py +623 -0
  16. {harness_sdk_python-0.17.0 → harness_sdk_python-0.19.0}/tests/test_settle.py +39 -0
  17. {harness_sdk_python-0.17.0 → harness_sdk_python-0.19.0}/tests/test_steer.py +19 -0
  18. {harness_sdk_python-0.17.0 → harness_sdk_python-0.19.0}/tests/test_stop_continue.py +52 -2
  19. {harness_sdk_python-0.17.0 → harness_sdk_python-0.19.0}/tests/test_stopping_window.py +35 -16
  20. {harness_sdk_python-0.17.0 → harness_sdk_python-0.19.0}/.gitignore +0 -0
  21. {harness_sdk_python-0.17.0 → harness_sdk_python-0.19.0}/README.md +0 -0
  22. {harness_sdk_python-0.17.0 → harness_sdk_python-0.19.0}/examples/__init__.py +0 -0
  23. {harness_sdk_python-0.17.0 → harness_sdk_python-0.19.0}/examples/runs_app.py +0 -0
  24. {harness_sdk_python-0.17.0 → harness_sdk_python-0.19.0}/src/harness_sdk/__init__.py +0 -0
  25. {harness_sdk_python-0.17.0 → harness_sdk_python-0.19.0}/src/harness_sdk/fenced_postgres.py +0 -0
  26. {harness_sdk_python-0.17.0 → harness_sdk_python-0.19.0}/src/harness_sdk/linear_thread.py +0 -0
  27. {harness_sdk_python-0.17.0 → harness_sdk_python-0.19.0}/tests/run_helpers.py +0 -0
  28. {harness_sdk_python-0.17.0 → harness_sdk_python-0.19.0}/tests/test_applied_visibility.py +0 -0
  29. {harness_sdk_python-0.17.0 → harness_sdk_python-0.19.0}/tests/test_branch_anchor.py +0 -0
  30. {harness_sdk_python-0.17.0 → harness_sdk_python-0.19.0}/tests/test_dispatching.py +0 -0
  31. {harness_sdk_python-0.17.0 → harness_sdk_python-0.19.0}/tests/test_linear_thread.py +0 -0
  32. {harness_sdk_python-0.17.0 → harness_sdk_python-0.19.0}/tests/test_meta.py +0 -0
  33. {harness_sdk_python-0.17.0 → harness_sdk_python-0.19.0}/tests/test_placement.py +0 -0
  34. {harness_sdk_python-0.17.0 → harness_sdk_python-0.19.0}/tests/test_prepare_hooks.py +0 -0
@@ -1,12 +1,12 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: harness-sdk-python
3
- Version: 0.17.0
3
+ Version: 0.19.0
4
4
  Summary: RunManager: the harness-sdk runs subsystem for Python Statewire hosts
5
5
  Project-URL: Repository, https://github.com/assistant-ui/harness-sdk
6
6
  License-Expression: MIT
7
7
  Requires-Python: <4.0,>=3.12
8
- Requires-Dist: pinned<0.12,>=0.11.0
9
- Requires-Dist: statewire<0.10,>=0.9.0
8
+ Requires-Dist: pinned<0.14,>=0.13.0
9
+ Requires-Dist: statewire<0.11,>=0.10.0
10
10
  Provides-Extra: deepagents
11
11
  Requires-Dist: deepagents>=0.6.12; extra == 'deepagents'
12
12
  Requires-Dist: langchain-core>=0.3; extra == 'deepagents'
@@ -1,11 +1,11 @@
1
1
  [project]
2
2
  name = "harness-sdk-python"
3
- version = "0.17.0"
3
+ version = "0.19.0"
4
4
  description = "RunManager: the harness-sdk runs subsystem for Python Statewire hosts"
5
5
  readme = "README.md"
6
6
  license = "MIT"
7
7
  requires-python = ">=3.12,<4.0"
8
- dependencies = ["statewire>=0.9.0,<0.10", "pinned>=0.11.0,<0.12"]
8
+ dependencies = ["statewire>=0.10.0,<0.11", "pinned>=0.13.0,<0.14"]
9
9
 
10
10
  [project.optional-dependencies]
11
11
  postgres = ["langgraph-checkpoint-postgres>=2.0.0"]
@@ -19,14 +19,21 @@ the unconfirmed batch record; ``ctx.applied()`` confirms it durably,
19
19
  ``steering.take()`` may refill it via deferred durable pops — at most one
20
20
  unconfirmed batch ever exists. Settle consumes a confirmed batch; a batch
21
21
  still unconfirmed at settle returns to the front of its lanes (after a crash,
22
- unconfirmed taken entries re-present there on their own). ``Complete`` with
23
- an unconfirmed batch settles as an error.
22
+ unconfirmed taken entries re-present there on their own). A batch already
23
+ durably confirmed redelivers in resume mode instead (``ctx.resume``): the
24
+ input batch must not be re-dispatched (``messages`` and the rollback surface
25
+ raise; meta and input outcomes stay readable) and the run continues from its
26
+ own continuation state (``ctx.state`` or an external checkpointer).
27
+ ``Complete`` with an unconfirmed batch settles as an error.
24
28
 
25
29
  ``epoch`` counts the entry's dispatches (not redeliveries); a ``run/stop``
26
30
  carrying an ``epoch`` other than the live one rejects. A mid-run stop or
27
31
  rewind opens a stopping window (``stopping: {reason}`` plus, for rewinds, the
28
- staged dispatch as ``nextDispatch``); on run end one envelope promotes
29
- ``nextDispatch`` into ``dispatching`` and clears the window.
32
+ staged dispatch as ``nextDispatch``) and ``status`` reads ``stopping`` until
33
+ the run settles; the ``run/stop`` verdict follows that settle and its flush,
34
+ so a settled stop covers every durable effect the runner enlisted. On run end
35
+ one envelope promotes ``nextDispatch`` into ``dispatching`` and clears the
36
+ window.
30
37
  """
31
38
 
32
39
  import asyncio
@@ -34,7 +41,7 @@ import logging
34
41
  from dataclasses import dataclass, field
35
42
  from typing import Any, AsyncIterator, Callable, Coroutine, Iterable, Protocol
36
43
 
37
- from pinned.trays import Tray, TrayDescriptor, TrayCtx
44
+ from pinned.trays import Tray, TrayDescriptor, TrayCtx, inbox
38
45
  from pinned.traystore import persistent, persistent_outbox, persistent_queue
39
46
  from statepatch import observable
40
47
  from statewire import StatewireReject
@@ -74,6 +81,15 @@ def _reject(reason: str, message: str) -> StatewireReject:
74
81
  return StatewireReject(message, payload={"reason": reason})
75
82
 
76
83
 
84
+ def _public(message: dict[str, Any]) -> dict[str, Any]:
85
+ """The message without its admission tag; the wire state is public."""
86
+ return {k: v for k, v in message.items() if k != "~admit"}
87
+
88
+
89
+ def _public_record(record: dict[str, Any]) -> dict[str, Any]:
90
+ return {**record, "messages": [_public(m) for m in record["messages"]]}
91
+
92
+
77
93
  def _run_id_of(params: Any) -> str:
78
94
  run_id = params.get("runId") if isinstance(params, dict) else None
79
95
  if not isinstance(run_id, str) or run_id == "":
@@ -187,6 +203,7 @@ class RunManager(Tray):
187
203
  queue = persistent_queue()
188
204
  steerQueue = persistent_queue()
189
205
  outbox = persistent_outbox() # {"record": ..., "lanes": [...]}: the in-flight dispatch
206
+ outcome = persistent(None) # the settled run's verdict, held until its follow-on lands
190
207
  batchApplied = persistent(False)
191
208
  confirmedTakes = persistent(0)
192
209
  runId = persistent(None)
@@ -199,6 +216,7 @@ class RunManager(Tray):
199
216
  stopReason = persistent(None)
200
217
  dispatchedIds = persistent([])
201
218
  dispatching = observable(None) # view: the unconfirmed batch record
219
+ steerView = observable([]) # view: the steer lane without admission tags
202
220
 
203
221
  state = observable(
204
222
  {
@@ -206,12 +224,13 @@ class RunManager(Tray):
206
224
  "status": status,
207
225
  "epoch": epoch,
208
226
  "queue": queue,
209
- "steerQueue": steerQueue,
227
+ "steerQueue": steerView,
210
228
  "dispatching": dispatching,
211
229
  "stopping": stopping,
212
230
  "nextDispatch": nextDispatch,
213
231
  "inputRequests": inputRequests,
214
232
  "error": error,
233
+ "stopReason": stopReason,
215
234
  }
216
235
  )
217
236
 
@@ -220,6 +239,7 @@ class RunManager(Tray):
220
239
  self._intake: list[Any] = []
221
240
  self._outcome: Any = None
222
241
  self._stop_staged = False
242
+ self._stop_waiters: list["asyncio.Future[Any]"] = []
223
243
  self._staged_rewind: _Rewind | None = None
224
244
  self._batch: list[tuple[str, dict[str, Any]]] = []
225
245
  self._pending_takes = 0
@@ -240,7 +260,10 @@ class RunManager(Tray):
240
260
  self._thread = api
241
261
  self._schedule = api.schedule
242
262
  self._hydrate_machine()
263
+ self._mirror_steer()
264
+ unsubscribe = self.steerQueue.subscribe(lambda _op: self._mirror_steer())
243
265
  yield
266
+ unsubscribe()
244
267
 
245
268
  class Thread(Protocol):
246
269
  """The thread projection RunManager reads; the host owns the tree."""
@@ -260,15 +283,36 @@ class RunManager(Tray):
260
283
 
261
284
  def _hydrate_machine(self) -> None:
262
285
  self.steerQueue.take_now(self.confirmedTakes.value)
286
+ settled = self.outcome.value
287
+ if settled is not None:
288
+ if self.outbox.value is not None and self.epoch.value != settled["epoch"]:
289
+ self.outcome.value = None # the follow-on dispatch landed: the settle is done
290
+ else:
291
+ self._replay_settle()
292
+ self._outcome = settled
263
293
  recorded = self.outbox.value
264
294
  if recorded is not None and not self.batchApplied.value:
265
295
  record = recorded["item"]["record"]
266
296
  lanes = recorded["item"]["lanes"]
267
297
  self._batch = list(zip(lanes, record["messages"]))
268
- self.dispatching.value = record
298
+ self.dispatching.value = _public_record(record)
269
299
  else:
270
300
  self.dispatching.value = None
301
+ if recorded is not None:
302
+ # heal a dispatch torn after its outbox write landed
303
+ self._clear_input()
304
+ self.error.value = None
305
+ self._set_status(
306
+ "stopping" if self.stopping.value is not None else "running"
307
+ )
271
308
  staged = self.nextDispatch.value
309
+ if (
310
+ staged is not None
311
+ and recorded is not None
312
+ and recorded["item"]["record"] == staged
313
+ ):
314
+ self.nextDispatch.value = None # torn promotion: the outbox carries it
315
+ staged = None
272
316
  if staged is not None:
273
317
  self._staged_rewind = _Rewind(
274
318
  staged["trigger"],
@@ -319,6 +363,8 @@ class RunManager(Tray):
319
363
  return self._lane(lane).value
320
364
 
321
365
  def _lane_of(self, message_id: str) -> str | None:
366
+ if self._is_dispatching(message_id):
367
+ return None # claimed by the staged batch; its lane entry is spoken for
322
368
  for lane in ("steerQueue", "queue"):
323
369
  if any(item["id"] == message_id for item in self._lane_items(lane)):
324
370
  return lane
@@ -330,6 +376,9 @@ class RunManager(Tray):
330
376
  def _is_dispatching(self, message_id: str) -> bool:
331
377
  return any(item["id"] == message_id for _, item in self._batch)
332
378
 
379
+ def _mirror_steer(self) -> None:
380
+ self.steerView.value = [_public(item) for item in self.steerQueue.value]
381
+
333
382
  def _sync_steering(self) -> None:
334
383
  ctx = self._run_live
335
384
  if ctx is None:
@@ -355,6 +404,12 @@ class RunManager(Tray):
355
404
  self._apply(entry, fx)
356
405
  return fx
357
406
 
407
+ def _resolve_stop_waiters(self) -> None:
408
+ waiters, self._stop_waiters = self._stop_waiters, []
409
+ for future in waiters:
410
+ if not future.done():
411
+ future.set_result(None)
412
+
358
413
  def _settle_applied(self, fx: _Effects) -> None:
359
414
  for future, applied, result in fx.staged_sends:
360
415
  applied()
@@ -369,15 +424,12 @@ class RunManager(Tray):
369
424
  fx = self._apply_intake()
370
425
  if not self._in_flight():
371
426
  self._stop_staged = False
372
- if isinstance(outcome, RunManager.Complete):
427
+ if outcome is not None and outcome["kind"] == "complete":
373
428
  self.dispatchedIds.value = []
374
429
  if self._staged_rewind is not None:
375
430
  rewind, self._staged_rewind = self._staged_rewind, None
376
- self._clear_input()
377
431
  if self._batch:
378
432
  self._unstage_batch()
379
- self.stopping.value = None
380
- self.nextDispatch.value = None
381
433
  if self.runId.value is None:
382
434
  self.runId.value = rewind.run_id
383
435
  self._dispatch(
@@ -392,6 +444,9 @@ class RunManager(Tray):
392
444
  self._settle_outcome(outcome)
393
445
  else:
394
446
  self._idle_action(fx)
447
+ if self.outcome.value is not None:
448
+ self.outcome.value = None # the follow-on's writes are enlisted ahead
449
+ self._resolve_stop_waiters()
395
450
  if self._in_flight() and (
396
451
  self._stop_staged or self._staged_rewind is not None
397
452
  ):
@@ -409,7 +464,9 @@ class RunManager(Tray):
409
464
  self._staged_rewind = None
410
465
  self.nextDispatch.value = None
411
466
  self.stopping.value = {"reason": "stop"}
412
- if not entry.future.done():
467
+ self._set_status("stopping")
468
+ self._stop_waiters.append(entry.future) # the verdict waits for the settle
469
+ elif not entry.future.done():
413
470
  entry.future.set_result(None)
414
471
  return
415
472
  if isinstance(entry, _Continue):
@@ -448,31 +505,35 @@ class RunManager(Tray):
448
505
  self._dispatch(self._continue_type(), [], root_meta=fx.continue_meta)
449
506
  elif status == "input-required":
450
507
  if self._batch:
451
- self._dispatch_staged(input_outcomes=self._take_input_outcomes())
508
+ self._dispatch_staged(input_outcomes=self._input_outcomes())
452
509
  elif fx.steer_added and self._lane_items("steerQueue"):
453
- self._pop_dispatchable(input_outcomes=self._take_input_outcomes())
510
+ self._pop_dispatchable(input_outcomes=self._input_outcomes())
454
511
  elif self._input_requests and len(self._input_answers) == len(
455
512
  self._input_requests
456
513
  ):
457
514
  self._dispatch(
458
- "input-resume", [], input_outcomes=self._take_input_outcomes()
515
+ "input-resume", [], input_outcomes=self._input_outcomes()
459
516
  )
460
517
 
461
- def _settle_outcome(self, outcome: Any) -> None:
462
- if isinstance(outcome, RunManager.Complete):
518
+ def _settle_outcome(self, outcome: dict[str, Any]) -> None:
519
+ kind = outcome["kind"]
520
+ if kind == "complete":
463
521
  if not self._pop_dispatchable():
464
522
  self._to_ready()
465
523
  return
466
- if isinstance(outcome, RunManager.InputRequired):
467
- self._input_requests = [dict(r) for r in outcome.requests]
524
+ if kind == "input-required":
525
+ requests = outcome["requests"]
526
+ self._input_requests = [dict(r) for r in requests]
468
527
  self._input_answers = {}
469
- self.inputRequests.value = [dict(r) for r in outcome.requests]
528
+ self.inputRequests.value = [dict(r) for r in requests]
470
529
  self._set_status("input-required")
471
530
  return
472
- status = "error" if isinstance(outcome, RunManager.Error) else "stopped"
473
- if outcome.dispatch_queue and self._pop_dispatchable():
531
+ if outcome["dispatchQueue"] and self._pop_dispatchable():
474
532
  return
475
- self._set_status(status)
533
+ error = outcome.get("error")
534
+ if error is not None:
535
+ self.error.value = error
536
+ self._set_status("error" if kind == "error" else "stopped")
476
537
 
477
538
  # ─── Dispatch and settle ────────────────────────────────
478
539
 
@@ -481,13 +542,11 @@ class RunManager(Tray):
481
542
  self._input_answers = {}
482
543
  self.inputRequests.value = None
483
544
 
484
- def _take_input_outcomes(self) -> tuple[tuple[dict[str, Any], Any, Any], ...]:
485
- outcomes = tuple(
545
+ def _input_outcomes(self) -> tuple[tuple[dict[str, Any], Any, Any], ...]:
546
+ return tuple(
486
547
  (request, *self._input_answers.get(request["id"], (None, None)))
487
548
  for request in self._input_requests
488
549
  )
489
- self._clear_input()
490
- return outcomes
491
550
 
492
551
  def _record_of(
493
552
  self,
@@ -530,21 +589,27 @@ class RunManager(Tray):
530
589
  root_meta=root_meta,
531
590
  input_outcomes=input_outcomes,
532
591
  )
592
+ item = {"record": record, "lanes": [lane for lane, _ in self._batch]}
533
593
  self.stopReason.value = None
534
594
  self.epoch.value = self.epoch.value + 1
535
- self.outbox.dispatch(
536
- {"record": record, "lanes": [lane for lane, _ in self._batch]}
537
- )
595
+ self.outbox.dispatch(item)
538
596
  self._fresh = True
597
+ self._reclaim(item) # the record owns its sources: consume them behind it
598
+ self.nextDispatch.value = None
599
+ self.stopping.value = None
600
+ self._clear_input()
539
601
  self.batchApplied.value = False
540
602
  self.error.value = None
541
603
  stripped = [
542
604
  {k: v for k, v in message.items() if k != "meta"} for message in messages
543
605
  ]
544
606
  if stripped:
545
- self.dispatchedIds.value = [m["id"] for m in stripped]
607
+ known = self.dispatchedIds.value
608
+ added = [m["id"] for m in stripped if m["id"] not in known]
609
+ if added:
610
+ self.dispatchedIds.value = [*known, *added]
546
611
  self._set_status("running")
547
- self.dispatching.value = record
612
+ self.dispatching.value = _public_record(record)
548
613
  self._idle.clear()
549
614
  self.kick()
550
615
 
@@ -577,12 +642,13 @@ class RunManager(Tray):
577
642
  def _make_run_ctx(self, item: Any) -> "RunManager.RunContext":
578
643
  record = item["record"]
579
644
  messages = [
580
- {k: v for k, v in message.items() if k != "meta"}
645
+ {k: v for k, v in message.items() if k not in ("meta", "~admit")}
581
646
  for message in record["messages"]
582
647
  ]
583
648
  ctx = RunManager.RunContext(
584
649
  trigger=record["trigger"],
585
- messages=tuple(messages),
650
+ _resume=self.batchApplied.value or self.confirmedTakes.value > 0,
651
+ _messages=tuple(messages),
586
652
  stop_requested=asyncio.Event(),
587
653
  _manager=self,
588
654
  _rollback_to=record["rollbackTo"] if "rollbackTo" in record else _ABSENT,
@@ -638,34 +704,81 @@ class RunManager(Tray):
638
704
  except asyncio.CancelledError:
639
705
  if self._closing or self.storage._retired is not None:
640
706
  raise # teardown: the un-acked outbox redelivers next activation
707
+ self.outcome.value = {
708
+ "kind": "cancel",
709
+ "epoch": self.epoch.value,
710
+ "dispatchQueue": False,
711
+ }
641
712
  self._settle(ctx)
642
713
  fx = self._apply_intake()
714
+ rewind = self._staged_rewind
715
+ if rewind is not None and not rewind.applied_sent:
716
+ # park the in-drain rewind durably; a cancel starts no dispatch
717
+ self.nextDispatch.value = self._record_of(
718
+ rewind.type,
719
+ list(rewind.messages),
720
+ rollback_to=rewind.rollback_to,
721
+ root_meta=rewind.root_meta,
722
+ )
723
+ rewind.applied()
724
+ rewind.applied_sent = True
643
725
  self._set_status("stopped")
726
+ self.outcome.value = None
727
+ self._resolve_stop_waiters()
644
728
  self._settle_applied(fx)
645
729
  self._idle.set()
646
730
  raise # staged intake settled; a cancelled run starts no dispatch
647
731
  except Exception as exc:
648
- self._settle(ctx)
649
- message = str(exc) or type(exc).__name__
650
- if isinstance(exc, StatewireReject):
651
- self._freeze(exc.message, exc.payload)
652
- else:
732
+ if not isinstance(exc, StatewireReject):
653
733
  logger.exception(
654
734
  "run %r crashed in %s",
655
735
  self.runId.value,
656
736
  getattr(fn, "__qualname__", repr(fn)),
657
737
  )
658
- self._freeze(message)
738
+ error = self._error_of(exc)
739
+ self.outcome.value = {
740
+ "kind": "error",
741
+ "epoch": self.epoch.value,
742
+ "dispatchQueue": False,
743
+ "error": error,
744
+ }
745
+ self._settle(ctx)
746
+ self._set_status("error")
747
+ self.error.value = error
659
748
  self._advance()
660
749
  return
750
+ self.outcome.value = self._outcome_record(outcome)
661
751
  self._settle(ctx)
662
- self._outcome = outcome
752
+ self._outcome = self.outcome.value
663
753
  self._advance()
664
754
 
755
+ def _error_of(self, exc: Exception) -> dict[str, Any]:
756
+ if isinstance(exc, StatewireReject):
757
+ if isinstance(exc.payload, dict):
758
+ return {**exc.payload, "message": exc.message}
759
+ if exc.payload is not None:
760
+ return {"message": exc.message, "payload": exc.payload}
761
+ return {"message": exc.message}
762
+ return {"message": str(exc) or type(exc).__name__}
763
+
764
+ def _outcome_record(self, outcome: Any) -> dict[str, Any]:
765
+ epoch = self.epoch.value
766
+ if isinstance(outcome, RunManager.Complete):
767
+ return {"kind": "complete", "epoch": epoch}
768
+ if isinstance(outcome, RunManager.InputRequired):
769
+ return {
770
+ "kind": "input-required",
771
+ "epoch": epoch,
772
+ "requests": [dict(r) for r in outcome.requests],
773
+ }
774
+ kind = "error" if isinstance(outcome, RunManager.Error) else "stop"
775
+ return {"kind": kind, "epoch": epoch, "dispatchQueue": outcome.dispatch_queue}
776
+
665
777
  def _settle(self, ctx: "RunManager.RunContext") -> None:
666
778
  if self._run_live is ctx:
667
779
  self._run_live = None
668
780
  if self._batch:
781
+ self._drop_dispatched(item["id"] for _, item in self._batch)
669
782
  self._unstage_batch()
670
783
  else:
671
784
  record = self.dispatching.value
@@ -674,13 +787,36 @@ class RunManager(Tray):
674
787
  and record["trigger"] != "steer"
675
788
  and record["messages"]
676
789
  ):
677
- self.dispatchedIds.value = []
790
+ # an unapplied rewind settled: its replacement never landed
791
+ self._drop_dispatched(m["id"] for m in record["messages"])
678
792
  self.dispatching.value = None
793
+ self._finish_settle()
794
+
795
+ def _drop_dispatched(self, ids: Iterable[str]) -> None:
796
+ dropped = set(ids)
797
+ kept = [i for i in self.dispatchedIds.value if i not in dropped]
798
+ if len(kept) != len(self.dispatchedIds.value):
799
+ self.dispatchedIds.value = kept
800
+
801
+ def _finish_settle(self) -> None:
679
802
  self.stopping.value = None
680
- self.batchApplied.value = False
681
803
  self.confirmedTakes.value = 0
804
+ self.batchApplied.value = False
682
805
  self._clear_dispatch_state() # commits confirmed takes; drops the run's checkpoint
683
- self.outbox.ack()
806
+ if self.outbox.value is not None:
807
+ self.outbox.ack()
808
+
809
+ def _replay_settle(self) -> None:
810
+ recorded = self.outbox.value
811
+ if recorded is not None and not self.batchApplied.value:
812
+ item = recorded["item"]
813
+ record = item["record"]
814
+ self._batch = list(zip(item["lanes"], record["messages"]))
815
+ self._drop_dispatched(m["id"] for m in record["messages"])
816
+ if self._batch:
817
+ self._unstage_batch()
818
+ self.dispatching.value = None
819
+ self._finish_settle()
684
820
 
685
821
  def _applied(self) -> None:
686
822
  if self.dispatching.value is None:
@@ -700,6 +836,10 @@ class RunManager(Tray):
700
836
  self.steerQueue.untake(self._pending_takes)
701
837
  self._pending_takes = 0
702
838
  return
839
+ for lane, item in taken: # claimed heads the dispatch never consumed
840
+ queue = self._lane(lane)
841
+ if queue.value and queue.peek()["id"] == item["id"]:
842
+ queue.popleft()
703
843
  for lane, item in reversed(taken):
704
844
  self._lane(lane).insert(0, item)
705
845
 
@@ -727,16 +867,22 @@ class RunManager(Tray):
727
867
 
728
868
  def _stage_batch(self, e: _Send) -> None:
729
869
  assert e.message is not None
730
- steer = self._lane_items("steerQueue") if e.lane == "steerQueue" else []
870
+ steer = (
871
+ [
872
+ item
873
+ for item in self._lane_items("steerQueue")
874
+ if not self._is_dispatching(item["id"])
875
+ ]
876
+ if e.lane == "steerQueue"
877
+ else []
878
+ )
731
879
  if len(self._batch) + len(steer) >= self.max_queued:
732
880
  raise _reject("queue-full", f"queue is full ({self.max_queued} messages)")
733
- if steer:
734
- self.steerQueue.value = []
735
- self._batch.extend(("steerQueue", item) for item in steer)
881
+ self._batch.extend(("steerQueue", item) for item in steer)
736
882
  self._batch.append((e.lane, self._stamped(e.message, e.meta)))
737
883
  self.dispatching.value = {
738
884
  "trigger": self._send_trigger(),
739
- "messages": [item for _, item in self._batch],
885
+ "messages": [_public(item) for _, item in self._batch],
740
886
  }
741
887
 
742
888
  def _dispatch_staged(
@@ -753,12 +899,11 @@ class RunManager(Tray):
753
899
  ) -> bool:
754
900
  steer = self._lane_items("steerQueue")
755
901
  if steer:
756
- self.steerQueue.value = []
757
902
  self._batch = [("steerQueue", item) for item in steer]
758
903
  self._dispatch("message-send", steer, input_outcomes=input_outcomes)
759
904
  return True
760
905
  if self._lane_items("queue"):
761
- item = self.queue.popleft()
906
+ item = self.queue.peek()
762
907
  self._batch = [("queue", item)]
763
908
  self._dispatch("message-send", [item], input_outcomes=input_outcomes)
764
909
  return True
@@ -768,6 +913,7 @@ class RunManager(Tray):
768
913
  self._staged_rewind = rewind
769
914
  if self._in_flight():
770
915
  self.stopping.value = {"reason": rewind.type}
916
+ self._set_status("stopping")
771
917
  self.nextDispatch.value = self._record_of(
772
918
  rewind.type,
773
919
  list(rewind.messages),
@@ -779,15 +925,6 @@ class RunManager(Tray):
779
925
  if not rewind.future.done():
780
926
  rewind.future.set_result(None)
781
927
 
782
- def _freeze(self, message: str, payload: Any = None) -> None:
783
- self._set_status("error")
784
- if isinstance(payload, dict):
785
- self.error.value = {**payload, "message": message}
786
- elif payload is not None:
787
- self.error.value = {"message": message, "payload": payload}
788
- else:
789
- self.error.value = {"message": message}
790
-
791
928
  # ─── Message and placement validation ───────────────────
792
929
 
793
930
  def _prepared(
@@ -1218,6 +1355,15 @@ class RunManager(Tray):
1218
1355
  ) -> Any:
1219
1356
  return await self._send("steerQueue", params, meta, applied)
1220
1357
 
1358
+ @inbox
1359
+ def admit_steer(self, message: dict[str, Any]) -> None:
1360
+ """Durably admit one steer message; a ``ctx.dispatch`` target."""
1361
+ if not isinstance(message, dict) or not isinstance(message.get("id"), str):
1362
+ raise TypeError(f"a steer message needs a string id, got {message!r}")
1363
+ self.steerQueue.append(self._stamped(message, message.get("meta")))
1364
+ self._sync_steering()
1365
+ self._schedule(self._advance)
1366
+
1221
1367
  async def dequeue(self, params: Any) -> None:
1222
1368
  message_id = params.get("messageId") if isinstance(params, dict) else None
1223
1369
  if not isinstance(message_id, str):
@@ -1225,7 +1371,7 @@ class RunManager(Tray):
1225
1371
  return await self._stage(_Dequeue(message_id))
1226
1372
 
1227
1373
  def _check_rewind_gate(self, command: str) -> None:
1228
- if self._status() == "running":
1374
+ if self._status() in ("running", "stopping"):
1229
1375
  if "rewind-during-run" not in self.capabilities:
1230
1376
  raise _reject(
1231
1377
  "capability-missing",
@@ -1299,7 +1445,11 @@ class RunManager(Tray):
1299
1445
  if epoch is not _ABSENT and (isinstance(epoch, bool) or not isinstance(epoch, int)):
1300
1446
  raise _reject("invalid-message", "epoch must be an integer")
1301
1447
  status = self._status()
1302
- if status != "running":
1448
+ stop_held = (self.stopping.value or {}).get("reason") == "stop"
1449
+ # a rewind-held window still admits a stop: it clears the staged rewind
1450
+ if status not in ("running", "stopping") or (
1451
+ status == "stopping" and stop_held
1452
+ ):
1303
1453
  raise _reject("wrong-state", f"run/stop is rejected in {status}")
1304
1454
  run_id = _run_id_of(params)
1305
1455
  if run_id != self.runId.value:
@@ -1444,20 +1594,29 @@ class RunManager(Tray):
1444
1594
  return ()
1445
1595
  manager._pending_takes += len(items)
1446
1596
  manager._batch = [("steerQueue", item) for item in items]
1447
- manager.dispatching.value = {"trigger": "steer", "messages": items}
1597
+ known = manager.dispatchedIds.value
1598
+ added = [item["id"] for item in items if item["id"] not in known]
1599
+ if added:
1600
+ manager.dispatchedIds.value = [*known, *added]
1601
+ manager.dispatching.value = {
1602
+ "trigger": "steer",
1603
+ "messages": [_public(item) for item in items],
1604
+ }
1448
1605
  self._ctx._message_meta.update(
1449
1606
  {item["id"]: item.get("meta") for item in items}
1450
1607
  )
1451
1608
  manager._sync_steering()
1452
1609
  return tuple(
1453
- {k: v for k, v in item.items() if k != "meta"} for item in items
1610
+ {k: v for k, v in item.items() if k not in ("meta", "~admit")}
1611
+ for item in items
1454
1612
  )
1455
1613
 
1456
1614
  @dataclass(frozen=True, eq=False)
1457
1615
  class RunContext:
1458
1616
  trigger: str
1459
- messages: tuple[dict[str, Any], ...]
1460
1617
  stop_requested: asyncio.Event
1618
+ _resume: bool
1619
+ _messages: tuple[dict[str, Any], ...]
1461
1620
  _manager: "RunManager"
1462
1621
  _rollback_to: Any
1463
1622
  _message_meta: dict[str, Any]
@@ -1471,6 +1630,24 @@ class RunManager(Tray):
1471
1630
  self, "steering", RunManager.Steering(asyncio.Event(), self)
1472
1631
  )
1473
1632
 
1633
+ @property
1634
+ def resume(self) -> bool:
1635
+ """True when the batch was already durably applied before this redelivery: do not re-apply it, continue from the app's own continuation state."""
1636
+ return self._resume
1637
+
1638
+ def _batch_only(self, name: str) -> None:
1639
+ if self._resume:
1640
+ raise RuntimeError(
1641
+ f"{name} is unavailable on a resume: the batch is already applied and"
1642
+ " must not be re-dispatched; continue from app-defined state"
1643
+ )
1644
+
1645
+ @property
1646
+ def messages(self) -> tuple[dict[str, Any], ...]:
1647
+ """The dispatched input batch; unavailable on a resume."""
1648
+ self._batch_only("messages")
1649
+ return self._messages
1650
+
1474
1651
  def meta(self, message_id: str | None = None) -> Any:
1475
1652
  """Op meta: a dispatched message's by id, or the run-starting op's with no id."""
1476
1653
  if message_id is None:
@@ -1490,10 +1667,12 @@ class RunManager(Tray):
1490
1667
 
1491
1668
  @property
1492
1669
  def has_rollback(self) -> bool:
1670
+ self._batch_only("has_rollback")
1493
1671
  return self._rollback_to is not _ABSENT
1494
1672
 
1495
1673
  @property
1496
1674
  def rollback_to(self) -> str | None:
1675
+ self._batch_only("rollback_to")
1497
1676
  if self._rollback_to is _ABSENT:
1498
1677
  raise AttributeError(
1499
1678
  "rollback_to is only present on rewind entries; check has_rollback"
@@ -1516,6 +1695,7 @@ class RunManager(Tray):
1516
1695
 
1517
1696
  def enlist(self, effect: Any) -> None:
1518
1697
  """Enlist an opaque durable effect ordered with the machine's writes."""
1698
+ self._ensure_active()
1519
1699
  self._tray_ctx.enlist(effect)
1520
1700
 
1521
1701
  def _ensure_active(self) -> None: