harness-sdk-python 0.17.0__tar.gz → 0.18.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (33) hide show
  1. {harness_sdk_python-0.17.0 → harness_sdk_python-0.18.0}/PKG-INFO +3 -3
  2. {harness_sdk_python-0.17.0 → harness_sdk_python-0.18.0}/pyproject.toml +2 -2
  3. {harness_sdk_python-0.17.0 → harness_sdk_python-0.18.0}/src/harness_sdk/run_manager.py +180 -52
  4. {harness_sdk_python-0.17.0 → harness_sdk_python-0.18.0}/tests/test_durability.py +26 -61
  5. {harness_sdk_python-0.17.0 → harness_sdk_python-0.18.0}/tests/test_edit_dispatched.py +2 -3
  6. {harness_sdk_python-0.17.0 → harness_sdk_python-0.18.0}/tests/test_edit_reload.py +6 -1
  7. {harness_sdk_python-0.17.0 → harness_sdk_python-0.18.0}/tests/test_enqueue.py +7 -2
  8. {harness_sdk_python-0.17.0 → harness_sdk_python-0.18.0}/tests/test_fenced_postgres.py +8 -1
  9. {harness_sdk_python-0.17.0 → harness_sdk_python-0.18.0}/tests/test_input_required.py +1 -12
  10. {harness_sdk_python-0.17.0 → harness_sdk_python-0.18.0}/tests/test_rewind_during_run.py +2 -3
  11. harness_sdk_python-0.18.0/tests/test_run_crash_sweep.py +619 -0
  12. {harness_sdk_python-0.17.0 → harness_sdk_python-0.18.0}/tests/test_settle.py +39 -0
  13. {harness_sdk_python-0.17.0 → harness_sdk_python-0.18.0}/tests/test_steer.py +19 -0
  14. {harness_sdk_python-0.17.0 → harness_sdk_python-0.18.0}/.gitignore +0 -0
  15. {harness_sdk_python-0.17.0 → harness_sdk_python-0.18.0}/README.md +0 -0
  16. {harness_sdk_python-0.17.0 → harness_sdk_python-0.18.0}/examples/__init__.py +0 -0
  17. {harness_sdk_python-0.17.0 → harness_sdk_python-0.18.0}/examples/runs_app.py +0 -0
  18. {harness_sdk_python-0.17.0 → harness_sdk_python-0.18.0}/src/harness_sdk/__init__.py +0 -0
  19. {harness_sdk_python-0.17.0 → harness_sdk_python-0.18.0}/src/harness_sdk/fenced_postgres.py +0 -0
  20. {harness_sdk_python-0.17.0 → harness_sdk_python-0.18.0}/src/harness_sdk/linear_thread.py +0 -0
  21. {harness_sdk_python-0.17.0 → harness_sdk_python-0.18.0}/tests/run_helpers.py +0 -0
  22. {harness_sdk_python-0.17.0 → harness_sdk_python-0.18.0}/tests/test_applied_visibility.py +0 -0
  23. {harness_sdk_python-0.17.0 → harness_sdk_python-0.18.0}/tests/test_batches.py +0 -0
  24. {harness_sdk_python-0.17.0 → harness_sdk_python-0.18.0}/tests/test_branch_anchor.py +0 -0
  25. {harness_sdk_python-0.17.0 → harness_sdk_python-0.18.0}/tests/test_dispatching.py +0 -0
  26. {harness_sdk_python-0.17.0 → harness_sdk_python-0.18.0}/tests/test_linear_thread.py +0 -0
  27. {harness_sdk_python-0.17.0 → harness_sdk_python-0.18.0}/tests/test_meta.py +0 -0
  28. {harness_sdk_python-0.17.0 → harness_sdk_python-0.18.0}/tests/test_outcomes.py +0 -0
  29. {harness_sdk_python-0.17.0 → harness_sdk_python-0.18.0}/tests/test_placement.py +0 -0
  30. {harness_sdk_python-0.17.0 → harness_sdk_python-0.18.0}/tests/test_prepare_hooks.py +0 -0
  31. {harness_sdk_python-0.17.0 → harness_sdk_python-0.18.0}/tests/test_restore.py +0 -0
  32. {harness_sdk_python-0.17.0 → harness_sdk_python-0.18.0}/tests/test_stop_continue.py +0 -0
  33. {harness_sdk_python-0.17.0 → harness_sdk_python-0.18.0}/tests/test_stopping_window.py +0 -0
@@ -1,12 +1,12 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: harness-sdk-python
3
- Version: 0.17.0
3
+ Version: 0.18.0
4
4
  Summary: RunManager: the harness-sdk runs subsystem for Python Statewire hosts
5
5
  Project-URL: Repository, https://github.com/assistant-ui/harness-sdk
6
6
  License-Expression: MIT
7
7
  Requires-Python: <4.0,>=3.12
8
- Requires-Dist: pinned<0.12,>=0.11.0
9
- Requires-Dist: statewire<0.10,>=0.9.0
8
+ Requires-Dist: pinned<0.13,>=0.12.0
9
+ Requires-Dist: statewire<0.10,>=0.9.2
10
10
  Provides-Extra: deepagents
11
11
  Requires-Dist: deepagents>=0.6.12; extra == 'deepagents'
12
12
  Requires-Dist: langchain-core>=0.3; extra == 'deepagents'
@@ -1,11 +1,11 @@
1
1
  [project]
2
2
  name = "harness-sdk-python"
3
- version = "0.17.0"
3
+ version = "0.18.0"
4
4
  description = "RunManager: the harness-sdk runs subsystem for Python Statewire hosts"
5
5
  readme = "README.md"
6
6
  license = "MIT"
7
7
  requires-python = ">=3.12,<4.0"
8
- dependencies = ["statewire>=0.9.0,<0.10", "pinned>=0.11.0,<0.12"]
8
+ dependencies = ["statewire>=0.9.2,<0.10", "pinned>=0.12.0,<0.13"]
9
9
 
10
10
  [project.optional-dependencies]
11
11
  postgres = ["langgraph-checkpoint-postgres>=2.0.0"]
@@ -19,8 +19,12 @@ the unconfirmed batch record; ``ctx.applied()`` confirms it durably,
19
19
  ``steering.take()`` may refill it via deferred durable pops — at most one
20
20
  unconfirmed batch ever exists. Settle consumes a confirmed batch; a batch
21
21
  still unconfirmed at settle returns to the front of its lanes (after a crash,
22
- unconfirmed taken entries re-present there on their own). ``Complete`` with
23
- an unconfirmed batch settles as an error.
22
+ unconfirmed taken entries re-present there on their own). A batch already
23
+ durably confirmed redelivers in resume mode instead (``ctx.resume``): the
24
+ input batch must not be re-dispatched (``messages`` and the rollback surface
25
+ raise; meta and input outcomes stay readable) and the run continues from its
26
+ own continuation state (``ctx.state`` or an external checkpointer).
27
+ ``Complete`` with an unconfirmed batch settles as an error.
24
28
 
25
29
  ``epoch`` counts the entry's dispatches (not redeliveries); a ``run/stop``
26
30
  carrying an ``epoch`` other than the live one rejects. A mid-run stop or
@@ -187,6 +191,7 @@ class RunManager(Tray):
187
191
  queue = persistent_queue()
188
192
  steerQueue = persistent_queue()
189
193
  outbox = persistent_outbox() # {"record": ..., "lanes": [...]}: the in-flight dispatch
194
+ outcome = persistent(None) # the settled run's verdict, held until its follow-on lands
190
195
  batchApplied = persistent(False)
191
196
  confirmedTakes = persistent(0)
192
197
  runId = persistent(None)
@@ -260,6 +265,13 @@ class RunManager(Tray):
260
265
 
261
266
  def _hydrate_machine(self) -> None:
262
267
  self.steerQueue.take_now(self.confirmedTakes.value)
268
+ settled = self.outcome.value
269
+ if settled is not None:
270
+ if self.outbox.value is not None and self.epoch.value != settled["epoch"]:
271
+ self.outcome.value = None # the follow-on dispatch landed: the settle is done
272
+ else:
273
+ self._replay_settle()
274
+ self._outcome = settled
263
275
  recorded = self.outbox.value
264
276
  if recorded is not None and not self.batchApplied.value:
265
277
  record = recorded["item"]["record"]
@@ -268,7 +280,19 @@ class RunManager(Tray):
268
280
  self.dispatching.value = record
269
281
  else:
270
282
  self.dispatching.value = None
283
+ if recorded is not None:
284
+ # heal a dispatch torn after its outbox write landed
285
+ self._clear_input()
286
+ self.error.value = None
287
+ self._set_status("running")
271
288
  staged = self.nextDispatch.value
289
+ if (
290
+ staged is not None
291
+ and recorded is not None
292
+ and recorded["item"]["record"] == staged
293
+ ):
294
+ self.nextDispatch.value = None # torn promotion: the outbox carries it
295
+ staged = None
272
296
  if staged is not None:
273
297
  self._staged_rewind = _Rewind(
274
298
  staged["trigger"],
@@ -319,6 +343,8 @@ class RunManager(Tray):
319
343
  return self._lane(lane).value
320
344
 
321
345
  def _lane_of(self, message_id: str) -> str | None:
346
+ if self._is_dispatching(message_id):
347
+ return None # claimed by the staged batch; its lane entry is spoken for
322
348
  for lane in ("steerQueue", "queue"):
323
349
  if any(item["id"] == message_id for item in self._lane_items(lane)):
324
350
  return lane
@@ -369,15 +395,12 @@ class RunManager(Tray):
369
395
  fx = self._apply_intake()
370
396
  if not self._in_flight():
371
397
  self._stop_staged = False
372
- if isinstance(outcome, RunManager.Complete):
398
+ if outcome is not None and outcome["kind"] == "complete":
373
399
  self.dispatchedIds.value = []
374
400
  if self._staged_rewind is not None:
375
401
  rewind, self._staged_rewind = self._staged_rewind, None
376
- self._clear_input()
377
402
  if self._batch:
378
403
  self._unstage_batch()
379
- self.stopping.value = None
380
- self.nextDispatch.value = None
381
404
  if self.runId.value is None:
382
405
  self.runId.value = rewind.run_id
383
406
  self._dispatch(
@@ -392,6 +415,8 @@ class RunManager(Tray):
392
415
  self._settle_outcome(outcome)
393
416
  else:
394
417
  self._idle_action(fx)
418
+ if self.outcome.value is not None:
419
+ self.outcome.value = None # the follow-on's writes are enlisted ahead
395
420
  if self._in_flight() and (
396
421
  self._stop_staged or self._staged_rewind is not None
397
422
  ):
@@ -448,31 +473,35 @@ class RunManager(Tray):
448
473
  self._dispatch(self._continue_type(), [], root_meta=fx.continue_meta)
449
474
  elif status == "input-required":
450
475
  if self._batch:
451
- self._dispatch_staged(input_outcomes=self._take_input_outcomes())
476
+ self._dispatch_staged(input_outcomes=self._input_outcomes())
452
477
  elif fx.steer_added and self._lane_items("steerQueue"):
453
- self._pop_dispatchable(input_outcomes=self._take_input_outcomes())
478
+ self._pop_dispatchable(input_outcomes=self._input_outcomes())
454
479
  elif self._input_requests and len(self._input_answers) == len(
455
480
  self._input_requests
456
481
  ):
457
482
  self._dispatch(
458
- "input-resume", [], input_outcomes=self._take_input_outcomes()
483
+ "input-resume", [], input_outcomes=self._input_outcomes()
459
484
  )
460
485
 
461
- def _settle_outcome(self, outcome: Any) -> None:
462
- if isinstance(outcome, RunManager.Complete):
486
+ def _settle_outcome(self, outcome: dict[str, Any]) -> None:
487
+ kind = outcome["kind"]
488
+ if kind == "complete":
463
489
  if not self._pop_dispatchable():
464
490
  self._to_ready()
465
491
  return
466
- if isinstance(outcome, RunManager.InputRequired):
467
- self._input_requests = [dict(r) for r in outcome.requests]
492
+ if kind == "input-required":
493
+ requests = outcome["requests"]
494
+ self._input_requests = [dict(r) for r in requests]
468
495
  self._input_answers = {}
469
- self.inputRequests.value = [dict(r) for r in outcome.requests]
496
+ self.inputRequests.value = [dict(r) for r in requests]
470
497
  self._set_status("input-required")
471
498
  return
472
- status = "error" if isinstance(outcome, RunManager.Error) else "stopped"
473
- if outcome.dispatch_queue and self._pop_dispatchable():
499
+ if outcome["dispatchQueue"] and self._pop_dispatchable():
474
500
  return
475
- self._set_status(status)
501
+ error = outcome.get("error")
502
+ if error is not None:
503
+ self.error.value = error
504
+ self._set_status("error" if kind == "error" else "stopped")
476
505
 
477
506
  # ─── Dispatch and settle ────────────────────────────────
478
507
 
@@ -481,13 +510,11 @@ class RunManager(Tray):
481
510
  self._input_answers = {}
482
511
  self.inputRequests.value = None
483
512
 
484
- def _take_input_outcomes(self) -> tuple[tuple[dict[str, Any], Any, Any], ...]:
485
- outcomes = tuple(
513
+ def _input_outcomes(self) -> tuple[tuple[dict[str, Any], Any, Any], ...]:
514
+ return tuple(
486
515
  (request, *self._input_answers.get(request["id"], (None, None)))
487
516
  for request in self._input_requests
488
517
  )
489
- self._clear_input()
490
- return outcomes
491
518
 
492
519
  def _record_of(
493
520
  self,
@@ -530,19 +557,25 @@ class RunManager(Tray):
530
557
  root_meta=root_meta,
531
558
  input_outcomes=input_outcomes,
532
559
  )
560
+ item = {"record": record, "lanes": [lane for lane, _ in self._batch]}
533
561
  self.stopReason.value = None
534
562
  self.epoch.value = self.epoch.value + 1
535
- self.outbox.dispatch(
536
- {"record": record, "lanes": [lane for lane, _ in self._batch]}
537
- )
563
+ self.outbox.dispatch(item)
538
564
  self._fresh = True
565
+ self._reclaim(item) # the record owns its sources: consume them behind it
566
+ self.nextDispatch.value = None
567
+ self.stopping.value = None
568
+ self._clear_input()
539
569
  self.batchApplied.value = False
540
570
  self.error.value = None
541
571
  stripped = [
542
572
  {k: v for k, v in message.items() if k != "meta"} for message in messages
543
573
  ]
544
574
  if stripped:
545
- self.dispatchedIds.value = [m["id"] for m in stripped]
575
+ known = self.dispatchedIds.value
576
+ added = [m["id"] for m in stripped if m["id"] not in known]
577
+ if added:
578
+ self.dispatchedIds.value = [*known, *added]
546
579
  self._set_status("running")
547
580
  self.dispatching.value = record
548
581
  self._idle.clear()
@@ -582,7 +615,8 @@ class RunManager(Tray):
582
615
  ]
583
616
  ctx = RunManager.RunContext(
584
617
  trigger=record["trigger"],
585
- messages=tuple(messages),
618
+ _resume=self.batchApplied.value or self.confirmedTakes.value > 0,
619
+ _messages=tuple(messages),
586
620
  stop_requested=asyncio.Event(),
587
621
  _manager=self,
588
622
  _rollback_to=record["rollbackTo"] if "rollbackTo" in record else _ABSENT,
@@ -638,34 +672,80 @@ class RunManager(Tray):
638
672
  except asyncio.CancelledError:
639
673
  if self._closing or self.storage._retired is not None:
640
674
  raise # teardown: the un-acked outbox redelivers next activation
675
+ self.outcome.value = {
676
+ "kind": "cancel",
677
+ "epoch": self.epoch.value,
678
+ "dispatchQueue": False,
679
+ }
641
680
  self._settle(ctx)
642
681
  fx = self._apply_intake()
682
+ rewind = self._staged_rewind
683
+ if rewind is not None and not rewind.applied_sent:
684
+ # park the in-drain rewind durably; a cancel starts no dispatch
685
+ self.nextDispatch.value = self._record_of(
686
+ rewind.type,
687
+ list(rewind.messages),
688
+ rollback_to=rewind.rollback_to,
689
+ root_meta=rewind.root_meta,
690
+ )
691
+ rewind.applied()
692
+ rewind.applied_sent = True
643
693
  self._set_status("stopped")
694
+ self.outcome.value = None
644
695
  self._settle_applied(fx)
645
696
  self._idle.set()
646
697
  raise # staged intake settled; a cancelled run starts no dispatch
647
698
  except Exception as exc:
648
- self._settle(ctx)
649
- message = str(exc) or type(exc).__name__
650
- if isinstance(exc, StatewireReject):
651
- self._freeze(exc.message, exc.payload)
652
- else:
699
+ if not isinstance(exc, StatewireReject):
653
700
  logger.exception(
654
701
  "run %r crashed in %s",
655
702
  self.runId.value,
656
703
  getattr(fn, "__qualname__", repr(fn)),
657
704
  )
658
- self._freeze(message)
705
+ error = self._error_of(exc)
706
+ self.outcome.value = {
707
+ "kind": "error",
708
+ "epoch": self.epoch.value,
709
+ "dispatchQueue": False,
710
+ "error": error,
711
+ }
712
+ self._settle(ctx)
713
+ self._set_status("error")
714
+ self.error.value = error
659
715
  self._advance()
660
716
  return
717
+ self.outcome.value = self._outcome_record(outcome)
661
718
  self._settle(ctx)
662
- self._outcome = outcome
719
+ self._outcome = self.outcome.value
663
720
  self._advance()
664
721
 
722
+ def _error_of(self, exc: Exception) -> dict[str, Any]:
723
+ if isinstance(exc, StatewireReject):
724
+ if isinstance(exc.payload, dict):
725
+ return {**exc.payload, "message": exc.message}
726
+ if exc.payload is not None:
727
+ return {"message": exc.message, "payload": exc.payload}
728
+ return {"message": exc.message}
729
+ return {"message": str(exc) or type(exc).__name__}
730
+
731
+ def _outcome_record(self, outcome: Any) -> dict[str, Any]:
732
+ epoch = self.epoch.value
733
+ if isinstance(outcome, RunManager.Complete):
734
+ return {"kind": "complete", "epoch": epoch}
735
+ if isinstance(outcome, RunManager.InputRequired):
736
+ return {
737
+ "kind": "input-required",
738
+ "epoch": epoch,
739
+ "requests": [dict(r) for r in outcome.requests],
740
+ }
741
+ kind = "error" if isinstance(outcome, RunManager.Error) else "stop"
742
+ return {"kind": kind, "epoch": epoch, "dispatchQueue": outcome.dispatch_queue}
743
+
665
744
  def _settle(self, ctx: "RunManager.RunContext") -> None:
666
745
  if self._run_live is ctx:
667
746
  self._run_live = None
668
747
  if self._batch:
748
+ self._drop_dispatched(item["id"] for _, item in self._batch)
669
749
  self._unstage_batch()
670
750
  else:
671
751
  record = self.dispatching.value
@@ -674,13 +754,36 @@ class RunManager(Tray):
674
754
  and record["trigger"] != "steer"
675
755
  and record["messages"]
676
756
  ):
677
- self.dispatchedIds.value = []
757
+ # an unapplied rewind settled: its replacement never landed
758
+ self._drop_dispatched(m["id"] for m in record["messages"])
678
759
  self.dispatching.value = None
760
+ self._finish_settle()
761
+
762
+ def _drop_dispatched(self, ids: Iterable[str]) -> None:
763
+ dropped = set(ids)
764
+ kept = [i for i in self.dispatchedIds.value if i not in dropped]
765
+ if len(kept) != len(self.dispatchedIds.value):
766
+ self.dispatchedIds.value = kept
767
+
768
+ def _finish_settle(self) -> None:
679
769
  self.stopping.value = None
680
- self.batchApplied.value = False
681
770
  self.confirmedTakes.value = 0
771
+ self.batchApplied.value = False
682
772
  self._clear_dispatch_state() # commits confirmed takes; drops the run's checkpoint
683
- self.outbox.ack()
773
+ if self.outbox.value is not None:
774
+ self.outbox.ack()
775
+
776
+ def _replay_settle(self) -> None:
777
+ recorded = self.outbox.value
778
+ if recorded is not None and not self.batchApplied.value:
779
+ item = recorded["item"]
780
+ record = item["record"]
781
+ self._batch = list(zip(item["lanes"], record["messages"]))
782
+ self._drop_dispatched(m["id"] for m in record["messages"])
783
+ if self._batch:
784
+ self._unstage_batch()
785
+ self.dispatching.value = None
786
+ self._finish_settle()
684
787
 
685
788
  def _applied(self) -> None:
686
789
  if self.dispatching.value is None:
@@ -700,6 +803,10 @@ class RunManager(Tray):
700
803
  self.steerQueue.untake(self._pending_takes)
701
804
  self._pending_takes = 0
702
805
  return
806
+ for lane, item in taken: # claimed heads the dispatch never consumed
807
+ queue = self._lane(lane)
808
+ if queue.value and queue.peek()["id"] == item["id"]:
809
+ queue.popleft()
703
810
  for lane, item in reversed(taken):
704
811
  self._lane(lane).insert(0, item)
705
812
 
@@ -727,12 +834,18 @@ class RunManager(Tray):
727
834
 
728
835
  def _stage_batch(self, e: _Send) -> None:
729
836
  assert e.message is not None
730
- steer = self._lane_items("steerQueue") if e.lane == "steerQueue" else []
837
+ steer = (
838
+ [
839
+ item
840
+ for item in self._lane_items("steerQueue")
841
+ if not self._is_dispatching(item["id"])
842
+ ]
843
+ if e.lane == "steerQueue"
844
+ else []
845
+ )
731
846
  if len(self._batch) + len(steer) >= self.max_queued:
732
847
  raise _reject("queue-full", f"queue is full ({self.max_queued} messages)")
733
- if steer:
734
- self.steerQueue.value = []
735
- self._batch.extend(("steerQueue", item) for item in steer)
848
+ self._batch.extend(("steerQueue", item) for item in steer)
736
849
  self._batch.append((e.lane, self._stamped(e.message, e.meta)))
737
850
  self.dispatching.value = {
738
851
  "trigger": self._send_trigger(),
@@ -753,12 +866,11 @@ class RunManager(Tray):
753
866
  ) -> bool:
754
867
  steer = self._lane_items("steerQueue")
755
868
  if steer:
756
- self.steerQueue.value = []
757
869
  self._batch = [("steerQueue", item) for item in steer]
758
870
  self._dispatch("message-send", steer, input_outcomes=input_outcomes)
759
871
  return True
760
872
  if self._lane_items("queue"):
761
- item = self.queue.popleft()
873
+ item = self.queue.peek()
762
874
  self._batch = [("queue", item)]
763
875
  self._dispatch("message-send", [item], input_outcomes=input_outcomes)
764
876
  return True
@@ -779,15 +891,6 @@ class RunManager(Tray):
779
891
  if not rewind.future.done():
780
892
  rewind.future.set_result(None)
781
893
 
782
- def _freeze(self, message: str, payload: Any = None) -> None:
783
- self._set_status("error")
784
- if isinstance(payload, dict):
785
- self.error.value = {**payload, "message": message}
786
- elif payload is not None:
787
- self.error.value = {"message": message, "payload": payload}
788
- else:
789
- self.error.value = {"message": message}
790
-
791
894
  # ─── Message and placement validation ───────────────────
792
895
 
793
896
  def _prepared(
@@ -1444,6 +1547,10 @@ class RunManager(Tray):
1444
1547
  return ()
1445
1548
  manager._pending_takes += len(items)
1446
1549
  manager._batch = [("steerQueue", item) for item in items]
1550
+ known = manager.dispatchedIds.value
1551
+ added = [item["id"] for item in items if item["id"] not in known]
1552
+ if added:
1553
+ manager.dispatchedIds.value = [*known, *added]
1447
1554
  manager.dispatching.value = {"trigger": "steer", "messages": items}
1448
1555
  self._ctx._message_meta.update(
1449
1556
  {item["id"]: item.get("meta") for item in items}
@@ -1456,8 +1563,9 @@ class RunManager(Tray):
1456
1563
  @dataclass(frozen=True, eq=False)
1457
1564
  class RunContext:
1458
1565
  trigger: str
1459
- messages: tuple[dict[str, Any], ...]
1460
1566
  stop_requested: asyncio.Event
1567
+ _resume: bool
1568
+ _messages: tuple[dict[str, Any], ...]
1461
1569
  _manager: "RunManager"
1462
1570
  _rollback_to: Any
1463
1571
  _message_meta: dict[str, Any]
@@ -1471,6 +1579,24 @@ class RunManager(Tray):
1471
1579
  self, "steering", RunManager.Steering(asyncio.Event(), self)
1472
1580
  )
1473
1581
 
1582
+ @property
1583
+ def resume(self) -> bool:
1584
+ """True when the batch was already durably applied before this redelivery: do not re-apply it, continue from the app's own continuation state."""
1585
+ return self._resume
1586
+
1587
+ def _batch_only(self, name: str) -> None:
1588
+ if self._resume:
1589
+ raise RuntimeError(
1590
+ f"{name} is unavailable on a resume: the batch is already applied and"
1591
+ " must not be re-dispatched; continue from app-defined state"
1592
+ )
1593
+
1594
+ @property
1595
+ def messages(self) -> tuple[dict[str, Any], ...]:
1596
+ """The dispatched input batch; unavailable on a resume."""
1597
+ self._batch_only("messages")
1598
+ return self._messages
1599
+
1474
1600
  def meta(self, message_id: str | None = None) -> Any:
1475
1601
  """Op meta: a dispatched message's by id, or the run-starting op's with no id."""
1476
1602
  if message_id is None:
@@ -1490,10 +1616,12 @@ class RunManager(Tray):
1490
1616
 
1491
1617
  @property
1492
1618
  def has_rollback(self) -> bool:
1619
+ self._batch_only("has_rollback")
1493
1620
  return self._rollback_to is not _ABSENT
1494
1621
 
1495
1622
  @property
1496
1623
  def rollback_to(self) -> str | None:
1624
+ self._batch_only("rollback_to")
1497
1625
  if self._rollback_to is _ABSENT:
1498
1626
  raise AttributeError(
1499
1627
  "rollback_to is only present on rewind entries; check has_rollback"
@@ -1,13 +1,12 @@
1
1
  """Contract: the durable seams of the run machine. A verdict never outruns the
2
2
  admission's flush; a crash redelivers the in-flight dispatch with its checkpoint
3
- and its epoch; a torn storage segment reconciles on the next activation; and the
4
- rich mid-flight state — the stopping window with its staged rewind, the applied
5
- batch, an unconfirmed steering take, and half-answered input requests — all
6
- hydrate back on the restarted machine.
3
+ and its epoch; and the rich mid-flight state the stopping window with its
4
+ staged rewind, the applied batch, an unconfirmed steering take, and
5
+ half-answered input requests — all hydrate back on the restarted machine.
6
+ Torn storage segments are swept exhaustively in test_run_crash_sweep.py.
7
7
  """
8
8
 
9
9
  import asyncio
10
- import contextlib
11
10
 
12
11
  from pinned.testing import BarrierStateStore, MemoryStateStore
13
12
  from run_helpers import (
@@ -38,26 +37,6 @@ def survivors(store: MemoryStateStore) -> MemoryStateStore:
38
37
  return seeded(MemoryStateStore(), store)
39
38
 
40
39
 
41
- class TornStore(MemoryStateStore):
42
- """Cuts the first batch writing under ``key`` right before that entry: the
43
- writes ahead of it land, the rest of the segment is lost."""
44
-
45
- def __init__(self, key: str) -> None:
46
- super().__init__()
47
- self._key = key
48
- self.tore = asyncio.Event()
49
-
50
- async def save_many(self, ns, id, entries, epoch) -> None:
51
- paths = list(entries)
52
- cut = next((i for i, path in enumerate(paths) if self._key in path), None)
53
- if self.tore.is_set() or cut is None:
54
- await super().save_many(ns, id, entries, epoch)
55
- return
56
- self.tore.set()
57
- await super().save_many(ns, id, {p: entries[p] for p in paths[:cut]}, epoch)
58
- raise RuntimeError("crash mid-segment")
59
-
60
-
61
40
  async def flush(host) -> None:
62
41
  """Await the machine's durable frontier: the crash point is everything before it."""
63
42
  await (await host.live.runs.get()).storage._frontier.wait()
@@ -114,39 +93,6 @@ async def test_a_crash_mid_run_redelivers_from_the_checkpoint():
114
93
  await drv2.wait_status("ready")
115
94
 
116
95
 
117
- async def test_a_torn_dispatch_segment_reconciles_on_the_next_activation():
118
- store = MemoryStateStore()
119
- script = Script()
120
- async with run_host(script, store=store) as (drv, host):
121
- drv.post("run/enqueue", add("m1"))
122
- call = await script.next_call()
123
- call.applied()
124
- await drv.command("run/enqueue", add("m2"), terminal=False)
125
- await flush(host)
126
- crashed = survivors(store)
127
-
128
- torn = seeded(TornStore("queue"), crashed)
129
- script2 = Script()
130
- with contextlib.suppress(Exception):
131
- async with run_host(script2, store=torn) as (drv2, host2):
132
- redelivered = await script2.next_call() # m1's batch is already confirmed
133
- redelivered.finish(RunManager.Complete()) # settling m1 tears on m2's pop
134
- await asyncio.wait_for(torn.tore.wait(), 5)
135
- crashed = survivors(torn)
136
-
137
- # m2's dispatch record landed; the queue entry it came from never cleared
138
- assert any(key[2][:3] == ("runs", "~", "queue") for key in crashed.states)
139
- script3 = Script()
140
- async with run_host(script3, store=crashed) as (drv3, host3):
141
- again = await script3.next_call()
142
- assert [m["id"] for m in again.ctx.messages] == ["m2"]
143
- again.applied()
144
- again.finish(RunManager.Complete())
145
- await drv3.wait_status("ready")
146
- assert queue_ids(drv3.replica, "queue") == []
147
- script3.no_call() # the redelivery reclaimed the lane entry: no second run
148
-
149
-
150
96
  async def test_a_stale_epoch_stop_rejects_across_a_redelivery():
151
97
  store = MemoryStateStore()
152
98
  script = Script()
@@ -223,12 +169,14 @@ async def test_a_staged_rewind_survives_a_crash_in_the_stopping_window():
223
169
  await drv2.wait_status("ready")
224
170
 
225
171
 
226
- async def test_an_applied_batch_is_not_requeued_after_a_crash():
172
+ async def test_an_applied_batch_redelivers_as_a_resume():
227
173
  store = MemoryStateStore()
228
174
  script = Script()
229
175
  async with run_host(script, store=store) as (drv, host):
230
176
  drv.post("run/enqueue", add("m1"))
231
177
  call = await script.next_call()
178
+ assert call.ctx.resume is False
179
+ call.ctx.set_state({"step": 1})
232
180
  call.applied()
233
181
  await flush(host)
234
182
  crashed = survivors(store)
@@ -236,7 +184,23 @@ async def test_an_applied_batch_is_not_requeued_after_a_crash():
236
184
  script2 = Script()
237
185
  async with run_host(script2, store=crashed) as (drv2, host2):
238
186
  redelivered = await script2.next_call()
239
- assert [m["id"] for m in redelivered.ctx.messages] == ["m1"]
187
+ assert redelivered.ctx.resume is True
188
+ assert redelivered.ctx.state == {"step": 1}
189
+ # informational reads of the applied batch stay open
190
+ assert redelivered.ctx.meta() is None
191
+ assert redelivered.ctx.meta("m1") is None
192
+ assert redelivered.ctx.input_outcomes == ()
193
+ for read in (
194
+ lambda: redelivered.ctx.messages,
195
+ lambda: redelivered.ctx.has_rollback,
196
+ lambda: redelivered.ctx.rollback_to,
197
+ ):
198
+ try:
199
+ read()
200
+ except RuntimeError:
201
+ pass
202
+ else:
203
+ raise AssertionError("a resume exposed a re-dispatchable batch")
240
204
  redelivered.finish(RunManager.Stop(dispatch_queue=True))
241
205
  await drv2.wait_status("stopped")
242
206
  assert queue_ids(drv2.replica, "queue") == []
@@ -259,9 +223,10 @@ async def test_an_unconfirmed_take_re_presents_on_the_restarted_steer_lane():
259
223
  script2 = Script()
260
224
  async with run_host(script2, store=crashed) as (drv2, host2):
261
225
  redelivered = await script2.next_call()
262
- assert [m["id"] for m in redelivered.ctx.messages] == ["m1"]
226
+ assert redelivered.ctx.resume is True
263
227
  await drv2.wait(lambda s: queue_ids(s, "steerQueue") == ["s1"])
264
228
  assert [m["id"] for m in redelivered.ctx.steering.take()] == ["s1"]
229
+ assert redelivered.ctx.meta("s1") is None # a take's meta stays readable
265
230
  redelivered.applied()
266
231
  redelivered.finish(RunManager.Complete())
267
232
  await drv2.wait_status("ready")
@@ -30,10 +30,9 @@ async def test_edit_of_dispatched_item_stops_rewinds_reruns(command):
30
30
  assert settled["type"] == "accepted"
31
31
 
32
32
 
33
- @pytest.mark.parametrize("capabilities", [(), ("rewind",)])
34
- async def test_edit_of_dispatched_item_requires_rewind_during_run(capabilities):
33
+ async def test_edit_of_dispatched_item_requires_rewind_during_run():
35
34
  script = Script()
36
- async with run_host(script, capabilities=capabilities) as (drv, host):
35
+ async with run_host(script, capabilities=("rewind",)) as (drv, host):
37
36
  drv.post("run/enqueue", add("m1"))
38
37
  (await script.next_call()).applied()
39
38
  script.thread["m1"] = {"parentId": None, "role": "user", "isLeaf": True}
@@ -110,7 +110,12 @@ async def test_edit_unknown_source_and_duplicate_replacement():
110
110
  "duplicate-id",
111
111
  )
112
112
  drv.post("run/edit", edit_params("u1", msg("u1", "v2"), anchor=None))
113
- await script.next_call()
113
+ call = await script.next_call()
114
+ assert call.ctx.trigger == "message-edit"
115
+ assert [(m["id"], m["parts"][0]["text"]) for m in call.ctx.messages] == [
116
+ ("u1", "v2")
117
+ ]
118
+ assert call.ctx.rollback_to is None
114
119
 
115
120
 
116
121
  async def test_edit_of_assistant_message_requires_assistant_edit():
@@ -13,8 +13,12 @@ async def test_ready_add_dispatches_message_send():
13
13
  assert [m["id"] for m in call.ctx.messages] == ["m1"]
14
14
  assert not call.ctx.has_rollback
15
15
  await drv.wait_status("running")
16
- assert "dispatch" not in drv.replica
16
+ assert drv.run["dispatching"] == {
17
+ "trigger": "message-send",
18
+ "messages": [msg("m1")],
19
+ }
17
20
  call.applied()
21
+ await drv.wait(lambda s: run_of(s)["dispatching"] is None)
18
22
  assert (await drv.res(seq))["type"] == "accepted"
19
23
  call.finish(RunManager.Complete())
20
24
  await drv.wait_status("ready")
@@ -142,12 +146,13 @@ async def test_move_of_dispatched_item():
142
146
  async with run_host(script) as (drv, host):
143
147
  drv.post("run/enqueue", add("m1"))
144
148
  (await script.next_call()).applied()
149
+ await drv.command("run/steer", add("s1"), terminal=False)
145
150
  assert_rejected(
146
151
  await drv.command("run/enqueue", target(messageId="m1"), terminal=False), "already-dispatched"
147
152
  )
148
153
  res = await drv.command("run/steer", target(messageId="m1"), terminal=False)
149
154
  assert res["type"] == "accepted"
150
- assert queue_ids(drv.replica, "steerQueue") == []
155
+ assert queue_ids(drv.replica, "steerQueue") == ["s1"]
151
156
  script.no_call()
152
157
 
153
158