harness-sdk-python 0.17.0__tar.gz → 0.18.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {harness_sdk_python-0.17.0 → harness_sdk_python-0.18.0}/PKG-INFO +3 -3
- {harness_sdk_python-0.17.0 → harness_sdk_python-0.18.0}/pyproject.toml +2 -2
- {harness_sdk_python-0.17.0 → harness_sdk_python-0.18.0}/src/harness_sdk/run_manager.py +180 -52
- {harness_sdk_python-0.17.0 → harness_sdk_python-0.18.0}/tests/test_durability.py +26 -61
- {harness_sdk_python-0.17.0 → harness_sdk_python-0.18.0}/tests/test_edit_dispatched.py +2 -3
- {harness_sdk_python-0.17.0 → harness_sdk_python-0.18.0}/tests/test_edit_reload.py +6 -1
- {harness_sdk_python-0.17.0 → harness_sdk_python-0.18.0}/tests/test_enqueue.py +7 -2
- {harness_sdk_python-0.17.0 → harness_sdk_python-0.18.0}/tests/test_fenced_postgres.py +8 -1
- {harness_sdk_python-0.17.0 → harness_sdk_python-0.18.0}/tests/test_input_required.py +1 -12
- {harness_sdk_python-0.17.0 → harness_sdk_python-0.18.0}/tests/test_rewind_during_run.py +2 -3
- harness_sdk_python-0.18.0/tests/test_run_crash_sweep.py +619 -0
- {harness_sdk_python-0.17.0 → harness_sdk_python-0.18.0}/tests/test_settle.py +39 -0
- {harness_sdk_python-0.17.0 → harness_sdk_python-0.18.0}/tests/test_steer.py +19 -0
- {harness_sdk_python-0.17.0 → harness_sdk_python-0.18.0}/.gitignore +0 -0
- {harness_sdk_python-0.17.0 → harness_sdk_python-0.18.0}/README.md +0 -0
- {harness_sdk_python-0.17.0 → harness_sdk_python-0.18.0}/examples/__init__.py +0 -0
- {harness_sdk_python-0.17.0 → harness_sdk_python-0.18.0}/examples/runs_app.py +0 -0
- {harness_sdk_python-0.17.0 → harness_sdk_python-0.18.0}/src/harness_sdk/__init__.py +0 -0
- {harness_sdk_python-0.17.0 → harness_sdk_python-0.18.0}/src/harness_sdk/fenced_postgres.py +0 -0
- {harness_sdk_python-0.17.0 → harness_sdk_python-0.18.0}/src/harness_sdk/linear_thread.py +0 -0
- {harness_sdk_python-0.17.0 → harness_sdk_python-0.18.0}/tests/run_helpers.py +0 -0
- {harness_sdk_python-0.17.0 → harness_sdk_python-0.18.0}/tests/test_applied_visibility.py +0 -0
- {harness_sdk_python-0.17.0 → harness_sdk_python-0.18.0}/tests/test_batches.py +0 -0
- {harness_sdk_python-0.17.0 → harness_sdk_python-0.18.0}/tests/test_branch_anchor.py +0 -0
- {harness_sdk_python-0.17.0 → harness_sdk_python-0.18.0}/tests/test_dispatching.py +0 -0
- {harness_sdk_python-0.17.0 → harness_sdk_python-0.18.0}/tests/test_linear_thread.py +0 -0
- {harness_sdk_python-0.17.0 → harness_sdk_python-0.18.0}/tests/test_meta.py +0 -0
- {harness_sdk_python-0.17.0 → harness_sdk_python-0.18.0}/tests/test_outcomes.py +0 -0
- {harness_sdk_python-0.17.0 → harness_sdk_python-0.18.0}/tests/test_placement.py +0 -0
- {harness_sdk_python-0.17.0 → harness_sdk_python-0.18.0}/tests/test_prepare_hooks.py +0 -0
- {harness_sdk_python-0.17.0 → harness_sdk_python-0.18.0}/tests/test_restore.py +0 -0
- {harness_sdk_python-0.17.0 → harness_sdk_python-0.18.0}/tests/test_stop_continue.py +0 -0
- {harness_sdk_python-0.17.0 → harness_sdk_python-0.18.0}/tests/test_stopping_window.py +0 -0
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: harness-sdk-python
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.18.0
|
|
4
4
|
Summary: RunManager: the harness-sdk runs subsystem for Python Statewire hosts
|
|
5
5
|
Project-URL: Repository, https://github.com/assistant-ui/harness-sdk
|
|
6
6
|
License-Expression: MIT
|
|
7
7
|
Requires-Python: <4.0,>=3.12
|
|
8
|
-
Requires-Dist: pinned<0.
|
|
9
|
-
Requires-Dist: statewire<0.10,>=0.9.
|
|
8
|
+
Requires-Dist: pinned<0.13,>=0.12.0
|
|
9
|
+
Requires-Dist: statewire<0.10,>=0.9.2
|
|
10
10
|
Provides-Extra: deepagents
|
|
11
11
|
Requires-Dist: deepagents>=0.6.12; extra == 'deepagents'
|
|
12
12
|
Requires-Dist: langchain-core>=0.3; extra == 'deepagents'
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "harness-sdk-python"
|
|
3
|
-
version = "0.
|
|
3
|
+
version = "0.18.0"
|
|
4
4
|
description = "RunManager: the harness-sdk runs subsystem for Python Statewire hosts"
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
license = "MIT"
|
|
7
7
|
requires-python = ">=3.12,<4.0"
|
|
8
|
-
dependencies = ["statewire>=0.9.
|
|
8
|
+
dependencies = ["statewire>=0.9.2,<0.10", "pinned>=0.12.0,<0.13"]
|
|
9
9
|
|
|
10
10
|
[project.optional-dependencies]
|
|
11
11
|
postgres = ["langgraph-checkpoint-postgres>=2.0.0"]
|
|
@@ -19,8 +19,12 @@ the unconfirmed batch record; ``ctx.applied()`` confirms it durably,
|
|
|
19
19
|
``steering.take()`` may refill it via deferred durable pops — at most one
|
|
20
20
|
unconfirmed batch ever exists. Settle consumes a confirmed batch; a batch
|
|
21
21
|
still unconfirmed at settle returns to the front of its lanes (after a crash,
|
|
22
|
-
unconfirmed taken entries re-present there on their own).
|
|
23
|
-
|
|
22
|
+
unconfirmed taken entries re-present there on their own). A batch already
|
|
23
|
+
durably confirmed redelivers in resume mode instead (``ctx.resume``): the
|
|
24
|
+
input batch must not be re-dispatched (``messages`` and the rollback surface
|
|
25
|
+
raise; meta and input outcomes stay readable) and the run continues from its
|
|
26
|
+
own continuation state (``ctx.state`` or an external checkpointer).
|
|
27
|
+
``Complete`` with an unconfirmed batch settles as an error.
|
|
24
28
|
|
|
25
29
|
``epoch`` counts the entry's dispatches (not redeliveries); a ``run/stop``
|
|
26
30
|
carrying an ``epoch`` other than the live one rejects. A mid-run stop or
|
|
@@ -187,6 +191,7 @@ class RunManager(Tray):
|
|
|
187
191
|
queue = persistent_queue()
|
|
188
192
|
steerQueue = persistent_queue()
|
|
189
193
|
outbox = persistent_outbox() # {"record": ..., "lanes": [...]}: the in-flight dispatch
|
|
194
|
+
outcome = persistent(None) # the settled run's verdict, held until its follow-on lands
|
|
190
195
|
batchApplied = persistent(False)
|
|
191
196
|
confirmedTakes = persistent(0)
|
|
192
197
|
runId = persistent(None)
|
|
@@ -260,6 +265,13 @@ class RunManager(Tray):
|
|
|
260
265
|
|
|
261
266
|
def _hydrate_machine(self) -> None:
|
|
262
267
|
self.steerQueue.take_now(self.confirmedTakes.value)
|
|
268
|
+
settled = self.outcome.value
|
|
269
|
+
if settled is not None:
|
|
270
|
+
if self.outbox.value is not None and self.epoch.value != settled["epoch"]:
|
|
271
|
+
self.outcome.value = None # the follow-on dispatch landed: the settle is done
|
|
272
|
+
else:
|
|
273
|
+
self._replay_settle()
|
|
274
|
+
self._outcome = settled
|
|
263
275
|
recorded = self.outbox.value
|
|
264
276
|
if recorded is not None and not self.batchApplied.value:
|
|
265
277
|
record = recorded["item"]["record"]
|
|
@@ -268,7 +280,19 @@ class RunManager(Tray):
|
|
|
268
280
|
self.dispatching.value = record
|
|
269
281
|
else:
|
|
270
282
|
self.dispatching.value = None
|
|
283
|
+
if recorded is not None:
|
|
284
|
+
# heal a dispatch torn after its outbox write landed
|
|
285
|
+
self._clear_input()
|
|
286
|
+
self.error.value = None
|
|
287
|
+
self._set_status("running")
|
|
271
288
|
staged = self.nextDispatch.value
|
|
289
|
+
if (
|
|
290
|
+
staged is not None
|
|
291
|
+
and recorded is not None
|
|
292
|
+
and recorded["item"]["record"] == staged
|
|
293
|
+
):
|
|
294
|
+
self.nextDispatch.value = None # torn promotion: the outbox carries it
|
|
295
|
+
staged = None
|
|
272
296
|
if staged is not None:
|
|
273
297
|
self._staged_rewind = _Rewind(
|
|
274
298
|
staged["trigger"],
|
|
@@ -319,6 +343,8 @@ class RunManager(Tray):
|
|
|
319
343
|
return self._lane(lane).value
|
|
320
344
|
|
|
321
345
|
def _lane_of(self, message_id: str) -> str | None:
|
|
346
|
+
if self._is_dispatching(message_id):
|
|
347
|
+
return None # claimed by the staged batch; its lane entry is spoken for
|
|
322
348
|
for lane in ("steerQueue", "queue"):
|
|
323
349
|
if any(item["id"] == message_id for item in self._lane_items(lane)):
|
|
324
350
|
return lane
|
|
@@ -369,15 +395,12 @@ class RunManager(Tray):
|
|
|
369
395
|
fx = self._apply_intake()
|
|
370
396
|
if not self._in_flight():
|
|
371
397
|
self._stop_staged = False
|
|
372
|
-
if
|
|
398
|
+
if outcome is not None and outcome["kind"] == "complete":
|
|
373
399
|
self.dispatchedIds.value = []
|
|
374
400
|
if self._staged_rewind is not None:
|
|
375
401
|
rewind, self._staged_rewind = self._staged_rewind, None
|
|
376
|
-
self._clear_input()
|
|
377
402
|
if self._batch:
|
|
378
403
|
self._unstage_batch()
|
|
379
|
-
self.stopping.value = None
|
|
380
|
-
self.nextDispatch.value = None
|
|
381
404
|
if self.runId.value is None:
|
|
382
405
|
self.runId.value = rewind.run_id
|
|
383
406
|
self._dispatch(
|
|
@@ -392,6 +415,8 @@ class RunManager(Tray):
|
|
|
392
415
|
self._settle_outcome(outcome)
|
|
393
416
|
else:
|
|
394
417
|
self._idle_action(fx)
|
|
418
|
+
if self.outcome.value is not None:
|
|
419
|
+
self.outcome.value = None # the follow-on's writes are enlisted ahead
|
|
395
420
|
if self._in_flight() and (
|
|
396
421
|
self._stop_staged or self._staged_rewind is not None
|
|
397
422
|
):
|
|
@@ -448,31 +473,35 @@ class RunManager(Tray):
|
|
|
448
473
|
self._dispatch(self._continue_type(), [], root_meta=fx.continue_meta)
|
|
449
474
|
elif status == "input-required":
|
|
450
475
|
if self._batch:
|
|
451
|
-
self._dispatch_staged(input_outcomes=self.
|
|
476
|
+
self._dispatch_staged(input_outcomes=self._input_outcomes())
|
|
452
477
|
elif fx.steer_added and self._lane_items("steerQueue"):
|
|
453
|
-
self._pop_dispatchable(input_outcomes=self.
|
|
478
|
+
self._pop_dispatchable(input_outcomes=self._input_outcomes())
|
|
454
479
|
elif self._input_requests and len(self._input_answers) == len(
|
|
455
480
|
self._input_requests
|
|
456
481
|
):
|
|
457
482
|
self._dispatch(
|
|
458
|
-
"input-resume", [], input_outcomes=self.
|
|
483
|
+
"input-resume", [], input_outcomes=self._input_outcomes()
|
|
459
484
|
)
|
|
460
485
|
|
|
461
|
-
def _settle_outcome(self, outcome: Any) -> None:
|
|
462
|
-
|
|
486
|
+
def _settle_outcome(self, outcome: dict[str, Any]) -> None:
|
|
487
|
+
kind = outcome["kind"]
|
|
488
|
+
if kind == "complete":
|
|
463
489
|
if not self._pop_dispatchable():
|
|
464
490
|
self._to_ready()
|
|
465
491
|
return
|
|
466
|
-
if
|
|
467
|
-
|
|
492
|
+
if kind == "input-required":
|
|
493
|
+
requests = outcome["requests"]
|
|
494
|
+
self._input_requests = [dict(r) for r in requests]
|
|
468
495
|
self._input_answers = {}
|
|
469
|
-
self.inputRequests.value = [dict(r) for r in
|
|
496
|
+
self.inputRequests.value = [dict(r) for r in requests]
|
|
470
497
|
self._set_status("input-required")
|
|
471
498
|
return
|
|
472
|
-
|
|
473
|
-
if outcome.dispatch_queue and self._pop_dispatchable():
|
|
499
|
+
if outcome["dispatchQueue"] and self._pop_dispatchable():
|
|
474
500
|
return
|
|
475
|
-
|
|
501
|
+
error = outcome.get("error")
|
|
502
|
+
if error is not None:
|
|
503
|
+
self.error.value = error
|
|
504
|
+
self._set_status("error" if kind == "error" else "stopped")
|
|
476
505
|
|
|
477
506
|
# ─── Dispatch and settle ────────────────────────────────
|
|
478
507
|
|
|
@@ -481,13 +510,11 @@ class RunManager(Tray):
|
|
|
481
510
|
self._input_answers = {}
|
|
482
511
|
self.inputRequests.value = None
|
|
483
512
|
|
|
484
|
-
def
|
|
485
|
-
|
|
513
|
+
def _input_outcomes(self) -> tuple[tuple[dict[str, Any], Any, Any], ...]:
|
|
514
|
+
return tuple(
|
|
486
515
|
(request, *self._input_answers.get(request["id"], (None, None)))
|
|
487
516
|
for request in self._input_requests
|
|
488
517
|
)
|
|
489
|
-
self._clear_input()
|
|
490
|
-
return outcomes
|
|
491
518
|
|
|
492
519
|
def _record_of(
|
|
493
520
|
self,
|
|
@@ -530,19 +557,25 @@ class RunManager(Tray):
|
|
|
530
557
|
root_meta=root_meta,
|
|
531
558
|
input_outcomes=input_outcomes,
|
|
532
559
|
)
|
|
560
|
+
item = {"record": record, "lanes": [lane for lane, _ in self._batch]}
|
|
533
561
|
self.stopReason.value = None
|
|
534
562
|
self.epoch.value = self.epoch.value + 1
|
|
535
|
-
self.outbox.dispatch(
|
|
536
|
-
{"record": record, "lanes": [lane for lane, _ in self._batch]}
|
|
537
|
-
)
|
|
563
|
+
self.outbox.dispatch(item)
|
|
538
564
|
self._fresh = True
|
|
565
|
+
self._reclaim(item) # the record owns its sources: consume them behind it
|
|
566
|
+
self.nextDispatch.value = None
|
|
567
|
+
self.stopping.value = None
|
|
568
|
+
self._clear_input()
|
|
539
569
|
self.batchApplied.value = False
|
|
540
570
|
self.error.value = None
|
|
541
571
|
stripped = [
|
|
542
572
|
{k: v for k, v in message.items() if k != "meta"} for message in messages
|
|
543
573
|
]
|
|
544
574
|
if stripped:
|
|
545
|
-
self.dispatchedIds.value
|
|
575
|
+
known = self.dispatchedIds.value
|
|
576
|
+
added = [m["id"] for m in stripped if m["id"] not in known]
|
|
577
|
+
if added:
|
|
578
|
+
self.dispatchedIds.value = [*known, *added]
|
|
546
579
|
self._set_status("running")
|
|
547
580
|
self.dispatching.value = record
|
|
548
581
|
self._idle.clear()
|
|
@@ -582,7 +615,8 @@ class RunManager(Tray):
|
|
|
582
615
|
]
|
|
583
616
|
ctx = RunManager.RunContext(
|
|
584
617
|
trigger=record["trigger"],
|
|
585
|
-
|
|
618
|
+
_resume=self.batchApplied.value or self.confirmedTakes.value > 0,
|
|
619
|
+
_messages=tuple(messages),
|
|
586
620
|
stop_requested=asyncio.Event(),
|
|
587
621
|
_manager=self,
|
|
588
622
|
_rollback_to=record["rollbackTo"] if "rollbackTo" in record else _ABSENT,
|
|
@@ -638,34 +672,80 @@ class RunManager(Tray):
|
|
|
638
672
|
except asyncio.CancelledError:
|
|
639
673
|
if self._closing or self.storage._retired is not None:
|
|
640
674
|
raise # teardown: the un-acked outbox redelivers next activation
|
|
675
|
+
self.outcome.value = {
|
|
676
|
+
"kind": "cancel",
|
|
677
|
+
"epoch": self.epoch.value,
|
|
678
|
+
"dispatchQueue": False,
|
|
679
|
+
}
|
|
641
680
|
self._settle(ctx)
|
|
642
681
|
fx = self._apply_intake()
|
|
682
|
+
rewind = self._staged_rewind
|
|
683
|
+
if rewind is not None and not rewind.applied_sent:
|
|
684
|
+
# park the in-drain rewind durably; a cancel starts no dispatch
|
|
685
|
+
self.nextDispatch.value = self._record_of(
|
|
686
|
+
rewind.type,
|
|
687
|
+
list(rewind.messages),
|
|
688
|
+
rollback_to=rewind.rollback_to,
|
|
689
|
+
root_meta=rewind.root_meta,
|
|
690
|
+
)
|
|
691
|
+
rewind.applied()
|
|
692
|
+
rewind.applied_sent = True
|
|
643
693
|
self._set_status("stopped")
|
|
694
|
+
self.outcome.value = None
|
|
644
695
|
self._settle_applied(fx)
|
|
645
696
|
self._idle.set()
|
|
646
697
|
raise # staged intake settled; a cancelled run starts no dispatch
|
|
647
698
|
except Exception as exc:
|
|
648
|
-
|
|
649
|
-
message = str(exc) or type(exc).__name__
|
|
650
|
-
if isinstance(exc, StatewireReject):
|
|
651
|
-
self._freeze(exc.message, exc.payload)
|
|
652
|
-
else:
|
|
699
|
+
if not isinstance(exc, StatewireReject):
|
|
653
700
|
logger.exception(
|
|
654
701
|
"run %r crashed in %s",
|
|
655
702
|
self.runId.value,
|
|
656
703
|
getattr(fn, "__qualname__", repr(fn)),
|
|
657
704
|
)
|
|
658
|
-
|
|
705
|
+
error = self._error_of(exc)
|
|
706
|
+
self.outcome.value = {
|
|
707
|
+
"kind": "error",
|
|
708
|
+
"epoch": self.epoch.value,
|
|
709
|
+
"dispatchQueue": False,
|
|
710
|
+
"error": error,
|
|
711
|
+
}
|
|
712
|
+
self._settle(ctx)
|
|
713
|
+
self._set_status("error")
|
|
714
|
+
self.error.value = error
|
|
659
715
|
self._advance()
|
|
660
716
|
return
|
|
717
|
+
self.outcome.value = self._outcome_record(outcome)
|
|
661
718
|
self._settle(ctx)
|
|
662
|
-
self._outcome = outcome
|
|
719
|
+
self._outcome = self.outcome.value
|
|
663
720
|
self._advance()
|
|
664
721
|
|
|
722
|
+
def _error_of(self, exc: Exception) -> dict[str, Any]:
|
|
723
|
+
if isinstance(exc, StatewireReject):
|
|
724
|
+
if isinstance(exc.payload, dict):
|
|
725
|
+
return {**exc.payload, "message": exc.message}
|
|
726
|
+
if exc.payload is not None:
|
|
727
|
+
return {"message": exc.message, "payload": exc.payload}
|
|
728
|
+
return {"message": exc.message}
|
|
729
|
+
return {"message": str(exc) or type(exc).__name__}
|
|
730
|
+
|
|
731
|
+
def _outcome_record(self, outcome: Any) -> dict[str, Any]:
|
|
732
|
+
epoch = self.epoch.value
|
|
733
|
+
if isinstance(outcome, RunManager.Complete):
|
|
734
|
+
return {"kind": "complete", "epoch": epoch}
|
|
735
|
+
if isinstance(outcome, RunManager.InputRequired):
|
|
736
|
+
return {
|
|
737
|
+
"kind": "input-required",
|
|
738
|
+
"epoch": epoch,
|
|
739
|
+
"requests": [dict(r) for r in outcome.requests],
|
|
740
|
+
}
|
|
741
|
+
kind = "error" if isinstance(outcome, RunManager.Error) else "stop"
|
|
742
|
+
return {"kind": kind, "epoch": epoch, "dispatchQueue": outcome.dispatch_queue}
|
|
743
|
+
|
|
665
744
|
def _settle(self, ctx: "RunManager.RunContext") -> None:
|
|
666
745
|
if self._run_live is ctx:
|
|
667
746
|
self._run_live = None
|
|
668
747
|
if self._batch:
|
|
748
|
+
self._drop_dispatched(item["id"] for _, item in self._batch)
|
|
669
749
|
self._unstage_batch()
|
|
670
750
|
else:
|
|
671
751
|
record = self.dispatching.value
|
|
@@ -674,13 +754,36 @@ class RunManager(Tray):
|
|
|
674
754
|
and record["trigger"] != "steer"
|
|
675
755
|
and record["messages"]
|
|
676
756
|
):
|
|
677
|
-
|
|
757
|
+
# an unapplied rewind settled: its replacement never landed
|
|
758
|
+
self._drop_dispatched(m["id"] for m in record["messages"])
|
|
678
759
|
self.dispatching.value = None
|
|
760
|
+
self._finish_settle()
|
|
761
|
+
|
|
762
|
+
def _drop_dispatched(self, ids: Iterable[str]) -> None:
|
|
763
|
+
dropped = set(ids)
|
|
764
|
+
kept = [i for i in self.dispatchedIds.value if i not in dropped]
|
|
765
|
+
if len(kept) != len(self.dispatchedIds.value):
|
|
766
|
+
self.dispatchedIds.value = kept
|
|
767
|
+
|
|
768
|
+
def _finish_settle(self) -> None:
|
|
679
769
|
self.stopping.value = None
|
|
680
|
-
self.batchApplied.value = False
|
|
681
770
|
self.confirmedTakes.value = 0
|
|
771
|
+
self.batchApplied.value = False
|
|
682
772
|
self._clear_dispatch_state() # commits confirmed takes; drops the run's checkpoint
|
|
683
|
-
self.outbox.
|
|
773
|
+
if self.outbox.value is not None:
|
|
774
|
+
self.outbox.ack()
|
|
775
|
+
|
|
776
|
+
def _replay_settle(self) -> None:
|
|
777
|
+
recorded = self.outbox.value
|
|
778
|
+
if recorded is not None and not self.batchApplied.value:
|
|
779
|
+
item = recorded["item"]
|
|
780
|
+
record = item["record"]
|
|
781
|
+
self._batch = list(zip(item["lanes"], record["messages"]))
|
|
782
|
+
self._drop_dispatched(m["id"] for m in record["messages"])
|
|
783
|
+
if self._batch:
|
|
784
|
+
self._unstage_batch()
|
|
785
|
+
self.dispatching.value = None
|
|
786
|
+
self._finish_settle()
|
|
684
787
|
|
|
685
788
|
def _applied(self) -> None:
|
|
686
789
|
if self.dispatching.value is None:
|
|
@@ -700,6 +803,10 @@ class RunManager(Tray):
|
|
|
700
803
|
self.steerQueue.untake(self._pending_takes)
|
|
701
804
|
self._pending_takes = 0
|
|
702
805
|
return
|
|
806
|
+
for lane, item in taken: # claimed heads the dispatch never consumed
|
|
807
|
+
queue = self._lane(lane)
|
|
808
|
+
if queue.value and queue.peek()["id"] == item["id"]:
|
|
809
|
+
queue.popleft()
|
|
703
810
|
for lane, item in reversed(taken):
|
|
704
811
|
self._lane(lane).insert(0, item)
|
|
705
812
|
|
|
@@ -727,12 +834,18 @@ class RunManager(Tray):
|
|
|
727
834
|
|
|
728
835
|
def _stage_batch(self, e: _Send) -> None:
|
|
729
836
|
assert e.message is not None
|
|
730
|
-
steer =
|
|
837
|
+
steer = (
|
|
838
|
+
[
|
|
839
|
+
item
|
|
840
|
+
for item in self._lane_items("steerQueue")
|
|
841
|
+
if not self._is_dispatching(item["id"])
|
|
842
|
+
]
|
|
843
|
+
if e.lane == "steerQueue"
|
|
844
|
+
else []
|
|
845
|
+
)
|
|
731
846
|
if len(self._batch) + len(steer) >= self.max_queued:
|
|
732
847
|
raise _reject("queue-full", f"queue is full ({self.max_queued} messages)")
|
|
733
|
-
|
|
734
|
-
self.steerQueue.value = []
|
|
735
|
-
self._batch.extend(("steerQueue", item) for item in steer)
|
|
848
|
+
self._batch.extend(("steerQueue", item) for item in steer)
|
|
736
849
|
self._batch.append((e.lane, self._stamped(e.message, e.meta)))
|
|
737
850
|
self.dispatching.value = {
|
|
738
851
|
"trigger": self._send_trigger(),
|
|
@@ -753,12 +866,11 @@ class RunManager(Tray):
|
|
|
753
866
|
) -> bool:
|
|
754
867
|
steer = self._lane_items("steerQueue")
|
|
755
868
|
if steer:
|
|
756
|
-
self.steerQueue.value = []
|
|
757
869
|
self._batch = [("steerQueue", item) for item in steer]
|
|
758
870
|
self._dispatch("message-send", steer, input_outcomes=input_outcomes)
|
|
759
871
|
return True
|
|
760
872
|
if self._lane_items("queue"):
|
|
761
|
-
item = self.queue.
|
|
873
|
+
item = self.queue.peek()
|
|
762
874
|
self._batch = [("queue", item)]
|
|
763
875
|
self._dispatch("message-send", [item], input_outcomes=input_outcomes)
|
|
764
876
|
return True
|
|
@@ -779,15 +891,6 @@ class RunManager(Tray):
|
|
|
779
891
|
if not rewind.future.done():
|
|
780
892
|
rewind.future.set_result(None)
|
|
781
893
|
|
|
782
|
-
def _freeze(self, message: str, payload: Any = None) -> None:
|
|
783
|
-
self._set_status("error")
|
|
784
|
-
if isinstance(payload, dict):
|
|
785
|
-
self.error.value = {**payload, "message": message}
|
|
786
|
-
elif payload is not None:
|
|
787
|
-
self.error.value = {"message": message, "payload": payload}
|
|
788
|
-
else:
|
|
789
|
-
self.error.value = {"message": message}
|
|
790
|
-
|
|
791
894
|
# ─── Message and placement validation ───────────────────
|
|
792
895
|
|
|
793
896
|
def _prepared(
|
|
@@ -1444,6 +1547,10 @@ class RunManager(Tray):
|
|
|
1444
1547
|
return ()
|
|
1445
1548
|
manager._pending_takes += len(items)
|
|
1446
1549
|
manager._batch = [("steerQueue", item) for item in items]
|
|
1550
|
+
known = manager.dispatchedIds.value
|
|
1551
|
+
added = [item["id"] for item in items if item["id"] not in known]
|
|
1552
|
+
if added:
|
|
1553
|
+
manager.dispatchedIds.value = [*known, *added]
|
|
1447
1554
|
manager.dispatching.value = {"trigger": "steer", "messages": items}
|
|
1448
1555
|
self._ctx._message_meta.update(
|
|
1449
1556
|
{item["id"]: item.get("meta") for item in items}
|
|
@@ -1456,8 +1563,9 @@ class RunManager(Tray):
|
|
|
1456
1563
|
@dataclass(frozen=True, eq=False)
|
|
1457
1564
|
class RunContext:
|
|
1458
1565
|
trigger: str
|
|
1459
|
-
messages: tuple[dict[str, Any], ...]
|
|
1460
1566
|
stop_requested: asyncio.Event
|
|
1567
|
+
_resume: bool
|
|
1568
|
+
_messages: tuple[dict[str, Any], ...]
|
|
1461
1569
|
_manager: "RunManager"
|
|
1462
1570
|
_rollback_to: Any
|
|
1463
1571
|
_message_meta: dict[str, Any]
|
|
@@ -1471,6 +1579,24 @@ class RunManager(Tray):
|
|
|
1471
1579
|
self, "steering", RunManager.Steering(asyncio.Event(), self)
|
|
1472
1580
|
)
|
|
1473
1581
|
|
|
1582
|
+
@property
|
|
1583
|
+
def resume(self) -> bool:
|
|
1584
|
+
"""True when the batch was already durably applied before this redelivery: do not re-apply it, continue from the app's own continuation state."""
|
|
1585
|
+
return self._resume
|
|
1586
|
+
|
|
1587
|
+
def _batch_only(self, name: str) -> None:
|
|
1588
|
+
if self._resume:
|
|
1589
|
+
raise RuntimeError(
|
|
1590
|
+
f"{name} is unavailable on a resume: the batch is already applied and"
|
|
1591
|
+
" must not be re-dispatched; continue from app-defined state"
|
|
1592
|
+
)
|
|
1593
|
+
|
|
1594
|
+
@property
|
|
1595
|
+
def messages(self) -> tuple[dict[str, Any], ...]:
|
|
1596
|
+
"""The dispatched input batch; unavailable on a resume."""
|
|
1597
|
+
self._batch_only("messages")
|
|
1598
|
+
return self._messages
|
|
1599
|
+
|
|
1474
1600
|
def meta(self, message_id: str | None = None) -> Any:
|
|
1475
1601
|
"""Op meta: a dispatched message's by id, or the run-starting op's with no id."""
|
|
1476
1602
|
if message_id is None:
|
|
@@ -1490,10 +1616,12 @@ class RunManager(Tray):
|
|
|
1490
1616
|
|
|
1491
1617
|
@property
|
|
1492
1618
|
def has_rollback(self) -> bool:
|
|
1619
|
+
self._batch_only("has_rollback")
|
|
1493
1620
|
return self._rollback_to is not _ABSENT
|
|
1494
1621
|
|
|
1495
1622
|
@property
|
|
1496
1623
|
def rollback_to(self) -> str | None:
|
|
1624
|
+
self._batch_only("rollback_to")
|
|
1497
1625
|
if self._rollback_to is _ABSENT:
|
|
1498
1626
|
raise AttributeError(
|
|
1499
1627
|
"rollback_to is only present on rewind entries; check has_rollback"
|
|
@@ -1,13 +1,12 @@
|
|
|
1
1
|
"""Contract: the durable seams of the run machine. A verdict never outruns the
|
|
2
2
|
admission's flush; a crash redelivers the in-flight dispatch with its checkpoint
|
|
3
|
-
and its epoch;
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
3
|
+
and its epoch; and the rich mid-flight state — the stopping window with its
|
|
4
|
+
staged rewind, the applied batch, an unconfirmed steering take, and
|
|
5
|
+
half-answered input requests — all hydrate back on the restarted machine.
|
|
6
|
+
Torn storage segments are swept exhaustively in test_run_crash_sweep.py.
|
|
7
7
|
"""
|
|
8
8
|
|
|
9
9
|
import asyncio
|
|
10
|
-
import contextlib
|
|
11
10
|
|
|
12
11
|
from pinned.testing import BarrierStateStore, MemoryStateStore
|
|
13
12
|
from run_helpers import (
|
|
@@ -38,26 +37,6 @@ def survivors(store: MemoryStateStore) -> MemoryStateStore:
|
|
|
38
37
|
return seeded(MemoryStateStore(), store)
|
|
39
38
|
|
|
40
39
|
|
|
41
|
-
class TornStore(MemoryStateStore):
|
|
42
|
-
"""Cuts the first batch writing under ``key`` right before that entry: the
|
|
43
|
-
writes ahead of it land, the rest of the segment is lost."""
|
|
44
|
-
|
|
45
|
-
def __init__(self, key: str) -> None:
|
|
46
|
-
super().__init__()
|
|
47
|
-
self._key = key
|
|
48
|
-
self.tore = asyncio.Event()
|
|
49
|
-
|
|
50
|
-
async def save_many(self, ns, id, entries, epoch) -> None:
|
|
51
|
-
paths = list(entries)
|
|
52
|
-
cut = next((i for i, path in enumerate(paths) if self._key in path), None)
|
|
53
|
-
if self.tore.is_set() or cut is None:
|
|
54
|
-
await super().save_many(ns, id, entries, epoch)
|
|
55
|
-
return
|
|
56
|
-
self.tore.set()
|
|
57
|
-
await super().save_many(ns, id, {p: entries[p] for p in paths[:cut]}, epoch)
|
|
58
|
-
raise RuntimeError("crash mid-segment")
|
|
59
|
-
|
|
60
|
-
|
|
61
40
|
async def flush(host) -> None:
|
|
62
41
|
"""Await the machine's durable frontier: the crash point is everything before it."""
|
|
63
42
|
await (await host.live.runs.get()).storage._frontier.wait()
|
|
@@ -114,39 +93,6 @@ async def test_a_crash_mid_run_redelivers_from_the_checkpoint():
|
|
|
114
93
|
await drv2.wait_status("ready")
|
|
115
94
|
|
|
116
95
|
|
|
117
|
-
async def test_a_torn_dispatch_segment_reconciles_on_the_next_activation():
|
|
118
|
-
store = MemoryStateStore()
|
|
119
|
-
script = Script()
|
|
120
|
-
async with run_host(script, store=store) as (drv, host):
|
|
121
|
-
drv.post("run/enqueue", add("m1"))
|
|
122
|
-
call = await script.next_call()
|
|
123
|
-
call.applied()
|
|
124
|
-
await drv.command("run/enqueue", add("m2"), terminal=False)
|
|
125
|
-
await flush(host)
|
|
126
|
-
crashed = survivors(store)
|
|
127
|
-
|
|
128
|
-
torn = seeded(TornStore("queue"), crashed)
|
|
129
|
-
script2 = Script()
|
|
130
|
-
with contextlib.suppress(Exception):
|
|
131
|
-
async with run_host(script2, store=torn) as (drv2, host2):
|
|
132
|
-
redelivered = await script2.next_call() # m1's batch is already confirmed
|
|
133
|
-
redelivered.finish(RunManager.Complete()) # settling m1 tears on m2's pop
|
|
134
|
-
await asyncio.wait_for(torn.tore.wait(), 5)
|
|
135
|
-
crashed = survivors(torn)
|
|
136
|
-
|
|
137
|
-
# m2's dispatch record landed; the queue entry it came from never cleared
|
|
138
|
-
assert any(key[2][:3] == ("runs", "~", "queue") for key in crashed.states)
|
|
139
|
-
script3 = Script()
|
|
140
|
-
async with run_host(script3, store=crashed) as (drv3, host3):
|
|
141
|
-
again = await script3.next_call()
|
|
142
|
-
assert [m["id"] for m in again.ctx.messages] == ["m2"]
|
|
143
|
-
again.applied()
|
|
144
|
-
again.finish(RunManager.Complete())
|
|
145
|
-
await drv3.wait_status("ready")
|
|
146
|
-
assert queue_ids(drv3.replica, "queue") == []
|
|
147
|
-
script3.no_call() # the redelivery reclaimed the lane entry: no second run
|
|
148
|
-
|
|
149
|
-
|
|
150
96
|
async def test_a_stale_epoch_stop_rejects_across_a_redelivery():
|
|
151
97
|
store = MemoryStateStore()
|
|
152
98
|
script = Script()
|
|
@@ -223,12 +169,14 @@ async def test_a_staged_rewind_survives_a_crash_in_the_stopping_window():
|
|
|
223
169
|
await drv2.wait_status("ready")
|
|
224
170
|
|
|
225
171
|
|
|
226
|
-
async def
|
|
172
|
+
async def test_an_applied_batch_redelivers_as_a_resume():
|
|
227
173
|
store = MemoryStateStore()
|
|
228
174
|
script = Script()
|
|
229
175
|
async with run_host(script, store=store) as (drv, host):
|
|
230
176
|
drv.post("run/enqueue", add("m1"))
|
|
231
177
|
call = await script.next_call()
|
|
178
|
+
assert call.ctx.resume is False
|
|
179
|
+
call.ctx.set_state({"step": 1})
|
|
232
180
|
call.applied()
|
|
233
181
|
await flush(host)
|
|
234
182
|
crashed = survivors(store)
|
|
@@ -236,7 +184,23 @@ async def test_an_applied_batch_is_not_requeued_after_a_crash():
|
|
|
236
184
|
script2 = Script()
|
|
237
185
|
async with run_host(script2, store=crashed) as (drv2, host2):
|
|
238
186
|
redelivered = await script2.next_call()
|
|
239
|
-
assert
|
|
187
|
+
assert redelivered.ctx.resume is True
|
|
188
|
+
assert redelivered.ctx.state == {"step": 1}
|
|
189
|
+
# informational reads of the applied batch stay open
|
|
190
|
+
assert redelivered.ctx.meta() is None
|
|
191
|
+
assert redelivered.ctx.meta("m1") is None
|
|
192
|
+
assert redelivered.ctx.input_outcomes == ()
|
|
193
|
+
for read in (
|
|
194
|
+
lambda: redelivered.ctx.messages,
|
|
195
|
+
lambda: redelivered.ctx.has_rollback,
|
|
196
|
+
lambda: redelivered.ctx.rollback_to,
|
|
197
|
+
):
|
|
198
|
+
try:
|
|
199
|
+
read()
|
|
200
|
+
except RuntimeError:
|
|
201
|
+
pass
|
|
202
|
+
else:
|
|
203
|
+
raise AssertionError("a resume exposed a re-dispatchable batch")
|
|
240
204
|
redelivered.finish(RunManager.Stop(dispatch_queue=True))
|
|
241
205
|
await drv2.wait_status("stopped")
|
|
242
206
|
assert queue_ids(drv2.replica, "queue") == []
|
|
@@ -259,9 +223,10 @@ async def test_an_unconfirmed_take_re_presents_on_the_restarted_steer_lane():
|
|
|
259
223
|
script2 = Script()
|
|
260
224
|
async with run_host(script2, store=crashed) as (drv2, host2):
|
|
261
225
|
redelivered = await script2.next_call()
|
|
262
|
-
assert
|
|
226
|
+
assert redelivered.ctx.resume is True
|
|
263
227
|
await drv2.wait(lambda s: queue_ids(s, "steerQueue") == ["s1"])
|
|
264
228
|
assert [m["id"] for m in redelivered.ctx.steering.take()] == ["s1"]
|
|
229
|
+
assert redelivered.ctx.meta("s1") is None # a take's meta stays readable
|
|
265
230
|
redelivered.applied()
|
|
266
231
|
redelivered.finish(RunManager.Complete())
|
|
267
232
|
await drv2.wait_status("ready")
|
|
@@ -30,10 +30,9 @@ async def test_edit_of_dispatched_item_stops_rewinds_reruns(command):
|
|
|
30
30
|
assert settled["type"] == "accepted"
|
|
31
31
|
|
|
32
32
|
|
|
33
|
-
|
|
34
|
-
async def test_edit_of_dispatched_item_requires_rewind_during_run(capabilities):
|
|
33
|
+
async def test_edit_of_dispatched_item_requires_rewind_during_run():
|
|
35
34
|
script = Script()
|
|
36
|
-
async with run_host(script, capabilities=
|
|
35
|
+
async with run_host(script, capabilities=("rewind",)) as (drv, host):
|
|
37
36
|
drv.post("run/enqueue", add("m1"))
|
|
38
37
|
(await script.next_call()).applied()
|
|
39
38
|
script.thread["m1"] = {"parentId": None, "role": "user", "isLeaf": True}
|
|
@@ -110,7 +110,12 @@ async def test_edit_unknown_source_and_duplicate_replacement():
|
|
|
110
110
|
"duplicate-id",
|
|
111
111
|
)
|
|
112
112
|
drv.post("run/edit", edit_params("u1", msg("u1", "v2"), anchor=None))
|
|
113
|
-
await script.next_call()
|
|
113
|
+
call = await script.next_call()
|
|
114
|
+
assert call.ctx.trigger == "message-edit"
|
|
115
|
+
assert [(m["id"], m["parts"][0]["text"]) for m in call.ctx.messages] == [
|
|
116
|
+
("u1", "v2")
|
|
117
|
+
]
|
|
118
|
+
assert call.ctx.rollback_to is None
|
|
114
119
|
|
|
115
120
|
|
|
116
121
|
async def test_edit_of_assistant_message_requires_assistant_edit():
|
|
@@ -13,8 +13,12 @@ async def test_ready_add_dispatches_message_send():
|
|
|
13
13
|
assert [m["id"] for m in call.ctx.messages] == ["m1"]
|
|
14
14
|
assert not call.ctx.has_rollback
|
|
15
15
|
await drv.wait_status("running")
|
|
16
|
-
assert "
|
|
16
|
+
assert drv.run["dispatching"] == {
|
|
17
|
+
"trigger": "message-send",
|
|
18
|
+
"messages": [msg("m1")],
|
|
19
|
+
}
|
|
17
20
|
call.applied()
|
|
21
|
+
await drv.wait(lambda s: run_of(s)["dispatching"] is None)
|
|
18
22
|
assert (await drv.res(seq))["type"] == "accepted"
|
|
19
23
|
call.finish(RunManager.Complete())
|
|
20
24
|
await drv.wait_status("ready")
|
|
@@ -142,12 +146,13 @@ async def test_move_of_dispatched_item():
|
|
|
142
146
|
async with run_host(script) as (drv, host):
|
|
143
147
|
drv.post("run/enqueue", add("m1"))
|
|
144
148
|
(await script.next_call()).applied()
|
|
149
|
+
await drv.command("run/steer", add("s1"), terminal=False)
|
|
145
150
|
assert_rejected(
|
|
146
151
|
await drv.command("run/enqueue", target(messageId="m1"), terminal=False), "already-dispatched"
|
|
147
152
|
)
|
|
148
153
|
res = await drv.command("run/steer", target(messageId="m1"), terminal=False)
|
|
149
154
|
assert res["type"] == "accepted"
|
|
150
|
-
assert queue_ids(drv.replica, "steerQueue") == []
|
|
155
|
+
assert queue_ids(drv.replica, "steerQueue") == ["s1"]
|
|
151
156
|
script.no_call()
|
|
152
157
|
|
|
153
158
|
|