harness-sdk-python 0.17.0__tar.gz → 0.19.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {harness_sdk_python-0.17.0 → harness_sdk_python-0.19.0}/PKG-INFO +3 -3
- {harness_sdk_python-0.17.0 → harness_sdk_python-0.19.0}/pyproject.toml +2 -2
- {harness_sdk_python-0.17.0 → harness_sdk_python-0.19.0}/src/harness_sdk/run_manager.py +245 -65
- harness_sdk_python-0.19.0/tests/test_admit_steer.py +118 -0
- {harness_sdk_python-0.17.0 → harness_sdk_python-0.19.0}/tests/test_batches.py +2 -14
- {harness_sdk_python-0.17.0 → harness_sdk_python-0.19.0}/tests/test_durability.py +82 -63
- {harness_sdk_python-0.17.0 → harness_sdk_python-0.19.0}/tests/test_edit_dispatched.py +2 -3
- {harness_sdk_python-0.17.0 → harness_sdk_python-0.19.0}/tests/test_edit_reload.py +6 -1
- {harness_sdk_python-0.17.0 → harness_sdk_python-0.19.0}/tests/test_enqueue.py +7 -2
- {harness_sdk_python-0.17.0 → harness_sdk_python-0.19.0}/tests/test_fenced_postgres.py +8 -1
- {harness_sdk_python-0.17.0 → harness_sdk_python-0.19.0}/tests/test_input_required.py +1 -12
- {harness_sdk_python-0.17.0 → harness_sdk_python-0.19.0}/tests/test_outcomes.py +1 -0
- {harness_sdk_python-0.17.0 → harness_sdk_python-0.19.0}/tests/test_restore.py +1 -0
- {harness_sdk_python-0.17.0 → harness_sdk_python-0.19.0}/tests/test_rewind_during_run.py +2 -3
- harness_sdk_python-0.19.0/tests/test_run_crash_sweep.py +623 -0
- {harness_sdk_python-0.17.0 → harness_sdk_python-0.19.0}/tests/test_settle.py +39 -0
- {harness_sdk_python-0.17.0 → harness_sdk_python-0.19.0}/tests/test_steer.py +19 -0
- {harness_sdk_python-0.17.0 → harness_sdk_python-0.19.0}/tests/test_stop_continue.py +52 -2
- {harness_sdk_python-0.17.0 → harness_sdk_python-0.19.0}/tests/test_stopping_window.py +35 -16
- {harness_sdk_python-0.17.0 → harness_sdk_python-0.19.0}/.gitignore +0 -0
- {harness_sdk_python-0.17.0 → harness_sdk_python-0.19.0}/README.md +0 -0
- {harness_sdk_python-0.17.0 → harness_sdk_python-0.19.0}/examples/__init__.py +0 -0
- {harness_sdk_python-0.17.0 → harness_sdk_python-0.19.0}/examples/runs_app.py +0 -0
- {harness_sdk_python-0.17.0 → harness_sdk_python-0.19.0}/src/harness_sdk/__init__.py +0 -0
- {harness_sdk_python-0.17.0 → harness_sdk_python-0.19.0}/src/harness_sdk/fenced_postgres.py +0 -0
- {harness_sdk_python-0.17.0 → harness_sdk_python-0.19.0}/src/harness_sdk/linear_thread.py +0 -0
- {harness_sdk_python-0.17.0 → harness_sdk_python-0.19.0}/tests/run_helpers.py +0 -0
- {harness_sdk_python-0.17.0 → harness_sdk_python-0.19.0}/tests/test_applied_visibility.py +0 -0
- {harness_sdk_python-0.17.0 → harness_sdk_python-0.19.0}/tests/test_branch_anchor.py +0 -0
- {harness_sdk_python-0.17.0 → harness_sdk_python-0.19.0}/tests/test_dispatching.py +0 -0
- {harness_sdk_python-0.17.0 → harness_sdk_python-0.19.0}/tests/test_linear_thread.py +0 -0
- {harness_sdk_python-0.17.0 → harness_sdk_python-0.19.0}/tests/test_meta.py +0 -0
- {harness_sdk_python-0.17.0 → harness_sdk_python-0.19.0}/tests/test_placement.py +0 -0
- {harness_sdk_python-0.17.0 → harness_sdk_python-0.19.0}/tests/test_prepare_hooks.py +0 -0
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: harness-sdk-python
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.19.0
|
|
4
4
|
Summary: RunManager: the harness-sdk runs subsystem for Python Statewire hosts
|
|
5
5
|
Project-URL: Repository, https://github.com/assistant-ui/harness-sdk
|
|
6
6
|
License-Expression: MIT
|
|
7
7
|
Requires-Python: <4.0,>=3.12
|
|
8
|
-
Requires-Dist: pinned<0.
|
|
9
|
-
Requires-Dist: statewire<0.
|
|
8
|
+
Requires-Dist: pinned<0.14,>=0.13.0
|
|
9
|
+
Requires-Dist: statewire<0.11,>=0.10.0
|
|
10
10
|
Provides-Extra: deepagents
|
|
11
11
|
Requires-Dist: deepagents>=0.6.12; extra == 'deepagents'
|
|
12
12
|
Requires-Dist: langchain-core>=0.3; extra == 'deepagents'
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "harness-sdk-python"
|
|
3
|
-
version = "0.
|
|
3
|
+
version = "0.19.0"
|
|
4
4
|
description = "RunManager: the harness-sdk runs subsystem for Python Statewire hosts"
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
license = "MIT"
|
|
7
7
|
requires-python = ">=3.12,<4.0"
|
|
8
|
-
dependencies = ["statewire>=0.
|
|
8
|
+
dependencies = ["statewire>=0.10.0,<0.11", "pinned>=0.13.0,<0.14"]
|
|
9
9
|
|
|
10
10
|
[project.optional-dependencies]
|
|
11
11
|
postgres = ["langgraph-checkpoint-postgres>=2.0.0"]
|
|
@@ -19,14 +19,21 @@ the unconfirmed batch record; ``ctx.applied()`` confirms it durably,
|
|
|
19
19
|
``steering.take()`` may refill it via deferred durable pops — at most one
|
|
20
20
|
unconfirmed batch ever exists. Settle consumes a confirmed batch; a batch
|
|
21
21
|
still unconfirmed at settle returns to the front of its lanes (after a crash,
|
|
22
|
-
unconfirmed taken entries re-present there on their own).
|
|
23
|
-
|
|
22
|
+
unconfirmed taken entries re-present there on their own). A batch already
|
|
23
|
+
durably confirmed redelivers in resume mode instead (``ctx.resume``): the
|
|
24
|
+
input batch must not be re-dispatched (``messages`` and the rollback surface
|
|
25
|
+
raise; meta and input outcomes stay readable) and the run continues from its
|
|
26
|
+
own continuation state (``ctx.state`` or an external checkpointer).
|
|
27
|
+
``Complete`` with an unconfirmed batch settles as an error.
|
|
24
28
|
|
|
25
29
|
``epoch`` counts the entry's dispatches (not redeliveries); a ``run/stop``
|
|
26
30
|
carrying an ``epoch`` other than the live one rejects. A mid-run stop or
|
|
27
31
|
rewind opens a stopping window (``stopping: {reason}`` plus, for rewinds, the
|
|
28
|
-
staged dispatch as ``nextDispatch``)
|
|
29
|
-
|
|
32
|
+
staged dispatch as ``nextDispatch``) and ``status`` reads ``stopping`` until
|
|
33
|
+
the run settles; the ``run/stop`` verdict follows that settle and its flush,
|
|
34
|
+
so a settled stop covers every durable effect the runner enlisted. On run end
|
|
35
|
+
one envelope promotes ``nextDispatch`` into ``dispatching`` and clears the
|
|
36
|
+
window.
|
|
30
37
|
"""
|
|
31
38
|
|
|
32
39
|
import asyncio
|
|
@@ -34,7 +41,7 @@ import logging
|
|
|
34
41
|
from dataclasses import dataclass, field
|
|
35
42
|
from typing import Any, AsyncIterator, Callable, Coroutine, Iterable, Protocol
|
|
36
43
|
|
|
37
|
-
from pinned.trays import Tray, TrayDescriptor, TrayCtx
|
|
44
|
+
from pinned.trays import Tray, TrayDescriptor, TrayCtx, inbox
|
|
38
45
|
from pinned.traystore import persistent, persistent_outbox, persistent_queue
|
|
39
46
|
from statepatch import observable
|
|
40
47
|
from statewire import StatewireReject
|
|
@@ -74,6 +81,15 @@ def _reject(reason: str, message: str) -> StatewireReject:
|
|
|
74
81
|
return StatewireReject(message, payload={"reason": reason})
|
|
75
82
|
|
|
76
83
|
|
|
84
|
+
def _public(message: dict[str, Any]) -> dict[str, Any]:
|
|
85
|
+
"""The message without its admission tag; the wire state is public."""
|
|
86
|
+
return {k: v for k, v in message.items() if k != "~admit"}
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def _public_record(record: dict[str, Any]) -> dict[str, Any]:
|
|
90
|
+
return {**record, "messages": [_public(m) for m in record["messages"]]}
|
|
91
|
+
|
|
92
|
+
|
|
77
93
|
def _run_id_of(params: Any) -> str:
|
|
78
94
|
run_id = params.get("runId") if isinstance(params, dict) else None
|
|
79
95
|
if not isinstance(run_id, str) or run_id == "":
|
|
@@ -187,6 +203,7 @@ class RunManager(Tray):
|
|
|
187
203
|
queue = persistent_queue()
|
|
188
204
|
steerQueue = persistent_queue()
|
|
189
205
|
outbox = persistent_outbox() # {"record": ..., "lanes": [...]}: the in-flight dispatch
|
|
206
|
+
outcome = persistent(None) # the settled run's verdict, held until its follow-on lands
|
|
190
207
|
batchApplied = persistent(False)
|
|
191
208
|
confirmedTakes = persistent(0)
|
|
192
209
|
runId = persistent(None)
|
|
@@ -199,6 +216,7 @@ class RunManager(Tray):
|
|
|
199
216
|
stopReason = persistent(None)
|
|
200
217
|
dispatchedIds = persistent([])
|
|
201
218
|
dispatching = observable(None) # view: the unconfirmed batch record
|
|
219
|
+
steerView = observable([]) # view: the steer lane without admission tags
|
|
202
220
|
|
|
203
221
|
state = observable(
|
|
204
222
|
{
|
|
@@ -206,12 +224,13 @@ class RunManager(Tray):
|
|
|
206
224
|
"status": status,
|
|
207
225
|
"epoch": epoch,
|
|
208
226
|
"queue": queue,
|
|
209
|
-
"steerQueue":
|
|
227
|
+
"steerQueue": steerView,
|
|
210
228
|
"dispatching": dispatching,
|
|
211
229
|
"stopping": stopping,
|
|
212
230
|
"nextDispatch": nextDispatch,
|
|
213
231
|
"inputRequests": inputRequests,
|
|
214
232
|
"error": error,
|
|
233
|
+
"stopReason": stopReason,
|
|
215
234
|
}
|
|
216
235
|
)
|
|
217
236
|
|
|
@@ -220,6 +239,7 @@ class RunManager(Tray):
|
|
|
220
239
|
self._intake: list[Any] = []
|
|
221
240
|
self._outcome: Any = None
|
|
222
241
|
self._stop_staged = False
|
|
242
|
+
self._stop_waiters: list["asyncio.Future[Any]"] = []
|
|
223
243
|
self._staged_rewind: _Rewind | None = None
|
|
224
244
|
self._batch: list[tuple[str, dict[str, Any]]] = []
|
|
225
245
|
self._pending_takes = 0
|
|
@@ -240,7 +260,10 @@ class RunManager(Tray):
|
|
|
240
260
|
self._thread = api
|
|
241
261
|
self._schedule = api.schedule
|
|
242
262
|
self._hydrate_machine()
|
|
263
|
+
self._mirror_steer()
|
|
264
|
+
unsubscribe = self.steerQueue.subscribe(lambda _op: self._mirror_steer())
|
|
243
265
|
yield
|
|
266
|
+
unsubscribe()
|
|
244
267
|
|
|
245
268
|
class Thread(Protocol):
|
|
246
269
|
"""The thread projection RunManager reads; the host owns the tree."""
|
|
@@ -260,15 +283,36 @@ class RunManager(Tray):
|
|
|
260
283
|
|
|
261
284
|
def _hydrate_machine(self) -> None:
|
|
262
285
|
self.steerQueue.take_now(self.confirmedTakes.value)
|
|
286
|
+
settled = self.outcome.value
|
|
287
|
+
if settled is not None:
|
|
288
|
+
if self.outbox.value is not None and self.epoch.value != settled["epoch"]:
|
|
289
|
+
self.outcome.value = None # the follow-on dispatch landed: the settle is done
|
|
290
|
+
else:
|
|
291
|
+
self._replay_settle()
|
|
292
|
+
self._outcome = settled
|
|
263
293
|
recorded = self.outbox.value
|
|
264
294
|
if recorded is not None and not self.batchApplied.value:
|
|
265
295
|
record = recorded["item"]["record"]
|
|
266
296
|
lanes = recorded["item"]["lanes"]
|
|
267
297
|
self._batch = list(zip(lanes, record["messages"]))
|
|
268
|
-
self.dispatching.value = record
|
|
298
|
+
self.dispatching.value = _public_record(record)
|
|
269
299
|
else:
|
|
270
300
|
self.dispatching.value = None
|
|
301
|
+
if recorded is not None:
|
|
302
|
+
# heal a dispatch torn after its outbox write landed
|
|
303
|
+
self._clear_input()
|
|
304
|
+
self.error.value = None
|
|
305
|
+
self._set_status(
|
|
306
|
+
"stopping" if self.stopping.value is not None else "running"
|
|
307
|
+
)
|
|
271
308
|
staged = self.nextDispatch.value
|
|
309
|
+
if (
|
|
310
|
+
staged is not None
|
|
311
|
+
and recorded is not None
|
|
312
|
+
and recorded["item"]["record"] == staged
|
|
313
|
+
):
|
|
314
|
+
self.nextDispatch.value = None # torn promotion: the outbox carries it
|
|
315
|
+
staged = None
|
|
272
316
|
if staged is not None:
|
|
273
317
|
self._staged_rewind = _Rewind(
|
|
274
318
|
staged["trigger"],
|
|
@@ -319,6 +363,8 @@ class RunManager(Tray):
|
|
|
319
363
|
return self._lane(lane).value
|
|
320
364
|
|
|
321
365
|
def _lane_of(self, message_id: str) -> str | None:
|
|
366
|
+
if self._is_dispatching(message_id):
|
|
367
|
+
return None # claimed by the staged batch; its lane entry is spoken for
|
|
322
368
|
for lane in ("steerQueue", "queue"):
|
|
323
369
|
if any(item["id"] == message_id for item in self._lane_items(lane)):
|
|
324
370
|
return lane
|
|
@@ -330,6 +376,9 @@ class RunManager(Tray):
|
|
|
330
376
|
def _is_dispatching(self, message_id: str) -> bool:
|
|
331
377
|
return any(item["id"] == message_id for _, item in self._batch)
|
|
332
378
|
|
|
379
|
+
def _mirror_steer(self) -> None:
|
|
380
|
+
self.steerView.value = [_public(item) for item in self.steerQueue.value]
|
|
381
|
+
|
|
333
382
|
def _sync_steering(self) -> None:
|
|
334
383
|
ctx = self._run_live
|
|
335
384
|
if ctx is None:
|
|
@@ -355,6 +404,12 @@ class RunManager(Tray):
|
|
|
355
404
|
self._apply(entry, fx)
|
|
356
405
|
return fx
|
|
357
406
|
|
|
407
|
+
def _resolve_stop_waiters(self) -> None:
|
|
408
|
+
waiters, self._stop_waiters = self._stop_waiters, []
|
|
409
|
+
for future in waiters:
|
|
410
|
+
if not future.done():
|
|
411
|
+
future.set_result(None)
|
|
412
|
+
|
|
358
413
|
def _settle_applied(self, fx: _Effects) -> None:
|
|
359
414
|
for future, applied, result in fx.staged_sends:
|
|
360
415
|
applied()
|
|
@@ -369,15 +424,12 @@ class RunManager(Tray):
|
|
|
369
424
|
fx = self._apply_intake()
|
|
370
425
|
if not self._in_flight():
|
|
371
426
|
self._stop_staged = False
|
|
372
|
-
if
|
|
427
|
+
if outcome is not None and outcome["kind"] == "complete":
|
|
373
428
|
self.dispatchedIds.value = []
|
|
374
429
|
if self._staged_rewind is not None:
|
|
375
430
|
rewind, self._staged_rewind = self._staged_rewind, None
|
|
376
|
-
self._clear_input()
|
|
377
431
|
if self._batch:
|
|
378
432
|
self._unstage_batch()
|
|
379
|
-
self.stopping.value = None
|
|
380
|
-
self.nextDispatch.value = None
|
|
381
433
|
if self.runId.value is None:
|
|
382
434
|
self.runId.value = rewind.run_id
|
|
383
435
|
self._dispatch(
|
|
@@ -392,6 +444,9 @@ class RunManager(Tray):
|
|
|
392
444
|
self._settle_outcome(outcome)
|
|
393
445
|
else:
|
|
394
446
|
self._idle_action(fx)
|
|
447
|
+
if self.outcome.value is not None:
|
|
448
|
+
self.outcome.value = None # the follow-on's writes are enlisted ahead
|
|
449
|
+
self._resolve_stop_waiters()
|
|
395
450
|
if self._in_flight() and (
|
|
396
451
|
self._stop_staged or self._staged_rewind is not None
|
|
397
452
|
):
|
|
@@ -409,7 +464,9 @@ class RunManager(Tray):
|
|
|
409
464
|
self._staged_rewind = None
|
|
410
465
|
self.nextDispatch.value = None
|
|
411
466
|
self.stopping.value = {"reason": "stop"}
|
|
412
|
-
|
|
467
|
+
self._set_status("stopping")
|
|
468
|
+
self._stop_waiters.append(entry.future) # the verdict waits for the settle
|
|
469
|
+
elif not entry.future.done():
|
|
413
470
|
entry.future.set_result(None)
|
|
414
471
|
return
|
|
415
472
|
if isinstance(entry, _Continue):
|
|
@@ -448,31 +505,35 @@ class RunManager(Tray):
|
|
|
448
505
|
self._dispatch(self._continue_type(), [], root_meta=fx.continue_meta)
|
|
449
506
|
elif status == "input-required":
|
|
450
507
|
if self._batch:
|
|
451
|
-
self._dispatch_staged(input_outcomes=self.
|
|
508
|
+
self._dispatch_staged(input_outcomes=self._input_outcomes())
|
|
452
509
|
elif fx.steer_added and self._lane_items("steerQueue"):
|
|
453
|
-
self._pop_dispatchable(input_outcomes=self.
|
|
510
|
+
self._pop_dispatchable(input_outcomes=self._input_outcomes())
|
|
454
511
|
elif self._input_requests and len(self._input_answers) == len(
|
|
455
512
|
self._input_requests
|
|
456
513
|
):
|
|
457
514
|
self._dispatch(
|
|
458
|
-
"input-resume", [], input_outcomes=self.
|
|
515
|
+
"input-resume", [], input_outcomes=self._input_outcomes()
|
|
459
516
|
)
|
|
460
517
|
|
|
461
|
-
def _settle_outcome(self, outcome: Any) -> None:
|
|
462
|
-
|
|
518
|
+
def _settle_outcome(self, outcome: dict[str, Any]) -> None:
|
|
519
|
+
kind = outcome["kind"]
|
|
520
|
+
if kind == "complete":
|
|
463
521
|
if not self._pop_dispatchable():
|
|
464
522
|
self._to_ready()
|
|
465
523
|
return
|
|
466
|
-
if
|
|
467
|
-
|
|
524
|
+
if kind == "input-required":
|
|
525
|
+
requests = outcome["requests"]
|
|
526
|
+
self._input_requests = [dict(r) for r in requests]
|
|
468
527
|
self._input_answers = {}
|
|
469
|
-
self.inputRequests.value = [dict(r) for r in
|
|
528
|
+
self.inputRequests.value = [dict(r) for r in requests]
|
|
470
529
|
self._set_status("input-required")
|
|
471
530
|
return
|
|
472
|
-
|
|
473
|
-
if outcome.dispatch_queue and self._pop_dispatchable():
|
|
531
|
+
if outcome["dispatchQueue"] and self._pop_dispatchable():
|
|
474
532
|
return
|
|
475
|
-
|
|
533
|
+
error = outcome.get("error")
|
|
534
|
+
if error is not None:
|
|
535
|
+
self.error.value = error
|
|
536
|
+
self._set_status("error" if kind == "error" else "stopped")
|
|
476
537
|
|
|
477
538
|
# ─── Dispatch and settle ────────────────────────────────
|
|
478
539
|
|
|
@@ -481,13 +542,11 @@ class RunManager(Tray):
|
|
|
481
542
|
self._input_answers = {}
|
|
482
543
|
self.inputRequests.value = None
|
|
483
544
|
|
|
484
|
-
def
|
|
485
|
-
|
|
545
|
+
def _input_outcomes(self) -> tuple[tuple[dict[str, Any], Any, Any], ...]:
|
|
546
|
+
return tuple(
|
|
486
547
|
(request, *self._input_answers.get(request["id"], (None, None)))
|
|
487
548
|
for request in self._input_requests
|
|
488
549
|
)
|
|
489
|
-
self._clear_input()
|
|
490
|
-
return outcomes
|
|
491
550
|
|
|
492
551
|
def _record_of(
|
|
493
552
|
self,
|
|
@@ -530,21 +589,27 @@ class RunManager(Tray):
|
|
|
530
589
|
root_meta=root_meta,
|
|
531
590
|
input_outcomes=input_outcomes,
|
|
532
591
|
)
|
|
592
|
+
item = {"record": record, "lanes": [lane for lane, _ in self._batch]}
|
|
533
593
|
self.stopReason.value = None
|
|
534
594
|
self.epoch.value = self.epoch.value + 1
|
|
535
|
-
self.outbox.dispatch(
|
|
536
|
-
{"record": record, "lanes": [lane for lane, _ in self._batch]}
|
|
537
|
-
)
|
|
595
|
+
self.outbox.dispatch(item)
|
|
538
596
|
self._fresh = True
|
|
597
|
+
self._reclaim(item) # the record owns its sources: consume them behind it
|
|
598
|
+
self.nextDispatch.value = None
|
|
599
|
+
self.stopping.value = None
|
|
600
|
+
self._clear_input()
|
|
539
601
|
self.batchApplied.value = False
|
|
540
602
|
self.error.value = None
|
|
541
603
|
stripped = [
|
|
542
604
|
{k: v for k, v in message.items() if k != "meta"} for message in messages
|
|
543
605
|
]
|
|
544
606
|
if stripped:
|
|
545
|
-
self.dispatchedIds.value
|
|
607
|
+
known = self.dispatchedIds.value
|
|
608
|
+
added = [m["id"] for m in stripped if m["id"] not in known]
|
|
609
|
+
if added:
|
|
610
|
+
self.dispatchedIds.value = [*known, *added]
|
|
546
611
|
self._set_status("running")
|
|
547
|
-
self.dispatching.value = record
|
|
612
|
+
self.dispatching.value = _public_record(record)
|
|
548
613
|
self._idle.clear()
|
|
549
614
|
self.kick()
|
|
550
615
|
|
|
@@ -577,12 +642,13 @@ class RunManager(Tray):
|
|
|
577
642
|
def _make_run_ctx(self, item: Any) -> "RunManager.RunContext":
|
|
578
643
|
record = item["record"]
|
|
579
644
|
messages = [
|
|
580
|
-
{k: v for k, v in message.items() if k
|
|
645
|
+
{k: v for k, v in message.items() if k not in ("meta", "~admit")}
|
|
581
646
|
for message in record["messages"]
|
|
582
647
|
]
|
|
583
648
|
ctx = RunManager.RunContext(
|
|
584
649
|
trigger=record["trigger"],
|
|
585
|
-
|
|
650
|
+
_resume=self.batchApplied.value or self.confirmedTakes.value > 0,
|
|
651
|
+
_messages=tuple(messages),
|
|
586
652
|
stop_requested=asyncio.Event(),
|
|
587
653
|
_manager=self,
|
|
588
654
|
_rollback_to=record["rollbackTo"] if "rollbackTo" in record else _ABSENT,
|
|
@@ -638,34 +704,81 @@ class RunManager(Tray):
|
|
|
638
704
|
except asyncio.CancelledError:
|
|
639
705
|
if self._closing or self.storage._retired is not None:
|
|
640
706
|
raise # teardown: the un-acked outbox redelivers next activation
|
|
707
|
+
self.outcome.value = {
|
|
708
|
+
"kind": "cancel",
|
|
709
|
+
"epoch": self.epoch.value,
|
|
710
|
+
"dispatchQueue": False,
|
|
711
|
+
}
|
|
641
712
|
self._settle(ctx)
|
|
642
713
|
fx = self._apply_intake()
|
|
714
|
+
rewind = self._staged_rewind
|
|
715
|
+
if rewind is not None and not rewind.applied_sent:
|
|
716
|
+
# park the in-drain rewind durably; a cancel starts no dispatch
|
|
717
|
+
self.nextDispatch.value = self._record_of(
|
|
718
|
+
rewind.type,
|
|
719
|
+
list(rewind.messages),
|
|
720
|
+
rollback_to=rewind.rollback_to,
|
|
721
|
+
root_meta=rewind.root_meta,
|
|
722
|
+
)
|
|
723
|
+
rewind.applied()
|
|
724
|
+
rewind.applied_sent = True
|
|
643
725
|
self._set_status("stopped")
|
|
726
|
+
self.outcome.value = None
|
|
727
|
+
self._resolve_stop_waiters()
|
|
644
728
|
self._settle_applied(fx)
|
|
645
729
|
self._idle.set()
|
|
646
730
|
raise # staged intake settled; a cancelled run starts no dispatch
|
|
647
731
|
except Exception as exc:
|
|
648
|
-
|
|
649
|
-
message = str(exc) or type(exc).__name__
|
|
650
|
-
if isinstance(exc, StatewireReject):
|
|
651
|
-
self._freeze(exc.message, exc.payload)
|
|
652
|
-
else:
|
|
732
|
+
if not isinstance(exc, StatewireReject):
|
|
653
733
|
logger.exception(
|
|
654
734
|
"run %r crashed in %s",
|
|
655
735
|
self.runId.value,
|
|
656
736
|
getattr(fn, "__qualname__", repr(fn)),
|
|
657
737
|
)
|
|
658
|
-
|
|
738
|
+
error = self._error_of(exc)
|
|
739
|
+
self.outcome.value = {
|
|
740
|
+
"kind": "error",
|
|
741
|
+
"epoch": self.epoch.value,
|
|
742
|
+
"dispatchQueue": False,
|
|
743
|
+
"error": error,
|
|
744
|
+
}
|
|
745
|
+
self._settle(ctx)
|
|
746
|
+
self._set_status("error")
|
|
747
|
+
self.error.value = error
|
|
659
748
|
self._advance()
|
|
660
749
|
return
|
|
750
|
+
self.outcome.value = self._outcome_record(outcome)
|
|
661
751
|
self._settle(ctx)
|
|
662
|
-
self._outcome = outcome
|
|
752
|
+
self._outcome = self.outcome.value
|
|
663
753
|
self._advance()
|
|
664
754
|
|
|
755
|
+
def _error_of(self, exc: Exception) -> dict[str, Any]:
|
|
756
|
+
if isinstance(exc, StatewireReject):
|
|
757
|
+
if isinstance(exc.payload, dict):
|
|
758
|
+
return {**exc.payload, "message": exc.message}
|
|
759
|
+
if exc.payload is not None:
|
|
760
|
+
return {"message": exc.message, "payload": exc.payload}
|
|
761
|
+
return {"message": exc.message}
|
|
762
|
+
return {"message": str(exc) or type(exc).__name__}
|
|
763
|
+
|
|
764
|
+
def _outcome_record(self, outcome: Any) -> dict[str, Any]:
|
|
765
|
+
epoch = self.epoch.value
|
|
766
|
+
if isinstance(outcome, RunManager.Complete):
|
|
767
|
+
return {"kind": "complete", "epoch": epoch}
|
|
768
|
+
if isinstance(outcome, RunManager.InputRequired):
|
|
769
|
+
return {
|
|
770
|
+
"kind": "input-required",
|
|
771
|
+
"epoch": epoch,
|
|
772
|
+
"requests": [dict(r) for r in outcome.requests],
|
|
773
|
+
}
|
|
774
|
+
kind = "error" if isinstance(outcome, RunManager.Error) else "stop"
|
|
775
|
+
return {"kind": kind, "epoch": epoch, "dispatchQueue": outcome.dispatch_queue}
|
|
776
|
+
|
|
665
777
|
def _settle(self, ctx: "RunManager.RunContext") -> None:
|
|
666
778
|
if self._run_live is ctx:
|
|
667
779
|
self._run_live = None
|
|
668
780
|
if self._batch:
|
|
781
|
+
self._drop_dispatched(item["id"] for _, item in self._batch)
|
|
669
782
|
self._unstage_batch()
|
|
670
783
|
else:
|
|
671
784
|
record = self.dispatching.value
|
|
@@ -674,13 +787,36 @@ class RunManager(Tray):
|
|
|
674
787
|
and record["trigger"] != "steer"
|
|
675
788
|
and record["messages"]
|
|
676
789
|
):
|
|
677
|
-
|
|
790
|
+
# an unapplied rewind settled: its replacement never landed
|
|
791
|
+
self._drop_dispatched(m["id"] for m in record["messages"])
|
|
678
792
|
self.dispatching.value = None
|
|
793
|
+
self._finish_settle()
|
|
794
|
+
|
|
795
|
+
def _drop_dispatched(self, ids: Iterable[str]) -> None:
|
|
796
|
+
dropped = set(ids)
|
|
797
|
+
kept = [i for i in self.dispatchedIds.value if i not in dropped]
|
|
798
|
+
if len(kept) != len(self.dispatchedIds.value):
|
|
799
|
+
self.dispatchedIds.value = kept
|
|
800
|
+
|
|
801
|
+
def _finish_settle(self) -> None:
|
|
679
802
|
self.stopping.value = None
|
|
680
|
-
self.batchApplied.value = False
|
|
681
803
|
self.confirmedTakes.value = 0
|
|
804
|
+
self.batchApplied.value = False
|
|
682
805
|
self._clear_dispatch_state() # commits confirmed takes; drops the run's checkpoint
|
|
683
|
-
self.outbox.
|
|
806
|
+
if self.outbox.value is not None:
|
|
807
|
+
self.outbox.ack()
|
|
808
|
+
|
|
809
|
+
def _replay_settle(self) -> None:
|
|
810
|
+
recorded = self.outbox.value
|
|
811
|
+
if recorded is not None and not self.batchApplied.value:
|
|
812
|
+
item = recorded["item"]
|
|
813
|
+
record = item["record"]
|
|
814
|
+
self._batch = list(zip(item["lanes"], record["messages"]))
|
|
815
|
+
self._drop_dispatched(m["id"] for m in record["messages"])
|
|
816
|
+
if self._batch:
|
|
817
|
+
self._unstage_batch()
|
|
818
|
+
self.dispatching.value = None
|
|
819
|
+
self._finish_settle()
|
|
684
820
|
|
|
685
821
|
def _applied(self) -> None:
|
|
686
822
|
if self.dispatching.value is None:
|
|
@@ -700,6 +836,10 @@ class RunManager(Tray):
|
|
|
700
836
|
self.steerQueue.untake(self._pending_takes)
|
|
701
837
|
self._pending_takes = 0
|
|
702
838
|
return
|
|
839
|
+
for lane, item in taken: # claimed heads the dispatch never consumed
|
|
840
|
+
queue = self._lane(lane)
|
|
841
|
+
if queue.value and queue.peek()["id"] == item["id"]:
|
|
842
|
+
queue.popleft()
|
|
703
843
|
for lane, item in reversed(taken):
|
|
704
844
|
self._lane(lane).insert(0, item)
|
|
705
845
|
|
|
@@ -727,16 +867,22 @@ class RunManager(Tray):
|
|
|
727
867
|
|
|
728
868
|
def _stage_batch(self, e: _Send) -> None:
|
|
729
869
|
assert e.message is not None
|
|
730
|
-
steer =
|
|
870
|
+
steer = (
|
|
871
|
+
[
|
|
872
|
+
item
|
|
873
|
+
for item in self._lane_items("steerQueue")
|
|
874
|
+
if not self._is_dispatching(item["id"])
|
|
875
|
+
]
|
|
876
|
+
if e.lane == "steerQueue"
|
|
877
|
+
else []
|
|
878
|
+
)
|
|
731
879
|
if len(self._batch) + len(steer) >= self.max_queued:
|
|
732
880
|
raise _reject("queue-full", f"queue is full ({self.max_queued} messages)")
|
|
733
|
-
|
|
734
|
-
self.steerQueue.value = []
|
|
735
|
-
self._batch.extend(("steerQueue", item) for item in steer)
|
|
881
|
+
self._batch.extend(("steerQueue", item) for item in steer)
|
|
736
882
|
self._batch.append((e.lane, self._stamped(e.message, e.meta)))
|
|
737
883
|
self.dispatching.value = {
|
|
738
884
|
"trigger": self._send_trigger(),
|
|
739
|
-
"messages": [item for _, item in self._batch],
|
|
885
|
+
"messages": [_public(item) for _, item in self._batch],
|
|
740
886
|
}
|
|
741
887
|
|
|
742
888
|
def _dispatch_staged(
|
|
@@ -753,12 +899,11 @@ class RunManager(Tray):
|
|
|
753
899
|
) -> bool:
|
|
754
900
|
steer = self._lane_items("steerQueue")
|
|
755
901
|
if steer:
|
|
756
|
-
self.steerQueue.value = []
|
|
757
902
|
self._batch = [("steerQueue", item) for item in steer]
|
|
758
903
|
self._dispatch("message-send", steer, input_outcomes=input_outcomes)
|
|
759
904
|
return True
|
|
760
905
|
if self._lane_items("queue"):
|
|
761
|
-
item = self.queue.
|
|
906
|
+
item = self.queue.peek()
|
|
762
907
|
self._batch = [("queue", item)]
|
|
763
908
|
self._dispatch("message-send", [item], input_outcomes=input_outcomes)
|
|
764
909
|
return True
|
|
@@ -768,6 +913,7 @@ class RunManager(Tray):
|
|
|
768
913
|
self._staged_rewind = rewind
|
|
769
914
|
if self._in_flight():
|
|
770
915
|
self.stopping.value = {"reason": rewind.type}
|
|
916
|
+
self._set_status("stopping")
|
|
771
917
|
self.nextDispatch.value = self._record_of(
|
|
772
918
|
rewind.type,
|
|
773
919
|
list(rewind.messages),
|
|
@@ -779,15 +925,6 @@ class RunManager(Tray):
|
|
|
779
925
|
if not rewind.future.done():
|
|
780
926
|
rewind.future.set_result(None)
|
|
781
927
|
|
|
782
|
-
def _freeze(self, message: str, payload: Any = None) -> None:
|
|
783
|
-
self._set_status("error")
|
|
784
|
-
if isinstance(payload, dict):
|
|
785
|
-
self.error.value = {**payload, "message": message}
|
|
786
|
-
elif payload is not None:
|
|
787
|
-
self.error.value = {"message": message, "payload": payload}
|
|
788
|
-
else:
|
|
789
|
-
self.error.value = {"message": message}
|
|
790
|
-
|
|
791
928
|
# ─── Message and placement validation ───────────────────
|
|
792
929
|
|
|
793
930
|
def _prepared(
|
|
@@ -1218,6 +1355,15 @@ class RunManager(Tray):
|
|
|
1218
1355
|
) -> Any:
|
|
1219
1356
|
return await self._send("steerQueue", params, meta, applied)
|
|
1220
1357
|
|
|
1358
|
+
@inbox
|
|
1359
|
+
def admit_steer(self, message: dict[str, Any]) -> None:
|
|
1360
|
+
"""Durably admit one steer message; a ``ctx.dispatch`` target."""
|
|
1361
|
+
if not isinstance(message, dict) or not isinstance(message.get("id"), str):
|
|
1362
|
+
raise TypeError(f"a steer message needs a string id, got {message!r}")
|
|
1363
|
+
self.steerQueue.append(self._stamped(message, message.get("meta")))
|
|
1364
|
+
self._sync_steering()
|
|
1365
|
+
self._schedule(self._advance)
|
|
1366
|
+
|
|
1221
1367
|
async def dequeue(self, params: Any) -> None:
|
|
1222
1368
|
message_id = params.get("messageId") if isinstance(params, dict) else None
|
|
1223
1369
|
if not isinstance(message_id, str):
|
|
@@ -1225,7 +1371,7 @@ class RunManager(Tray):
|
|
|
1225
1371
|
return await self._stage(_Dequeue(message_id))
|
|
1226
1372
|
|
|
1227
1373
|
def _check_rewind_gate(self, command: str) -> None:
|
|
1228
|
-
if self._status()
|
|
1374
|
+
if self._status() in ("running", "stopping"):
|
|
1229
1375
|
if "rewind-during-run" not in self.capabilities:
|
|
1230
1376
|
raise _reject(
|
|
1231
1377
|
"capability-missing",
|
|
@@ -1299,7 +1445,11 @@ class RunManager(Tray):
|
|
|
1299
1445
|
if epoch is not _ABSENT and (isinstance(epoch, bool) or not isinstance(epoch, int)):
|
|
1300
1446
|
raise _reject("invalid-message", "epoch must be an integer")
|
|
1301
1447
|
status = self._status()
|
|
1302
|
-
|
|
1448
|
+
stop_held = (self.stopping.value or {}).get("reason") == "stop"
|
|
1449
|
+
# a rewind-held window still admits a stop: it clears the staged rewind
|
|
1450
|
+
if status not in ("running", "stopping") or (
|
|
1451
|
+
status == "stopping" and stop_held
|
|
1452
|
+
):
|
|
1303
1453
|
raise _reject("wrong-state", f"run/stop is rejected in {status}")
|
|
1304
1454
|
run_id = _run_id_of(params)
|
|
1305
1455
|
if run_id != self.runId.value:
|
|
@@ -1444,20 +1594,29 @@ class RunManager(Tray):
|
|
|
1444
1594
|
return ()
|
|
1445
1595
|
manager._pending_takes += len(items)
|
|
1446
1596
|
manager._batch = [("steerQueue", item) for item in items]
|
|
1447
|
-
manager.
|
|
1597
|
+
known = manager.dispatchedIds.value
|
|
1598
|
+
added = [item["id"] for item in items if item["id"] not in known]
|
|
1599
|
+
if added:
|
|
1600
|
+
manager.dispatchedIds.value = [*known, *added]
|
|
1601
|
+
manager.dispatching.value = {
|
|
1602
|
+
"trigger": "steer",
|
|
1603
|
+
"messages": [_public(item) for item in items],
|
|
1604
|
+
}
|
|
1448
1605
|
self._ctx._message_meta.update(
|
|
1449
1606
|
{item["id"]: item.get("meta") for item in items}
|
|
1450
1607
|
)
|
|
1451
1608
|
manager._sync_steering()
|
|
1452
1609
|
return tuple(
|
|
1453
|
-
{k: v for k, v in item.items() if k
|
|
1610
|
+
{k: v for k, v in item.items() if k not in ("meta", "~admit")}
|
|
1611
|
+
for item in items
|
|
1454
1612
|
)
|
|
1455
1613
|
|
|
1456
1614
|
@dataclass(frozen=True, eq=False)
|
|
1457
1615
|
class RunContext:
|
|
1458
1616
|
trigger: str
|
|
1459
|
-
messages: tuple[dict[str, Any], ...]
|
|
1460
1617
|
stop_requested: asyncio.Event
|
|
1618
|
+
_resume: bool
|
|
1619
|
+
_messages: tuple[dict[str, Any], ...]
|
|
1461
1620
|
_manager: "RunManager"
|
|
1462
1621
|
_rollback_to: Any
|
|
1463
1622
|
_message_meta: dict[str, Any]
|
|
@@ -1471,6 +1630,24 @@ class RunManager(Tray):
|
|
|
1471
1630
|
self, "steering", RunManager.Steering(asyncio.Event(), self)
|
|
1472
1631
|
)
|
|
1473
1632
|
|
|
1633
|
+
@property
|
|
1634
|
+
def resume(self) -> bool:
|
|
1635
|
+
"""True when the batch was already durably applied before this redelivery: do not re-apply it, continue from the app's own continuation state."""
|
|
1636
|
+
return self._resume
|
|
1637
|
+
|
|
1638
|
+
def _batch_only(self, name: str) -> None:
|
|
1639
|
+
if self._resume:
|
|
1640
|
+
raise RuntimeError(
|
|
1641
|
+
f"{name} is unavailable on a resume: the batch is already applied and"
|
|
1642
|
+
" must not be re-dispatched; continue from app-defined state"
|
|
1643
|
+
)
|
|
1644
|
+
|
|
1645
|
+
@property
|
|
1646
|
+
def messages(self) -> tuple[dict[str, Any], ...]:
|
|
1647
|
+
"""The dispatched input batch; unavailable on a resume."""
|
|
1648
|
+
self._batch_only("messages")
|
|
1649
|
+
return self._messages
|
|
1650
|
+
|
|
1474
1651
|
def meta(self, message_id: str | None = None) -> Any:
|
|
1475
1652
|
"""Op meta: a dispatched message's by id, or the run-starting op's with no id."""
|
|
1476
1653
|
if message_id is None:
|
|
@@ -1490,10 +1667,12 @@ class RunManager(Tray):
|
|
|
1490
1667
|
|
|
1491
1668
|
@property
|
|
1492
1669
|
def has_rollback(self) -> bool:
|
|
1670
|
+
self._batch_only("has_rollback")
|
|
1493
1671
|
return self._rollback_to is not _ABSENT
|
|
1494
1672
|
|
|
1495
1673
|
@property
|
|
1496
1674
|
def rollback_to(self) -> str | None:
|
|
1675
|
+
self._batch_only("rollback_to")
|
|
1497
1676
|
if self._rollback_to is _ABSENT:
|
|
1498
1677
|
raise AttributeError(
|
|
1499
1678
|
"rollback_to is only present on rewind entries; check has_rollback"
|
|
@@ -1516,6 +1695,7 @@ class RunManager(Tray):
|
|
|
1516
1695
|
|
|
1517
1696
|
def enlist(self, effect: Any) -> None:
|
|
1518
1697
|
"""Enlist an opaque durable effect ordered with the machine's writes."""
|
|
1698
|
+
self._ensure_active()
|
|
1519
1699
|
self._tray_ctx.enlist(effect)
|
|
1520
1700
|
|
|
1521
1701
|
def _ensure_active(self) -> None:
|