harness-sdk-python 0.18.0__tar.gz → 0.19.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {harness_sdk_python-0.18.0 → harness_sdk_python-0.19.0}/PKG-INFO +3 -3
- {harness_sdk_python-0.18.0 → harness_sdk_python-0.19.0}/pyproject.toml +2 -2
- {harness_sdk_python-0.18.0 → harness_sdk_python-0.19.0}/src/harness_sdk/run_manager.py +66 -14
- harness_sdk_python-0.19.0/tests/test_admit_steer.py +118 -0
- {harness_sdk_python-0.18.0 → harness_sdk_python-0.19.0}/tests/test_batches.py +2 -14
- {harness_sdk_python-0.18.0 → harness_sdk_python-0.19.0}/tests/test_durability.py +56 -2
- {harness_sdk_python-0.18.0 → harness_sdk_python-0.19.0}/tests/test_outcomes.py +1 -0
- {harness_sdk_python-0.18.0 → harness_sdk_python-0.19.0}/tests/test_restore.py +1 -0
- {harness_sdk_python-0.18.0 → harness_sdk_python-0.19.0}/tests/test_run_crash_sweep.py +6 -2
- {harness_sdk_python-0.18.0 → harness_sdk_python-0.19.0}/tests/test_stop_continue.py +52 -2
- {harness_sdk_python-0.18.0 → harness_sdk_python-0.19.0}/tests/test_stopping_window.py +35 -16
- {harness_sdk_python-0.18.0 → harness_sdk_python-0.19.0}/.gitignore +0 -0
- {harness_sdk_python-0.18.0 → harness_sdk_python-0.19.0}/README.md +0 -0
- {harness_sdk_python-0.18.0 → harness_sdk_python-0.19.0}/examples/__init__.py +0 -0
- {harness_sdk_python-0.18.0 → harness_sdk_python-0.19.0}/examples/runs_app.py +0 -0
- {harness_sdk_python-0.18.0 → harness_sdk_python-0.19.0}/src/harness_sdk/__init__.py +0 -0
- {harness_sdk_python-0.18.0 → harness_sdk_python-0.19.0}/src/harness_sdk/fenced_postgres.py +0 -0
- {harness_sdk_python-0.18.0 → harness_sdk_python-0.19.0}/src/harness_sdk/linear_thread.py +0 -0
- {harness_sdk_python-0.18.0 → harness_sdk_python-0.19.0}/tests/run_helpers.py +0 -0
- {harness_sdk_python-0.18.0 → harness_sdk_python-0.19.0}/tests/test_applied_visibility.py +0 -0
- {harness_sdk_python-0.18.0 → harness_sdk_python-0.19.0}/tests/test_branch_anchor.py +0 -0
- {harness_sdk_python-0.18.0 → harness_sdk_python-0.19.0}/tests/test_dispatching.py +0 -0
- {harness_sdk_python-0.18.0 → harness_sdk_python-0.19.0}/tests/test_edit_dispatched.py +0 -0
- {harness_sdk_python-0.18.0 → harness_sdk_python-0.19.0}/tests/test_edit_reload.py +0 -0
- {harness_sdk_python-0.18.0 → harness_sdk_python-0.19.0}/tests/test_enqueue.py +0 -0
- {harness_sdk_python-0.18.0 → harness_sdk_python-0.19.0}/tests/test_fenced_postgres.py +0 -0
- {harness_sdk_python-0.18.0 → harness_sdk_python-0.19.0}/tests/test_input_required.py +0 -0
- {harness_sdk_python-0.18.0 → harness_sdk_python-0.19.0}/tests/test_linear_thread.py +0 -0
- {harness_sdk_python-0.18.0 → harness_sdk_python-0.19.0}/tests/test_meta.py +0 -0
- {harness_sdk_python-0.18.0 → harness_sdk_python-0.19.0}/tests/test_placement.py +0 -0
- {harness_sdk_python-0.18.0 → harness_sdk_python-0.19.0}/tests/test_prepare_hooks.py +0 -0
- {harness_sdk_python-0.18.0 → harness_sdk_python-0.19.0}/tests/test_rewind_during_run.py +0 -0
- {harness_sdk_python-0.18.0 → harness_sdk_python-0.19.0}/tests/test_settle.py +0 -0
- {harness_sdk_python-0.18.0 → harness_sdk_python-0.19.0}/tests/test_steer.py +0 -0
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: harness-sdk-python
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.19.0
|
|
4
4
|
Summary: RunManager: the harness-sdk runs subsystem for Python Statewire hosts
|
|
5
5
|
Project-URL: Repository, https://github.com/assistant-ui/harness-sdk
|
|
6
6
|
License-Expression: MIT
|
|
7
7
|
Requires-Python: <4.0,>=3.12
|
|
8
|
-
Requires-Dist: pinned<0.
|
|
9
|
-
Requires-Dist: statewire<0.
|
|
8
|
+
Requires-Dist: pinned<0.14,>=0.13.0
|
|
9
|
+
Requires-Dist: statewire<0.11,>=0.10.0
|
|
10
10
|
Provides-Extra: deepagents
|
|
11
11
|
Requires-Dist: deepagents>=0.6.12; extra == 'deepagents'
|
|
12
12
|
Requires-Dist: langchain-core>=0.3; extra == 'deepagents'
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "harness-sdk-python"
|
|
3
|
-
version = "0.
|
|
3
|
+
version = "0.19.0"
|
|
4
4
|
description = "RunManager: the harness-sdk runs subsystem for Python Statewire hosts"
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
license = "MIT"
|
|
7
7
|
requires-python = ">=3.12,<4.0"
|
|
8
|
-
dependencies = ["statewire>=0.
|
|
8
|
+
dependencies = ["statewire>=0.10.0,<0.11", "pinned>=0.13.0,<0.14"]
|
|
9
9
|
|
|
10
10
|
[project.optional-dependencies]
|
|
11
11
|
postgres = ["langgraph-checkpoint-postgres>=2.0.0"]
|
|
@@ -29,8 +29,11 @@ own continuation state (``ctx.state`` or an external checkpointer).
|
|
|
29
29
|
``epoch`` counts the entry's dispatches (not redeliveries); a ``run/stop``
|
|
30
30
|
carrying an ``epoch`` other than the live one rejects. A mid-run stop or
|
|
31
31
|
rewind opens a stopping window (``stopping: {reason}`` plus, for rewinds, the
|
|
32
|
-
staged dispatch as ``nextDispatch``)
|
|
33
|
-
|
|
32
|
+
staged dispatch as ``nextDispatch``) and ``status`` reads ``stopping`` until
|
|
33
|
+
the run settles; the ``run/stop`` verdict follows that settle and its flush,
|
|
34
|
+
so a settled stop covers every durable effect the runner enlisted. On run end
|
|
35
|
+
one envelope promotes ``nextDispatch`` into ``dispatching`` and clears the
|
|
36
|
+
window.
|
|
34
37
|
"""
|
|
35
38
|
|
|
36
39
|
import asyncio
|
|
@@ -38,7 +41,7 @@ import logging
|
|
|
38
41
|
from dataclasses import dataclass, field
|
|
39
42
|
from typing import Any, AsyncIterator, Callable, Coroutine, Iterable, Protocol
|
|
40
43
|
|
|
41
|
-
from pinned.trays import Tray, TrayDescriptor, TrayCtx
|
|
44
|
+
from pinned.trays import Tray, TrayDescriptor, TrayCtx, inbox
|
|
42
45
|
from pinned.traystore import persistent, persistent_outbox, persistent_queue
|
|
43
46
|
from statepatch import observable
|
|
44
47
|
from statewire import StatewireReject
|
|
@@ -78,6 +81,15 @@ def _reject(reason: str, message: str) -> StatewireReject:
|
|
|
78
81
|
return StatewireReject(message, payload={"reason": reason})
|
|
79
82
|
|
|
80
83
|
|
|
84
|
+
def _public(message: dict[str, Any]) -> dict[str, Any]:
|
|
85
|
+
"""The message without its admission tag; the wire state is public."""
|
|
86
|
+
return {k: v for k, v in message.items() if k != "~admit"}
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def _public_record(record: dict[str, Any]) -> dict[str, Any]:
|
|
90
|
+
return {**record, "messages": [_public(m) for m in record["messages"]]}
|
|
91
|
+
|
|
92
|
+
|
|
81
93
|
def _run_id_of(params: Any) -> str:
|
|
82
94
|
run_id = params.get("runId") if isinstance(params, dict) else None
|
|
83
95
|
if not isinstance(run_id, str) or run_id == "":
|
|
@@ -204,6 +216,7 @@ class RunManager(Tray):
|
|
|
204
216
|
stopReason = persistent(None)
|
|
205
217
|
dispatchedIds = persistent([])
|
|
206
218
|
dispatching = observable(None) # view: the unconfirmed batch record
|
|
219
|
+
steerView = observable([]) # view: the steer lane without admission tags
|
|
207
220
|
|
|
208
221
|
state = observable(
|
|
209
222
|
{
|
|
@@ -211,12 +224,13 @@ class RunManager(Tray):
|
|
|
211
224
|
"status": status,
|
|
212
225
|
"epoch": epoch,
|
|
213
226
|
"queue": queue,
|
|
214
|
-
"steerQueue":
|
|
227
|
+
"steerQueue": steerView,
|
|
215
228
|
"dispatching": dispatching,
|
|
216
229
|
"stopping": stopping,
|
|
217
230
|
"nextDispatch": nextDispatch,
|
|
218
231
|
"inputRequests": inputRequests,
|
|
219
232
|
"error": error,
|
|
233
|
+
"stopReason": stopReason,
|
|
220
234
|
}
|
|
221
235
|
)
|
|
222
236
|
|
|
@@ -225,6 +239,7 @@ class RunManager(Tray):
|
|
|
225
239
|
self._intake: list[Any] = []
|
|
226
240
|
self._outcome: Any = None
|
|
227
241
|
self._stop_staged = False
|
|
242
|
+
self._stop_waiters: list["asyncio.Future[Any]"] = []
|
|
228
243
|
self._staged_rewind: _Rewind | None = None
|
|
229
244
|
self._batch: list[tuple[str, dict[str, Any]]] = []
|
|
230
245
|
self._pending_takes = 0
|
|
@@ -245,7 +260,10 @@ class RunManager(Tray):
|
|
|
245
260
|
self._thread = api
|
|
246
261
|
self._schedule = api.schedule
|
|
247
262
|
self._hydrate_machine()
|
|
263
|
+
self._mirror_steer()
|
|
264
|
+
unsubscribe = self.steerQueue.subscribe(lambda _op: self._mirror_steer())
|
|
248
265
|
yield
|
|
266
|
+
unsubscribe()
|
|
249
267
|
|
|
250
268
|
class Thread(Protocol):
|
|
251
269
|
"""The thread projection RunManager reads; the host owns the tree."""
|
|
@@ -277,14 +295,16 @@ class RunManager(Tray):
|
|
|
277
295
|
record = recorded["item"]["record"]
|
|
278
296
|
lanes = recorded["item"]["lanes"]
|
|
279
297
|
self._batch = list(zip(lanes, record["messages"]))
|
|
280
|
-
self.dispatching.value = record
|
|
298
|
+
self.dispatching.value = _public_record(record)
|
|
281
299
|
else:
|
|
282
300
|
self.dispatching.value = None
|
|
283
301
|
if recorded is not None:
|
|
284
302
|
# heal a dispatch torn after its outbox write landed
|
|
285
303
|
self._clear_input()
|
|
286
304
|
self.error.value = None
|
|
287
|
-
self._set_status(
|
|
305
|
+
self._set_status(
|
|
306
|
+
"stopping" if self.stopping.value is not None else "running"
|
|
307
|
+
)
|
|
288
308
|
staged = self.nextDispatch.value
|
|
289
309
|
if (
|
|
290
310
|
staged is not None
|
|
@@ -356,6 +376,9 @@ class RunManager(Tray):
|
|
|
356
376
|
def _is_dispatching(self, message_id: str) -> bool:
|
|
357
377
|
return any(item["id"] == message_id for _, item in self._batch)
|
|
358
378
|
|
|
379
|
+
def _mirror_steer(self) -> None:
|
|
380
|
+
self.steerView.value = [_public(item) for item in self.steerQueue.value]
|
|
381
|
+
|
|
359
382
|
def _sync_steering(self) -> None:
|
|
360
383
|
ctx = self._run_live
|
|
361
384
|
if ctx is None:
|
|
@@ -381,6 +404,12 @@ class RunManager(Tray):
|
|
|
381
404
|
self._apply(entry, fx)
|
|
382
405
|
return fx
|
|
383
406
|
|
|
407
|
+
def _resolve_stop_waiters(self) -> None:
|
|
408
|
+
waiters, self._stop_waiters = self._stop_waiters, []
|
|
409
|
+
for future in waiters:
|
|
410
|
+
if not future.done():
|
|
411
|
+
future.set_result(None)
|
|
412
|
+
|
|
384
413
|
def _settle_applied(self, fx: _Effects) -> None:
|
|
385
414
|
for future, applied, result in fx.staged_sends:
|
|
386
415
|
applied()
|
|
@@ -417,6 +446,7 @@ class RunManager(Tray):
|
|
|
417
446
|
self._idle_action(fx)
|
|
418
447
|
if self.outcome.value is not None:
|
|
419
448
|
self.outcome.value = None # the follow-on's writes are enlisted ahead
|
|
449
|
+
self._resolve_stop_waiters()
|
|
420
450
|
if self._in_flight() and (
|
|
421
451
|
self._stop_staged or self._staged_rewind is not None
|
|
422
452
|
):
|
|
@@ -434,7 +464,9 @@ class RunManager(Tray):
|
|
|
434
464
|
self._staged_rewind = None
|
|
435
465
|
self.nextDispatch.value = None
|
|
436
466
|
self.stopping.value = {"reason": "stop"}
|
|
437
|
-
|
|
467
|
+
self._set_status("stopping")
|
|
468
|
+
self._stop_waiters.append(entry.future) # the verdict waits for the settle
|
|
469
|
+
elif not entry.future.done():
|
|
438
470
|
entry.future.set_result(None)
|
|
439
471
|
return
|
|
440
472
|
if isinstance(entry, _Continue):
|
|
@@ -577,7 +609,7 @@ class RunManager(Tray):
|
|
|
577
609
|
if added:
|
|
578
610
|
self.dispatchedIds.value = [*known, *added]
|
|
579
611
|
self._set_status("running")
|
|
580
|
-
self.dispatching.value = record
|
|
612
|
+
self.dispatching.value = _public_record(record)
|
|
581
613
|
self._idle.clear()
|
|
582
614
|
self.kick()
|
|
583
615
|
|
|
@@ -610,7 +642,7 @@ class RunManager(Tray):
|
|
|
610
642
|
def _make_run_ctx(self, item: Any) -> "RunManager.RunContext":
|
|
611
643
|
record = item["record"]
|
|
612
644
|
messages = [
|
|
613
|
-
{k: v for k, v in message.items() if k
|
|
645
|
+
{k: v for k, v in message.items() if k not in ("meta", "~admit")}
|
|
614
646
|
for message in record["messages"]
|
|
615
647
|
]
|
|
616
648
|
ctx = RunManager.RunContext(
|
|
@@ -692,6 +724,7 @@ class RunManager(Tray):
|
|
|
692
724
|
rewind.applied_sent = True
|
|
693
725
|
self._set_status("stopped")
|
|
694
726
|
self.outcome.value = None
|
|
727
|
+
self._resolve_stop_waiters()
|
|
695
728
|
self._settle_applied(fx)
|
|
696
729
|
self._idle.set()
|
|
697
730
|
raise # staged intake settled; a cancelled run starts no dispatch
|
|
@@ -849,7 +882,7 @@ class RunManager(Tray):
|
|
|
849
882
|
self._batch.append((e.lane, self._stamped(e.message, e.meta)))
|
|
850
883
|
self.dispatching.value = {
|
|
851
884
|
"trigger": self._send_trigger(),
|
|
852
|
-
"messages": [item for _, item in self._batch],
|
|
885
|
+
"messages": [_public(item) for _, item in self._batch],
|
|
853
886
|
}
|
|
854
887
|
|
|
855
888
|
def _dispatch_staged(
|
|
@@ -880,6 +913,7 @@ class RunManager(Tray):
|
|
|
880
913
|
self._staged_rewind = rewind
|
|
881
914
|
if self._in_flight():
|
|
882
915
|
self.stopping.value = {"reason": rewind.type}
|
|
916
|
+
self._set_status("stopping")
|
|
883
917
|
self.nextDispatch.value = self._record_of(
|
|
884
918
|
rewind.type,
|
|
885
919
|
list(rewind.messages),
|
|
@@ -1321,6 +1355,15 @@ class RunManager(Tray):
|
|
|
1321
1355
|
) -> Any:
|
|
1322
1356
|
return await self._send("steerQueue", params, meta, applied)
|
|
1323
1357
|
|
|
1358
|
+
@inbox
|
|
1359
|
+
def admit_steer(self, message: dict[str, Any]) -> None:
|
|
1360
|
+
"""Durably admit one steer message; a ``ctx.dispatch`` target."""
|
|
1361
|
+
if not isinstance(message, dict) or not isinstance(message.get("id"), str):
|
|
1362
|
+
raise TypeError(f"a steer message needs a string id, got {message!r}")
|
|
1363
|
+
self.steerQueue.append(self._stamped(message, message.get("meta")))
|
|
1364
|
+
self._sync_steering()
|
|
1365
|
+
self._schedule(self._advance)
|
|
1366
|
+
|
|
1324
1367
|
async def dequeue(self, params: Any) -> None:
|
|
1325
1368
|
message_id = params.get("messageId") if isinstance(params, dict) else None
|
|
1326
1369
|
if not isinstance(message_id, str):
|
|
@@ -1328,7 +1371,7 @@ class RunManager(Tray):
|
|
|
1328
1371
|
return await self._stage(_Dequeue(message_id))
|
|
1329
1372
|
|
|
1330
1373
|
def _check_rewind_gate(self, command: str) -> None:
|
|
1331
|
-
if self._status()
|
|
1374
|
+
if self._status() in ("running", "stopping"):
|
|
1332
1375
|
if "rewind-during-run" not in self.capabilities:
|
|
1333
1376
|
raise _reject(
|
|
1334
1377
|
"capability-missing",
|
|
@@ -1402,7 +1445,11 @@ class RunManager(Tray):
|
|
|
1402
1445
|
if epoch is not _ABSENT and (isinstance(epoch, bool) or not isinstance(epoch, int)):
|
|
1403
1446
|
raise _reject("invalid-message", "epoch must be an integer")
|
|
1404
1447
|
status = self._status()
|
|
1405
|
-
|
|
1448
|
+
stop_held = (self.stopping.value or {}).get("reason") == "stop"
|
|
1449
|
+
# a rewind-held window still admits a stop: it clears the staged rewind
|
|
1450
|
+
if status not in ("running", "stopping") or (
|
|
1451
|
+
status == "stopping" and stop_held
|
|
1452
|
+
):
|
|
1406
1453
|
raise _reject("wrong-state", f"run/stop is rejected in {status}")
|
|
1407
1454
|
run_id = _run_id_of(params)
|
|
1408
1455
|
if run_id != self.runId.value:
|
|
@@ -1551,13 +1598,17 @@ class RunManager(Tray):
|
|
|
1551
1598
|
added = [item["id"] for item in items if item["id"] not in known]
|
|
1552
1599
|
if added:
|
|
1553
1600
|
manager.dispatchedIds.value = [*known, *added]
|
|
1554
|
-
manager.dispatching.value = {
|
|
1601
|
+
manager.dispatching.value = {
|
|
1602
|
+
"trigger": "steer",
|
|
1603
|
+
"messages": [_public(item) for item in items],
|
|
1604
|
+
}
|
|
1555
1605
|
self._ctx._message_meta.update(
|
|
1556
1606
|
{item["id"]: item.get("meta") for item in items}
|
|
1557
1607
|
)
|
|
1558
1608
|
manager._sync_steering()
|
|
1559
1609
|
return tuple(
|
|
1560
|
-
{k: v for k, v in item.items() if k
|
|
1610
|
+
{k: v for k, v in item.items() if k not in ("meta", "~admit")}
|
|
1611
|
+
for item in items
|
|
1561
1612
|
)
|
|
1562
1613
|
|
|
1563
1614
|
@dataclass(frozen=True, eq=False)
|
|
@@ -1644,6 +1695,7 @@ class RunManager(Tray):
|
|
|
1644
1695
|
|
|
1645
1696
|
def enlist(self, effect: Any) -> None:
|
|
1646
1697
|
"""Enlist an opaque durable effect ordered with the machine's writes."""
|
|
1698
|
+
self._ensure_active()
|
|
1647
1699
|
self._tray_ctx.enlist(effect)
|
|
1648
1700
|
|
|
1649
1701
|
def _ensure_active(self) -> None:
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
"""Contract: ``RunManager.admit_steer`` is the ``ctx.dispatch`` target for
|
|
2
|
+
durable steering — a pooled command admits one steer message exactly once
|
|
3
|
+
across a crash redelivery, and the public seams (``steering.take()``,
|
|
4
|
+
``ctx.messages``, the replicated state view) never carry the admission tag
|
|
5
|
+
the durable queue record keeps for redelivery dedup."""
|
|
6
|
+
|
|
7
|
+
import asyncio
|
|
8
|
+
import copy
|
|
9
|
+
|
|
10
|
+
from pinned import PinnedWorker
|
|
11
|
+
from pinned.testing import MemoryStateStore
|
|
12
|
+
from run_helpers import Script, add, msg
|
|
13
|
+
from statewire_helpers import apply_ops, attach_of, post_frame, statewire_client
|
|
14
|
+
|
|
15
|
+
from harness_sdk import RunManager, run_manager
|
|
16
|
+
from statewire import Statewire, command
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def make_host(script: Script):
|
|
20
|
+
class Host(Statewire):
|
|
21
|
+
gate = asyncio.Event()
|
|
22
|
+
live = None
|
|
23
|
+
|
|
24
|
+
async def lifespan(self):
|
|
25
|
+
type(self).live = self
|
|
26
|
+
self.state = {}
|
|
27
|
+
yield
|
|
28
|
+
|
|
29
|
+
async def get_message_meta(self, message_id):
|
|
30
|
+
return await script.get_message_meta(message_id)
|
|
31
|
+
|
|
32
|
+
@run_manager()
|
|
33
|
+
async def runs(self, ctx):
|
|
34
|
+
return await script.run(ctx)
|
|
35
|
+
|
|
36
|
+
@command("run/steerDurable")
|
|
37
|
+
async def steer_durable(self, params, *, ctx):
|
|
38
|
+
runs = await self.runs.get()
|
|
39
|
+
outcome = await ctx.dispatch(runs.admit_steer, params["message"])
|
|
40
|
+
await type(self).gate.wait()
|
|
41
|
+
return {"outcome": outcome}
|
|
42
|
+
|
|
43
|
+
@command("run/enqueue")
|
|
44
|
+
async def run_enqueue(self, params, *, ctx):
|
|
45
|
+
runs = await self.runs.get()
|
|
46
|
+
return await runs.enqueue(params, applied=ctx.applied)
|
|
47
|
+
|
|
48
|
+
return Host
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def statement(seq, method, params):
|
|
52
|
+
return {"method": method, "params": [params], "seq": seq}
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
async def cmds_until(stream, entry, limit=10):
|
|
56
|
+
seen = []
|
|
57
|
+
for _ in range(limit):
|
|
58
|
+
event = await stream.next_event()
|
|
59
|
+
seen.extend(event.get("cmd", []))
|
|
60
|
+
if entry in seen:
|
|
61
|
+
return seen
|
|
62
|
+
raise AssertionError(f"{entry} never rode the stream: {seen}")
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
async def test_admit_steer_admits_exactly_once_and_the_run_sees_no_tag():
|
|
66
|
+
store = MemoryStateStore()
|
|
67
|
+
script = Script()
|
|
68
|
+
host = make_host(script)
|
|
69
|
+
async with statewire_client(host, worker=PinnedWorker(state_store=store)) as (
|
|
70
|
+
app,
|
|
71
|
+
client,
|
|
72
|
+
):
|
|
73
|
+
async with attach_of(app, {"headers": {}}) as stream:
|
|
74
|
+
await stream.next_event()
|
|
75
|
+
frame = {
|
|
76
|
+
"cmd": [statement(1, "run/steerDurable", {"message": msg("s1")})]
|
|
77
|
+
}
|
|
78
|
+
assert (await post_frame(client, frame)).status_code == 200
|
|
79
|
+
machine = await host.live.runs.get()
|
|
80
|
+
while not machine.steerQueue.value:
|
|
81
|
+
await asyncio.sleep(0)
|
|
82
|
+
await host.live._frontier.wait()
|
|
83
|
+
|
|
84
|
+
script2 = Script()
|
|
85
|
+
host2 = make_host(script2)
|
|
86
|
+
host2.gate.set()
|
|
87
|
+
async with statewire_client(host2, worker=PinnedWorker(state_store=store)) as (
|
|
88
|
+
app,
|
|
89
|
+
client,
|
|
90
|
+
):
|
|
91
|
+
async with attach_of(app, {"headers": {}}) as stream:
|
|
92
|
+
replica: dict = {}
|
|
93
|
+
apply_ops(replica, copy.deepcopy(await stream.next_event()))
|
|
94
|
+
# replay of the redelivered seq re-emits its recorded outcome
|
|
95
|
+
frame = {
|
|
96
|
+
"cmd": [statement(1, "run/steerDurable", {"message": msg("s1")})]
|
|
97
|
+
}
|
|
98
|
+
assert (await post_frame(client, frame)).status_code == 200
|
|
99
|
+
await cmds_until(
|
|
100
|
+
stream,
|
|
101
|
+
{"seq": 1, "type": "result", "payload": {"outcome": "admitted"}},
|
|
102
|
+
)
|
|
103
|
+
machine = await host2.live.runs.get()
|
|
104
|
+
# the durable record keeps the dedup tag; the state view never shows it
|
|
105
|
+
(entry,) = machine.steerQueue.value
|
|
106
|
+
assert entry["~admit"] == ["c1", 1]
|
|
107
|
+
assert replica["runs"][0]["steerQueue"] == [msg("s1")]
|
|
108
|
+
|
|
109
|
+
frame = {"cmd": [statement(2, "run/enqueue", add("m1"))]}
|
|
110
|
+
assert (await post_frame(client, frame)).status_code == 200
|
|
111
|
+
call = await script2.next_call()
|
|
112
|
+
call.applied()
|
|
113
|
+
(taken,) = call.ctx.steering.take()
|
|
114
|
+
assert taken == msg("s1")
|
|
115
|
+
call.applied()
|
|
116
|
+
call.finish(RunManager.Complete())
|
|
117
|
+
await cmds_until(stream, {"seq": 2, "dur": True})
|
|
118
|
+
assert machine.steerQueue.value == []
|
|
@@ -59,27 +59,15 @@ async def test_steer_and_stop_in_one_batch_nets_to_stop():
|
|
|
59
59
|
[("run/steer", add("s1")), ("run/stop", {"runId": await drv.run_id()})]
|
|
60
60
|
)
|
|
61
61
|
assert (await drv.res(first))["type"] == "accepted"
|
|
62
|
-
assert (await drv.res(first + 1))["type"] == "
|
|
62
|
+
assert (await drv.res(first + 1, terminal=False))["type"] == "pending"
|
|
63
63
|
await asyncio.wait_for(call.ctx.stop_requested.wait(), 5)
|
|
64
64
|
call.finish(RunManager.Stop(dispatch_queue=False))
|
|
65
|
+
assert (await drv.res(first + 1))["type"] == "accepted"
|
|
65
66
|
await drv.wait_status("stopped")
|
|
66
67
|
assert queue_ids(drv.replica, "steerQueue") == ["s1"]
|
|
67
68
|
script.no_call()
|
|
68
69
|
|
|
69
70
|
|
|
70
|
-
async def test_stop_settles_at_the_stopping_write():
|
|
71
|
-
script = Script()
|
|
72
|
-
async with run_host(script) as (drv, host):
|
|
73
|
-
drv.post("run/enqueue", add("m1"))
|
|
74
|
-
call = await script.next_call()
|
|
75
|
-
call.applied()
|
|
76
|
-
settled = await drv.stop()
|
|
77
|
-
assert settled["type"] == "accepted"
|
|
78
|
-
assert drv.replica["status"] == "running"
|
|
79
|
-
call.finish(RunManager.Stop(dispatch_queue=False))
|
|
80
|
-
await drv.wait_status("stopped")
|
|
81
|
-
|
|
82
|
-
|
|
83
71
|
class TailScript(Script):
|
|
84
72
|
"""get_message_meta suspends, pushing staging past the handler prefix."""
|
|
85
73
|
|
|
@@ -110,9 +110,11 @@ async def test_a_stale_epoch_stop_rejects_across_a_redelivery():
|
|
|
110
110
|
assert redelivered.ctx.attempts == 2
|
|
111
111
|
await drv2.wait(lambda s: run_of(s)["epoch"] == 1) # deliveries never bump it
|
|
112
112
|
assert_rejected(await drv2.stop({"epoch": 2}, terminal=False), "wrong-state")
|
|
113
|
-
|
|
114
|
-
assert
|
|
113
|
+
pending = await drv2.stop({"epoch": 1}, terminal=False)
|
|
114
|
+
assert pending["type"] == "pending"
|
|
115
|
+
await asyncio.wait_for(redelivered.ctx.stop_requested.wait(), 5)
|
|
115
116
|
redelivered.finish(RunManager.Stop(dispatch_queue=False))
|
|
117
|
+
assert (await drv2.res(pending["seq"]))["type"] == "accepted"
|
|
116
118
|
await drv2.wait_status("stopped")
|
|
117
119
|
|
|
118
120
|
|
|
@@ -159,6 +161,7 @@ async def test_a_staged_rewind_survives_a_crash_in_the_stopping_window():
|
|
|
159
161
|
async with run_host(script2, store=crashed, capabilities=REWIND) as (drv2, host2):
|
|
160
162
|
redelivered = await script2.next_call()
|
|
161
163
|
assert redelivered.ctx.stop_requested.is_set()
|
|
164
|
+
assert drv2.replica["status"] == "stopping" # the window hydrates with its status
|
|
162
165
|
redelivered.finish(RunManager.Stop(dispatch_queue=False))
|
|
163
166
|
rewind = await script2.next_call()
|
|
164
167
|
assert rewind.ctx.trigger == "message-edit"
|
|
@@ -169,6 +172,30 @@ async def test_a_staged_rewind_survives_a_crash_in_the_stopping_window():
|
|
|
169
172
|
await drv2.wait_status("ready")
|
|
170
173
|
|
|
171
174
|
|
|
175
|
+
async def test_the_stopping_status_survives_a_crash_in_the_stop_window():
|
|
176
|
+
store = MemoryStateStore()
|
|
177
|
+
script = Script()
|
|
178
|
+
async with run_host(script, store=store) as (drv, host):
|
|
179
|
+
drv.post("run/enqueue", add("m1"))
|
|
180
|
+
call = await script.next_call()
|
|
181
|
+
call.applied()
|
|
182
|
+
await drv.stop(terminal=False)
|
|
183
|
+
await asyncio.wait_for(call.ctx.stop_requested.wait(), 5)
|
|
184
|
+
await drv.wait_status("stopping")
|
|
185
|
+
await flush(host)
|
|
186
|
+
crashed = survivors(store)
|
|
187
|
+
|
|
188
|
+
script2 = Script()
|
|
189
|
+
async with run_host(script2, store=crashed) as (drv2, host2):
|
|
190
|
+
assert drv2.replica["status"] == "stopping"
|
|
191
|
+
assert run_of(drv2.replica)["stopping"] == {"reason": "stop"}
|
|
192
|
+
redelivered = await script2.next_call()
|
|
193
|
+
assert redelivered.ctx.stop_requested.is_set()
|
|
194
|
+
redelivered.finish(RunManager.Stop(dispatch_queue=False))
|
|
195
|
+
await drv2.wait_status("stopped")
|
|
196
|
+
assert run_of(drv2.replica)["stopping"] is None
|
|
197
|
+
|
|
198
|
+
|
|
172
199
|
async def test_an_applied_batch_redelivers_as_a_resume():
|
|
173
200
|
store = MemoryStateStore()
|
|
174
201
|
script = Script()
|
|
@@ -269,3 +296,30 @@ async def test_input_answers_survive_a_restart_mid_answering():
|
|
|
269
296
|
resumed.applied()
|
|
270
297
|
resumed.finish(RunManager.Complete())
|
|
271
298
|
await drv2.wait_status("ready")
|
|
299
|
+
|
|
300
|
+
|
|
301
|
+
async def test_the_stop_reason_survives_a_restart_until_the_next_dispatch():
|
|
302
|
+
store = MemoryStateStore()
|
|
303
|
+
script = Script()
|
|
304
|
+
async with run_host(script, store=store) as (drv, host):
|
|
305
|
+
drv.post("run/enqueue", add("m1"))
|
|
306
|
+
call = await script.next_call()
|
|
307
|
+
call.applied()
|
|
308
|
+
await drv.stop({"reason": "user-stop"}, terminal=False)
|
|
309
|
+
call.finish(RunManager.Stop(dispatch_queue=False))
|
|
310
|
+
await drv.wait_status("stopped")
|
|
311
|
+
assert drv.run["stopReason"] == "user-stop"
|
|
312
|
+
await flush(host)
|
|
313
|
+
crashed = survivors(store)
|
|
314
|
+
|
|
315
|
+
script2 = Script()
|
|
316
|
+
async with run_host(script2, store=crashed) as (drv2, host2):
|
|
317
|
+
assert drv2.replica["status"] == "stopped"
|
|
318
|
+
assert drv2.run["stopReason"] == "user-stop"
|
|
319
|
+
drv2.post("run/enqueue", add("m2"))
|
|
320
|
+
drained = await script2.next_call()
|
|
321
|
+
assert drained.ctx.trigger == "stop-continue"
|
|
322
|
+
await drv2.wait(lambda s: run_of(s)["stopReason"] is None)
|
|
323
|
+
drained.applied()
|
|
324
|
+
drained.finish(RunManager.Complete())
|
|
325
|
+
await drv2.wait_status("ready")
|
|
@@ -5,8 +5,8 @@ limit; each scenario is swept over every crash point. Whatever prefix
|
|
|
5
5
|
survived, the restarted machine must satisfy: an accepted message or edit is
|
|
6
6
|
either consumed by a durably settled dispatch or delivered exactly once after
|
|
7
7
|
the restart; an accepted input answer reaches a run's input_outcomes; status
|
|
8
|
-
"input-required" always carries inputRequests;
|
|
9
|
-
a run.
|
|
8
|
+
"input-required" always carries inputRequests; an open stop window and status
|
|
9
|
+
"stopping" imply each other; and a fresh send still reaches a run.
|
|
10
10
|
|
|
11
11
|
The redelivery lands on the correct arm of the run callback: a batch not yet
|
|
12
12
|
durably applied re-presents in dispatch mode with the full record, a durably
|
|
@@ -266,6 +266,10 @@ async def flush(runs) -> None:
|
|
|
266
266
|
def check_wire(runs) -> None:
|
|
267
267
|
if runs.status.value == "input-required":
|
|
268
268
|
assert runs.inputRequests.value, "input-required without inputRequests"
|
|
269
|
+
if runs.stopping.value is not None:
|
|
270
|
+
assert runs.status.value == "stopping", "an open stop window without its status"
|
|
271
|
+
elif runs.status.value == "stopping":
|
|
272
|
+
raise AssertionError("stopping status without an open stop window")
|
|
269
273
|
|
|
270
274
|
|
|
271
275
|
async def settle_call(runs, call, seen: set[str]) -> None:
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import asyncio
|
|
2
2
|
|
|
3
3
|
import pytest
|
|
4
|
-
from run_helpers import Script, add, assert_rejected, run_host, run_of
|
|
4
|
+
from run_helpers import Script, add, assert_rejected, machine, run_host, run_of
|
|
5
5
|
|
|
6
6
|
from harness_sdk import RunManager
|
|
7
7
|
|
|
@@ -15,7 +15,7 @@ async def test_stop_is_not_optimistic():
|
|
|
15
15
|
pending = await drv.stop(terminal=False)
|
|
16
16
|
assert pending["type"] == "pending"
|
|
17
17
|
await asyncio.wait_for(call.ctx.stop_requested.wait(), 5)
|
|
18
|
-
|
|
18
|
+
await drv.wait_status("stopping")
|
|
19
19
|
call.finish(RunManager.Stop(dispatch_queue=False))
|
|
20
20
|
settled = await drv.res(pending["seq"])
|
|
21
21
|
assert settled["type"] == "accepted"
|
|
@@ -45,12 +45,15 @@ async def test_stop_reason_is_visible_on_the_start_context():
|
|
|
45
45
|
call = await script.next_call()
|
|
46
46
|
call.applied()
|
|
47
47
|
assert call.ctx.stop_reason is None
|
|
48
|
+
await drv.run_id()
|
|
49
|
+
assert drv.run["stopReason"] is None
|
|
48
50
|
pending = await drv.stop({"reason": "user-stop"}, terminal=False)
|
|
49
51
|
await asyncio.wait_for(call.ctx.stop_requested.wait(), 5)
|
|
50
52
|
assert call.ctx.stop_reason == "user-stop"
|
|
51
53
|
call.finish(RunManager.Stop(dispatch_queue=False))
|
|
52
54
|
assert (await drv.res(pending["seq"]))["type"] == "accepted"
|
|
53
55
|
await drv.wait_status("stopped")
|
|
56
|
+
assert drv.run["stopReason"] == "user-stop"
|
|
54
57
|
|
|
55
58
|
|
|
56
59
|
async def test_stop_reason_resets_for_the_next_run():
|
|
@@ -64,6 +67,7 @@ async def test_stop_reason_resets_for_the_next_run():
|
|
|
64
67
|
call.finish(RunManager.Stop(dispatch_queue=True))
|
|
65
68
|
drain = await script.next_call()
|
|
66
69
|
assert drain.ctx.stop_reason is None
|
|
70
|
+
await drv.wait(lambda s: run_of(s)["stopReason"] is None)
|
|
67
71
|
|
|
68
72
|
|
|
69
73
|
async def test_stop_rejects_malformed_params():
|
|
@@ -195,6 +199,52 @@ async def test_stop_settles_even_when_the_run_completes_normally():
|
|
|
195
199
|
await drv.wait_status("ready")
|
|
196
200
|
|
|
197
201
|
|
|
202
|
+
async def test_stop_verdict_awaits_the_settle_and_the_checkpoint_drain():
|
|
203
|
+
script = Script()
|
|
204
|
+
async with run_host(script) as (drv, host):
|
|
205
|
+
drv.post("run/enqueue", add("m1"))
|
|
206
|
+
call = await script.next_call()
|
|
207
|
+
call.applied()
|
|
208
|
+
release = asyncio.Event()
|
|
209
|
+
durable = asyncio.Event()
|
|
210
|
+
|
|
211
|
+
async def checkpoint():
|
|
212
|
+
await release.wait()
|
|
213
|
+
durable.set()
|
|
214
|
+
|
|
215
|
+
call.ctx.enlist(checkpoint())
|
|
216
|
+
runs = await machine(host.live)
|
|
217
|
+
verdict = asyncio.ensure_future(
|
|
218
|
+
runs.stop({"runId": await drv.run_id()}, applied=lambda: None)
|
|
219
|
+
)
|
|
220
|
+
await asyncio.wait_for(call.ctx.stop_requested.wait(), 5)
|
|
221
|
+
await asyncio.sleep(0.02)
|
|
222
|
+
assert not verdict.done() # the run is still winding down
|
|
223
|
+
call.finish(RunManager.Stop(dispatch_queue=False))
|
|
224
|
+
await asyncio.wait_for(runs.idle.wait(), 5)
|
|
225
|
+
assert runs.status.value == "stopped"
|
|
226
|
+
await asyncio.sleep(0.02)
|
|
227
|
+
assert not verdict.done() # the checkpoint drain is still open
|
|
228
|
+
release.set()
|
|
229
|
+
assert await asyncio.wait_for(verdict, 5) is None
|
|
230
|
+
assert durable.is_set() # every enlisted checkpoint durable before the verdict
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
async def test_a_late_enlist_after_the_run_settled_raises():
|
|
234
|
+
script = Script()
|
|
235
|
+
async with run_host(script) as (drv, host):
|
|
236
|
+
drv.post("run/enqueue", add("m1"))
|
|
237
|
+
call = await script.next_call()
|
|
238
|
+
await drv.run_id()
|
|
239
|
+
call.applied()
|
|
240
|
+
call.finish(RunManager.Complete())
|
|
241
|
+
await drv.wait(lambda s: run_of(s)["runId"] is None)
|
|
242
|
+
straggler = asyncio.sleep(0)
|
|
243
|
+
with pytest.raises(RuntimeError, match="settled"):
|
|
244
|
+
call.ctx.enlist(straggler)
|
|
245
|
+
straggler.close()
|
|
246
|
+
|
|
247
|
+
|
|
198
248
|
@pytest.mark.parametrize(
|
|
199
249
|
"end,continue_type",
|
|
200
250
|
[("error", "error-continue"), ("stopped", "stop-continue")],
|
|
@@ -1,10 +1,12 @@
|
|
|
1
1
|
"""Contract: the stopping window. Three openers — run/stop, mid-run
|
|
2
2
|
edit/reload, and the enqueue/steer edit-of-dispatched form — write
|
|
3
|
-
``stopping: {reason}`` (plus the staged rewind as ``nextDispatch``) and
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
3
|
+
``stopping: {reason}`` (plus the staged rewind as ``nextDispatch``) and flip
|
|
4
|
+
``status`` to "stopping" in the same envelope. Rewind commands settle
|
|
5
|
+
accepted with that write; the run/stop verdict follows the run's settle. On
|
|
6
|
+
run end one envelope promotes ``nextDispatch`` into ``dispatching`` and
|
|
7
|
+
clears the window; a later stop/rewind replaces the staged dispatch, whose
|
|
8
|
+
command already settled at its admission write. A second run/stop inside the
|
|
9
|
+
window rejects wrong-state."""
|
|
8
10
|
|
|
9
11
|
import asyncio
|
|
10
12
|
import copy
|
|
@@ -13,6 +15,7 @@ import pytest
|
|
|
13
15
|
from run_helpers import (
|
|
14
16
|
Script,
|
|
15
17
|
add,
|
|
18
|
+
assert_rejected,
|
|
16
19
|
edit_params,
|
|
17
20
|
msg,
|
|
18
21
|
queue_ids,
|
|
@@ -57,26 +60,40 @@ async def start_run(drv, script, *, parent="p0"):
|
|
|
57
60
|
return call
|
|
58
61
|
|
|
59
62
|
|
|
60
|
-
async def
|
|
63
|
+
async def test_stop_opens_the_window_and_flips_status_to_stopping():
|
|
61
64
|
script = Script()
|
|
62
65
|
async with run_host(script) as (drv, host):
|
|
63
66
|
call = await start_run(drv, script)
|
|
64
|
-
|
|
65
|
-
assert
|
|
66
|
-
|
|
67
|
+
pending = await drv.stop(terminal=False)
|
|
68
|
+
assert pending["type"] == "pending"
|
|
69
|
+
await drv.wait_status("stopping")
|
|
67
70
|
assert drv.run["stopping"] == {"reason": "stop"}
|
|
68
71
|
assert drv.run["nextDispatch"] is None
|
|
69
72
|
opened = next(
|
|
70
|
-
state
|
|
71
|
-
for env, state in states(drv)
|
|
72
|
-
if any(r["seq"] == settled["seq"] for r in env.get("res", []))
|
|
73
|
+
state for _, state in states(drv) if state.get("status") == "stopping"
|
|
73
74
|
)
|
|
74
75
|
assert window_of(opened) == ({"reason": "stop"}, None)
|
|
75
76
|
call.finish(RunManager.Stop(dispatch_queue=False))
|
|
77
|
+
assert (await drv.res(pending["seq"]))["type"] == "accepted"
|
|
76
78
|
await drv.wait_status("stopped")
|
|
77
79
|
assert drv.run["stopping"] is None
|
|
78
80
|
|
|
79
81
|
|
|
82
|
+
async def test_a_second_stop_inside_the_window_rejects_wrong_state():
|
|
83
|
+
script = Script()
|
|
84
|
+
async with run_host(script) as (drv, host):
|
|
85
|
+
call = await start_run(drv, script)
|
|
86
|
+
run_id = await drv.run_id()
|
|
87
|
+
pending = await drv.stop(terminal=False)
|
|
88
|
+
await drv.wait_status("stopping")
|
|
89
|
+
assert_rejected(
|
|
90
|
+
await drv.command("run/stop", {"runId": run_id}), "wrong-state"
|
|
91
|
+
)
|
|
92
|
+
call.finish(RunManager.Stop(dispatch_queue=False))
|
|
93
|
+
assert (await drv.res(pending["seq"]))["type"] == "accepted"
|
|
94
|
+
await drv.wait_status("stopped")
|
|
95
|
+
|
|
96
|
+
|
|
80
97
|
async def test_mid_run_edit_stages_the_window_and_promotes_in_one_envelope():
|
|
81
98
|
script = Script()
|
|
82
99
|
async with run_host(script, capabilities=("rewind", "rewind-during-run")) as (
|
|
@@ -87,7 +104,7 @@ async def test_mid_run_edit_stages_the_window_and_promotes_in_one_envelope():
|
|
|
87
104
|
seq = drv.post("run/edit", edit_params("m1", msg("m2"), anchor="p0"))
|
|
88
105
|
await asyncio.wait_for(call.ctx.stop_requested.wait(), 5)
|
|
89
106
|
await drv.wait(lambda s: run_of(s)["stopping"] is not None)
|
|
90
|
-
assert drv.replica["status"] == "
|
|
107
|
+
assert drv.replica["status"] == "stopping"
|
|
91
108
|
assert drv.run["stopping"] == {"reason": "message-edit"}
|
|
92
109
|
staged = drv.run["nextDispatch"]
|
|
93
110
|
assert staged == {
|
|
@@ -161,7 +178,7 @@ async def test_edit_of_dispatched_message_opens_the_window(command):
|
|
|
161
178
|
call = await start_run(drv, script)
|
|
162
179
|
seq = drv.post(command, target(message=msg("m1", "edited")))
|
|
163
180
|
await drv.wait(lambda s: run_of(s)["stopping"] is not None)
|
|
164
|
-
assert drv.replica["status"] == "
|
|
181
|
+
assert drv.replica["status"] == "stopping"
|
|
165
182
|
assert drv.run["stopping"] == {"reason": "message-edit"}
|
|
166
183
|
assert drv.run["nextDispatch"] == {
|
|
167
184
|
"trigger": "message-edit",
|
|
@@ -210,11 +227,12 @@ async def test_stop_clears_a_staged_rewind():
|
|
|
210
227
|
seq = drv.post("run/edit", edit_params("m1", msg("m2"), anchor="p0"))
|
|
211
228
|
await drv.wait(lambda s: run_of(s)["nextDispatch"] is not None)
|
|
212
229
|
assert (await drv.res(seq))["type"] == "accepted"
|
|
213
|
-
|
|
214
|
-
assert
|
|
230
|
+
pending = await drv.stop(terminal=False)
|
|
231
|
+
assert pending["type"] == "pending"
|
|
215
232
|
await drv.wait(lambda s: run_of(s)["stopping"] == {"reason": "stop"})
|
|
216
233
|
assert drv.run["nextDispatch"] is None
|
|
217
234
|
call.finish(RunManager.Stop(dispatch_queue=False))
|
|
235
|
+
assert (await drv.res(pending["seq"]))["type"] == "accepted"
|
|
218
236
|
await drv.wait_status("stopped")
|
|
219
237
|
script.no_call()
|
|
220
238
|
|
|
@@ -253,6 +271,7 @@ ENTRY_KEYS = {
|
|
|
253
271
|
"nextDispatch",
|
|
254
272
|
"inputRequests",
|
|
255
273
|
"error",
|
|
274
|
+
"stopReason",
|
|
256
275
|
}
|
|
257
276
|
|
|
258
277
|
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|