manta-node 0.6b5.dev771__py3-none-any.whl → 0.6b5.dev783__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- manta_node/cli/_version_generated.py +3 -3
- manta_node/infrastructure/mqtt/command_handler.py +67 -4
- manta_node/supervisor/executor.py +6 -1
- manta_node/supervisor/inflight.py +7 -4
- manta_node/supervisor/lifecycle.py +69 -29
- manta_node/supervisor/process.py +117 -5
- manta_node/supervisor/records.py +9 -0
- manta_node/supervisor/switch.py +16 -11
- {manta_node-0.6b5.dev771.dist-info → manta_node-0.6b5.dev783.dist-info}/METADATA +1 -1
- {manta_node-0.6b5.dev771.dist-info → manta_node-0.6b5.dev783.dist-info}/RECORD +14 -14
- {manta_node-0.6b5.dev771.dist-info → manta_node-0.6b5.dev783.dist-info}/WHEEL +0 -0
- {manta_node-0.6b5.dev771.dist-info → manta_node-0.6b5.dev783.dist-info}/entry_points.txt +0 -0
- {manta_node-0.6b5.dev771.dist-info → manta_node-0.6b5.dev783.dist-info}/licenses/LICENSE +0 -0
- {manta_node-0.6b5.dev771.dist-info → manta_node-0.6b5.dev783.dist-info}/top_level.txt +0 -0
|
@@ -4,7 +4,7 @@ This file is auto-generated at build time by scripts/generate_version.py.
|
|
|
4
4
|
Do not edit manually.
|
|
5
5
|
"""
|
|
6
6
|
|
|
7
|
-
__version__ = "2026.10.
|
|
8
|
-
__build_date__ = "2026.10.
|
|
9
|
-
__commit_id__ = "
|
|
7
|
+
__version__ = "2026.10.08.05fbabb8"
|
|
8
|
+
__build_date__ = "2026.10.08"
|
|
9
|
+
__commit_id__ = "05fbabb8"
|
|
10
10
|
__package_version__ = "0.6b5" # release-series base, moved by scripts/bump_version.py
|
|
@@ -62,6 +62,11 @@ MAX_MQTT_CONNECT_FAILURES = 3
|
|
|
62
62
|
REPLAY_POLL_SECONDS = 2.0
|
|
63
63
|
REPLAY_TIMEOUT_SECONDS = 120.0
|
|
64
64
|
|
|
65
|
+
# Every update must end in one outcome (B-24, mechanism §6.6): these name the
|
|
66
|
+
# reasons the node reports for a malformed command and an unavailable supervisor.
|
|
67
|
+
MALFORMED_COMMAND_REASON = "malformed command"
|
|
68
|
+
SUPERVISOR_UNAVAILABLE_REASON = "the node's supervisor is unavailable"
|
|
69
|
+
|
|
65
70
|
|
|
66
71
|
def _node_update_state(state) -> NodeUpdateState:
|
|
67
72
|
"""Map a seam state to the wire enum; they mirror 1:1 (manta-common#291)."""
|
|
@@ -759,8 +764,18 @@ class CommandHandler(Collector, MqttBase):
|
|
|
759
764
|
before relaying (ADR-0042 D5) and reports the refusal, so the platform's
|
|
760
765
|
one-path rule holds even if a command reaches a host- or fleet-updated
|
|
761
766
|
node, or one whose path is unknown.
|
|
767
|
+
|
|
768
|
+
Every ending is reported (B-24, mechanism §6.6): a malformed command is
|
|
769
|
+
`REFUSED` "malformed command" when a request id can be read, and a
|
|
770
|
+
missing or failing seam is `FAILED` "the node's supervisor is
|
|
771
|
+
unavailable". Neither raises: nothing changed (NF-3), and the platform
|
|
772
|
+
must see one outcome rather than wait out its deadline.
|
|
762
773
|
"""
|
|
763
|
-
|
|
774
|
+
try:
|
|
775
|
+
request = self._parse_update(payload)
|
|
776
|
+
except MantaMQTTError as exc:
|
|
777
|
+
await self._refuse_malformed(payload, exc)
|
|
778
|
+
return
|
|
764
779
|
# One update path per node (ADR-0042 D5). Only a platform-path node takes
|
|
765
780
|
# a platform command; the platform fails closed on every other path, and
|
|
766
781
|
# if one arrives anyway, refuse it here and report the refusal rather
|
|
@@ -774,13 +789,61 @@ class CommandHandler(Collector, MqttBase):
|
|
|
774
789
|
)
|
|
775
790
|
)
|
|
776
791
|
return
|
|
777
|
-
|
|
792
|
+
try:
|
|
793
|
+
outcome = await self.relay_update(request)
|
|
794
|
+
except MantaNodeError:
|
|
795
|
+
await self._publish_failed(
|
|
796
|
+
request.request_id, SUPERVISOR_UNAVAILABLE_REASON
|
|
797
|
+
)
|
|
798
|
+
return
|
|
778
799
|
if outcome is None:
|
|
779
|
-
|
|
780
|
-
|
|
800
|
+
await self._publish_failed(
|
|
801
|
+
request.request_id, SUPERVISOR_UNAVAILABLE_REASON
|
|
781
802
|
)
|
|
803
|
+
return
|
|
782
804
|
await self.publish_update_status(outcome)
|
|
783
805
|
|
|
806
|
+
async def _refuse_malformed(self, payload: bytes, exc: Exception) -> None:
|
|
807
|
+
"""Report a malformed command as `REFUSED`, when a request id can be read.
|
|
808
|
+
|
|
809
|
+
The Manager correlates the outcome by request id, so a command we cannot
|
|
810
|
+
accept still gets a `REFUSED` "malformed command" when the id survived
|
|
811
|
+
the parse. A payload that carries none has nothing to correlate and is
|
|
812
|
+
dropped with a log line.
|
|
813
|
+
"""
|
|
814
|
+
request_id = self._recover_request_id(payload)
|
|
815
|
+
if request_id is None:
|
|
816
|
+
self.tracer.warning(
|
|
817
|
+
f"Dropping a malformed update_node payload with no readable"
|
|
818
|
+
f" request id: {exc}"
|
|
819
|
+
)
|
|
820
|
+
return
|
|
821
|
+
self.tracer.warning(
|
|
822
|
+
f"Refusing a malformed update_node command {request_id}: {exc}"
|
|
823
|
+
)
|
|
824
|
+
await self.publish_update_status(
|
|
825
|
+
SwitchOutcome(
|
|
826
|
+
request_id=request_id,
|
|
827
|
+
state=SeamState.REFUSED,
|
|
828
|
+
reason=MALFORMED_COMMAND_REASON,
|
|
829
|
+
)
|
|
830
|
+
)
|
|
831
|
+
|
|
832
|
+
@staticmethod
|
|
833
|
+
def _recover_request_id(payload: bytes) -> str | None:
|
|
834
|
+
"""The request id from a payload we could not accept, if one is readable."""
|
|
835
|
+
try:
|
|
836
|
+
command = MqNodeUpdate().parse(payload)
|
|
837
|
+
except Exception:
|
|
838
|
+
return None
|
|
839
|
+
return command.request_id or None
|
|
840
|
+
|
|
841
|
+
async def _publish_failed(self, request_id: str, reason: str) -> None:
|
|
842
|
+
"""Report a `FAILED` outcome for a request that changed nothing (NF-3)."""
|
|
843
|
+
await self.publish_update_status(
|
|
844
|
+
SwitchOutcome(request_id=request_id, state=SeamState.FAILED, reason=reason)
|
|
845
|
+
)
|
|
846
|
+
|
|
784
847
|
@staticmethod
|
|
785
848
|
def _parse_update(payload: bytes) -> UpdateRequest:
|
|
786
849
|
"""Validate an `MqNodeUpdate` and turn it into the seam's request.
|
|
@@ -96,6 +96,7 @@ def make_executor(
|
|
|
96
96
|
bounds: Bounds | None = None,
|
|
97
97
|
verify: Callable[[str, str | None], None] = verify_candidate,
|
|
98
98
|
install: Callable[[str], str] = version_tree.install_version,
|
|
99
|
+
switch_lock: threading.Lock | None = None,
|
|
99
100
|
) -> Callable[[UpdateRequest], Awaitable[SwitchOutcome]]:
|
|
100
101
|
"""Build the seam handler: a request in, the lifecycle's outcome out.
|
|
101
102
|
|
|
@@ -108,11 +109,15 @@ def make_executor(
|
|
|
108
109
|
concurrent switches would race both. ``SeamServer`` dedupes by ``request_id``
|
|
109
110
|
only; a second, differently-ided request while one is switching is refused
|
|
110
111
|
here rather than run.
|
|
112
|
+
|
|
113
|
+
``switch_lock`` lets the supervisor share this "a switch is in flight" fact:
|
|
114
|
+
its crash-loop revert takes the same lock, so a switch's outcome and a
|
|
115
|
+
crash-loop revert never both decide the same update.
|
|
111
116
|
"""
|
|
112
117
|
bounds = bounds or load_bounds()
|
|
113
118
|
# One switch at a time. Held across the whole lifecycle, so a second request
|
|
114
119
|
# (any request_id) is refused while one is in flight, not run concurrently.
|
|
115
|
-
switching = threading.Lock()
|
|
120
|
+
switching = switch_lock if switch_lock is not None else threading.Lock()
|
|
116
121
|
|
|
117
122
|
def fetch(target: str) -> str:
|
|
118
123
|
# Pull-only (NF-10). A version dir the fleet path already materialised
|
|
@@ -18,7 +18,7 @@ import time
|
|
|
18
18
|
from dataclasses import dataclass
|
|
19
19
|
from pathlib import Path
|
|
20
20
|
|
|
21
|
-
from .records import read_json, write_json_atomic
|
|
21
|
+
from .records import TREE_LOCK, read_json, write_json_atomic
|
|
22
22
|
|
|
23
23
|
INFLIGHT_NAME = "inflight.json"
|
|
24
24
|
# The last terminal outcome, written when a switch resolves and read when a
|
|
@@ -69,7 +69,8 @@ class InFlight:
|
|
|
69
69
|
|
|
70
70
|
|
|
71
71
|
def write_inflight(record: InFlight, path: Path | None = None) -> None:
|
|
72
|
-
|
|
72
|
+
with TREE_LOCK:
|
|
73
|
+
write_json_atomic(path or inflight_path(), record.to_dict())
|
|
73
74
|
|
|
74
75
|
|
|
75
76
|
def read_inflight(path: Path | None = None) -> InFlight | None:
|
|
@@ -84,7 +85,8 @@ def read_inflight(path: Path | None = None) -> InFlight | None:
|
|
|
84
85
|
|
|
85
86
|
|
|
86
87
|
def clear_inflight(path: Path | None = None) -> None:
|
|
87
|
-
|
|
88
|
+
with TREE_LOCK:
|
|
89
|
+
(path or inflight_path()).unlink(missing_ok=True)
|
|
88
90
|
|
|
89
91
|
|
|
90
92
|
def deadline_from_now(window: float) -> float:
|
|
@@ -125,7 +127,8 @@ class UpdateOutcome:
|
|
|
125
127
|
|
|
126
128
|
|
|
127
129
|
def write_outcome(outcome: UpdateOutcome, path: Path | None = None) -> None:
|
|
128
|
-
|
|
130
|
+
with TREE_LOCK:
|
|
131
|
+
write_json_atomic(path or outcome_path(), outcome.to_dict())
|
|
129
132
|
|
|
130
133
|
|
|
131
134
|
def read_outcome(path: Path | None = None) -> UpdateOutcome | None:
|
|
@@ -17,6 +17,7 @@ supervisor.
|
|
|
17
17
|
|
|
18
18
|
from __future__ import annotations
|
|
19
19
|
|
|
20
|
+
import sys
|
|
20
21
|
import time
|
|
21
22
|
from collections.abc import Callable
|
|
22
23
|
from dataclasses import dataclass
|
|
@@ -128,15 +129,17 @@ def run_switch(
|
|
|
128
129
|
nonlocal candidate_python
|
|
129
130
|
candidate_python = fetch(request.target)
|
|
130
131
|
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
132
|
+
cause = _within(bounds.fetch, do_fetch)
|
|
133
|
+
if cause is not None:
|
|
134
|
+
return _step_failed(
|
|
135
|
+
request, "fetch", "fetch exceeded its bound (no switch)", cause
|
|
134
136
|
)
|
|
135
137
|
|
|
136
138
|
# VERIFY — the candidate must be usable in its OWN environment (NF-2): the
|
|
137
139
|
# fetcher named the interpreter the candidate will run from.
|
|
138
|
-
|
|
139
|
-
|
|
140
|
+
cause = _within(bounds.verify, lambda: verify(candidate_dir, candidate_python))
|
|
141
|
+
if cause is not None:
|
|
142
|
+
return _step_failed(request, "verify", "verification failed (no switch)", cause)
|
|
140
143
|
|
|
141
144
|
# FREE SPACE — refuse before staging when there is no room for a full version
|
|
142
145
|
# dir (NF-12, §6.2). Refusing here, before any write, is what stops a
|
|
@@ -156,8 +159,11 @@ def run_switch(
|
|
|
156
159
|
candidate_dir, candidate_python
|
|
157
160
|
)
|
|
158
161
|
|
|
159
|
-
|
|
160
|
-
|
|
162
|
+
cause = _within(bounds.stage, do_stage)
|
|
163
|
+
if cause is not None:
|
|
164
|
+
return _step_failed(
|
|
165
|
+
request, "stage", "stage exceeded its bound or failed", cause
|
|
166
|
+
)
|
|
161
167
|
target = staged["target"]
|
|
162
168
|
|
|
163
169
|
record = InFlight(
|
|
@@ -169,25 +175,26 @@ def run_switch(
|
|
|
169
175
|
write_inflight(record)
|
|
170
176
|
|
|
171
177
|
# SWITCH — the commit point (NF-1), bounded.
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
clear_inflight()
|
|
175
|
-
_reclaim_candidate(bounds, target, last_good_dir)
|
|
176
|
-
return _outcome(
|
|
177
|
-
request, SeamState.FAILED, "switch exceeded its bound (repoint failed)"
|
|
178
|
-
)
|
|
179
|
-
except VersionTreeError as exc:
|
|
178
|
+
cause = _within(bounds.switch, lambda: version_tree.commit(target))
|
|
179
|
+
if cause is not None:
|
|
180
180
|
clear_inflight()
|
|
181
181
|
_reclaim_candidate(bounds, target, last_good_dir)
|
|
182
|
-
return
|
|
182
|
+
return _step_failed(request, "switch", "switch failed (repoint failed)", cause)
|
|
183
183
|
|
|
184
184
|
# RESTART onto the committed version, then wait for the return window. A
|
|
185
185
|
# restart that raises is a failed switch, not a crash: surface DOWN and clear
|
|
186
186
|
# the record rather than letting the exception escape.
|
|
187
|
-
|
|
187
|
+
cause = _within(bounds.switch, restart)
|
|
188
|
+
if cause is not None:
|
|
188
189
|
clear_inflight()
|
|
189
190
|
_reclaim_candidate(bounds, target, last_good_dir)
|
|
190
|
-
return
|
|
191
|
+
return _step_failed(
|
|
192
|
+
request,
|
|
193
|
+
"restart",
|
|
194
|
+
"restart failed after the switch",
|
|
195
|
+
cause,
|
|
196
|
+
state=SeamState.DOWN,
|
|
197
|
+
)
|
|
191
198
|
|
|
192
199
|
if is_back(record.deadline_epoch):
|
|
193
200
|
clear_inflight()
|
|
@@ -215,12 +222,19 @@ def run_switch(
|
|
|
215
222
|
)
|
|
216
223
|
try:
|
|
217
224
|
version_tree.commit(version_tree.versions_dir() / last_good)
|
|
218
|
-
|
|
219
|
-
|
|
225
|
+
cause = _within(bounds.switch, restart)
|
|
226
|
+
if cause is not None:
|
|
227
|
+
raise LifecycleError(f"restart after revert failed: {cause}")
|
|
220
228
|
except (VersionTreeError, LifecycleError) as exc:
|
|
221
229
|
clear_inflight()
|
|
222
230
|
_reclaim_candidate(bounds, target, last_good_dir)
|
|
223
|
-
return
|
|
231
|
+
return _step_failed(
|
|
232
|
+
request,
|
|
233
|
+
"revert",
|
|
234
|
+
f"revert failed: {exc}",
|
|
235
|
+
str(exc),
|
|
236
|
+
state=SeamState.DOWN,
|
|
237
|
+
)
|
|
224
238
|
clear_inflight()
|
|
225
239
|
# The candidate was superseded the moment the switch did not land: it is
|
|
226
240
|
# reclaimed even though the switch failed, so repeated failing switches do
|
|
@@ -257,17 +271,43 @@ def _reclaim_candidate(
|
|
|
257
271
|
version_tree.reclaim(bounds.retention_dirs, protect)
|
|
258
272
|
|
|
259
273
|
|
|
260
|
-
def _within(bound: float, step: Callable[[], None]) ->
|
|
261
|
-
"""Run ``step``;
|
|
274
|
+
def _within(bound: float, step: Callable[[], None]) -> str | None:
|
|
275
|
+
"""Run ``step``; return None if it completed within ``bound``, else the cause.
|
|
262
276
|
|
|
263
277
|
A step that raises is a failure of that state, not a crash: before the switch
|
|
264
|
-
it leaves the node on its current version (NF-3). The
|
|
265
|
-
|
|
266
|
-
|
|
278
|
+
it leaves the node on its current version (NF-3). The return names the
|
|
279
|
+
failure — the exception, or the bound it exceeded — so the caller can record
|
|
280
|
+
the stage and its cause. The time bound is enforced by the step itself (a
|
|
281
|
+
subprocess timeout / an HTTP timeout); this records the comparison so the
|
|
282
|
+
caller does not have to.
|
|
267
283
|
"""
|
|
268
284
|
start = time.monotonic()
|
|
269
285
|
try:
|
|
270
286
|
step()
|
|
271
|
-
except Exception:
|
|
272
|
-
return
|
|
273
|
-
|
|
287
|
+
except Exception as exc:
|
|
288
|
+
return f"{type(exc).__name__}: {exc}"
|
|
289
|
+
elapsed = time.monotonic() - start
|
|
290
|
+
if elapsed > bound:
|
|
291
|
+
return f"exceeded its {bound}s bound ({elapsed:.1f}s)"
|
|
292
|
+
return None
|
|
293
|
+
|
|
294
|
+
|
|
295
|
+
def _step_failed(
|
|
296
|
+
request: UpdateRequest,
|
|
297
|
+
stage: str,
|
|
298
|
+
reason: str,
|
|
299
|
+
cause: str,
|
|
300
|
+
*,
|
|
301
|
+
state: SeamState = SeamState.FAILED,
|
|
302
|
+
) -> SwitchOutcome:
|
|
303
|
+
"""Build a failure outcome naming the stage and cause, and log it.
|
|
304
|
+
|
|
305
|
+
``detail`` always carries ``stage`` and ``cause`` together, and the failed
|
|
306
|
+
stage always logs a line to stderr (the supervisor's journal) before
|
|
307
|
+
returning, so the outcome and the journal name the same failure.
|
|
308
|
+
"""
|
|
309
|
+
print(
|
|
310
|
+
f"supervisor: {stage} failed for request {request.request_id}: {cause}",
|
|
311
|
+
file=sys.stderr,
|
|
312
|
+
)
|
|
313
|
+
return _outcome(request, state, reason, stage=stage, cause=cause)
|
manta_node/supervisor/process.py
CHANGED
|
@@ -44,15 +44,27 @@ from ..seam import (
|
|
|
44
44
|
from . import switch as version_tree
|
|
45
45
|
from .bounds import load_bounds
|
|
46
46
|
from .executor import ChildControl, make_executor
|
|
47
|
-
from .inflight import
|
|
47
|
+
from .inflight import (
|
|
48
|
+
InFlight,
|
|
49
|
+
UpdateOutcome,
|
|
50
|
+
clear_inflight,
|
|
51
|
+
read_inflight,
|
|
52
|
+
write_outcome,
|
|
53
|
+
)
|
|
48
54
|
from .lifecycle import MIN_FREE_BYTES
|
|
49
55
|
from .records import (
|
|
56
|
+
TREE_LOCK,
|
|
50
57
|
supervisor_record,
|
|
51
58
|
supervisor_record_path,
|
|
52
59
|
write_json_atomic,
|
|
53
60
|
)
|
|
54
61
|
from .seam import SeamServer, SocketSeamClient
|
|
55
|
-
from .switch import
|
|
62
|
+
from .switch import (
|
|
63
|
+
VersionTreeError,
|
|
64
|
+
boot_python,
|
|
65
|
+
bootstrap_current,
|
|
66
|
+
installed_node_version,
|
|
67
|
+
)
|
|
56
68
|
|
|
57
69
|
# A child that exits more than RESTART_LIMIT times within RESTART_WINDOW_SECONDS
|
|
58
70
|
# is a crash loop: the supervisor stops restarting and exits non-zero, so systemd
|
|
@@ -112,6 +124,11 @@ class Supervisor:
|
|
|
112
124
|
# a replay-capable child reports the outcome either way. Consumed by the
|
|
113
125
|
# one planned respawn it belongs to.
|
|
114
126
|
self._pending_update_request_id: str | None = None
|
|
127
|
+
# The executor's single-switch lock, held for the whole lifecycle. The
|
|
128
|
+
# crash-loop branch takes it non-blocking to tell "a seam switch is in
|
|
129
|
+
# flight": then that switch owns the update and polls the child itself,
|
|
130
|
+
# so the revert must not race its decision.
|
|
131
|
+
self._switch_lock = threading.Lock()
|
|
115
132
|
# The seam server, served in a background thread while the child runs.
|
|
116
133
|
self._seam_thread: threading.Thread | None = None
|
|
117
134
|
self._seam_server: SeamServer | None = None
|
|
@@ -168,6 +185,74 @@ class Supervisor:
|
|
|
168
185
|
if record is not None:
|
|
169
186
|
self._pending_update_request_id = record.request_id
|
|
170
187
|
|
|
188
|
+
def _resume_or_revert_on_start(self) -> None:
|
|
189
|
+
"""Act on a left-over in-flight record when the supervisor starts (D9).
|
|
190
|
+
|
|
191
|
+
An expired window means the return was never observed while nothing
|
|
192
|
+
supervised: revert to the last good and record it. An open window keeps
|
|
193
|
+
the candidate current — the boot that follows carries the request id and
|
|
194
|
+
replays (NF-8) — and the return is observed at the platform's deadline
|
|
195
|
+
(mechanism §6.6); a later start past the window reverts.
|
|
196
|
+
"""
|
|
197
|
+
record = read_inflight()
|
|
198
|
+
if record is None:
|
|
199
|
+
return
|
|
200
|
+
if record.expired():
|
|
201
|
+
self._revert(record, "the window passed before the supervisor came back")
|
|
202
|
+
return
|
|
203
|
+
self._pending_update_request_id = record.request_id
|
|
204
|
+
|
|
205
|
+
def _revert(self, record: InFlight, reason: str) -> bool:
|
|
206
|
+
"""Commit the last good version and record a REVERTED outcome (D9/D12).
|
|
207
|
+
|
|
208
|
+
The boot that follows carries the request id, so a replay-capable node
|
|
209
|
+
reports the revert (NF-8). Returns False when the record was already
|
|
210
|
+
resolved by another path, when there is no valid last good to revert to,
|
|
211
|
+
or when the commit fails; the outcome is then DOWN, since the node
|
|
212
|
+
cannot be confirmed on any known-good version. Held under the shared
|
|
213
|
+
``TREE_LOCK`` (the same lock ``version_tree.commit`` and the record
|
|
214
|
+
writers take, so a seam switch on the executor thread cannot interleave)
|
|
215
|
+
and re-reading the record inside it, so a stale snapshot can never commit
|
|
216
|
+
or overwrite a record another path already resolved.
|
|
217
|
+
"""
|
|
218
|
+
with TREE_LOCK:
|
|
219
|
+
current = read_inflight()
|
|
220
|
+
if current is None or current.request_id != record.request_id:
|
|
221
|
+
return False # already resolved by another path
|
|
222
|
+
record = current
|
|
223
|
+
self._pending_update_request_id = record.request_id
|
|
224
|
+
last_good = record.last_good
|
|
225
|
+
target = version_tree.versions_dir() / last_good if last_good else None
|
|
226
|
+
if target is None or version_tree.read_manifest(target) is None:
|
|
227
|
+
self._record_outcome(
|
|
228
|
+
record, SeamState.DOWN, "no last-good version to revert to"
|
|
229
|
+
)
|
|
230
|
+
return False
|
|
231
|
+
try:
|
|
232
|
+
version_tree.commit(target)
|
|
233
|
+
except (VersionTreeError, OSError) as exc:
|
|
234
|
+
self._record_outcome(record, SeamState.DOWN, f"revert failed: {exc}")
|
|
235
|
+
return False
|
|
236
|
+
self._record_outcome(
|
|
237
|
+
record, SeamState.REVERTED, reason, reverted_to=last_good
|
|
238
|
+
)
|
|
239
|
+
return True
|
|
240
|
+
|
|
241
|
+
@staticmethod
|
|
242
|
+
def _record_outcome(
|
|
243
|
+
record: InFlight, state: SeamState, reason: str, **detail
|
|
244
|
+
) -> None:
|
|
245
|
+
"""Write the request's terminal outcome and clear the in-flight record."""
|
|
246
|
+
write_outcome(
|
|
247
|
+
UpdateOutcome(
|
|
248
|
+
request_id=record.request_id,
|
|
249
|
+
state=state.value,
|
|
250
|
+
reason=reason,
|
|
251
|
+
detail=detail,
|
|
252
|
+
)
|
|
253
|
+
)
|
|
254
|
+
clear_inflight()
|
|
255
|
+
|
|
171
256
|
def _restart_child(self) -> None:
|
|
172
257
|
"""The seam executor's restart: wind the child down so the loop respawns it.
|
|
173
258
|
|
|
@@ -192,6 +277,7 @@ class Supervisor:
|
|
|
192
277
|
handler = make_executor(
|
|
193
278
|
self._child_control(),
|
|
194
279
|
bounds=load_bounds(),
|
|
280
|
+
switch_lock=self._switch_lock,
|
|
195
281
|
)
|
|
196
282
|
server = SeamServer()
|
|
197
283
|
self._seam_server = server
|
|
@@ -277,9 +363,10 @@ class Supervisor:
|
|
|
277
363
|
def run(self) -> int:
|
|
278
364
|
signal.signal(signal.SIGTERM, self._on_signal)
|
|
279
365
|
signal.signal(signal.SIGINT, self._on_signal)
|
|
280
|
-
#
|
|
281
|
-
#
|
|
282
|
-
|
|
366
|
+
# D9: re-evaluate a left-over in-flight record when the supervisor starts.
|
|
367
|
+
# An expired window reverts to the last good; an open one keeps the
|
|
368
|
+
# candidate and lets the boot replay. Either way the boot carries the id.
|
|
369
|
+
self._resume_or_revert_on_start()
|
|
283
370
|
# 2c no-op switch: materialise the current code as the first version dir
|
|
284
371
|
# and commit it, so the child boots from `current`. Idempotent across
|
|
285
372
|
# restarts; a host with no version skips this entirely.
|
|
@@ -320,6 +407,31 @@ class Supervisor:
|
|
|
320
407
|
]
|
|
321
408
|
self._restarts.append(now)
|
|
322
409
|
if len(self._restarts) > RESTART_LIMIT:
|
|
410
|
+
# D12: a candidate that crash-loops is reverted at once rather
|
|
411
|
+
# than left down on the bad build. An update is in flight only
|
|
412
|
+
# while the in-flight record is present.
|
|
413
|
+
if not self._switch_lock.acquire(blocking=False):
|
|
414
|
+
# A seam switch is in flight: it owns the update and its
|
|
415
|
+
# return probe decides, so keep the child supervised rather
|
|
416
|
+
# than reverting under it. Its is_back sees the dead child and
|
|
417
|
+
# reverts quickly; do not enforce the limit here.
|
|
418
|
+
self._restarts = []
|
|
419
|
+
self._write_record(None, "restarting")
|
|
420
|
+
time.sleep(RESTART_BACKOFF_SECONDS)
|
|
421
|
+
continue
|
|
422
|
+
try:
|
|
423
|
+
record = read_inflight()
|
|
424
|
+
reverted = record is not None and self._revert(
|
|
425
|
+
record, "the candidate crash-looped past the restart limit"
|
|
426
|
+
)
|
|
427
|
+
finally:
|
|
428
|
+
self._switch_lock.release()
|
|
429
|
+
if reverted:
|
|
430
|
+
# The revert landed; respawn onto the last good with a fresh
|
|
431
|
+
# budget. A revert that could not land (DOWN) falls through:
|
|
432
|
+
# a fresh budget would only repeat a known-bad build.
|
|
433
|
+
self._restarts = []
|
|
434
|
+
continue
|
|
323
435
|
self._write_record(None, "crash-loop")
|
|
324
436
|
print(
|
|
325
437
|
f"supervisor: child exited {len(self._restarts)} times in "
|
manta_node/supervisor/records.py
CHANGED
|
@@ -18,6 +18,7 @@ from __future__ import annotations
|
|
|
18
18
|
|
|
19
19
|
import json
|
|
20
20
|
import os
|
|
21
|
+
import threading
|
|
21
22
|
from datetime import datetime, timezone
|
|
22
23
|
from pathlib import Path
|
|
23
24
|
|
|
@@ -25,6 +26,14 @@ INSTANCE_DIR = Path.home() / ".manta" / "nodes" / "instances"
|
|
|
25
26
|
|
|
26
27
|
SUPERVISOR_SUFFIX = ".supervisor.json"
|
|
27
28
|
|
|
29
|
+
# Guards the shared version-tree state two threads can touch at once: the
|
|
30
|
+
# supervisor's supervision loop (its crash-loop / start revert) and a seam
|
|
31
|
+
# switch running on the executor's worker thread. Every commit of `current` and
|
|
32
|
+
# every write to the in-flight/outcome records takes it, so their filesystem
|
|
33
|
+
# sequences never interleave. Re-entrant: a caller may hold it across several
|
|
34
|
+
# locked operations.
|
|
35
|
+
TREE_LOCK = threading.RLock()
|
|
36
|
+
|
|
28
37
|
|
|
29
38
|
def supervisor_record_path(alias: str) -> Path:
|
|
30
39
|
"""The supervisor record for ``alias`` (the pid of record)."""
|
manta_node/supervisor/switch.py
CHANGED
|
@@ -31,7 +31,7 @@ from importlib.metadata import PackageNotFoundError, version
|
|
|
31
31
|
from pathlib import Path
|
|
32
32
|
|
|
33
33
|
from ..seam import is_safe_version_name
|
|
34
|
-
from .records import read_json, write_json_atomic
|
|
34
|
+
from .records import TREE_LOCK, read_json, write_json_atomic
|
|
35
35
|
|
|
36
36
|
# The version tree and the pointer live under the node's state dir, beside the
|
|
37
37
|
# instances, never inside a version dir (NF-6).
|
|
@@ -156,15 +156,19 @@ def read_manifest(version_dir: Path) -> Version | None:
|
|
|
156
156
|
def materialise_version(version: str, python: str) -> Path:
|
|
157
157
|
"""Create an immutable version dir for ``version``; return its path.
|
|
158
158
|
|
|
159
|
-
Idempotent: an existing dir
|
|
160
|
-
|
|
159
|
+
Idempotent: an existing dir is returned unchanged when its identity matches
|
|
160
|
+
``version`` — the manifest's node version, or the legacy pair id it was
|
|
161
|
+
materialised under. The fleet path still names a candidate by its pair
|
|
162
|
+
(`manta-node@X+manta-common-core@Y`) and pre-materialises that dir, so
|
|
163
|
+
matching the pair too is what keeps its apply from being rejected as "not a
|
|
164
|
+
valid version dir" (#520).
|
|
161
165
|
"""
|
|
162
166
|
if not is_safe_version_name(version):
|
|
163
167
|
raise VersionTreeError(f"refusing an unsafe version dir name: {version!r}")
|
|
164
168
|
target = versions_dir() / version
|
|
165
169
|
if target.exists():
|
|
166
170
|
existing = read_manifest(target)
|
|
167
|
-
if existing and existing.version
|
|
171
|
+
if existing is not None and version in (existing.version, existing.pair):
|
|
168
172
|
return target
|
|
169
173
|
raise VersionTreeError(
|
|
170
174
|
f"{target} exists but is not a valid version dir for {version!r}"
|
|
@@ -249,13 +253,14 @@ def commit(target: Path) -> None:
|
|
|
249
253
|
"""
|
|
250
254
|
if read_manifest(target) is None:
|
|
251
255
|
raise VersionTreeError(f"refusing to commit an invalid version dir: {target}")
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
256
|
+
with TREE_LOCK:
|
|
257
|
+
versions_dir().mkdir(parents=True, exist_ok=True)
|
|
258
|
+
link = current_link()
|
|
259
|
+
link.parent.mkdir(parents=True, exist_ok=True)
|
|
260
|
+
tmp = link.with_name(link.name + ".tmp")
|
|
261
|
+
tmp.unlink(missing_ok=True)
|
|
262
|
+
os.symlink(_link_target(target), tmp)
|
|
263
|
+
os.replace(tmp, link)
|
|
259
264
|
|
|
260
265
|
|
|
261
266
|
def selected() -> Path | None:
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: manta_node
|
|
3
|
-
Version: 0.6b5.
|
|
3
|
+
Version: 0.6b5.dev783
|
|
4
4
|
Summary: Node side software of the Manta platform. It is responsible for the communication with the Manta Core and the execution of the tasks.
|
|
5
5
|
Author-email: Hugo Miralles <hugo.miralles@manta-tech.io>, Matthew Thompson <matthew.thompson@manta-tech.io>
|
|
6
6
|
License-Expression: AGPL-3.0-or-later
|
|
@@ -7,7 +7,7 @@ manta_node/tasks.py,sha256=o837Hi-KFVJg9LF5iOtE8oD6UuTOfrEPWR-JOdhRGeY,6983
|
|
|
7
7
|
manta_node/update_path.py,sha256=6N_bN9z8MqnyWWfGTe1-aJFijf2tSy_IvO6VHvNSQVU,1667
|
|
8
8
|
manta_node/utils.py,sha256=Hw4Iw2jYQYsfVLVJuBMt-_rMFBeml3nCDE2jiiXc8YM,1155
|
|
9
9
|
manta_node/cli/__init__.py,sha256=arzO3mXDJ-E2yeeytbZfN7mCswwxdQ43_QjyeL2E5yE,134
|
|
10
|
-
manta_node/cli/_version_generated.py,sha256=
|
|
10
|
+
manta_node/cli/_version_generated.py,sha256=m3MylUbtMYYQiLv0mYdjiR_-m2XdvXi3nndOIIDTPI0,332
|
|
11
11
|
manta_node/cli/config_manager.py,sha256=BCju9zqzraviL3KrzRnB-vZQUY__Q5kuEqb7EdB8YdI,4409
|
|
12
12
|
manta_node/cli/main.py,sha256=dZa6zrypWBvpN8hd4qpxevvlyNFhxKINCg1537NJ36w,4318
|
|
13
13
|
manta_node/cli/version.py,sha256=L4lNTz-Uw_ejumy1tXJcJhcjhb-5FRcRQ6ogZM5N1Xo,5136
|
|
@@ -42,7 +42,7 @@ manta_node/infrastructure/metrics/__init__.py,sha256=ODGAj9s63F5jba2c3dQRMfNK6I4
|
|
|
42
42
|
manta_node/infrastructure/metrics/collector.py,sha256=9ESw__Pc2tV6ldG5yhos2IS7T5Td59KxTIwDRZc_G4w,3067
|
|
43
43
|
manta_node/infrastructure/metrics/metrics_collector.py,sha256=osIYrNzU3g9WCAxVQ14isM9YEQ3eJ7F3QgKBbpgeMJs,13077
|
|
44
44
|
manta_node/infrastructure/mqtt/__init__.py,sha256=riXo1sXgO4YV6DHCmWRqK61BAmuGLdHiVdQFunAJadA,140
|
|
45
|
-
manta_node/infrastructure/mqtt/command_handler.py,sha256=
|
|
45
|
+
manta_node/infrastructure/mqtt/command_handler.py,sha256=rEXdIK3EYQJibi9-2xczsmrgJDMznK_v0Dfszu-LOzM,38517
|
|
46
46
|
manta_node/infrastructure/network/__init__.py,sha256=On3XLgycFQoc-fBCa5hFuFDfEJjmFy7Wl2CK3D2LkdI,478
|
|
47
47
|
manta_node/infrastructure/network/event_registry.py,sha256=iWEfUJRzDnWjOrDinpYqVyIUpGKJjp7RZkjMDPfpplM,1661
|
|
48
48
|
manta_node/infrastructure/network/factory.py,sha256=-LvHbBp5nBpQ7fyQT1T0pvK3aczZqWc9QNDW37brBeU,3897
|
|
@@ -55,17 +55,17 @@ manta_node/infrastructure/security/auth_agent.py,sha256=WkPTg_g0gix7xomb_on_Ft3T
|
|
|
55
55
|
manta_node/infrastructure/security/token_provider.py,sha256=u3bcbQYNBskFzunO-wOSZpGkDN_CU1BCB5HUbZTC4Vg,1832
|
|
56
56
|
manta_node/supervisor/__init__.py,sha256=kgdoNce1tM4e7jvdxkdmnNJV1L92-NI2cRfYSdm9vsw,542
|
|
57
57
|
manta_node/supervisor/bounds.py,sha256=MlUfj0tYY-US348JsxYA-Ph5pSpgoOPlRdVTwmrPtZA,3181
|
|
58
|
-
manta_node/supervisor/executor.py,sha256=
|
|
59
|
-
manta_node/supervisor/inflight.py,sha256=
|
|
60
|
-
manta_node/supervisor/lifecycle.py,sha256=
|
|
61
|
-
manta_node/supervisor/process.py,sha256=
|
|
62
|
-
manta_node/supervisor/records.py,sha256=
|
|
58
|
+
manta_node/supervisor/executor.py,sha256=S4X7SyaRzYXwj4XD3yS-KNyWfvqUpbCVHHZzejG7trs,8476
|
|
59
|
+
manta_node/supervisor/inflight.py,sha256=mM8qNoBqTwg0WP-SHxkPznwSF8gIlyA28dDUtVFaLl8,4598
|
|
60
|
+
manta_node/supervisor/lifecycle.py,sha256=_-Tzb3Ipu_RERC5T8DeHyy7BS5RypIc8rIz2GvKRG2s,12000
|
|
61
|
+
manta_node/supervisor/process.py,sha256=7cy1OtwNtu6K3lbiBMRA4fzb25a0hE4AADDb-xUmXjA,25896
|
|
62
|
+
manta_node/supervisor/records.py,sha256=O1YjNN2bH1xzOJo_3l1G6FRvWo1CmdAfvSDJBj6sKIM,6858
|
|
63
63
|
manta_node/supervisor/seam.py,sha256=4kczRku5Ctn4xRvXS-pyzaDFau7LnX5bYOn6IRNKRyE,9665
|
|
64
64
|
manta_node/supervisor/sequence.py,sha256=eYu7rrk1ecc4Y2jQhhJ7CopPz3m0lOsFLJj5lT2PwBw,1685
|
|
65
|
-
manta_node/supervisor/switch.py,sha256=
|
|
66
|
-
manta_node-0.6b5.
|
|
67
|
-
manta_node-0.6b5.
|
|
68
|
-
manta_node-0.6b5.
|
|
69
|
-
manta_node-0.6b5.
|
|
70
|
-
manta_node-0.6b5.
|
|
71
|
-
manta_node-0.6b5.
|
|
65
|
+
manta_node/supervisor/switch.py,sha256=PbWLNnRwQFhOBDAUSLcDwZsIW5jPHq_zNKIyS7Hd8C0,22342
|
|
66
|
+
manta_node-0.6b5.dev783.dist-info/licenses/LICENSE,sha256=9mBgLhiB9goQPQjg9XQJ6FjgVxYw9Q8lyOZYkPGwF_c,34484
|
|
67
|
+
manta_node-0.6b5.dev783.dist-info/METADATA,sha256=QpF9kkPRM3JA3cONAHjq9DZQqmIYjsvhCueElWJjGb0,4937
|
|
68
|
+
manta_node-0.6b5.dev783.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
|
|
69
|
+
manta_node-0.6b5.dev783.dist-info/entry_points.txt,sha256=uCwggRApalLEsYs1YxwTBN0-e9oMJAkMoBp01HaSdoY,110
|
|
70
|
+
manta_node-0.6b5.dev783.dist-info/top_level.txt,sha256=nbT5J5WETuXM9UPoQckl29uUOq8wMYdRb9QDx5kAFeo,11
|
|
71
|
+
manta_node-0.6b5.dev783.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|