manta-node 0.6b5.dev703__py3-none-any.whl → 0.6b5.dev705__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- manta_node/cli/_version_generated.py +2 -2
- manta_node/infrastructure/mqtt/command_handler.py +34 -0
- manta_node/node_orchestrator.py +31 -1
- manta_node/supervisor/bounds.py +88 -0
- manta_node/supervisor/executor.py +165 -0
- manta_node/supervisor/inflight.py +82 -0
- manta_node/supervisor/lifecycle.py +207 -0
- manta_node/supervisor/process.py +172 -5
- manta_node/supervisor/seam.py +12 -8
- manta_node/supervisor/switch.py +27 -5
- {manta_node-0.6b5.dev703.dist-info → manta_node-0.6b5.dev705.dist-info}/METADATA +1 -1
- {manta_node-0.6b5.dev703.dist-info → manta_node-0.6b5.dev705.dist-info}/RECORD +16 -12
- {manta_node-0.6b5.dev703.dist-info → manta_node-0.6b5.dev705.dist-info}/WHEEL +0 -0
- {manta_node-0.6b5.dev703.dist-info → manta_node-0.6b5.dev705.dist-info}/entry_points.txt +0 -0
- {manta_node-0.6b5.dev703.dist-info → manta_node-0.6b5.dev705.dist-info}/licenses/LICENSE +0 -0
- {manta_node-0.6b5.dev703.dist-info → manta_node-0.6b5.dev705.dist-info}/top_level.txt +0 -0
|
@@ -4,7 +4,7 @@ This file is auto-generated at build time by scripts/generate_version.py.
|
|
|
4
4
|
Do not edit manually.
|
|
5
5
|
"""
|
|
6
6
|
|
|
7
|
-
__version__ = "2026.09.24.
|
|
7
|
+
__version__ = "2026.09.24.d60eab69"
|
|
8
8
|
__build_date__ = "2026.09.24"
|
|
9
|
-
__commit_id__ = "
|
|
9
|
+
__commit_id__ = "d60eab69"
|
|
10
10
|
__package_version__ = "0.6b5"
|
|
@@ -621,6 +621,8 @@ class CommandHandler(Collector, MqttBase):
|
|
|
621
621
|
await self.collect_information(payload)
|
|
622
622
|
elif key == "stop_node":
|
|
623
623
|
await self.stop_node(payload)
|
|
624
|
+
elif key == "update_node":
|
|
625
|
+
await self.update_node(payload)
|
|
624
626
|
else:
|
|
625
627
|
self.tracer.error(f"Not allowed method: {key}")
|
|
626
628
|
raise MantaMQTTError(
|
|
@@ -698,6 +700,38 @@ class CommandHandler(Collector, MqttBase):
|
|
|
698
700
|
self.tracer.exception(f"Seam relay failed: {exc}")
|
|
699
701
|
raise MantaNodeError(f"Supervisor seam unavailable: {exc}") from exc
|
|
700
702
|
|
|
703
|
+
async def update_node(self, payload: bytes):
|
|
704
|
+
"""Handle an `update_node` command from the Manager (the Manager path).
|
|
705
|
+
|
|
706
|
+
The Manager declares a pair and the node relays it to its supervisor over
|
|
707
|
+
the local seam. The payload carries the target pair and the fields the
|
|
708
|
+
seam's `UpdateRequest` needs; the node computes no ordering (mechanism
|
|
709
|
+
§6.2). With no supervisor the relay is inert, and the node reports that
|
|
710
|
+
rather than applying anything itself.
|
|
711
|
+
"""
|
|
712
|
+
try:
|
|
713
|
+
data = json.loads(payload)
|
|
714
|
+
request = UpdateRequest(
|
|
715
|
+
request_id=data["request_id"],
|
|
716
|
+
target=data["target"],
|
|
717
|
+
force=bool(data.get("force", False)),
|
|
718
|
+
declared_sequence=data.get("declared_sequence"),
|
|
719
|
+
)
|
|
720
|
+
except (ValueError, UnicodeDecodeError, KeyError, TypeError) as exc:
|
|
721
|
+
# A malformed payload — invalid JSON, missing request_id/target, or a
|
|
722
|
+
# wrong shape — is reported the same way, not left to escape as a
|
|
723
|
+
# bare KeyError past the dispatch's MantaError handling.
|
|
724
|
+
raise MantaMQTTError(f"Malformed update_node payload: {exc}") from exc
|
|
725
|
+
outcome = await self.relay_update(request)
|
|
726
|
+
if outcome is None:
|
|
727
|
+
raise MantaNodeError(
|
|
728
|
+
"update_node received but no supervisor serves the seam"
|
|
729
|
+
)
|
|
730
|
+
await self.publish_message(
|
|
731
|
+
"manager/update_node_status",
|
|
732
|
+
outcome.to_bytes(),
|
|
733
|
+
)
|
|
734
|
+
|
|
701
735
|
async def on_task_setup_failed(self, task: MqttTask, exc: Exception):
|
|
702
736
|
"""
|
|
703
737
|
Publish a FAILED task update when task setup fails.
|
manta_node/node_orchestrator.py
CHANGED
|
@@ -38,6 +38,8 @@ from .infrastructure.mqtt.command_handler import CommandHandler
|
|
|
38
38
|
from .infrastructure.process.executor import ProcessExecutor
|
|
39
39
|
from .infrastructure.security.auth_agent import AuthAgent
|
|
40
40
|
from .infrastructure.security.token_provider import InMemoryTokenProvider
|
|
41
|
+
from .seam import seam_socket_path
|
|
42
|
+
from .supervisor.seam import SocketSeamClient
|
|
41
43
|
from .task_manager import TaskManager
|
|
42
44
|
|
|
43
45
|
__all__ = ["Node", "NodeFatalError"]
|
|
@@ -83,6 +85,18 @@ def _execution_mode_from_config(config) -> ExecutionMode:
|
|
|
83
85
|
)
|
|
84
86
|
|
|
85
87
|
|
|
88
|
+
def _seam_client_if_served():
|
|
89
|
+
"""The node's seam client, when a supervisor serves the socket; else None.
|
|
90
|
+
|
|
91
|
+
The node relays an update to its supervisor over the local seam (2d). When
|
|
92
|
+
the socket is absent — no supervisor owns this node — the relay stays inert
|
|
93
|
+
and the node behaves exactly as before (the pre-2d path).
|
|
94
|
+
"""
|
|
95
|
+
if seam_socket_path().exists():
|
|
96
|
+
return SocketSeamClient()
|
|
97
|
+
return None
|
|
98
|
+
|
|
99
|
+
|
|
86
100
|
def _build_executor(config) -> TaskExecutor:
|
|
87
101
|
"""Build the ``TaskExecutor`` backend from the execution-mode config."""
|
|
88
102
|
mode = config.execution.mode
|
|
@@ -589,6 +603,10 @@ class Node:
|
|
|
589
603
|
ping_interval=self.config.mqtt.ping_interval,
|
|
590
604
|
max_concurrent_tasks=self.config.tasks.max_concurrent,
|
|
591
605
|
task_timeout=self.config.tasks.task_timeout,
|
|
606
|
+
# The local seam to a supervisor, when one owns this node: the
|
|
607
|
+
# relay's client for the Manager path (2d). Absent when no
|
|
608
|
+
# supervisor serves the socket, so the relay stays inert.
|
|
609
|
+
seam_client=_seam_client_if_served(),
|
|
592
610
|
)
|
|
593
611
|
|
|
594
612
|
# Start the server for docker communications through
|
|
@@ -668,7 +686,19 @@ class Node:
|
|
|
668
686
|
and self.command_loop is not None
|
|
669
687
|
and not self.command_loop.cancelled()
|
|
670
688
|
):
|
|
671
|
-
|
|
689
|
+
# A restart the supervisor makes for an update carries the
|
|
690
|
+
# request id in the env; the DISCONNECTED it publishes is then
|
|
691
|
+
# attributed to the update, not read as a fleet disconnect
|
|
692
|
+
# (NF-8). The manager correlates the id; the node does not
|
|
693
|
+
# treat this as an unexpected loss.
|
|
694
|
+
updating = os.environ.get("MANTA_NODE_UPDATING")
|
|
695
|
+
if updating:
|
|
696
|
+
self.tracer.info(
|
|
697
|
+
f"Restarting for in-flight update {updating};"
|
|
698
|
+
" publishing DISCONNECTED attributed to the update"
|
|
699
|
+
)
|
|
700
|
+
else:
|
|
701
|
+
self.tracer.info("Publishing node status to manager")
|
|
672
702
|
await self.command_handler.client.publish(
|
|
673
703
|
"manager/set_node_status",
|
|
674
704
|
NodeStatus(
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
"""The switch lifecycle's bounds — config defaults, operator-overridable.
|
|
2
|
+
|
|
3
|
+
Mechanism §8 "Bounds (config defaults)". These are **runtime config in the
|
|
4
|
+
supervisor's own file** (``~/.manta/supervisor.toml``), not install-time
|
|
5
|
+
constants: an operator can widen or tighten them per host. The defaults are the
|
|
6
|
+
ones the design settled:
|
|
7
|
+
|
|
8
|
+
- return window **40 s** (the fleet's proven 4x10 s, sustained);
|
|
9
|
+
- per-state in-flight bounds: fetch <= 300 s, verify <= 60 s, stage <= 120 s,
|
|
10
|
+
switch <= 30 s (~600 s total);
|
|
11
|
+
- retention: current + last good, <= 3 dirs.
|
|
12
|
+
|
|
13
|
+
A bound is a **ceiling on one state**, not a promise the state takes that long:
|
|
14
|
+
the executor moves on as soon as the state completes, and treats exceeding the
|
|
15
|
+
bound as a failure of that state (which, before the switch, leaves the node on
|
|
16
|
+
its current version — NF-3).
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
from __future__ import annotations
|
|
20
|
+
|
|
21
|
+
from dataclasses import dataclass
|
|
22
|
+
from pathlib import Path
|
|
23
|
+
|
|
24
|
+
try:
|
|
25
|
+
import tomllib
|
|
26
|
+
except ImportError: # Python 3.10 has no tomllib; the node floor allows it.
|
|
27
|
+
import tomli as tomllib
|
|
28
|
+
|
|
29
|
+
# Bounds live beside the node's other supervisor state, never in a version tree.
|
|
30
|
+
SUPERVISOR_CONFIG_NAME = "supervisor.toml"
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def supervisor_config_path() -> Path:
|
|
34
|
+
return Path.home() / ".manta" / SUPERVISOR_CONFIG_NAME
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
@dataclass(frozen=True)
|
|
38
|
+
class Bounds:
|
|
39
|
+
"""The lifecycle's time bounds (seconds) and the retention bound (dirs)."""
|
|
40
|
+
|
|
41
|
+
return_window: float = 40.0
|
|
42
|
+
fetch: float = 300.0
|
|
43
|
+
verify: float = 60.0
|
|
44
|
+
stage: float = 120.0
|
|
45
|
+
switch: float = 30.0
|
|
46
|
+
retention_dirs: int = 3
|
|
47
|
+
|
|
48
|
+
def for_state(self, state: str) -> float:
|
|
49
|
+
"""The in-flight bound for a lifecycle state name."""
|
|
50
|
+
return getattr(self, state)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def load_bounds(path: Path | None = None) -> Bounds:
|
|
54
|
+
"""Load bounds from the supervisor's config, falling back to the defaults.
|
|
55
|
+
|
|
56
|
+
A missing file, a partial table, or an unreadable/invalid one all fall back
|
|
57
|
+
to the defaults per key rather than failing: this runs on the update path, and
|
|
58
|
+
refusing an update over a typo in a bound would turn a cosmetic config error
|
|
59
|
+
into a stuck fleet. An unknown key is ignored (forward-compatible).
|
|
60
|
+
"""
|
|
61
|
+
path = path or supervisor_config_path()
|
|
62
|
+
try:
|
|
63
|
+
raw = tomllib.loads(path.read_text())
|
|
64
|
+
except (FileNotFoundError, OSError, tomllib.TOMLDecodeError):
|
|
65
|
+
return Bounds()
|
|
66
|
+
|
|
67
|
+
table = raw.get("bounds", {})
|
|
68
|
+
if not isinstance(table, dict):
|
|
69
|
+
return Bounds()
|
|
70
|
+
|
|
71
|
+
known = set(Bounds.__dataclass_fields__)
|
|
72
|
+
values = {}
|
|
73
|
+
for key, value in table.items():
|
|
74
|
+
if key not in known:
|
|
75
|
+
continue
|
|
76
|
+
# The numeric bounds must be numbers, the retention bound an int. A
|
|
77
|
+
# wrong-typed value falls back to the default for that key rather than
|
|
78
|
+
# reaching the dataclass, where it would only fail on use.
|
|
79
|
+
if key == "retention_dirs":
|
|
80
|
+
if isinstance(value, int) and not isinstance(value, bool) and value > 0:
|
|
81
|
+
values[key] = value
|
|
82
|
+
elif (
|
|
83
|
+
isinstance(value, (int, float))
|
|
84
|
+
and not isinstance(value, bool)
|
|
85
|
+
and value > 0
|
|
86
|
+
):
|
|
87
|
+
values[key] = float(value)
|
|
88
|
+
return Bounds(**values)
|
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
"""The supervisor's real seam executor: adapters that drive the switch lifecycle.
|
|
2
|
+
|
|
3
|
+
Mechanism §6.4's "executor" side. ``run_switch`` (``lifecycle.py``) is the state
|
|
4
|
+
machine; this module supplies its four verbs as concrete system actions:
|
|
5
|
+
|
|
6
|
+
- **fetch** — pull the candidate pair from PyPI into a version dir (NF-10, pull
|
|
7
|
+
only; ADR-0042 D2 says PyPI by default). At this stage the fetch materialises a
|
|
8
|
+
version dir naming the candidate and its interpreter; the wheel install itself is
|
|
9
|
+
the deploy-side apply (manta-deploy#1172), so the executor keeps the *decision*
|
|
10
|
+
and the *record* honest without duplicating the installer.
|
|
11
|
+
- **verify** — NF-2: import the candidate entrypoint and call the reporting path in
|
|
12
|
+
the candidate's own environment. The full closure check is the same subprocess
|
|
13
|
+
check; a failure leaves the node on its current version.
|
|
14
|
+
- **restart** — restart the child onto the committed version (the process manager
|
|
15
|
+
supplies the handle).
|
|
16
|
+
- **is_back** — wait out the return window for sustained liveness on the new
|
|
17
|
+
version (NF-8), polling the child's liveness.
|
|
18
|
+
|
|
19
|
+
The supervisor is infrastructure: it decides nothing about versions, drift,
|
|
20
|
+
fleets or outcomes (D6). It performs the verbs the node relayed.
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
from __future__ import annotations
|
|
24
|
+
|
|
25
|
+
import asyncio
|
|
26
|
+
import subprocess
|
|
27
|
+
import sys
|
|
28
|
+
import threading
|
|
29
|
+
import time
|
|
30
|
+
from collections.abc import Awaitable, Callable
|
|
31
|
+
from dataclasses import dataclass
|
|
32
|
+
|
|
33
|
+
from ..seam import SeamState, SwitchOutcome, UpdateRequest
|
|
34
|
+
from . import switch as version_tree
|
|
35
|
+
from .bounds import Bounds, load_bounds
|
|
36
|
+
from .lifecycle import run_switch
|
|
37
|
+
|
|
38
|
+
# How often the return probe checks child liveness within the window.
|
|
39
|
+
RETURN_POLL_SECONDS = 2.0
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def verify_candidate(candidate_dir: str, python: str | None = None) -> None:
|
|
43
|
+
"""Prove a candidate imports and reports its build, in its own environment.
|
|
44
|
+
|
|
45
|
+
NF-2 is three parts (imports, the reporting call, the full closure); this
|
|
46
|
+
invokes the candidate's own interpreter to import ``manta_node`` and call the
|
|
47
|
+
reporting path, which exercises all three within that interpreter's
|
|
48
|
+
environment. A non-zero exit means the candidate is not usable, and the caller
|
|
49
|
+
leaves the node on its current version (NF-3).
|
|
50
|
+
"""
|
|
51
|
+
interpreter = python or sys.executable
|
|
52
|
+
try:
|
|
53
|
+
check = subprocess.run(
|
|
54
|
+
[
|
|
55
|
+
interpreter,
|
|
56
|
+
"-c",
|
|
57
|
+
(
|
|
58
|
+
"import manta_node;"
|
|
59
|
+
" from manta_node.cli.version import get_full_version_info;"
|
|
60
|
+
" get_full_version_info()"
|
|
61
|
+
),
|
|
62
|
+
],
|
|
63
|
+
capture_output=True,
|
|
64
|
+
text=True,
|
|
65
|
+
timeout=60,
|
|
66
|
+
check=False,
|
|
67
|
+
)
|
|
68
|
+
except (OSError, subprocess.TimeoutExpired) as exc:
|
|
69
|
+
# A missing interpreter or a hung import is a verification failure, not a
|
|
70
|
+
# crash: the caller leaves the node on its current version (NF-3).
|
|
71
|
+
raise RuntimeError(
|
|
72
|
+
f"candidate {candidate_dir} failed verification: {exc!r}"
|
|
73
|
+
) from exc
|
|
74
|
+
if check.returncode != 0:
|
|
75
|
+
raise RuntimeError(
|
|
76
|
+
f"candidate {candidate_dir} failed verification: {check.stderr.strip()}"
|
|
77
|
+
)
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
@dataclass
|
|
81
|
+
class ChildControl:
|
|
82
|
+
"""The supervisor's handle on the child, for restart and liveness."""
|
|
83
|
+
|
|
84
|
+
restart: Callable[[], None]
|
|
85
|
+
is_alive: Callable[[], bool]
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def make_executor(
|
|
89
|
+
control: ChildControl,
|
|
90
|
+
*,
|
|
91
|
+
bounds: Bounds | None = None,
|
|
92
|
+
verify: Callable[[str, str | None], None] = verify_candidate,
|
|
93
|
+
child_python: str | None = None,
|
|
94
|
+
) -> Callable[[UpdateRequest], Awaitable[SwitchOutcome]]:
|
|
95
|
+
"""Build the seam handler: a request in, the lifecycle's outcome out.
|
|
96
|
+
|
|
97
|
+
``fetch`` names the candidate's interpreter (the version dir materialise is
|
|
98
|
+
the deploy apply's job); ``verify`` runs in that interpreter's environment
|
|
99
|
+
(NF-2); ``restart`` and ``is_back`` come from the child control.
|
|
100
|
+
|
|
101
|
+
The handler is async and runs ``run_switch`` in a worker thread — but only
|
|
102
|
+
**one at a time**: a switch restarts the child and moves ``current``, so two
|
|
103
|
+
concurrent switches would race both. ``SeamServer`` dedupes by ``request_id``
|
|
104
|
+
only; a second, differently-ided request while one is switching is refused
|
|
105
|
+
here rather than run.
|
|
106
|
+
"""
|
|
107
|
+
bounds = bounds or load_bounds()
|
|
108
|
+
# One switch at a time. Held across the whole lifecycle, so a second request
|
|
109
|
+
# (any request_id) is refused while one is in flight, not run concurrently.
|
|
110
|
+
switching = threading.Lock()
|
|
111
|
+
|
|
112
|
+
def fetch(target: str) -> str:
|
|
113
|
+
# Pull-only (NF-10). If the deploy apply has already materialised the
|
|
114
|
+
# candidate's version dir (with its interpreter), use that interpreter —
|
|
115
|
+
# so verification runs in the candidate's own environment, not merely the
|
|
116
|
+
# supervisor's. Otherwise the candidate will run from the interpreter the
|
|
117
|
+
# supervisor boots children with. The wheel install is the deploy apply;
|
|
118
|
+
# the version dir is (re)materialised at STAGE.
|
|
119
|
+
manifest = version_tree.read_manifest(version_tree.versions_dir() / target)
|
|
120
|
+
if manifest is not None:
|
|
121
|
+
return manifest.python
|
|
122
|
+
return child_python or sys.executable
|
|
123
|
+
|
|
124
|
+
def restart() -> None:
|
|
125
|
+
control.restart()
|
|
126
|
+
|
|
127
|
+
def is_back(deadline: float) -> bool:
|
|
128
|
+
# Sustained liveness on the new version, held past the window (NF-8): the
|
|
129
|
+
# child must be alive at the deadline, not merely once.
|
|
130
|
+
while time.time() < deadline:
|
|
131
|
+
if not control.is_alive():
|
|
132
|
+
return False
|
|
133
|
+
time.sleep(RETURN_POLL_SECONDS)
|
|
134
|
+
return control.is_alive()
|
|
135
|
+
|
|
136
|
+
async def handler(request: UpdateRequest) -> SwitchOutcome:
|
|
137
|
+
# Refuse a second switch while one is in flight (NF-5): a switch restarts
|
|
138
|
+
# the child and moves `current`, so concurrency there is a race, not a
|
|
139
|
+
# queue. `acquire(blocking=False)` makes the check atomic.
|
|
140
|
+
if not switching.acquire(blocking=False):
|
|
141
|
+
return SwitchOutcome(
|
|
142
|
+
request_id=request.request_id,
|
|
143
|
+
state=SeamState.REFUSED,
|
|
144
|
+
reason="a switch is already in flight",
|
|
145
|
+
)
|
|
146
|
+
try:
|
|
147
|
+
# The lifecycle is synchronous (it blocks through the return window);
|
|
148
|
+
# run it in a worker thread so it does not block the seam's event
|
|
149
|
+
# loop, which is serving the socket the request arrived on.
|
|
150
|
+
return await asyncio.to_thread(
|
|
151
|
+
run_switch,
|
|
152
|
+
request,
|
|
153
|
+
fetch=fetch,
|
|
154
|
+
verify=verify,
|
|
155
|
+
restart=restart,
|
|
156
|
+
is_back=is_back,
|
|
157
|
+
bounds=bounds,
|
|
158
|
+
)
|
|
159
|
+
finally:
|
|
160
|
+
switching.release()
|
|
161
|
+
|
|
162
|
+
return handler
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
__all__ = ["ChildControl", "SeamState", "make_executor", "verify_candidate"]
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
"""The durable in-flight record — the one place the deadline and last-good live.
|
|
2
|
+
|
|
3
|
+
Mechanism §6.4, NF-6's supervisor carve-out. When a switch is in flight the
|
|
4
|
+
supervisor writes this record: the **candidate** being made current, the
|
|
5
|
+
**last-good** version to revert to, and the **deadline** (the return window's
|
|
6
|
+
end). It is **supervisor-written and supervisor-read** — the node never touches
|
|
7
|
+
it — so a supervisor restart re-evaluates it rather than losing the update in
|
|
8
|
+
progress (NF-1/NF-8).
|
|
9
|
+
|
|
10
|
+
It lives in the state dir beside the instances and the version tree, never inside
|
|
11
|
+
a version dir (NF-6). It is cleared once the switch resolves (reported or
|
|
12
|
+
reverted); its presence is the "an update is in flight" fact.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
import time
|
|
18
|
+
from dataclasses import dataclass
|
|
19
|
+
from pathlib import Path
|
|
20
|
+
|
|
21
|
+
from .records import read_json, write_json_atomic
|
|
22
|
+
|
|
23
|
+
INFLIGHT_NAME = "inflight.json"
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def inflight_path() -> Path:
|
|
27
|
+
return Path.home() / ".manta" / "nodes" / INFLIGHT_NAME
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
@dataclass(frozen=True)
|
|
31
|
+
class InFlight:
|
|
32
|
+
"""The durable record of a switch in progress."""
|
|
33
|
+
|
|
34
|
+
request_id: str
|
|
35
|
+
candidate: str
|
|
36
|
+
last_good: str | None
|
|
37
|
+
deadline_epoch: float
|
|
38
|
+
|
|
39
|
+
def to_dict(self) -> dict:
|
|
40
|
+
return {
|
|
41
|
+
"request_id": self.request_id,
|
|
42
|
+
"candidate": self.candidate,
|
|
43
|
+
"last_good": self.last_good,
|
|
44
|
+
"deadline_epoch": self.deadline_epoch,
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
@classmethod
|
|
48
|
+
def from_dict(cls, data: dict) -> InFlight:
|
|
49
|
+
return cls(
|
|
50
|
+
request_id=data["request_id"],
|
|
51
|
+
candidate=data["candidate"],
|
|
52
|
+
last_good=data.get("last_good"),
|
|
53
|
+
deadline_epoch=float(data["deadline_epoch"]),
|
|
54
|
+
)
|
|
55
|
+
|
|
56
|
+
def expired(self, now: float | None = None) -> bool:
|
|
57
|
+
"""True once the return window (the deadline) has passed."""
|
|
58
|
+
return (now if now is not None else time.time()) > self.deadline_epoch
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def write_inflight(record: InFlight, path: Path | None = None) -> None:
|
|
62
|
+
write_json_atomic(path or inflight_path(), record.to_dict())
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def read_inflight(path: Path | None = None) -> InFlight | None:
|
|
66
|
+
"""The in-flight record, or None when no switch is in flight."""
|
|
67
|
+
data = read_json(path or inflight_path())
|
|
68
|
+
if not data:
|
|
69
|
+
return None
|
|
70
|
+
try:
|
|
71
|
+
return InFlight.from_dict(data)
|
|
72
|
+
except (KeyError, TypeError, ValueError):
|
|
73
|
+
return None
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def clear_inflight(path: Path | None = None) -> None:
|
|
77
|
+
(path or inflight_path()).unlink(missing_ok=True)
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def deadline_from_now(window: float) -> float:
|
|
81
|
+
"""The deadline for a window that starts now (the window is about the switch)."""
|
|
82
|
+
return time.time() + window
|
|
@@ -0,0 +1,207 @@
|
|
|
1
|
+
"""The switch lifecycle — fetch, verify, stage, switch, restart, revert.
|
|
2
|
+
|
|
3
|
+
Mechanism §6, ADR-0041 NF-1/NF-2/NF-3/NF-4/NF-8. This is the **real executor**
|
|
4
|
+
the seam serves (the 2b/2c stub is replaced): a candidate is fetched, **verified
|
|
5
|
+
in its own environment** before it can become current, staged into its own version
|
|
6
|
+
dir, made current by the atomic pointer (the commit point, 2c), and the child is
|
|
7
|
+
restarted. If the child does not come back **within the return window**, the
|
|
8
|
+
supervisor **reverts** to the last good version, so a bad update is
|
|
9
|
+
non-destructive and locally reversible without SSH (NF-3/NF-4).
|
|
10
|
+
|
|
11
|
+
The steps are injected (`fetch`, `verify`, `restart`, `is_back`) so the state
|
|
12
|
+
machine is tested against the contract in isolation — the supervisor's verbs are
|
|
13
|
+
system actions, and the machine decides nothing about versions (D6). The real
|
|
14
|
+
adapters (PyPI fetch, import/closure verification, child restart) are wired by the
|
|
15
|
+
supervisor.
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
import time
|
|
21
|
+
from collections.abc import Callable
|
|
22
|
+
from dataclasses import dataclass
|
|
23
|
+
from typing import Protocol
|
|
24
|
+
|
|
25
|
+
from ..seam import SeamState, SwitchOutcome, UpdateRequest
|
|
26
|
+
from . import switch as version_tree
|
|
27
|
+
from .bounds import Bounds
|
|
28
|
+
from .inflight import InFlight, clear_inflight, deadline_from_now, write_inflight
|
|
29
|
+
from .switch import VersionTreeError
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
class LifecycleError(Exception):
|
|
33
|
+
"""A lifecycle step failed; the node stays on its current version (NF-3)."""
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
@dataclass
|
|
37
|
+
class SwitchContext:
|
|
38
|
+
"""The mutable state of one switch attempt."""
|
|
39
|
+
|
|
40
|
+
request: UpdateRequest
|
|
41
|
+
bounds: Bounds
|
|
42
|
+
last_good: str | None
|
|
43
|
+
candidate_dir: str | None = None
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
class Fetcher(Protocol):
|
|
47
|
+
"""Fetch a candidate pair; return the interpreter it will run from.
|
|
48
|
+
|
|
49
|
+
The interpreter names the candidate's **own environment**, which VERIFY runs
|
|
50
|
+
in (NF-2). The version dir is materialised at STAGE, after verification.
|
|
51
|
+
"""
|
|
52
|
+
|
|
53
|
+
def __call__(self, target: str) -> str: ...
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
class Verifier(Protocol):
|
|
57
|
+
"""Prove the candidate usable in its own environment (NF-2)."""
|
|
58
|
+
|
|
59
|
+
def __call__(self, candidate_dir: str, python: str) -> None: ...
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
class Restarter(Protocol):
|
|
63
|
+
"""Restart the child onto the committed version."""
|
|
64
|
+
|
|
65
|
+
def __call__(self) -> None: ...
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
class ReturnProbe(Protocol):
|
|
69
|
+
"""Whether the child has come back (sustained) after the switch (NF-8)."""
|
|
70
|
+
|
|
71
|
+
def __call__(self, deadline: float) -> bool: ...
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def _outcome(
|
|
75
|
+
request: UpdateRequest, state: SeamState, reason: str, **detail
|
|
76
|
+
) -> SwitchOutcome:
|
|
77
|
+
return SwitchOutcome(
|
|
78
|
+
request_id=request.request_id, state=state, reason=reason, detail=detail
|
|
79
|
+
)
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def run_switch(
|
|
83
|
+
request: UpdateRequest,
|
|
84
|
+
*,
|
|
85
|
+
fetch: Fetcher,
|
|
86
|
+
verify: Verifier,
|
|
87
|
+
restart: Restarter,
|
|
88
|
+
is_back: ReturnProbe,
|
|
89
|
+
bounds: Bounds | None = None,
|
|
90
|
+
now: Callable[[], float] = time.time,
|
|
91
|
+
) -> SwitchOutcome:
|
|
92
|
+
"""Run the lifecycle for one request and return its terminal outcome.
|
|
93
|
+
|
|
94
|
+
Terminal states: ``REPORTING`` (came back on the new version), ``REVERTED``
|
|
95
|
+
(no return within the window, reverted to the last good), ``FAILED`` (a
|
|
96
|
+
pre-switch step failed; nothing changed), ``DOWN`` (the switch happened and
|
|
97
|
+
the revert itself failed — the node is not running, surfaced honestly).
|
|
98
|
+
|
|
99
|
+
The in-flight record is written before the switch and cleared when the
|
|
100
|
+
lifecycle resolves, so a supervisor restart mid-window can re-evaluate it.
|
|
101
|
+
"""
|
|
102
|
+
bounds = bounds or Bounds()
|
|
103
|
+
last_good_dir = version_tree.selected()
|
|
104
|
+
last_good = last_good_dir.name if last_good_dir else None
|
|
105
|
+
candidate_dir = request.target
|
|
106
|
+
|
|
107
|
+
# FETCH — bounded; the fetcher returns the candidate's own interpreter.
|
|
108
|
+
candidate_python: str | None = None
|
|
109
|
+
|
|
110
|
+
def do_fetch() -> None:
|
|
111
|
+
nonlocal candidate_python
|
|
112
|
+
candidate_python = fetch(request.target)
|
|
113
|
+
|
|
114
|
+
if not _within(bounds.fetch, do_fetch):
|
|
115
|
+
return _outcome(
|
|
116
|
+
request, SeamState.FAILED, "fetch exceeded its bound (no switch)"
|
|
117
|
+
)
|
|
118
|
+
|
|
119
|
+
# VERIFY — the candidate must be usable in its OWN environment (NF-2): the
|
|
120
|
+
# fetcher named the interpreter the candidate will run from.
|
|
121
|
+
if not _within(bounds.verify, lambda: verify(candidate_dir, candidate_python)):
|
|
122
|
+
return _outcome(request, SeamState.FAILED, "verification failed (no switch)")
|
|
123
|
+
|
|
124
|
+
# STAGE — materialise the candidate's version dir (idempotent), bounded.
|
|
125
|
+
staged: dict = {}
|
|
126
|
+
|
|
127
|
+
def do_stage() -> None:
|
|
128
|
+
staged["target"] = version_tree.materialise_version(
|
|
129
|
+
candidate_dir, candidate_python
|
|
130
|
+
)
|
|
131
|
+
|
|
132
|
+
if not _within(bounds.stage, do_stage):
|
|
133
|
+
return _outcome(request, SeamState.FAILED, "stage exceeded its bound or failed")
|
|
134
|
+
target = staged["target"]
|
|
135
|
+
|
|
136
|
+
record = InFlight(
|
|
137
|
+
request_id=request.request_id,
|
|
138
|
+
candidate=candidate_dir,
|
|
139
|
+
last_good=last_good,
|
|
140
|
+
deadline_epoch=deadline_from_now(bounds.return_window),
|
|
141
|
+
)
|
|
142
|
+
write_inflight(record)
|
|
143
|
+
|
|
144
|
+
# SWITCH — the commit point (NF-1), bounded.
|
|
145
|
+
try:
|
|
146
|
+
if not _within(bounds.switch, lambda: version_tree.commit(target)):
|
|
147
|
+
clear_inflight()
|
|
148
|
+
return _outcome(
|
|
149
|
+
request, SeamState.FAILED, "switch exceeded its bound (repoint failed)"
|
|
150
|
+
)
|
|
151
|
+
except VersionTreeError as exc:
|
|
152
|
+
clear_inflight()
|
|
153
|
+
return _outcome(request, SeamState.FAILED, f"switch failed: {exc}")
|
|
154
|
+
|
|
155
|
+
# RESTART onto the committed version, then wait for the return window. A
|
|
156
|
+
# restart that raises is a failed switch, not a crash: surface DOWN and clear
|
|
157
|
+
# the record rather than letting the exception escape.
|
|
158
|
+
if not _within(bounds.switch, restart):
|
|
159
|
+
clear_inflight()
|
|
160
|
+
return _outcome(request, SeamState.DOWN, "restart failed after the switch")
|
|
161
|
+
|
|
162
|
+
if is_back(record.deadline_epoch):
|
|
163
|
+
clear_inflight()
|
|
164
|
+
return _outcome(
|
|
165
|
+
request,
|
|
166
|
+
SeamState.REPORTING,
|
|
167
|
+
"came back on the new version",
|
|
168
|
+
candidate=candidate_dir,
|
|
169
|
+
)
|
|
170
|
+
|
|
171
|
+
# No return within the window: revert to the last good (NF-3/NF-4/NF-8).
|
|
172
|
+
if last_good is None:
|
|
173
|
+
# Nothing to revert to: surface DOWN rather than claim success.
|
|
174
|
+
clear_inflight()
|
|
175
|
+
return _outcome(
|
|
176
|
+
request, SeamState.DOWN, "no return and no last-good version to revert to"
|
|
177
|
+
)
|
|
178
|
+
try:
|
|
179
|
+
version_tree.commit(version_tree.versions_dir() / last_good)
|
|
180
|
+
if not _within(bounds.switch, restart):
|
|
181
|
+
raise LifecycleError("restart after revert failed")
|
|
182
|
+
except (VersionTreeError, LifecycleError) as exc:
|
|
183
|
+
clear_inflight()
|
|
184
|
+
return _outcome(request, SeamState.DOWN, f"revert failed: {exc}")
|
|
185
|
+
clear_inflight()
|
|
186
|
+
return _outcome(
|
|
187
|
+
request,
|
|
188
|
+
SeamState.REVERTED,
|
|
189
|
+
"no return within the window; reverted",
|
|
190
|
+
reverted_to=last_good,
|
|
191
|
+
)
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
def _within(bound: float, step: Callable[[], None]) -> bool:
|
|
195
|
+
"""Run ``step``; True if it completed, False if it raised or exceeded ``bound``.
|
|
196
|
+
|
|
197
|
+
A step that raises is a failure of that state, not a crash: before the switch
|
|
198
|
+
it leaves the node on its current version (NF-3). The time bound is enforced
|
|
199
|
+
by the step itself (a subprocess timeout / an HTTP timeout); this records the
|
|
200
|
+
comparison so the caller does not have to.
|
|
201
|
+
"""
|
|
202
|
+
start = time.monotonic()
|
|
203
|
+
try:
|
|
204
|
+
step()
|
|
205
|
+
except Exception:
|
|
206
|
+
return False
|
|
207
|
+
return (time.monotonic() - start) <= bound
|
manta_node/supervisor/process.py
CHANGED
|
@@ -23,19 +23,37 @@ wrong one.
|
|
|
23
23
|
from __future__ import annotations
|
|
24
24
|
|
|
25
25
|
import argparse
|
|
26
|
+
import asyncio
|
|
26
27
|
import os
|
|
27
28
|
import signal
|
|
28
29
|
import subprocess
|
|
29
30
|
import sys
|
|
31
|
+
import threading
|
|
30
32
|
import time
|
|
33
|
+
import uuid
|
|
31
34
|
from pathlib import Path
|
|
32
35
|
|
|
36
|
+
from ..seam import (
|
|
37
|
+
SeamError,
|
|
38
|
+
SeamState,
|
|
39
|
+
SwitchOutcome,
|
|
40
|
+
UpdateRequest,
|
|
41
|
+
seam_socket_path,
|
|
42
|
+
)
|
|
43
|
+
from .bounds import load_bounds
|
|
44
|
+
from .executor import ChildControl, make_executor
|
|
45
|
+
from .inflight import read_inflight
|
|
33
46
|
from .records import (
|
|
34
47
|
supervisor_record,
|
|
35
48
|
supervisor_record_path,
|
|
36
49
|
write_json_atomic,
|
|
37
50
|
)
|
|
38
|
-
from .
|
|
51
|
+
from .seam import SeamServer, SocketSeamClient
|
|
52
|
+
from .switch import boot_python, bootstrap_current, installed_pair_id
|
|
53
|
+
|
|
54
|
+
# The child's env carries this when the supervisor restarts it for an in-flight
|
|
55
|
+
# update, so the node can attribute the DISCONNECTED it publishes on the restart.
|
|
56
|
+
UPDATING_ENV = "MANTA_NODE_UPDATING"
|
|
39
57
|
|
|
40
58
|
# A child that exits more than RESTART_LIMIT times within RESTART_WINDOW_SECONDS
|
|
41
59
|
# is a crash loop: the supervisor stops restarting and exits non-zero, so systemd
|
|
@@ -51,6 +69,9 @@ STDOUT_MAX_BYTES = 10 * 1024 * 1024
|
|
|
51
69
|
# How long a terminal stop waits for the child to wind down before SIGKILL.
|
|
52
70
|
STOP_GRACE_SECONDS = 10
|
|
53
71
|
|
|
72
|
+
# How long to wait for the seam socket to appear after the server thread starts.
|
|
73
|
+
SEAM_STARTUP_TIMEOUT_SECONDS = 5
|
|
74
|
+
|
|
54
75
|
|
|
55
76
|
def _log_dir() -> Path:
|
|
56
77
|
return Path.home() / ".manta" / "logs" / "nodes"
|
|
@@ -81,6 +102,13 @@ class Supervisor:
|
|
|
81
102
|
self._stop = False
|
|
82
103
|
self._child: subprocess.Popen | None = None
|
|
83
104
|
self._restarts: list[float] = []
|
|
105
|
+
# Set by the seam executor when it wants the child restarted onto the
|
|
106
|
+
# committed version: the supervision loop respawns it and does NOT count
|
|
107
|
+
# the exit toward the crash-loop limit (a planned restart, not a crash).
|
|
108
|
+
self._restart_requested = threading.Event()
|
|
109
|
+
# The seam server, served in a background thread while the child runs.
|
|
110
|
+
self._seam_thread: threading.Thread | None = None
|
|
111
|
+
self._seam_server: SeamServer | None = None
|
|
84
112
|
|
|
85
113
|
def _write_record(self, child_pid: int | None, state: str) -> None:
|
|
86
114
|
write_json_atomic(
|
|
@@ -119,6 +147,61 @@ class Supervisor:
|
|
|
119
147
|
return self.child_python
|
|
120
148
|
return boot_python(self.child_python)
|
|
121
149
|
|
|
150
|
+
def _child_alive(self) -> bool:
|
|
151
|
+
return self._child is not None and self._child.poll() is None
|
|
152
|
+
|
|
153
|
+
def _restart_child(self) -> None:
|
|
154
|
+
"""The seam executor's restart: wind the child down so the loop respawns it.
|
|
155
|
+
|
|
156
|
+
Marking the request first means the supervision loop treats the exit as
|
|
157
|
+
planned (respawn from `current`) rather than a crash toward the limit.
|
|
158
|
+
"""
|
|
159
|
+
self._restart_requested.set()
|
|
160
|
+
self._signal_child(signal.SIGTERM)
|
|
161
|
+
|
|
162
|
+
def _child_control(self) -> ChildControl:
|
|
163
|
+
return ChildControl(restart=self._restart_child, is_alive=self._child_alive)
|
|
164
|
+
|
|
165
|
+
def _start_seam(self) -> None:
|
|
166
|
+
"""Serve the seam on its socket in a background thread.
|
|
167
|
+
|
|
168
|
+
The supervision loop is synchronous (it blocks on the child); the seam
|
|
169
|
+
server is asyncio. A daemon thread with its own event loop lets the two
|
|
170
|
+
run side by side — requests arrive while the child is supervised. The
|
|
171
|
+
handler is the real lifecycle, driven by the child control.
|
|
172
|
+
"""
|
|
173
|
+
handler = make_executor(
|
|
174
|
+
self._child_control(),
|
|
175
|
+
bounds=load_bounds(),
|
|
176
|
+
child_python=self.child_python,
|
|
177
|
+
)
|
|
178
|
+
server = SeamServer()
|
|
179
|
+
self._seam_server = server
|
|
180
|
+
self._seam_thread = threading.Thread(
|
|
181
|
+
target=self._serve_seam, args=(server, handler), daemon=True
|
|
182
|
+
)
|
|
183
|
+
self._seam_thread.start()
|
|
184
|
+
# Wait for the socket to appear, so a trigger issued right after startup
|
|
185
|
+
# finds a served socket rather than racing the bind.
|
|
186
|
+
for _ in range(SEAM_STARTUP_TIMEOUT_SECONDS * 100):
|
|
187
|
+
if seam_socket_path().exists():
|
|
188
|
+
return
|
|
189
|
+
time.sleep(0.01)
|
|
190
|
+
print(
|
|
191
|
+
"supervisor: seam socket did not appear within the startup window",
|
|
192
|
+
file=sys.stderr,
|
|
193
|
+
)
|
|
194
|
+
|
|
195
|
+
def _serve_seam(self, server: SeamServer, handler) -> None:
|
|
196
|
+
loop = asyncio.new_event_loop()
|
|
197
|
+
asyncio.set_event_loop(loop)
|
|
198
|
+
try:
|
|
199
|
+
loop.run_until_complete(server.start(handler))
|
|
200
|
+
except Exception as exc: # the seam must not take the supervisor down
|
|
201
|
+
print(f"supervisor: seam server stopped: {exc!r}", file=sys.stderr)
|
|
202
|
+
finally:
|
|
203
|
+
loop.close()
|
|
204
|
+
|
|
122
205
|
def _spawn(self) -> subprocess.Popen:
|
|
123
206
|
# `--subprocess` is the existing hidden flag: the child does not write an
|
|
124
207
|
# instance file, so exactly one writer (the supervisor) owns the records.
|
|
@@ -148,9 +231,21 @@ class Supervisor:
|
|
|
148
231
|
stdout_path = log_dir / f"{self.alias}.supervisor.stdout"
|
|
149
232
|
if stdout_path.exists() and stdout_path.stat().st_size >= STDOUT_MAX_BYTES:
|
|
150
233
|
stdout_path.replace(stdout_path.with_name(f"{stdout_path.name}.1"))
|
|
234
|
+
# Mark the child when an update is in flight, so the node's DISCONNECTED
|
|
235
|
+
# on this restart is attributed to the update, not read as a fleet
|
|
236
|
+
# disconnect (NF-8, B-22). The node cannot read the supervisor's in-flight
|
|
237
|
+
# record (NF-6), so the supervisor passes the request id in the env.
|
|
238
|
+
env = None
|
|
239
|
+
record = read_inflight()
|
|
240
|
+
if record is not None:
|
|
241
|
+
env = {**os.environ, UPDATING_ENV: record.request_id}
|
|
151
242
|
with open(stdout_path, "a") as stream:
|
|
152
243
|
child = subprocess.Popen(
|
|
153
|
-
cmd,
|
|
244
|
+
cmd,
|
|
245
|
+
stdout=stream,
|
|
246
|
+
stderr=stream,
|
|
247
|
+
start_new_session=True,
|
|
248
|
+
env=env,
|
|
154
249
|
)
|
|
155
250
|
# Publish the pid BEFORE returning: a signal that fires between here and
|
|
156
251
|
# the caller's `_stop` check must be able to reach the child, not a gone
|
|
@@ -166,6 +261,10 @@ class Supervisor:
|
|
|
166
261
|
# restarts; a host with no version pair skips this entirely.
|
|
167
262
|
if self.version_pair:
|
|
168
263
|
bootstrap_current(self.version_pair, self.child_python)
|
|
264
|
+
# Serve the seam on a version-tree node: a switch is meaningful only
|
|
265
|
+
# there, and it is where the executor has a tree to act on. The socket
|
|
266
|
+
# now exists while the supervisor runs, so a trigger is served.
|
|
267
|
+
self._start_seam()
|
|
169
268
|
self._write_record(None, "starting")
|
|
170
269
|
|
|
171
270
|
while not self._stop:
|
|
@@ -184,6 +283,13 @@ class Supervisor:
|
|
|
184
283
|
if self._stop:
|
|
185
284
|
break
|
|
186
285
|
|
|
286
|
+
# A restart the seam executor asked for is planned, not a crash: it
|
|
287
|
+
# respawns from `current` and does not count toward the limit.
|
|
288
|
+
if self._restart_requested.is_set():
|
|
289
|
+
self._restart_requested.clear()
|
|
290
|
+
self._write_record(None, "restarting")
|
|
291
|
+
continue
|
|
292
|
+
|
|
187
293
|
now = time.monotonic()
|
|
188
294
|
self._restarts = [
|
|
189
295
|
t for t in self._restarts if now - t < RESTART_WINDOW_SECONDS
|
|
@@ -209,11 +315,54 @@ class Supervisor:
|
|
|
209
315
|
self._child.wait(timeout=STOP_GRACE_SECONDS)
|
|
210
316
|
except subprocess.TimeoutExpired:
|
|
211
317
|
self._signal_child(signal.SIGKILL)
|
|
318
|
+
# Close the seam socket so a later supervisor is not left with a stale one
|
|
319
|
+
# to unlink. The server thread is a daemon and ends with the process.
|
|
320
|
+
seam_socket_path().unlink(missing_ok=True)
|
|
212
321
|
supervisor_record_path(self.alias).unlink(missing_ok=True)
|
|
213
322
|
return 0
|
|
214
323
|
|
|
215
324
|
|
|
325
|
+
async def _apply_over_seam(request: UpdateRequest) -> SwitchOutcome:
|
|
326
|
+
"""Submit one update request to a running supervisor over the seam."""
|
|
327
|
+
return await SocketSeamClient().request_update(request)
|
|
328
|
+
|
|
329
|
+
|
|
330
|
+
def apply_command(argv: list[str]) -> int:
|
|
331
|
+
"""`manta-supervisor apply <pair>` — the fleet path's local trigger (2d).
|
|
332
|
+
|
|
333
|
+
The fleet tooling runs this as the node's service user; it writes an
|
|
334
|
+
``UpdateRequest`` to the supervisor's socket and prints the outcome. This is
|
|
335
|
+
the named trigger for the fleet path; the Manager path (Phase 3) submits the
|
|
336
|
+
same request from the node's MQTT relay instead.
|
|
337
|
+
"""
|
|
338
|
+
parser = argparse.ArgumentParser(
|
|
339
|
+
prog="manta-supervisor apply",
|
|
340
|
+
description="Ask a running supervisor to apply a pair (writes the seam).",
|
|
341
|
+
)
|
|
342
|
+
parser.add_argument("pair", help="the pair (or set id) to apply")
|
|
343
|
+
parser.add_argument("--force", action="store_true", help="override the guard")
|
|
344
|
+
parser.add_argument("--request-id", help="idempotency key (fresh uuid by default)")
|
|
345
|
+
args = parser.parse_args(argv)
|
|
346
|
+
|
|
347
|
+
request = UpdateRequest(
|
|
348
|
+
request_id=args.request_id or uuid.uuid4().hex,
|
|
349
|
+
target=args.pair,
|
|
350
|
+
force=args.force,
|
|
351
|
+
)
|
|
352
|
+
try:
|
|
353
|
+
outcome = asyncio.run(_apply_over_seam(request))
|
|
354
|
+
except SeamError as exc:
|
|
355
|
+
print(f"manta-supervisor apply: {exc}", file=sys.stderr)
|
|
356
|
+
return 1
|
|
357
|
+
print(f"{outcome.state.value}: {outcome.reason}")
|
|
358
|
+
return 0 if outcome.state not in (SeamState.FAILED, SeamState.DOWN) else 1
|
|
359
|
+
|
|
360
|
+
|
|
216
361
|
def main(argv: list[str] | None = None) -> int:
|
|
362
|
+
argv = list(sys.argv[1:] if argv is None else argv)
|
|
363
|
+
if argv and argv[0] == "apply":
|
|
364
|
+
return apply_command(argv[1:])
|
|
365
|
+
|
|
217
366
|
parser = argparse.ArgumentParser(
|
|
218
367
|
prog="manta-supervisor",
|
|
219
368
|
description="Own a manta_node child and its restart (ADR-0041 boundary B1).",
|
|
@@ -230,16 +379,34 @@ def main(argv: list[str] | None = None) -> int:
|
|
|
230
379
|
parser.add_argument(
|
|
231
380
|
"--pair",
|
|
232
381
|
help=(
|
|
233
|
-
"the pair this node runs, for the immutable version tree (2c)
|
|
234
|
-
"
|
|
382
|
+
"the pair this node runs, for the immutable version tree (2c)."
|
|
383
|
+
" Defaults to the pair derived from the installed distributions, so"
|
|
384
|
+
" provisioning does not name a version: boot materialises the"
|
|
385
|
+
" in-place code as that pair's version dir, and the fleet apply later"
|
|
386
|
+
" installs a different pair into its own dir."
|
|
387
|
+
),
|
|
388
|
+
)
|
|
389
|
+
parser.add_argument(
|
|
390
|
+
"--version-tree",
|
|
391
|
+
action="store_true",
|
|
392
|
+
help=(
|
|
393
|
+
"enable the version tree: serve the seam and boot the child from"
|
|
394
|
+
" `current`, using --pair or the derived pair. Off by default, so a"
|
|
395
|
+
" node runs the pre-2c path until the deploy enables it."
|
|
235
396
|
),
|
|
236
397
|
)
|
|
237
398
|
args = parser.parse_args(argv)
|
|
399
|
+
version_pair = None
|
|
400
|
+
if args.version_tree:
|
|
401
|
+
version_pair = args.pair or installed_pair_id()
|
|
402
|
+
elif args.pair:
|
|
403
|
+
# --pair alone still opts in, for a hand-run supervisor.
|
|
404
|
+
version_pair = args.pair
|
|
238
405
|
return Supervisor(
|
|
239
406
|
args.alias,
|
|
240
407
|
args.config or args.alias,
|
|
241
408
|
child_python=args.python,
|
|
242
|
-
version_pair=
|
|
409
|
+
version_pair=version_pair,
|
|
243
410
|
).run()
|
|
244
411
|
|
|
245
412
|
|
manta_node/supervisor/seam.py
CHANGED
|
@@ -13,11 +13,11 @@ the 2b acceptance: the code can ship, be idle, and change nothing.
|
|
|
13
13
|
``0600`` on the socket and its parent directory. Local and same-user only
|
|
14
14
|
(NF-9/NF-10).
|
|
15
15
|
|
|
16
|
-
**What "execute" means here.**
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
16
|
+
**What "execute" means here.** The server is transport and idempotency only: it
|
|
17
|
+
hands each new request to the handler and returns the handler's outcome. The
|
|
18
|
+
supervisor wires the **real lifecycle** (``lifecycle.run_switch`` via
|
|
19
|
+
``executor.make_executor``) as that handler; ``stub_handler`` remains for a
|
|
20
|
+
supervisor with no version tree, answering ``REQUESTED`` without acting.
|
|
21
21
|
"""
|
|
22
22
|
|
|
23
23
|
from __future__ import annotations
|
|
@@ -91,8 +91,8 @@ class SeamServer:
|
|
|
91
91
|
await writer.wait_closed()
|
|
92
92
|
|
|
93
93
|
async def _answer(self, request: UpdateRequest, handler) -> SwitchOutcome:
|
|
94
|
-
# Idempotency: a repeat of a
|
|
95
|
-
#
|
|
94
|
+
# Idempotency: a repeat of a COMPLETED request returns the same outcome; a
|
|
95
|
+
# repeat still in flight is refused, not run twice (NF-5).
|
|
96
96
|
if request.request_id in self._answered:
|
|
97
97
|
return self._answered[request.request_id]
|
|
98
98
|
if request.request_id in self._in_flight:
|
|
@@ -106,7 +106,11 @@ class SeamServer:
|
|
|
106
106
|
outcome = await handler(request)
|
|
107
107
|
finally:
|
|
108
108
|
self._in_flight.discard(request.request_id)
|
|
109
|
-
|
|
109
|
+
# A REFUSED is not a completed request: caching it would make a later
|
|
110
|
+
# retry of the same id return the stale refusal instead of being tried
|
|
111
|
+
# (the idempotency contract is about *completed* requests).
|
|
112
|
+
if outcome.state is not SeamState.REFUSED:
|
|
113
|
+
self._answered[request.request_id] = outcome
|
|
110
114
|
return outcome
|
|
111
115
|
|
|
112
116
|
async def start(self, handler) -> None:
|
manta_node/supervisor/switch.py
CHANGED
|
@@ -8,11 +8,11 @@ and the child restart resolves deterministically to the version ``current`` name
|
|
|
8
8
|
and a failed repoint leaves the previous version selected. There is no
|
|
9
9
|
half-applied state.
|
|
10
10
|
|
|
11
|
-
**
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
11
|
+
**Bootstrapping is a no-op switch.** ``bootstrap_current`` materialises the
|
|
12
|
+
*current* code as the first version dir and points ``current`` at it, so a node
|
|
13
|
+
runs the same code from a version dir. ``materialise_version``/``commit`` are also
|
|
14
|
+
the primitives the real switch lifecycle drives (``lifecycle.py``); a version dir
|
|
15
|
+
holds the pair's identity and the interpreter that runs it.
|
|
16
16
|
|
|
17
17
|
**State is never inside the tree** (NF-6). The tree lives beside the node's state
|
|
18
18
|
(``~/.manta/nodes/``), and a version dir contains only code-side markers — never
|
|
@@ -24,6 +24,7 @@ from __future__ import annotations
|
|
|
24
24
|
import os
|
|
25
25
|
from dataclasses import dataclass
|
|
26
26
|
from datetime import datetime, timezone
|
|
27
|
+
from importlib.metadata import PackageNotFoundError, version
|
|
27
28
|
from pathlib import Path
|
|
28
29
|
|
|
29
30
|
from .records import read_json, write_json_atomic
|
|
@@ -42,6 +43,27 @@ def versions_dir() -> Path:
|
|
|
42
43
|
return Path.home() / ".manta" / "nodes" / VERSIONS_DIR_NAME
|
|
43
44
|
|
|
44
45
|
|
|
46
|
+
def installed_pair_id() -> str:
|
|
47
|
+
"""The pair id of the code currently installed, from distribution metadata.
|
|
48
|
+
|
|
49
|
+
The pair is named `<node-package>@<version>+<common-package>@<version>`, so a
|
|
50
|
+
version dir is self-describing and two different pairs never collide. Read
|
|
51
|
+
from installed metadata (stdlib only — the supervisor imports no
|
|
52
|
+
`manta_common`): this is the pair the in-place venv holds, which boot
|
|
53
|
+
materialises as the first version dir.
|
|
54
|
+
"""
|
|
55
|
+
|
|
56
|
+
def safe(name: str) -> str:
|
|
57
|
+
try:
|
|
58
|
+
return version(name)
|
|
59
|
+
except PackageNotFoundError:
|
|
60
|
+
return "unknown"
|
|
61
|
+
|
|
62
|
+
return (
|
|
63
|
+
f"manta-node@{safe('manta_node')}+manta-common-core@{safe('manta-common-core')}"
|
|
64
|
+
)
|
|
65
|
+
|
|
66
|
+
|
|
45
67
|
def current_link() -> Path:
|
|
46
68
|
return Path.home() / ".manta" / "nodes" / CURRENT_LINK_NAME
|
|
47
69
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: manta_node
|
|
3
|
-
Version: 0.6b5.
|
|
3
|
+
Version: 0.6b5.dev705
|
|
4
4
|
Summary: Node side software of the Manta platform. It is responsible for the communication with the Manta Core and the execution of the tasks.
|
|
5
5
|
Author-email: Hugo Miralles <hugo.miralles@manta-tech.io>, Matthew Thompson <matthew.thompson@manta-tech.io>
|
|
6
6
|
License-Expression: AGPL-3.0-or-later
|
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
manta_node/__init__.py,sha256=bgGWXH3yQVhNf0v1ZGUBAy5Y51fqb0Y-EaLnocbcino,153
|
|
2
2
|
manta_node/__main__.py,sha256=SjrJyuMg_ZFKV6WOVe3MrtWM8Qmu065dkKZYl4n2YLw,1665
|
|
3
|
-
manta_node/node_orchestrator.py,sha256=
|
|
3
|
+
manta_node/node_orchestrator.py,sha256=nfpNCM8wgnihKhZZ36GIizmyyaOdEOBhdfn7PqqTCrQ,29591
|
|
4
4
|
manta_node/seam.py,sha256=wAI3uL1iuV6pmDOaBW7WZi7_V7-NsHQDP6C_gFIjCiA,4747
|
|
5
5
|
manta_node/task_manager.py,sha256=xRE-yKdUW0FFR8vtX1jDwl1AxTbqVD_JS7QVR62GA00,38002
|
|
6
6
|
manta_node/tasks.py,sha256=o837Hi-KFVJg9LF5iOtE8oD6UuTOfrEPWR-JOdhRGeY,6983
|
|
7
7
|
manta_node/utils.py,sha256=Hw4Iw2jYQYsfVLVJuBMt-_rMFBeml3nCDE2jiiXc8YM,1155
|
|
8
8
|
manta_node/cli/__init__.py,sha256=arzO3mXDJ-E2yeeytbZfN7mCswwxdQ43_QjyeL2E5yE,134
|
|
9
|
-
manta_node/cli/_version_generated.py,sha256=
|
|
9
|
+
manta_node/cli/_version_generated.py,sha256=Avtgxf4CTYxrwjUmb8TM92DAZRNRqrzV-eNId81M60o,275
|
|
10
10
|
manta_node/cli/config_manager.py,sha256=BCju9zqzraviL3KrzRnB-vZQUY__Q5kuEqb7EdB8YdI,4409
|
|
11
11
|
manta_node/cli/main.py,sha256=dZa6zrypWBvpN8hd4qpxevvlyNFhxKINCg1537NJ36w,4318
|
|
12
12
|
manta_node/cli/version.py,sha256=3lrnIWpOosszBN-VsCBVLxDgpvCC7IB0aDIKx_3e7hI,3115
|
|
@@ -41,7 +41,7 @@ manta_node/infrastructure/metrics/__init__.py,sha256=ODGAj9s63F5jba2c3dQRMfNK6I4
|
|
|
41
41
|
manta_node/infrastructure/metrics/collector.py,sha256=9ESw__Pc2tV6ldG5yhos2IS7T5Td59KxTIwDRZc_G4w,3067
|
|
42
42
|
manta_node/infrastructure/metrics/metrics_collector.py,sha256=TI_9tRoIUunGx_BstVavK6VElig7Wc5mJ0kB4bFumVs,11048
|
|
43
43
|
manta_node/infrastructure/mqtt/__init__.py,sha256=riXo1sXgO4YV6DHCmWRqK61BAmuGLdHiVdQFunAJadA,140
|
|
44
|
-
manta_node/infrastructure/mqtt/command_handler.py,sha256=
|
|
44
|
+
manta_node/infrastructure/mqtt/command_handler.py,sha256=aGJMPF8uAdeEw5xfE_WlL3-ROiSV_LCUXf5EpmZrbtk,29728
|
|
45
45
|
manta_node/infrastructure/network/__init__.py,sha256=On3XLgycFQoc-fBCa5hFuFDfEJjmFy7Wl2CK3D2LkdI,478
|
|
46
46
|
manta_node/infrastructure/network/event_registry.py,sha256=iWEfUJRzDnWjOrDinpYqVyIUpGKJjp7RZkjMDPfpplM,1661
|
|
47
47
|
manta_node/infrastructure/network/factory.py,sha256=-LvHbBp5nBpQ7fyQT1T0pvK3aczZqWc9QNDW37brBeU,3897
|
|
@@ -53,13 +53,17 @@ manta_node/infrastructure/security/__init__.py,sha256=lI6vkbUmfmt0m363iO4HzuzVNo
|
|
|
53
53
|
manta_node/infrastructure/security/auth_agent.py,sha256=WkPTg_g0gix7xomb_on_Ft3TNX0FpeqS7gwgJW9mTvk,11037
|
|
54
54
|
manta_node/infrastructure/security/token_provider.py,sha256=u3bcbQYNBskFzunO-wOSZpGkDN_CU1BCB5HUbZTC4Vg,1832
|
|
55
55
|
manta_node/supervisor/__init__.py,sha256=kgdoNce1tM4e7jvdxkdmnNJV1L92-NI2cRfYSdm9vsw,542
|
|
56
|
-
manta_node/supervisor/
|
|
56
|
+
manta_node/supervisor/bounds.py,sha256=MlUfj0tYY-US348JsxYA-Ph5pSpgoOPlRdVTwmrPtZA,3181
|
|
57
|
+
manta_node/supervisor/executor.py,sha256=a0JafVoTmtbIu87nwUtj08KAC-Q-u0Ajdn_ptyu9yoc,6833
|
|
58
|
+
manta_node/supervisor/inflight.py,sha256=JOqwXYM928fpcQtYdoK7596eAg28KM70bh85cyQgX3Q,2663
|
|
59
|
+
manta_node/supervisor/lifecycle.py,sha256=JE74H0ODt5_y997kliiVDiQAf1laYptNjkrnnfGdgK0,7411
|
|
60
|
+
manta_node/supervisor/process.py,sha256=mWd2C4-S8NpupcLp0YbBEYmqQUQbrfMAZi5TF9UQ_cM,17504
|
|
57
61
|
manta_node/supervisor/records.py,sha256=fUlL1SfmWpGnCbJ9956cuy0H3a7-aEF-B-naqbJiYoU,6405
|
|
58
|
-
manta_node/supervisor/seam.py,sha256=
|
|
59
|
-
manta_node/supervisor/switch.py,sha256=
|
|
60
|
-
manta_node-0.6b5.
|
|
61
|
-
manta_node-0.6b5.
|
|
62
|
-
manta_node-0.6b5.
|
|
63
|
-
manta_node-0.6b5.
|
|
64
|
-
manta_node-0.6b5.
|
|
65
|
-
manta_node-0.6b5.
|
|
62
|
+
manta_node/supervisor/seam.py,sha256=HomMFTDbc0dZQ9VrfGVhQxjiU7E23-zwyjZnLsnq6dc,8031
|
|
63
|
+
manta_node/supervisor/switch.py,sha256=BUSvMdWyUfZmeEVSBmQ7wLNRD7troPxZ3pdg8tOGXo4,7433
|
|
64
|
+
manta_node-0.6b5.dev705.dist-info/licenses/LICENSE,sha256=9mBgLhiB9goQPQjg9XQJ6FjgVxYw9Q8lyOZYkPGwF_c,34484
|
|
65
|
+
manta_node-0.6b5.dev705.dist-info/METADATA,sha256=ajleN4PRTTgRVihXNc2WTZEatE_cgnmro32xQBBd1JY,4231
|
|
66
|
+
manta_node-0.6b5.dev705.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
|
|
67
|
+
manta_node-0.6b5.dev705.dist-info/entry_points.txt,sha256=uCwggRApalLEsYs1YxwTBN0-e9oMJAkMoBp01HaSdoY,110
|
|
68
|
+
manta_node-0.6b5.dev705.dist-info/top_level.txt,sha256=nbT5J5WETuXM9UPoQckl29uUOq8wMYdRb9QDx5kAFeo,11
|
|
69
|
+
manta_node-0.6b5.dev705.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|