manta-node 0.6b5.dev697__py3-none-any.whl → 0.6b5.dev699__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- manta_node/cli/_version_generated.py +3 -3
- manta_node/cli/commands/status.py +45 -69
- manta_node/cli/commands/stop.py +69 -48
- manta_node/supervisor/__init__.py +11 -0
- manta_node/supervisor/process.py +210 -0
- manta_node/supervisor/records.py +167 -0
- {manta_node-0.6b5.dev697.dist-info → manta_node-0.6b5.dev699.dist-info}/METADATA +1 -1
- {manta_node-0.6b5.dev697.dist-info → manta_node-0.6b5.dev699.dist-info}/RECORD +12 -9
- {manta_node-0.6b5.dev697.dist-info → manta_node-0.6b5.dev699.dist-info}/entry_points.txt +1 -0
- {manta_node-0.6b5.dev697.dist-info → manta_node-0.6b5.dev699.dist-info}/WHEEL +0 -0
- {manta_node-0.6b5.dev697.dist-info → manta_node-0.6b5.dev699.dist-info}/licenses/LICENSE +0 -0
- {manta_node-0.6b5.dev697.dist-info → manta_node-0.6b5.dev699.dist-info}/top_level.txt +0 -0
|
@@ -4,7 +4,7 @@ This file is auto-generated at build time by scripts/generate_version.py.
|
|
|
4
4
|
Do not edit manually.
|
|
5
5
|
"""
|
|
6
6
|
|
|
7
|
-
__version__ = "2026.09.
|
|
8
|
-
__build_date__ = "2026.09.
|
|
9
|
-
__commit_id__ = "
|
|
7
|
+
__version__ = "2026.09.24.c33eff33"
|
|
8
|
+
__build_date__ = "2026.09.24"
|
|
9
|
+
__commit_id__ = "c33eff33"
|
|
10
10
|
__package_version__ = "0.6b5"
|
|
@@ -2,88 +2,64 @@
|
|
|
2
2
|
|
|
3
3
|
import argparse
|
|
4
4
|
import contextlib
|
|
5
|
-
import json
|
|
6
|
-
import os
|
|
7
5
|
from datetime import datetime
|
|
8
|
-
from pathlib import Path
|
|
9
6
|
|
|
10
7
|
import psutil
|
|
11
8
|
from rich.console import Console
|
|
12
9
|
from rich.table import Table
|
|
13
10
|
|
|
11
|
+
from ...supervisor.records import list_instances
|
|
12
|
+
|
|
14
13
|
console = Console()
|
|
15
14
|
|
|
16
15
|
|
|
17
16
|
def find_running_instances():
|
|
18
|
-
"""
|
|
17
|
+
"""Running instances for display, with the supervisor record read-through.
|
|
19
18
|
|
|
20
|
-
|
|
19
|
+
Under a supervisor the pid of record is the supervisor's, and only the
|
|
20
|
+
supervisor record exists (the child runs with ``--subprocess``); without one
|
|
21
|
+
it is the node's own instance record. ``list_instances`` resolves both.
|
|
22
|
+
"""
|
|
21
23
|
instances = []
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
24
|
+
for record in list_instances():
|
|
25
|
+
pid = record["pid"]
|
|
26
|
+
# The workload is the child under a supervisor, the node itself without
|
|
27
|
+
# one: `status` shows its cpu/memory, while `pid` stays the pid a stop
|
|
28
|
+
# signals (the supervisor's, when present).
|
|
29
|
+
workload_pid = record.get("workload_pid") or pid
|
|
30
|
+
cpu_percent = None
|
|
31
|
+
memory_mb = None
|
|
32
|
+
create_time = None
|
|
33
|
+
status = "running"
|
|
27
34
|
try:
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
# Add additional info from JSON
|
|
58
|
-
alias = instance_data.get("alias", instance_id)
|
|
59
|
-
config_name = instance_data.get("config_name", "unknown")
|
|
60
|
-
start_time_str = instance_data.get("start_time")
|
|
61
|
-
if start_time_str and not create_time:
|
|
62
|
-
with contextlib.suppress(ValueError, TypeError):
|
|
63
|
-
create_time = datetime.fromisoformat(start_time_str)
|
|
64
|
-
|
|
65
|
-
instances.append(
|
|
66
|
-
{
|
|
67
|
-
"instance_id": instance_id,
|
|
68
|
-
"alias": alias,
|
|
69
|
-
"config_name": config_name,
|
|
70
|
-
"pid": pid,
|
|
71
|
-
"status": status,
|
|
72
|
-
"cpu_percent": cpu_percent,
|
|
73
|
-
"memory_mb": memory_mb,
|
|
74
|
-
"start_time": create_time,
|
|
75
|
-
"instance_file": instance_file,
|
|
76
|
-
}
|
|
77
|
-
)
|
|
78
|
-
|
|
79
|
-
except OSError:
|
|
80
|
-
# Process no longer exists, remove stale instance file
|
|
81
|
-
instance_file.unlink(missing_ok=True)
|
|
82
|
-
|
|
83
|
-
except (ValueError, FileNotFoundError):
|
|
84
|
-
# Invalid instance file, remove it
|
|
85
|
-
instance_file.unlink(missing_ok=True)
|
|
86
|
-
|
|
35
|
+
process = psutil.Process(workload_pid)
|
|
36
|
+
cpu_percent = process.cpu_percent()
|
|
37
|
+
memory_mb = process.memory_info().rss // (1024 * 1024)
|
|
38
|
+
create_time = datetime.fromtimestamp(process.create_time())
|
|
39
|
+
except ImportError:
|
|
40
|
+
pass
|
|
41
|
+
except psutil.NoSuchProcess:
|
|
42
|
+
# Disappeared between the liveness check and here; skip it.
|
|
43
|
+
continue
|
|
44
|
+
|
|
45
|
+
start_time_str = record.get("start_time")
|
|
46
|
+
if start_time_str and not create_time:
|
|
47
|
+
with contextlib.suppress(ValueError, TypeError):
|
|
48
|
+
create_time = datetime.fromisoformat(start_time_str)
|
|
49
|
+
|
|
50
|
+
instances.append(
|
|
51
|
+
{
|
|
52
|
+
"instance_id": record["instance_id"],
|
|
53
|
+
"alias": record["alias"],
|
|
54
|
+
"config_name": record["config_name"],
|
|
55
|
+
"pid": pid,
|
|
56
|
+
"status": status,
|
|
57
|
+
"cpu_percent": cpu_percent,
|
|
58
|
+
"memory_mb": memory_mb,
|
|
59
|
+
"start_time": create_time,
|
|
60
|
+
"instance_file": record["instance_file"],
|
|
61
|
+
}
|
|
62
|
+
)
|
|
87
63
|
return instances
|
|
88
64
|
|
|
89
65
|
|
manta_node/cli/commands/stop.py
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
"""Node stop command implementation."""
|
|
2
2
|
|
|
3
3
|
import argparse
|
|
4
|
-
import json
|
|
5
4
|
import os
|
|
6
5
|
import signal
|
|
7
6
|
import time
|
|
8
|
-
from pathlib import Path
|
|
9
7
|
|
|
10
8
|
from rich.console import Console
|
|
11
9
|
|
|
10
|
+
from ...supervisor.records import list_instances
|
|
11
|
+
|
|
12
12
|
console = Console()
|
|
13
13
|
|
|
14
14
|
|
|
@@ -28,45 +28,23 @@ def print_warning(message: str):
|
|
|
28
28
|
|
|
29
29
|
|
|
30
30
|
def find_running_instances():
|
|
31
|
-
"""
|
|
32
|
-
|
|
33
|
-
instances_dir = Path.home() / ".manta" / "nodes" / "instances"
|
|
34
|
-
instances = []
|
|
35
|
-
|
|
36
|
-
if not instances_dir.exists():
|
|
37
|
-
return instances
|
|
38
|
-
|
|
39
|
-
for instance_file in instances_dir.glob("*.json"):
|
|
40
|
-
try:
|
|
41
|
-
with open(instance_file, "r") as f:
|
|
42
|
-
instance_data = json.load(f)
|
|
43
|
-
|
|
44
|
-
pid = instance_data.get("pid")
|
|
45
|
-
instance_id = instance_data.get("instance_id", instance_file.stem)
|
|
46
|
-
alias = instance_data.get("alias", instance_id)
|
|
47
|
-
|
|
48
|
-
# Check if process is still running
|
|
49
|
-
try:
|
|
50
|
-
# Send signal 0 to check if process exists
|
|
51
|
-
|
|
52
|
-
os.kill(pid, 0)
|
|
53
|
-
instances.append(
|
|
54
|
-
{
|
|
55
|
-
"instance_id": instance_id,
|
|
56
|
-
"alias": alias,
|
|
57
|
-
"pid": pid,
|
|
58
|
-
"instance_file": instance_file,
|
|
59
|
-
}
|
|
60
|
-
)
|
|
61
|
-
except OSError:
|
|
62
|
-
# Process no longer exists, remove stale instance file
|
|
63
|
-
instance_file.unlink(missing_ok=True)
|
|
64
|
-
|
|
65
|
-
except (ValueError, FileNotFoundError):
|
|
66
|
-
# Invalid instance file, remove it
|
|
67
|
-
instance_file.unlink(missing_ok=True)
|
|
31
|
+
"""Running instances to stop, with the supervisor record read-through.
|
|
68
32
|
|
|
69
|
-
|
|
33
|
+
The pid returned for each alias is the **pid of record**: the supervisor's
|
|
34
|
+
when one is present, else the node's own. Signalling it stops the supervisor
|
|
35
|
+
(which winds the child down) or the node directly — never both.
|
|
36
|
+
"""
|
|
37
|
+
return [
|
|
38
|
+
{
|
|
39
|
+
"instance_id": record["instance_id"],
|
|
40
|
+
"alias": record["alias"],
|
|
41
|
+
"pid": record["pid"],
|
|
42
|
+
"child_pid": record.get("child_pid"),
|
|
43
|
+
"source": record["source"],
|
|
44
|
+
"instance_file": record["instance_file"],
|
|
45
|
+
}
|
|
46
|
+
for record in list_instances()
|
|
47
|
+
]
|
|
70
48
|
|
|
71
49
|
|
|
72
50
|
def stop_instance_by_pid(
|
|
@@ -85,15 +63,53 @@ def stop_instance_by_pid(
|
|
|
85
63
|
pid = instance["pid"]
|
|
86
64
|
instance_id = instance["instance_id"]
|
|
87
65
|
alias = instance.get("alias", instance_id)
|
|
66
|
+
# A supervised node's pid of record is the SUPERVISOR, and its child leads a
|
|
67
|
+
# SEPARATE process group (`start_new_session=True`), so `killpg` on the
|
|
68
|
+
# supervisor never reaches the child. A bare SIGKILL of the supervisor cannot
|
|
69
|
+
# be forwarded and would orphan the child. So: SIGTERM the supervisor, which
|
|
70
|
+
# forwards to the child and exits; to force, also SIGKILL the recorded child
|
|
71
|
+
# pid, then the supervisor.
|
|
72
|
+
owned_by_supervisor = instance.get("source") == "supervisor"
|
|
73
|
+
child_pid = instance.get("child_pid")
|
|
74
|
+
|
|
75
|
+
def signal_supervisor(signum: int) -> None:
|
|
76
|
+
_signal_pid(pid, signum)
|
|
77
|
+
|
|
78
|
+
def signal_child(signum: int) -> None:
|
|
79
|
+
# The child is its own session: signal the group it leads, so any task
|
|
80
|
+
# grandchildren it spawned go too. Fall back to the pid if the group is
|
|
81
|
+
# gone or not ours.
|
|
82
|
+
if not child_pid:
|
|
83
|
+
return
|
|
84
|
+
try:
|
|
85
|
+
os.killpg(os.getpgid(child_pid), signum)
|
|
86
|
+
except (ProcessLookupError, PermissionError):
|
|
87
|
+
_signal_pid(child_pid, signum)
|
|
88
|
+
|
|
89
|
+
def _signal_pid(target: int | None, signum: int) -> None:
|
|
90
|
+
if not target:
|
|
91
|
+
return
|
|
92
|
+
try:
|
|
93
|
+
os.kill(target, signum)
|
|
94
|
+
except ProcessLookupError:
|
|
95
|
+
pass
|
|
88
96
|
|
|
89
97
|
try:
|
|
90
98
|
if force:
|
|
91
|
-
|
|
92
|
-
|
|
99
|
+
if owned_by_supervisor:
|
|
100
|
+
# Force: SIGTERM the supervisor (it forwards and exits), then
|
|
101
|
+
# SIGKILL the child directly and the supervisor — neither can be
|
|
102
|
+
# orphaned. A plain SIGKILL of the supervisor alone would orphan
|
|
103
|
+
# the child, which leads its own session.
|
|
104
|
+
signal_supervisor(signal.SIGTERM)
|
|
105
|
+
signal_child(signal.SIGKILL)
|
|
106
|
+
signal_supervisor(signal.SIGKILL)
|
|
107
|
+
else:
|
|
108
|
+
_signal_pid(pid, signal.SIGKILL)
|
|
93
109
|
console.print(f"Force killed node '{alias}' (PID: {pid})")
|
|
94
110
|
else:
|
|
95
111
|
# Graceful shutdown with SIGTERM
|
|
96
|
-
|
|
112
|
+
signal_supervisor(signal.SIGTERM)
|
|
97
113
|
console.print(f"Sent termination signal to node '{alias}' (PID: {pid})")
|
|
98
114
|
|
|
99
115
|
# Wait for process to terminate gracefully
|
|
@@ -111,21 +127,26 @@ def stop_instance_by_pid(
|
|
|
111
127
|
console.print(f"[green]Node '{alias}' stopped gracefully[/green]")
|
|
112
128
|
break
|
|
113
129
|
else:
|
|
114
|
-
# Timeout reached,
|
|
130
|
+
# Timeout reached: force the child and the supervisor, never the
|
|
131
|
+
# supervisor alone (which would orphan the child).
|
|
115
132
|
print_warning(
|
|
116
133
|
f"Node '{alias}' did not stop gracefully, sending SIGKILL..."
|
|
117
134
|
)
|
|
118
|
-
|
|
135
|
+
signal_child(signal.SIGKILL)
|
|
136
|
+
signal_supervisor(signal.SIGKILL)
|
|
119
137
|
console.print(f"Force killed node '{alias}' after timeout")
|
|
120
138
|
|
|
121
|
-
#
|
|
122
|
-
|
|
139
|
+
# Only the node's own instance record is the CLI's to remove; a
|
|
140
|
+
# supervisor record is cleared by the supervisor as it exits.
|
|
141
|
+
if not owned_by_supervisor:
|
|
142
|
+
instance["instance_file"].unlink(missing_ok=True)
|
|
123
143
|
return True
|
|
124
144
|
|
|
125
145
|
except OSError as e:
|
|
126
146
|
if e.errno == 3: # No such process
|
|
127
147
|
# Process already dead, just remove instance file
|
|
128
|
-
|
|
148
|
+
if not owned_by_supervisor:
|
|
149
|
+
instance["instance_file"].unlink(missing_ok=True)
|
|
129
150
|
print_warning(f"Node '{alias}' was already stopped")
|
|
130
151
|
return True
|
|
131
152
|
else:
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
"""The node supervisor — infrastructure that owns the node child (ADR-0041 B1).
|
|
2
|
+
|
|
3
|
+
The supervisor is a long-lived process that starts ``manta_node start <alias>``
|
|
4
|
+
as its child and owns the child's restart. It is **infrastructure, not domain**:
|
|
5
|
+
it makes no decision about versions, drift, fleets or outcomes, and it imports
|
|
6
|
+
nothing domain-related from ``manta_node`` (pinned by a guard test). Making a
|
|
7
|
+
version current, verifying a candidate and rolling back are later steps (2c/2d);
|
|
8
|
+
this package is the process manager only.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
__all__ = []
|
|
@@ -0,0 +1,210 @@
|
|
|
1
|
+
"""The supervisor process manager — starts the node child and owns its restart.
|
|
2
|
+
|
|
3
|
+
The systemd unit's ``ExecStart`` becomes ``manta-supervisor <alias>``; the
|
|
4
|
+
supervisor starts ``manta_node start <alias>`` as its child, records itself as the
|
|
5
|
+
pid of record, restarts the child under a **bounded** policy, and treats a stop
|
|
6
|
+
signal as **terminal**. Two restart owners never coexist: the unit restarts only
|
|
7
|
+
the supervisor, the supervisor restarts only the child.
|
|
8
|
+
|
|
9
|
+
Spike A (manta-node-python#483) proved the shape this implements; the prototype
|
|
10
|
+
is manta-node-python#484. The guarantee that a supervisor killed outright leaves
|
|
11
|
+
no orphan is the unit's ``KillMode=control-group`` (systemd reaps the cgroup);
|
|
12
|
+
``start_new_session=True`` below is the belt-and-braces process-group path for a
|
|
13
|
+
supervisor stopped outside systemd.
|
|
14
|
+
|
|
15
|
+
**Placement.** At 2a the supervisor runs from the node's own venv, because the
|
|
16
|
+
child's interpreter *is* the supervisor's there. The migration's "supervisor
|
|
17
|
+
installed outside the node venv" is satisfied at 2c, when the child runs from the
|
|
18
|
+
version tree; ``--python`` (``child_python``) selects that interpreter, so the
|
|
19
|
+
supervisor can then live in its own venv without running the node code from the
|
|
20
|
+
wrong one.
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
from __future__ import annotations
|
|
24
|
+
|
|
25
|
+
import argparse
|
|
26
|
+
import os
|
|
27
|
+
import signal
|
|
28
|
+
import subprocess
|
|
29
|
+
import sys
|
|
30
|
+
import time
|
|
31
|
+
from pathlib import Path
|
|
32
|
+
|
|
33
|
+
from .records import (
|
|
34
|
+
supervisor_record,
|
|
35
|
+
supervisor_record_path,
|
|
36
|
+
write_json_atomic,
|
|
37
|
+
)
|
|
38
|
+
|
|
39
|
+
# A child that exits more than RESTART_LIMIT times within RESTART_WINDOW_SECONDS
|
|
40
|
+
# is a crash loop: the supervisor stops restarting and exits non-zero, so systemd
|
|
41
|
+
# sees a failed unit rather than an infinitely restarting one.
|
|
42
|
+
RESTART_LIMIT = 5
|
|
43
|
+
RESTART_WINDOW_SECONDS = 120
|
|
44
|
+
RESTART_BACKOFF_SECONDS = 2
|
|
45
|
+
|
|
46
|
+
# Bound on the supervisor's stdout stream. A long-lived supervisor reopens this
|
|
47
|
+
# on every restart, so without a cap a crash loop grows it without limit.
|
|
48
|
+
STDOUT_MAX_BYTES = 10 * 1024 * 1024
|
|
49
|
+
|
|
50
|
+
# How long a terminal stop waits for the child to wind down before SIGKILL.
|
|
51
|
+
STOP_GRACE_SECONDS = 10
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _log_dir() -> Path:
|
|
55
|
+
return Path.home() / ".manta" / "logs" / "nodes"
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
class Supervisor:
|
|
59
|
+
"""Runs one node child, restarts it (bounded), and stops it terminally."""
|
|
60
|
+
|
|
61
|
+
def __init__(
|
|
62
|
+
self, alias: str, config: str, child_python: str | None = None
|
|
63
|
+
) -> None:
|
|
64
|
+
self.alias = alias
|
|
65
|
+
self.config = config
|
|
66
|
+
# The interpreter that runs the child. Today the supervisor and the node
|
|
67
|
+
# code share one venv, so this defaults to the supervisor's own; at 2c
|
|
68
|
+
# the child runs from the version tree's interpreter, a DIFFERENT venv
|
|
69
|
+
# from the supervisor's, and the unit passes that interpreter here.
|
|
70
|
+
self.child_python = child_python or sys.executable
|
|
71
|
+
self._stop = False
|
|
72
|
+
self._child: subprocess.Popen | None = None
|
|
73
|
+
self._restarts: list[float] = []
|
|
74
|
+
|
|
75
|
+
def _write_record(self, child_pid: int | None, state: str) -> None:
|
|
76
|
+
write_json_atomic(
|
|
77
|
+
supervisor_record_path(self.alias),
|
|
78
|
+
supervisor_record(self.alias, os.getpid(), child_pid, state),
|
|
79
|
+
)
|
|
80
|
+
|
|
81
|
+
def _signal_child(self, signum: int) -> None:
|
|
82
|
+
"""Signal the child's process group, falling back to the child alone.
|
|
83
|
+
|
|
84
|
+
The child is spawned with ``start_new_session=True``, so it leads its own
|
|
85
|
+
group; signalling the group takes any grandchildren with it.
|
|
86
|
+
"""
|
|
87
|
+
if not self._child or self._child.poll() is not None:
|
|
88
|
+
return
|
|
89
|
+
try:
|
|
90
|
+
os.killpg(os.getpgid(self._child.pid), signum)
|
|
91
|
+
except (ProcessLookupError, PermissionError):
|
|
92
|
+
self._child.send_signal(signum)
|
|
93
|
+
|
|
94
|
+
def _on_signal(self, signum: int, _frame) -> None:
|
|
95
|
+
# A stop is terminal: remember it and forward it. The main loop must not
|
|
96
|
+
# restart once this is set.
|
|
97
|
+
self._stop = True
|
|
98
|
+
self._signal_child(signal.SIGTERM)
|
|
99
|
+
|
|
100
|
+
def _spawn(self) -> subprocess.Popen:
|
|
101
|
+
# `--subprocess` is the existing hidden flag: the child does not write an
|
|
102
|
+
# instance file, so exactly one writer (the supervisor) owns the records.
|
|
103
|
+
# `--foreground` keeps it attached to this process rather than detaching.
|
|
104
|
+
# `--alias` is passed explicitly: `--config` names the TOML to load, and
|
|
105
|
+
# the alias the operator sees must be the supervisor's, not whatever
|
|
106
|
+
# `identity.alias` the TOML happens to carry.
|
|
107
|
+
cmd = [
|
|
108
|
+
self.child_python,
|
|
109
|
+
"-m",
|
|
110
|
+
"manta_node",
|
|
111
|
+
"start",
|
|
112
|
+
self.config,
|
|
113
|
+
"--foreground",
|
|
114
|
+
"--subprocess",
|
|
115
|
+
"--alias",
|
|
116
|
+
self.alias,
|
|
117
|
+
]
|
|
118
|
+
log_dir = _log_dir()
|
|
119
|
+
log_dir.mkdir(parents=True, exist_ok=True)
|
|
120
|
+
# One writer per path: the supervisor owns the child's stdout/stderr
|
|
121
|
+
# stream. The child's own rotating file log is separate and untouched.
|
|
122
|
+
# A long-lived supervisor reopens this on every restart, so it is bounded
|
|
123
|
+
# like the child's own rotating handler: roll once past the cap, keeping
|
|
124
|
+
# one previous run. Roll BEFORE the open, and never while a writer holds
|
|
125
|
+
# it — a rename does not retarget an already-open fd.
|
|
126
|
+
stdout_path = log_dir / f"{self.alias}.supervisor.stdout"
|
|
127
|
+
if stdout_path.exists() and stdout_path.stat().st_size >= STDOUT_MAX_BYTES:
|
|
128
|
+
stdout_path.replace(stdout_path.with_name(f"{stdout_path.name}.1"))
|
|
129
|
+
with open(stdout_path, "a") as stream:
|
|
130
|
+
child = subprocess.Popen(
|
|
131
|
+
cmd, stdout=stream, stderr=stream, start_new_session=True
|
|
132
|
+
)
|
|
133
|
+
# Publish the pid BEFORE returning: a signal that fires between here and
|
|
134
|
+
# the caller's `_stop` check must be able to reach the child, not a gone
|
|
135
|
+
# one.
|
|
136
|
+
self._child = child
|
|
137
|
+
return child
|
|
138
|
+
|
|
139
|
+
def run(self) -> int:
|
|
140
|
+
signal.signal(signal.SIGTERM, self._on_signal)
|
|
141
|
+
signal.signal(signal.SIGINT, self._on_signal)
|
|
142
|
+
self._write_record(None, "starting")
|
|
143
|
+
|
|
144
|
+
while not self._stop:
|
|
145
|
+
# `_spawn` publishes `self._child` before returning, so a signal
|
|
146
|
+
# during the fork reaches the new pid; one arriving just after is
|
|
147
|
+
# caught by the `_stop` re-check below.
|
|
148
|
+
child = self._spawn()
|
|
149
|
+
if self._stop:
|
|
150
|
+
# A stop arrived while we forked: forward it and do not restart.
|
|
151
|
+
# Without this the loop would block on a child never signalled.
|
|
152
|
+
self._signal_child(signal.SIGTERM)
|
|
153
|
+
self._write_record(child.pid, "running")
|
|
154
|
+
child.wait()
|
|
155
|
+
rc = child.returncode
|
|
156
|
+
|
|
157
|
+
if self._stop:
|
|
158
|
+
break
|
|
159
|
+
|
|
160
|
+
now = time.monotonic()
|
|
161
|
+
self._restarts = [
|
|
162
|
+
t for t in self._restarts if now - t < RESTART_WINDOW_SECONDS
|
|
163
|
+
]
|
|
164
|
+
self._restarts.append(now)
|
|
165
|
+
if len(self._restarts) > RESTART_LIMIT:
|
|
166
|
+
self._write_record(None, "crash-loop")
|
|
167
|
+
print(
|
|
168
|
+
f"supervisor: child exited {len(self._restarts)} times in "
|
|
169
|
+
f"{RESTART_WINDOW_SECONDS}s (rc={rc}); refusing to restart",
|
|
170
|
+
file=sys.stderr,
|
|
171
|
+
)
|
|
172
|
+
return 1
|
|
173
|
+
|
|
174
|
+
self._write_record(None, "restarting")
|
|
175
|
+
time.sleep(RESTART_BACKOFF_SECONDS)
|
|
176
|
+
|
|
177
|
+
# Terminal stop: wind the child down, then clear our record so `status`
|
|
178
|
+
# does not report a dead supervisor.
|
|
179
|
+
if self._child and self._child.poll() is None:
|
|
180
|
+
self._signal_child(signal.SIGTERM)
|
|
181
|
+
try:
|
|
182
|
+
self._child.wait(timeout=STOP_GRACE_SECONDS)
|
|
183
|
+
except subprocess.TimeoutExpired:
|
|
184
|
+
self._signal_child(signal.SIGKILL)
|
|
185
|
+
supervisor_record_path(self.alias).unlink(missing_ok=True)
|
|
186
|
+
return 0
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
def main(argv: list[str] | None = None) -> int:
|
|
190
|
+
parser = argparse.ArgumentParser(
|
|
191
|
+
prog="manta-supervisor",
|
|
192
|
+
description="Own a manta_node child and its restart (ADR-0041 boundary B1).",
|
|
193
|
+
)
|
|
194
|
+
parser.add_argument("alias", help="node alias (the config name it starts)")
|
|
195
|
+
parser.add_argument("--config", help="config name (defaults to the alias)")
|
|
196
|
+
parser.add_argument(
|
|
197
|
+
"--python",
|
|
198
|
+
help=(
|
|
199
|
+
"interpreter that runs the node child (defaults to the supervisor's"
|
|
200
|
+
" own; set it when the child runs from a separate version tree, 2c)"
|
|
201
|
+
),
|
|
202
|
+
)
|
|
203
|
+
args = parser.parse_args(argv)
|
|
204
|
+
return Supervisor(
|
|
205
|
+
args.alias, args.config or args.alias, child_python=args.python
|
|
206
|
+
).run()
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
if __name__ == "__main__":
|
|
210
|
+
sys.exit(main())
|
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
"""The two on-disk records the supervisor and the node keep for one instance.
|
|
2
|
+
|
|
3
|
+
Under the split (ADR-0041 boundary B1) there are two processes and two records:
|
|
4
|
+
|
|
5
|
+
- the **node instance record**, ``<id>.json`` — node-owned content (alias,
|
|
6
|
+
config, ``log_file``, manager host). The running node writes it; the
|
|
7
|
+
supervisor never does.
|
|
8
|
+
- the **supervisor record**, ``<alias>.supervisor.json`` — supervisor-owned, and
|
|
9
|
+
the **pid of record**. Under ``--subprocess`` the child writes no instance
|
|
10
|
+
record (``start.py`` skips it), so on a supervised node this is the only record
|
|
11
|
+
present and the CLI's ``status``/``stop`` must read it.
|
|
12
|
+
|
|
13
|
+
This module has no dependency on the node's domain: the supervisor is
|
|
14
|
+
infrastructure, and a guard test pins that (``tests/.../test_supervisor_is_infrastructure``).
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
from __future__ import annotations
|
|
18
|
+
|
|
19
|
+
import json
|
|
20
|
+
import os
|
|
21
|
+
from datetime import datetime, timezone
|
|
22
|
+
from pathlib import Path
|
|
23
|
+
|
|
24
|
+
INSTANCE_DIR = Path.home() / ".manta" / "nodes" / "instances"
|
|
25
|
+
|
|
26
|
+
SUPERVISOR_SUFFIX = ".supervisor.json"
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def supervisor_record_path(alias: str) -> Path:
|
|
30
|
+
"""The supervisor record for ``alias`` (the pid of record)."""
|
|
31
|
+
return INSTANCE_DIR / f"{alias}{SUPERVISOR_SUFFIX}"
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def write_json_atomic(path: Path, data: dict) -> None:
|
|
35
|
+
"""Write ``data`` as JSON so no reader ever sees a half-written file.
|
|
36
|
+
|
|
37
|
+
A tmp file beside the target plus ``replace`` — an atomic rename on the same
|
|
38
|
+
filesystem. A reader either sees the previous record or the new one.
|
|
39
|
+
"""
|
|
40
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
41
|
+
tmp = path.with_suffix(path.suffix + ".tmp")
|
|
42
|
+
tmp.write_text(json.dumps(data, indent=2))
|
|
43
|
+
tmp.replace(path)
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def read_json(path: Path) -> dict | None:
|
|
47
|
+
"""Read a JSON object, or None if it is absent or unreadable/invalid."""
|
|
48
|
+
try:
|
|
49
|
+
return json.loads(path.read_text())
|
|
50
|
+
except (FileNotFoundError, ValueError, OSError):
|
|
51
|
+
return None
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def supervisor_record(
|
|
55
|
+
alias: str, supervisor_pid: int, child_pid: int | None, state: str
|
|
56
|
+
) -> dict:
|
|
57
|
+
"""The supervisor record's shape."""
|
|
58
|
+
return {
|
|
59
|
+
"alias": alias,
|
|
60
|
+
"supervisor_pid": supervisor_pid,
|
|
61
|
+
"child_pid": child_pid,
|
|
62
|
+
"state": state,
|
|
63
|
+
"start_time": datetime.now(timezone.utc).isoformat(),
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def is_alive(pid: int | None) -> bool:
|
|
68
|
+
"""True when ``pid`` names a live process."""
|
|
69
|
+
if not pid:
|
|
70
|
+
return False
|
|
71
|
+
try:
|
|
72
|
+
os.kill(pid, 0)
|
|
73
|
+
except OSError:
|
|
74
|
+
return False
|
|
75
|
+
return True
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def pid_of_record(alias: str) -> tuple[int | None, str]:
|
|
79
|
+
"""(pid, source) for ``alias``: the supervisor record wins when it is live.
|
|
80
|
+
|
|
81
|
+
The pid of record is the one a stop signals. Under the split that is the
|
|
82
|
+
supervisor's; the instance record (the node's own pid) is the fallback for a
|
|
83
|
+
node running without a supervisor.
|
|
84
|
+
"""
|
|
85
|
+
sup = read_json(supervisor_record_path(alias))
|
|
86
|
+
if sup and is_alive(sup.get("supervisor_pid")):
|
|
87
|
+
return sup["supervisor_pid"], "supervisor"
|
|
88
|
+
|
|
89
|
+
# Fall back to the node instance record: the supervisor may be absent (a node
|
|
90
|
+
# on the pre-split path) or dead. Match on the record's own alias, not the
|
|
91
|
+
# filename stem, which carries the per-start instance id.
|
|
92
|
+
for record in sorted(INSTANCE_DIR.glob("*.json")):
|
|
93
|
+
if record.name.endswith(SUPERVISOR_SUFFIX):
|
|
94
|
+
continue
|
|
95
|
+
data = read_json(record)
|
|
96
|
+
if data and data.get("alias") == alias and is_alive(data.get("pid")):
|
|
97
|
+
return data["pid"], "instance"
|
|
98
|
+
return None, "none"
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def list_instances() -> list[dict]:
|
|
102
|
+
"""Live instances for the CLI, one per alias: the pid of record and its source.
|
|
103
|
+
|
|
104
|
+
A supervised node has only a supervisor record; a pre-split node has only an
|
|
105
|
+
instance record. A stale record (the pid is gone) is not returned, and its
|
|
106
|
+
file is removed — the same cleanup the CLI did before the split.
|
|
107
|
+
"""
|
|
108
|
+
# Collect the two kinds separately, then resolve per alias with an EXPLICIT
|
|
109
|
+
# rule — never by filename/glob order. A supervisor record always wins over a
|
|
110
|
+
# node instance record for the same alias (as `pid_of_record` does); relying
|
|
111
|
+
# on sort order would let a legacy instance record silently override a live
|
|
112
|
+
# supervisor record whenever its config name sorts after the alias.
|
|
113
|
+
supervisors: dict[str, dict] = {}
|
|
114
|
+
instances: dict[str, dict] = {}
|
|
115
|
+
|
|
116
|
+
for record in sorted(INSTANCE_DIR.glob("*.json")):
|
|
117
|
+
data = read_json(record)
|
|
118
|
+
if data is None:
|
|
119
|
+
record.unlink(missing_ok=True)
|
|
120
|
+
continue
|
|
121
|
+
|
|
122
|
+
if record.name.endswith(SUPERVISOR_SUFFIX):
|
|
123
|
+
alias = data.get("alias")
|
|
124
|
+
pid = data.get("supervisor_pid")
|
|
125
|
+
entry = {
|
|
126
|
+
"alias": alias,
|
|
127
|
+
"instance_id": alias,
|
|
128
|
+
"config_name": alias,
|
|
129
|
+
"pid": pid,
|
|
130
|
+
# The child is the workload the operator cares about: `status`
|
|
131
|
+
# reports ITS cpu/memory, not the thin supervisor's. `pid` stays
|
|
132
|
+
# the pid of record (the supervisor) — what a stop signals.
|
|
133
|
+
"child_pid": data.get("child_pid"),
|
|
134
|
+
"workload_pid": data.get("child_pid"),
|
|
135
|
+
"source": "supervisor",
|
|
136
|
+
"start_time": data.get("start_time"),
|
|
137
|
+
"instance_file": record,
|
|
138
|
+
}
|
|
139
|
+
bucket = supervisors
|
|
140
|
+
else:
|
|
141
|
+
instance_id = data.get("instance_id", record.stem)
|
|
142
|
+
alias = data.get("alias", instance_id)
|
|
143
|
+
pid = data.get("pid")
|
|
144
|
+
entry = {
|
|
145
|
+
"alias": alias,
|
|
146
|
+
"instance_id": instance_id,
|
|
147
|
+
"config_name": data.get("config_name", "unknown"),
|
|
148
|
+
"pid": pid,
|
|
149
|
+
"child_pid": None,
|
|
150
|
+
"workload_pid": pid,
|
|
151
|
+
"source": "instance",
|
|
152
|
+
"start_time": data.get("start_time"),
|
|
153
|
+
"instance_file": record,
|
|
154
|
+
}
|
|
155
|
+
bucket = instances
|
|
156
|
+
|
|
157
|
+
if not alias or not is_alive(pid):
|
|
158
|
+
# A stale record is not a running instance: remove it so the next
|
|
159
|
+
# `status` does not keep reporting a dead pid.
|
|
160
|
+
record.unlink(missing_ok=True)
|
|
161
|
+
continue
|
|
162
|
+
|
|
163
|
+
bucket[alias] = entry
|
|
164
|
+
|
|
165
|
+
# Explicit precedence: the supervisor record wins for an alias that has both.
|
|
166
|
+
merged = {**instances, **supervisors}
|
|
167
|
+
return [merged[alias] for alias in sorted(merged)]
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: manta_node
|
|
3
|
-
Version: 0.6b5.
|
|
3
|
+
Version: 0.6b5.dev699
|
|
4
4
|
Summary: Node side software of the Manta platform. It is responsible for the communication with the Manta Core and the execution of the tasks.
|
|
5
5
|
Author-email: Hugo Miralles <hugo.miralles@manta-tech.io>, Matthew Thompson <matthew.thompson@manta-tech.io>
|
|
6
6
|
License-Expression: AGPL-3.0-or-later
|
|
@@ -5,7 +5,7 @@ manta_node/task_manager.py,sha256=xRE-yKdUW0FFR8vtX1jDwl1AxTbqVD_JS7QVR62GA00,38
|
|
|
5
5
|
manta_node/tasks.py,sha256=o837Hi-KFVJg9LF5iOtE8oD6UuTOfrEPWR-JOdhRGeY,6983
|
|
6
6
|
manta_node/utils.py,sha256=Hw4Iw2jYQYsfVLVJuBMt-_rMFBeml3nCDE2jiiXc8YM,1155
|
|
7
7
|
manta_node/cli/__init__.py,sha256=arzO3mXDJ-E2yeeytbZfN7mCswwxdQ43_QjyeL2E5yE,134
|
|
8
|
-
manta_node/cli/_version_generated.py,sha256=
|
|
8
|
+
manta_node/cli/_version_generated.py,sha256=170TvSZcSvg_7ZR2FQR00hnIEL-ogAwT4LU4dOtsmm4,275
|
|
9
9
|
manta_node/cli/config_manager.py,sha256=BCju9zqzraviL3KrzRnB-vZQUY__Q5kuEqb7EdB8YdI,4409
|
|
10
10
|
manta_node/cli/main.py,sha256=dZa6zrypWBvpN8hd4qpxevvlyNFhxKINCg1537NJ36w,4318
|
|
11
11
|
manta_node/cli/version.py,sha256=3lrnIWpOosszBN-VsCBVLxDgpvCC7IB0aDIKx_3e7hI,3115
|
|
@@ -15,8 +15,8 @@ manta_node/cli/commands/config.py,sha256=BiarCD0S8rcNYbio4RLzDvUS7A50s5dd5F4TiZq
|
|
|
15
15
|
manta_node/cli/commands/logs.py,sha256=cXOSdwhDIJAiKMmJYdzrnpapIxudxVdMzYT-KAWDKVk,5259
|
|
16
16
|
manta_node/cli/commands/network.py,sha256=QqncD6vbNlg65ptQdFYexEUMALQm8UwcftbW3sfQrjw,42070
|
|
17
17
|
manta_node/cli/commands/start.py,sha256=HuIovoL3a1yQuEIIZMmvRUu3mq1cKJlFZgBf-U2zS7A,25637
|
|
18
|
-
manta_node/cli/commands/status.py,sha256=
|
|
19
|
-
manta_node/cli/commands/stop.py,sha256=
|
|
18
|
+
manta_node/cli/commands/status.py,sha256=I3z4bee0uqobEhA4akJ3OErRJ7_UcieW6tvLP3pyZ-A,5608
|
|
19
|
+
manta_node/cli/commands/stop.py,sha256=bGsvDrTiIvDN20HMsXKQYg83iWRaAnhbJ8xcalJwre0,9117
|
|
20
20
|
manta_node/domain/__init__.py,sha256=uyJXDf0QgfGCsXLMggqKCB5QV9oneYJ4iM-REaJGXDM,217
|
|
21
21
|
manta_node/domain/task_lifecycle.py,sha256=CmaK6OL4KNSy81YYosINduVQQW3LjycAwmfpRmXrldk,3323
|
|
22
22
|
manta_node/domain/value_objects/__init__.py,sha256=Sut2I8iSNNYBtI42-tfKXJ8aod2HV_k_N_uM_KWwjqs,135
|
|
@@ -51,9 +51,12 @@ manta_node/infrastructure/process/executor.py,sha256=Ab1XvNuelEGCD5KdZLgYYnpjYZ5
|
|
|
51
51
|
manta_node/infrastructure/security/__init__.py,sha256=lI6vkbUmfmt0m363iO4HzuzVNoQAc7Clp2R5vAhZbgE,64
|
|
52
52
|
manta_node/infrastructure/security/auth_agent.py,sha256=WkPTg_g0gix7xomb_on_Ft3TNX0FpeqS7gwgJW9mTvk,11037
|
|
53
53
|
manta_node/infrastructure/security/token_provider.py,sha256=u3bcbQYNBskFzunO-wOSZpGkDN_CU1BCB5HUbZTC4Vg,1832
|
|
54
|
-
manta_node
|
|
55
|
-
manta_node
|
|
56
|
-
manta_node
|
|
57
|
-
manta_node-0.6b5.
|
|
58
|
-
manta_node-0.6b5.
|
|
59
|
-
manta_node-0.6b5.
|
|
54
|
+
manta_node/supervisor/__init__.py,sha256=kgdoNce1tM4e7jvdxkdmnNJV1L92-NI2cRfYSdm9vsw,542
|
|
55
|
+
manta_node/supervisor/process.py,sha256=is3phcIKF9AdRORqXK7_rwiiqCF_4t4txXu-vGtUOE8,8655
|
|
56
|
+
manta_node/supervisor/records.py,sha256=fUlL1SfmWpGnCbJ9956cuy0H3a7-aEF-B-naqbJiYoU,6405
|
|
57
|
+
manta_node-0.6b5.dev699.dist-info/licenses/LICENSE,sha256=9mBgLhiB9goQPQjg9XQJ6FjgVxYw9Q8lyOZYkPGwF_c,34484
|
|
58
|
+
manta_node-0.6b5.dev699.dist-info/METADATA,sha256=ydzPrhwmNrMaeIIvaXzsr2QRoBk2O6L9ZoM0zlfS3tc,4231
|
|
59
|
+
manta_node-0.6b5.dev699.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
|
|
60
|
+
manta_node-0.6b5.dev699.dist-info/entry_points.txt,sha256=uCwggRApalLEsYs1YxwTBN0-e9oMJAkMoBp01HaSdoY,110
|
|
61
|
+
manta_node-0.6b5.dev699.dist-info/top_level.txt,sha256=nbT5J5WETuXM9UPoQckl29uUOq8wMYdRb9QDx5kAFeo,11
|
|
62
|
+
manta_node-0.6b5.dev699.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|