sentinel-pm 0.2.6__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sentinel_cli/__init__.py +29 -0
- sentinel_cli/cluster.py +80 -0
- sentinel_cli/common.py +16 -0
- sentinel_cli/daemon.py +255 -0
- sentinel_cli/group.py +332 -0
- sentinel_cli/main.py +488 -0
- sentinel_cli/metrics.py +97 -0
- sentinel_cli/port.py +86 -0
- sentinel_cli/py.typed +0 -0
- sentinel_cli/remote.py +144 -0
- sentinel_cli/startup.py +318 -0
- sentinel_core/__init__.py +1 -0
- sentinel_core/cluster.py +121 -0
- sentinel_core/dependencies.py +106 -0
- sentinel_core/env.py +149 -0
- sentinel_core/format.py +31 -0
- sentinel_core/health.py +81 -0
- sentinel_core/logs.py +176 -0
- sentinel_core/metrics.py +164 -0
- sentinel_core/metrics_server.py +44 -0
- sentinel_core/models.py +221 -0
- sentinel_core/options.py +118 -0
- sentinel_core/process.py +650 -0
- sentinel_core/py.typed +0 -0
- sentinel_core/registries.py +218 -0
- sentinel_core/remote.py +76 -0
- sentinel_core/restart_monitor.py +217 -0
- sentinel_core/state.py +181 -0
- sentinel_core/state_store.py +149 -0
- sentinel_pm-0.2.6.dist-info/METADATA +825 -0
- sentinel_pm-0.2.6.dist-info/RECORD +34 -0
- sentinel_pm-0.2.6.dist-info/WHEEL +4 -0
- sentinel_pm-0.2.6.dist-info/entry_points.txt +2 -0
- sentinel_pm-0.2.6.dist-info/licenses/LICENSE +20 -0
sentinel_cli/__init__.py
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
"""CLI interface for Sentinel"""
|
|
2
|
+
|
|
3
|
+
import typer
|
|
4
|
+
|
|
5
|
+
from .cluster import cluster_app
|
|
6
|
+
from .daemon import daemon_app
|
|
7
|
+
from .group import group_app
|
|
8
|
+
from .main import register_main_commands
|
|
9
|
+
from .metrics import metrics_app
|
|
10
|
+
from .port import port_app
|
|
11
|
+
from .remote import remote_app
|
|
12
|
+
from .startup import startup_app
|
|
13
|
+
|
|
14
|
+
app = typer.Typer(
|
|
15
|
+
name="sentinel",
|
|
16
|
+
help="A lightweight process orchestrator CLI",
|
|
17
|
+
no_args_is_help=True,
|
|
18
|
+
)
|
|
19
|
+
|
|
20
|
+
register_main_commands(app)
|
|
21
|
+
app.add_typer(port_app, name="port")
|
|
22
|
+
app.add_typer(cluster_app, name="cluster")
|
|
23
|
+
app.add_typer(group_app, name="group")
|
|
24
|
+
app.add_typer(daemon_app, name="daemon")
|
|
25
|
+
app.add_typer(startup_app, name="startup")
|
|
26
|
+
app.add_typer(metrics_app, name="metrics")
|
|
27
|
+
app.add_typer(remote_app, name="remote")
|
|
28
|
+
|
|
29
|
+
__all__ = ["app"]
|
sentinel_cli/cluster.py
ADDED
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
"""Cluster management commands"""
|
|
2
|
+
|
|
3
|
+
from typing import Annotated
|
|
4
|
+
|
|
5
|
+
import typer
|
|
6
|
+
from rich.console import Console
|
|
7
|
+
from rich.markup import escape
|
|
8
|
+
from rich.table import Table
|
|
9
|
+
|
|
10
|
+
from sentinel_core.cluster import cluster_members, cluster_spec, scale_cluster
|
|
11
|
+
from sentinel_core.models import ProcessInfo
|
|
12
|
+
from sentinel_core.process import ProcessIdentityError, managed_process
|
|
13
|
+
|
|
14
|
+
from .common import load_state
|
|
15
|
+
|
|
16
|
+
console = Console()
|
|
17
|
+
cluster_app = typer.Typer(
|
|
18
|
+
name="cluster",
|
|
19
|
+
help="Process cluster management",
|
|
20
|
+
no_args_is_help=True,
|
|
21
|
+
)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def _running(instance: ProcessInfo) -> bool:
|
|
25
|
+
try:
|
|
26
|
+
return managed_process(instance) is not None
|
|
27
|
+
except ProcessIdentityError:
|
|
28
|
+
return False
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
@cluster_app.command("list")
|
|
32
|
+
def cluster_list() -> None:
|
|
33
|
+
"""List clusters, their instance ranges and how many instances are running"""
|
|
34
|
+
state = load_state()
|
|
35
|
+
names = sorted({info.cluster for info in state.list_processes() if info.cluster})
|
|
36
|
+
|
|
37
|
+
if not names:
|
|
38
|
+
console.print("[dim]No clusters found[/]")
|
|
39
|
+
return
|
|
40
|
+
|
|
41
|
+
table = Table(show_header=True, header_style="bold")
|
|
42
|
+
table.add_column("CLUSTER", style="bold")
|
|
43
|
+
table.add_column("RUNNING", justify="right")
|
|
44
|
+
table.add_column("INSTANCES", justify="right")
|
|
45
|
+
table.add_column("RANGE", justify="right")
|
|
46
|
+
|
|
47
|
+
for name in names:
|
|
48
|
+
members = cluster_members(state, name)
|
|
49
|
+
running = sum(1 for member in members if _running(member))
|
|
50
|
+
table.add_row(
|
|
51
|
+
escape(name),
|
|
52
|
+
str(running),
|
|
53
|
+
str(len(members)),
|
|
54
|
+
cluster_spec(members).describe(),
|
|
55
|
+
)
|
|
56
|
+
|
|
57
|
+
console.print(table)
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
@cluster_app.command("scale")
|
|
61
|
+
def cluster_scale(
|
|
62
|
+
name: Annotated[str, typer.Argument(help="Cluster name")],
|
|
63
|
+
count: Annotated[int, typer.Argument(help="Desired number of instances")],
|
|
64
|
+
force: Annotated[bool, typer.Option("--force", "-f", help="Force kill scaled-down instances")] = False,
|
|
65
|
+
) -> None:
|
|
66
|
+
"""Grow or shrink a cluster within its recorded range"""
|
|
67
|
+
state = load_state()
|
|
68
|
+
|
|
69
|
+
try:
|
|
70
|
+
result = scale_cluster(state, name, count, force=force)
|
|
71
|
+
except ValueError as error:
|
|
72
|
+
console.print(f"[red]✗[/] {escape(str(error))}")
|
|
73
|
+
raise typer.Exit(1) from None
|
|
74
|
+
|
|
75
|
+
for info in result.stopped:
|
|
76
|
+
console.print(f"[green]✓[/] Stopped [bold]{escape(info.name)}[/]")
|
|
77
|
+
for info in result.started:
|
|
78
|
+
console.print(f"[green]✓[/] Started [bold]{escape(info.name)}[/] (id: {info.id}, pid: {info.pid})")
|
|
79
|
+
|
|
80
|
+
console.print(f"[green]✓[/] Cluster [bold]{escape(name)}[/] now runs {count} instance(s)")
|
sentinel_cli/common.py
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
"""Shared CLI helpers"""
|
|
2
|
+
|
|
3
|
+
from rich.console import Console
|
|
4
|
+
from rich.markup import escape
|
|
5
|
+
|
|
6
|
+
from sentinel_core.state import State
|
|
7
|
+
|
|
8
|
+
console = Console(stderr=True)
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def load_state() -> State:
|
|
12
|
+
"""Construct the CLI state, surfacing any state-load warnings."""
|
|
13
|
+
state = State()
|
|
14
|
+
for warning in state.load_warnings:
|
|
15
|
+
console.print(f"[red]✗[/] {escape(warning)}")
|
|
16
|
+
return state
|
sentinel_cli/daemon.py
ADDED
|
@@ -0,0 +1,255 @@
|
|
|
1
|
+
"""Daemon commands for continuous process monitoring"""
|
|
2
|
+
|
|
3
|
+
import fcntl
|
|
4
|
+
import logging
|
|
5
|
+
import os
|
|
6
|
+
import shutil
|
|
7
|
+
import signal
|
|
8
|
+
import subprocess
|
|
9
|
+
import sys
|
|
10
|
+
import time
|
|
11
|
+
from collections.abc import Generator
|
|
12
|
+
from contextlib import contextmanager
|
|
13
|
+
from logging.handlers import RotatingFileHandler
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
|
|
16
|
+
import psutil
|
|
17
|
+
import typer
|
|
18
|
+
from rich.console import Console
|
|
19
|
+
from rich.markup import escape
|
|
20
|
+
|
|
21
|
+
from sentinel_core.logs import DEFAULT_LOG_ROTATION_BACKUPS, DEFAULT_LOG_ROTATION_MAX_BYTES
|
|
22
|
+
from sentinel_core.restart_monitor import RestartMonitor
|
|
23
|
+
from sentinel_core.state import STATE_DIR, ProcessInfo
|
|
24
|
+
|
|
25
|
+
console = Console()
|
|
26
|
+
daemon_app = typer.Typer(name="daemon", help="Manage the restart monitor daemon", no_args_is_help=True)
|
|
27
|
+
|
|
28
|
+
logger = logging.getLogger(__name__)
|
|
29
|
+
|
|
30
|
+
DAEMON_PID_FILE: Path = STATE_DIR / "daemon.pid"
|
|
31
|
+
|
|
32
|
+
DAEMON_RUN_ARG = "monitor"
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _daemon_subprocess_env() -> dict[str, str]:
|
|
36
|
+
"""Environment for the spawned daemon, carrying module search visibility.
|
|
37
|
+
|
|
38
|
+
Distributions such as Nix wrap the console script and add their store
|
|
39
|
+
site directories to sys.path at startup, so a bare
|
|
40
|
+
`python -m sentinel_cli.daemon` child cannot import the package or its
|
|
41
|
+
dependencies. Propagating the current interpreter's importable path
|
|
42
|
+
entries keeps the daemon importable.
|
|
43
|
+
"""
|
|
44
|
+
env = {**os.environ, "SENTINEL_STATE_DIR": str(STATE_DIR)}
|
|
45
|
+
extra_paths = [p for p in sys.path if p and Path(p).is_dir() and p not in (os.getcwd(), "")]
|
|
46
|
+
if extra_paths:
|
|
47
|
+
existing = env.get("PYTHONPATH", "")
|
|
48
|
+
env["PYTHONPATH"] = os.pathsep.join([*extra_paths, existing]) if existing else os.pathsep.join(extra_paths)
|
|
49
|
+
return env
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def _get_daemon_pid() -> int | None:
|
|
53
|
+
"""Read the daemon PID from the pid file, or None if not running."""
|
|
54
|
+
if not DAEMON_PID_FILE.exists():
|
|
55
|
+
return None
|
|
56
|
+
|
|
57
|
+
try:
|
|
58
|
+
pid = int(DAEMON_PID_FILE.read_text().strip())
|
|
59
|
+
except ValueError, OSError:
|
|
60
|
+
# A half-written file belongs to a daemon that is still starting; the
|
|
61
|
+
# caller polls, so leave the child's file alone instead of deleting it.
|
|
62
|
+
return None
|
|
63
|
+
|
|
64
|
+
try:
|
|
65
|
+
os.kill(pid, 0)
|
|
66
|
+
except OSError:
|
|
67
|
+
DAEMON_PID_FILE.unlink(missing_ok=True)
|
|
68
|
+
return None
|
|
69
|
+
|
|
70
|
+
if not _is_daemon_process(pid):
|
|
71
|
+
DAEMON_PID_FILE.unlink(missing_ok=True)
|
|
72
|
+
return None
|
|
73
|
+
|
|
74
|
+
return pid
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def _is_daemon_process(pid: int) -> bool:
|
|
78
|
+
"""Guard against PID reuse: verify the pid actually runs the sentinel daemon."""
|
|
79
|
+
try:
|
|
80
|
+
cmdline = psutil.Process(pid).cmdline()
|
|
81
|
+
except psutil.Error:
|
|
82
|
+
return False
|
|
83
|
+
return any("sentinel_cli.daemon" in part for part in cmdline)
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def _write_daemon_pid(pid: int) -> None:
|
|
87
|
+
"""Publish the pid atomically so a poll never sees a truncated file."""
|
|
88
|
+
STATE_DIR.mkdir(mode=0o700, parents=True, exist_ok=True)
|
|
89
|
+
temporary = DAEMON_PID_FILE.with_name(f"{DAEMON_PID_FILE.name}.tmp")
|
|
90
|
+
temporary.write_text(str(pid))
|
|
91
|
+
os.replace(temporary, DAEMON_PID_FILE)
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def _rotate_daemon_log(source: str, destination: str) -> None:
|
|
95
|
+
"""Rotate the log in place, keeping the inherited stderr descriptor valid.
|
|
96
|
+
|
|
97
|
+
Copying and then truncating keeps stderr pointing at the live file, at the
|
|
98
|
+
cost of any bytes written between the two steps.
|
|
99
|
+
"""
|
|
100
|
+
shutil.copy2(source, destination)
|
|
101
|
+
with open(source, "r+b") as active:
|
|
102
|
+
active.truncate(0)
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def _configure_daemon_logging() -> None:
|
|
106
|
+
STATE_DIR.mkdir(mode=0o700, parents=True, exist_ok=True)
|
|
107
|
+
handler = RotatingFileHandler(
|
|
108
|
+
STATE_DIR / "daemon.log",
|
|
109
|
+
maxBytes=DEFAULT_LOG_ROTATION_MAX_BYTES,
|
|
110
|
+
backupCount=DEFAULT_LOG_ROTATION_BACKUPS,
|
|
111
|
+
)
|
|
112
|
+
handler.rotator = _rotate_daemon_log
|
|
113
|
+
logging.basicConfig(
|
|
114
|
+
level=logging.INFO,
|
|
115
|
+
format="%(asctime)s %(levelname)s %(name)s: %(message)s",
|
|
116
|
+
handlers=[handler],
|
|
117
|
+
)
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
@contextmanager
|
|
121
|
+
def _daemon_lock(name: str, blocking: bool = True) -> Generator[None]:
|
|
122
|
+
STATE_DIR.mkdir(mode=0o700, parents=True, exist_ok=True)
|
|
123
|
+
with (STATE_DIR / name).open("a") as lock:
|
|
124
|
+
fcntl.flock(lock, fcntl.LOCK_EX | (0 if blocking else fcntl.LOCK_NB))
|
|
125
|
+
try:
|
|
126
|
+
yield
|
|
127
|
+
finally:
|
|
128
|
+
fcntl.flock(lock, fcntl.LOCK_UN)
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def _daemon_main_loop() -> None:
|
|
132
|
+
with _daemon_lock("daemon.lock", blocking=False):
|
|
133
|
+
_run_monitor()
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def _run_monitor() -> None:
|
|
137
|
+
"""Main loop for the daemon process (runs in background)."""
|
|
138
|
+
_configure_daemon_logging()
|
|
139
|
+
|
|
140
|
+
should_exit = False
|
|
141
|
+
|
|
142
|
+
def signal_handler(signum: int, frame: object) -> None:
|
|
143
|
+
nonlocal should_exit
|
|
144
|
+
should_exit = True
|
|
145
|
+
|
|
146
|
+
signal.signal(signal.SIGTERM, signal_handler)
|
|
147
|
+
signal.signal(signal.SIGINT, signal_handler)
|
|
148
|
+
|
|
149
|
+
monitor = RestartMonitor(check_interval=5.0)
|
|
150
|
+
|
|
151
|
+
def on_restart(old_info: ProcessInfo, new_info: ProcessInfo) -> None:
|
|
152
|
+
logger.info(f"Restarted {new_info.name} (old_pid: {old_info.pid}, new_pid: {new_info.pid})")
|
|
153
|
+
|
|
154
|
+
monitor.set_restart_callback(on_restart)
|
|
155
|
+
monitor.start()
|
|
156
|
+
_write_daemon_pid(os.getpid())
|
|
157
|
+
logger.info(f"Daemon started (pid: {os.getpid()})")
|
|
158
|
+
|
|
159
|
+
try:
|
|
160
|
+
while not should_exit:
|
|
161
|
+
time.sleep(1)
|
|
162
|
+
finally:
|
|
163
|
+
monitor.stop()
|
|
164
|
+
if DAEMON_PID_FILE.exists() and DAEMON_PID_FILE.read_text().strip() == str(os.getpid()):
|
|
165
|
+
DAEMON_PID_FILE.unlink(missing_ok=True)
|
|
166
|
+
logger.info("Daemon stopped")
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
@daemon_app.command()
|
|
170
|
+
def start() -> None:
|
|
171
|
+
"""Start the restart monitor daemon"""
|
|
172
|
+
with _daemon_lock("daemon-control.lock"):
|
|
173
|
+
existing_pid = _get_daemon_pid()
|
|
174
|
+
if existing_pid:
|
|
175
|
+
console.print(f"[yellow]⚠[/] Daemon already running (pid: {existing_pid})")
|
|
176
|
+
return
|
|
177
|
+
try:
|
|
178
|
+
with (STATE_DIR / "daemon.log").open("a") as stderr:
|
|
179
|
+
proc = subprocess.Popen(
|
|
180
|
+
[sys.executable, "-m", "sentinel_cli.daemon", DAEMON_RUN_ARG],
|
|
181
|
+
start_new_session=True,
|
|
182
|
+
stdin=subprocess.DEVNULL,
|
|
183
|
+
stdout=subprocess.DEVNULL,
|
|
184
|
+
stderr=stderr,
|
|
185
|
+
env=_daemon_subprocess_env(),
|
|
186
|
+
)
|
|
187
|
+
except OSError as e:
|
|
188
|
+
console.print(f"[red]✗[/] Failed to start daemon: {escape(str(e))}")
|
|
189
|
+
raise typer.Exit(1) from None
|
|
190
|
+
|
|
191
|
+
deadline = time.monotonic() + 10
|
|
192
|
+
while proc.poll() is None and time.monotonic() < deadline:
|
|
193
|
+
# Only the child publishes its PID, after initialization succeeds.
|
|
194
|
+
if _get_daemon_pid() == proc.pid:
|
|
195
|
+
console.print(f"[green]✓[/] Started daemon (pid: {proc.pid})")
|
|
196
|
+
return
|
|
197
|
+
time.sleep(0.05)
|
|
198
|
+
if proc.poll() is None:
|
|
199
|
+
proc.kill()
|
|
200
|
+
proc.wait()
|
|
201
|
+
console.print(f"[red]✗[/] Daemon failed to start. See {escape(str(STATE_DIR / 'daemon.log'))}")
|
|
202
|
+
raise typer.Exit(1)
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
@daemon_app.command()
|
|
206
|
+
def stop() -> None:
|
|
207
|
+
"""Stop the restart monitor daemon"""
|
|
208
|
+
with _daemon_lock("daemon-control.lock"):
|
|
209
|
+
pid = _get_daemon_pid()
|
|
210
|
+
if not pid:
|
|
211
|
+
console.print("[dim]Daemon is not running[/]")
|
|
212
|
+
return
|
|
213
|
+
try:
|
|
214
|
+
proc = psutil.Process(pid)
|
|
215
|
+
proc.terminate()
|
|
216
|
+
# The daemon leaves its loop within a second but may spend up to ten
|
|
217
|
+
# seconds joining the monitor thread.
|
|
218
|
+
try:
|
|
219
|
+
proc.wait(timeout=15)
|
|
220
|
+
except psutil.TimeoutExpired:
|
|
221
|
+
proc.kill()
|
|
222
|
+
proc.wait(timeout=5)
|
|
223
|
+
except psutil.NoSuchProcess:
|
|
224
|
+
pass
|
|
225
|
+
except (OSError, psutil.Error) as e:
|
|
226
|
+
console.print(f"[red]✗[/] Failed to stop daemon: {escape(str(e))}")
|
|
227
|
+
raise typer.Exit(1) from None
|
|
228
|
+
console.print(f"[green]✓[/] Stopped daemon (pid: {pid})")
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
@daemon_app.command()
|
|
232
|
+
def status() -> None:
|
|
233
|
+
"""Show daemon status"""
|
|
234
|
+
pid = _get_daemon_pid()
|
|
235
|
+
if pid:
|
|
236
|
+
console.print(f"[green]●[/] Daemon is running (pid: {pid})")
|
|
237
|
+
else:
|
|
238
|
+
console.print("[dim]○[/] Daemon is not running")
|
|
239
|
+
|
|
240
|
+
|
|
241
|
+
def is_daemon_running() -> bool:
|
|
242
|
+
"""Check if the daemon is currently running."""
|
|
243
|
+
return _get_daemon_pid() is not None
|
|
244
|
+
|
|
245
|
+
|
|
246
|
+
def main() -> None:
|
|
247
|
+
"""Module entry point: run the background monitor loop or the CLI app."""
|
|
248
|
+
if sys.argv[1:2] == [DAEMON_RUN_ARG]:
|
|
249
|
+
_daemon_main_loop()
|
|
250
|
+
else:
|
|
251
|
+
daemon_app()
|
|
252
|
+
|
|
253
|
+
|
|
254
|
+
if __name__ == "__main__":
|
|
255
|
+
main()
|