hoist-cli 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- hoist/__init__.py +4 -0
- hoist/__main__.py +4 -0
- hoist/cli.py +1672 -0
- hoist/config.py +131 -0
- hoist/containers.py +781 -0
- hoist/dash/__init__.py +0 -0
- hoist/dash/collect.py +497 -0
- hoist/dash/fleet.py +140 -0
- hoist/dash/jobs.py +111 -0
- hoist/dash/server.py +979 -0
- hoist/dash/static/app.css +595 -0
- hoist/dash/static/app.js +437 -0
- hoist/dash/static/app.ui.js +501 -0
- hoist/dash/static/app.views.js +1057 -0
- hoist/dash/static/favicon.svg +1 -0
- hoist/dash/static/fonts/IBMPlexMono-400.woff2 +0 -0
- hoist/dash/static/fonts/IBMPlexMono-500.woff2 +0 -0
- hoist/dash/static/fonts/IBMPlexSans.woff2 +0 -0
- hoist/dash/static/fonts/OFL.txt +93 -0
- hoist/dash/static/fonts/fonts.css +3 -0
- hoist/dash/static/index.html +41 -0
- hoist/dash/static/login.html +41 -0
- hoist/dash/static/vendor/XTERM-LICENSE +21 -0
- hoist/dash/static/vendor/addon-fit.js +2 -0
- hoist/dash/static/vendor/xterm.css +218 -0
- hoist/dash/static/vendor/xterm.js +2 -0
- hoist/dash/team.py +241 -0
- hoist/dash/terminal.py +164 -0
- hoist/detect.py +159 -0
- hoist/front.py +457 -0
- hoist/notify.py +111 -0
- hoist/qr.py +417 -0
- hoist/service.py +210 -0
- hoist/source.py +410 -0
- hoist/tunnel.py +376 -0
- hoist/ui.py +101 -0
- hoist_cli-0.1.0.dist-info/METADATA +524 -0
- hoist_cli-0.1.0.dist-info/RECORD +42 -0
- hoist_cli-0.1.0.dist-info/WHEEL +5 -0
- hoist_cli-0.1.0.dist-info/entry_points.txt +2 -0
- hoist_cli-0.1.0.dist-info/licenses/LICENSE +21 -0
- hoist_cli-0.1.0.dist-info/top_level.txt +1 -0
hoist/dash/__init__.py
ADDED
|
File without changes
|
hoist/dash/collect.py
ADDED
|
@@ -0,0 +1,497 @@
|
|
|
1
|
+
"""Everything the dashboard shows, gathered off the request path.
|
|
2
|
+
|
|
3
|
+
A background sampler reads systemd, /proc and the tunnel config every few
|
|
4
|
+
seconds and probes each public URL every half minute, so a page load is a
|
|
5
|
+
dictionary lookup rather than a dozen subprocesses. History is kept in
|
|
6
|
+
memory only: an hour of samples, enough to see what a deploy did.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import json
|
|
12
|
+
import os
|
|
13
|
+
import shutil
|
|
14
|
+
import socket
|
|
15
|
+
import subprocess
|
|
16
|
+
import threading
|
|
17
|
+
import time
|
|
18
|
+
import urllib.error
|
|
19
|
+
import urllib.request
|
|
20
|
+
from collections import deque
|
|
21
|
+
from pathlib import Path
|
|
22
|
+
|
|
23
|
+
from .. import config, containers, detect, notify, service, source, tunnel
|
|
24
|
+
|
|
25
|
+
SAMPLE_EVERY = 3.0 # seconds between resource samples while someone is watching
|
|
26
|
+
IDLE_EVERY = 30.0 # ...and when nobody has the dashboard open (each sample spawns
|
|
27
|
+
# systemctl and docker, which was most of the idle CPU)
|
|
28
|
+
WATCHING_FOR = 30.0 # a client counts as watching for this long after its last request
|
|
29
|
+
PROBE_EVERY = 30.0 # seconds between public URL checks while they are being used:
|
|
30
|
+
# someone is watching, or a notification webhook needs downtime alerts.
|
|
31
|
+
# Otherwise nothing reads the results, so they are not taken (they
|
|
32
|
+
# were most of the idle CPU).
|
|
33
|
+
KEEP_SAMPLES = 1200 # one hour at SAMPLE_EVERY
|
|
34
|
+
KEEP_PROBES = 120 # one hour at PROBE_EVERY
|
|
35
|
+
|
|
36
|
+
UNIT_PROPS = (
|
|
37
|
+
"Id", "ActiveState", "SubState", "MainPID", "NRestarts",
|
|
38
|
+
"ActiveEnterTimestamp", "MemoryCurrent", "CPUUsageNSec",
|
|
39
|
+
)
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
# --- low-level readers ----------------------------------------------------------
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def _int(value: str | None) -> int | None:
|
|
46
|
+
try:
|
|
47
|
+
number = int(value or "")
|
|
48
|
+
except ValueError:
|
|
49
|
+
return None
|
|
50
|
+
# systemd reports "not tracked" as UINT64_MAX
|
|
51
|
+
return None if number >= 2**63 else number
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def unit_props(units: list[str]) -> dict[str, dict[str, str]]:
|
|
55
|
+
"""`systemctl --user show` for many units in one call."""
|
|
56
|
+
if not units:
|
|
57
|
+
return {}
|
|
58
|
+
result = subprocess.run(
|
|
59
|
+
["systemctl", "--user", "show", "-p", ",".join(UNIT_PROPS), *units],
|
|
60
|
+
capture_output=True, text=True,
|
|
61
|
+
)
|
|
62
|
+
out: dict[str, dict[str, str]] = {}
|
|
63
|
+
for block in result.stdout.strip().split("\n\n"):
|
|
64
|
+
props = dict(line.split("=", 1) for line in block.splitlines() if "=" in line)
|
|
65
|
+
if props.get("Id"):
|
|
66
|
+
out[props["Id"]] = props
|
|
67
|
+
return out
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def _cpu_times() -> tuple[int, int]:
|
|
71
|
+
"""(busy, total) jiffies from the first line of /proc/stat."""
|
|
72
|
+
with open("/proc/stat") as fh:
|
|
73
|
+
fields = [int(x) for x in fh.readline().split()[1:]]
|
|
74
|
+
idle = fields[3] + (fields[4] if len(fields) > 4 else 0)
|
|
75
|
+
total = sum(fields[:8])
|
|
76
|
+
return total - idle, total
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def net_bytes() -> tuple[int, int]:
|
|
80
|
+
"""Total received/sent bytes on real interfaces (not loopback or Docker's)."""
|
|
81
|
+
rx = tx = 0
|
|
82
|
+
with open("/proc/net/dev") as fh:
|
|
83
|
+
for line in fh.readlines()[2:]:
|
|
84
|
+
name, _, data = line.partition(":")
|
|
85
|
+
name = name.strip()
|
|
86
|
+
if name == "lo" or name.startswith(("veth", "docker", "br-", "virbr")):
|
|
87
|
+
continue
|
|
88
|
+
fields = data.split()
|
|
89
|
+
rx += int(fields[0])
|
|
90
|
+
tx += int(fields[8])
|
|
91
|
+
return rx, tx
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def meminfo() -> dict[str, int]:
|
|
95
|
+
info: dict[str, int] = {}
|
|
96
|
+
with open("/proc/meminfo") as fh:
|
|
97
|
+
for line in fh:
|
|
98
|
+
key, _, rest = line.partition(":")
|
|
99
|
+
info[key] = int(rest.split()[0]) * 1024
|
|
100
|
+
return info
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def uptime_seconds() -> float:
|
|
104
|
+
with open("/proc/uptime") as fh:
|
|
105
|
+
return float(fh.read().split()[0])
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def _since(timestamp: str) -> float | None:
|
|
109
|
+
"""Parse systemd's 'Sat 2026-11-07 16:11:56 IST' into epoch seconds."""
|
|
110
|
+
if not timestamp or timestamp == "n/a":
|
|
111
|
+
return None
|
|
112
|
+
parts = timestamp.split()
|
|
113
|
+
try: # systemd prints local time; drop the weekday and zone name
|
|
114
|
+
parsed = time.strptime(" ".join(parts[1:3]), "%Y-%m-%d %H:%M:%S")
|
|
115
|
+
except (ValueError, IndexError):
|
|
116
|
+
return None
|
|
117
|
+
return time.mktime(parsed)
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
CGROUP = Path("/sys/fs/cgroup/system.slice")
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def container_usage(apps: dict) -> dict[str, dict]:
|
|
124
|
+
"""Memory and CPU of the containers behind database and Compose apps.
|
|
125
|
+
|
|
126
|
+
Their systemd unit only holds the docker client, so its own figures are
|
|
127
|
+
the client's; the real numbers are in each container's cgroup.
|
|
128
|
+
"""
|
|
129
|
+
wanted = {n for n, a in apps.items() if a.get("kind") in ("db", "compose")}
|
|
130
|
+
if not wanted or not containers.available():
|
|
131
|
+
return {}
|
|
132
|
+
listing = containers.run("ps", "--no-trunc", "--format",
|
|
133
|
+
'{{.ID}}|{{.Names}}|{{.Label "com.docker.compose.project"}}', timeout=10)
|
|
134
|
+
usage: dict[str, dict] = {}
|
|
135
|
+
for line in listing.stdout.splitlines():
|
|
136
|
+
cid, _, rest = line.partition("|")
|
|
137
|
+
cname, _, project = rest.partition("|")
|
|
138
|
+
owner = None
|
|
139
|
+
for name in wanted:
|
|
140
|
+
if cname == containers.container_name(name) or project == containers.compose_project(name):
|
|
141
|
+
owner = name
|
|
142
|
+
if owner is None:
|
|
143
|
+
continue
|
|
144
|
+
scope = CGROUP / f"docker-{cid}.scope"
|
|
145
|
+
try:
|
|
146
|
+
memory = int((scope / "memory.current").read_text())
|
|
147
|
+
cpu_us = next(int(l.split()[1]) for l in (scope / "cpu.stat").read_text().splitlines()
|
|
148
|
+
if l.startswith("usage_usec"))
|
|
149
|
+
except (OSError, ValueError, StopIteration):
|
|
150
|
+
continue
|
|
151
|
+
entry = usage.setdefault(owner, {"memory": 0, "cpu_ns": 0, "containers": 0})
|
|
152
|
+
entry["memory"] += memory
|
|
153
|
+
entry["cpu_ns"] += cpu_us * 1000
|
|
154
|
+
entry["containers"] += 1
|
|
155
|
+
return usage
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def routes() -> dict:
|
|
159
|
+
"""Every ingress rule in the tunnel config, hoist's own and hand-written."""
|
|
160
|
+
try:
|
|
161
|
+
path = tunnel.find_config()
|
|
162
|
+
text = path.read_text()
|
|
163
|
+
except (tunnel.TunnelError, OSError) as exc:
|
|
164
|
+
return {"config": None, "error": str(exc), "rules": [], "tunnel": None, "domain": None}
|
|
165
|
+
_, entries, _ = tunnel.parse(text)
|
|
166
|
+
rules = []
|
|
167
|
+
for entry in entries:
|
|
168
|
+
service_line = next((ln.split("service:", 1)[1].strip() for ln in entry.lines
|
|
169
|
+
if "service:" in ln), "")
|
|
170
|
+
rules.append({
|
|
171
|
+
"hostname": entry.hostname,
|
|
172
|
+
"service": service_line,
|
|
173
|
+
"managed": entry.managed,
|
|
174
|
+
"catchall": entry.is_catchall,
|
|
175
|
+
})
|
|
176
|
+
return {"config": str(path), "rules": rules, "tunnel": tunnel.tunnel_id(text),
|
|
177
|
+
"domain": tunnel.default_domain(text), "error": None}
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
def probe(url: str, timeout: float = 8.0) -> dict:
|
|
181
|
+
"""One public request: status code and time to first byte."""
|
|
182
|
+
started = time.monotonic()
|
|
183
|
+
request = urllib.request.Request(url, headers={"User-Agent": "hoist-dash/1"})
|
|
184
|
+
try:
|
|
185
|
+
with urllib.request.urlopen(request, timeout=timeout) as response:
|
|
186
|
+
response.read(1)
|
|
187
|
+
code = response.status
|
|
188
|
+
except urllib.error.HTTPError as exc:
|
|
189
|
+
code = exc.code
|
|
190
|
+
except Exception as exc: # DNS, TLS, timeout
|
|
191
|
+
return {"t": time.time(), "code": 0, "ms": None, "up": False,
|
|
192
|
+
"error": str(getattr(exc, "reason", exc))[:120]}
|
|
193
|
+
ms = round((time.monotonic() - started) * 1000)
|
|
194
|
+
# 4xx is the app answering (a 404 at / is normal for an API); 5xx and
|
|
195
|
+
# Cloudflare's 52x/530 mean the origin is not there.
|
|
196
|
+
return {"t": time.time(), "code": code, "ms": ms, "up": code < 500, "error": None}
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
def journal(unit: str, lines: int = 300) -> list[dict]:
|
|
200
|
+
result = subprocess.run(
|
|
201
|
+
["journalctl", "--user", "-u", unit, "-n", str(lines), "--no-pager", "-o", "json",
|
|
202
|
+
"--output-fields=MESSAGE,PRIORITY,_PID"],
|
|
203
|
+
capture_output=True, text=True, errors="replace",
|
|
204
|
+
)
|
|
205
|
+
return [entry for entry in (parse_journal_line(ln) for ln in result.stdout.splitlines()) if entry]
|
|
206
|
+
|
|
207
|
+
|
|
208
|
+
def parse_journal_line(line: str) -> dict | None:
|
|
209
|
+
try:
|
|
210
|
+
raw = json.loads(line)
|
|
211
|
+
except json.JSONDecodeError:
|
|
212
|
+
return None
|
|
213
|
+
message = raw.get("MESSAGE", "")
|
|
214
|
+
if isinstance(message, list): # journald stores non-UTF-8 as a byte array
|
|
215
|
+
message = bytes(message).decode("utf-8", "replace")
|
|
216
|
+
try:
|
|
217
|
+
ts = int(raw.get("__REALTIME_TIMESTAMP", "0")) / 1e6
|
|
218
|
+
except ValueError:
|
|
219
|
+
ts = 0.0
|
|
220
|
+
try:
|
|
221
|
+
priority = int(raw.get("PRIORITY", 6))
|
|
222
|
+
except (TypeError, ValueError):
|
|
223
|
+
priority = 6
|
|
224
|
+
return {"t": ts, "msg": message, "p": priority}
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
# --- the sampler -------------------------------------------------------------------
|
|
228
|
+
|
|
229
|
+
|
|
230
|
+
class Monitor(threading.Thread):
|
|
231
|
+
"""Keeps a fresh snapshot and an hour of history, in memory."""
|
|
232
|
+
|
|
233
|
+
def __init__(self) -> None:
|
|
234
|
+
super().__init__(daemon=True, name="hoist-dash-monitor")
|
|
235
|
+
self.lock = threading.Lock()
|
|
236
|
+
self.wake = threading.Event()
|
|
237
|
+
self.system_series: deque = deque(maxlen=KEEP_SAMPLES)
|
|
238
|
+
self.app_series: dict[str, deque] = {}
|
|
239
|
+
self.probes: dict[str, deque] = {}
|
|
240
|
+
self._last_cpu: tuple[int, int] | None = None
|
|
241
|
+
self._last_net: tuple[int, int, float] | None = None
|
|
242
|
+
self._last_unit_cpu: dict[str, tuple[int, float]] = {}
|
|
243
|
+
self._last_probe = 0.0
|
|
244
|
+
self._down: dict[str, int] = {} # consecutive failed public checks per app
|
|
245
|
+
self._alerted: set[str] = set()
|
|
246
|
+
self.snapshot: dict = {}
|
|
247
|
+
self.started = time.time()
|
|
248
|
+
self.last_watched = 0.0
|
|
249
|
+
self.refresh()
|
|
250
|
+
|
|
251
|
+
# Call after an action so the next poll reflects it without waiting.
|
|
252
|
+
def nudge(self) -> None:
|
|
253
|
+
self.wake.set()
|
|
254
|
+
|
|
255
|
+
def watched(self) -> bool:
|
|
256
|
+
return time.time() - self.last_watched <= WATCHING_FOR
|
|
257
|
+
|
|
258
|
+
def probes_wanted(self) -> bool:
|
|
259
|
+
"""Public URL checks feed the dashboard and the downtime alerts, nothing else."""
|
|
260
|
+
return self.watched() or bool((config.load().get("notify") or {}).get("url"))
|
|
261
|
+
|
|
262
|
+
def touch(self) -> None:
|
|
263
|
+
"""A browser asked for data: sample at full rate for the next while."""
|
|
264
|
+
was_idle = not self.watched()
|
|
265
|
+
self.last_watched = time.time()
|
|
266
|
+
if was_idle:
|
|
267
|
+
self._last_probe = 0.0 # the first viewer gets fresh checks, not hour-old ones
|
|
268
|
+
self.wake.set() # ...and does not wait out an idle interval
|
|
269
|
+
|
|
270
|
+
def interval(self) -> float:
|
|
271
|
+
return SAMPLE_EVERY if self.watched() else IDLE_EVERY
|
|
272
|
+
|
|
273
|
+
def run(self) -> None:
|
|
274
|
+
while True:
|
|
275
|
+
self.wake.wait(self.interval())
|
|
276
|
+
self.wake.clear()
|
|
277
|
+
try:
|
|
278
|
+
self.refresh()
|
|
279
|
+
except Exception as exc: # never let one bad read kill the sampler
|
|
280
|
+
print(f"hoist dash: sampler error: {exc}")
|
|
281
|
+
|
|
282
|
+
def refresh(self) -> None:
|
|
283
|
+
state = config.load()
|
|
284
|
+
apps = state.get("apps", {})
|
|
285
|
+
now = time.time()
|
|
286
|
+
|
|
287
|
+
units = [service.unit_name(n) for n, a in apps.items() if not a.get("adopted")]
|
|
288
|
+
timers = [f"{source.sync_unit_base(n)}.timer" for n, a in apps.items() if a.get("repo")]
|
|
289
|
+
props = unit_props(units + timers)
|
|
290
|
+
|
|
291
|
+
busy, total = _cpu_times()
|
|
292
|
+
cpu = None
|
|
293
|
+
if self._last_cpu and total > self._last_cpu[1]:
|
|
294
|
+
cpu = 100.0 * (busy - self._last_cpu[0]) / (total - self._last_cpu[1])
|
|
295
|
+
self._last_cpu = (busy, total)
|
|
296
|
+
rx, tx = net_bytes()
|
|
297
|
+
rx_rate = tx_rate = None
|
|
298
|
+
if self._last_net and now > self._last_net[2]:
|
|
299
|
+
span = now - self._last_net[2]
|
|
300
|
+
rx_rate = max(0, (rx - self._last_net[0]) / span)
|
|
301
|
+
tx_rate = max(0, (tx - self._last_net[1]) / span)
|
|
302
|
+
self._last_net = (rx, tx, now)
|
|
303
|
+
mem = meminfo()
|
|
304
|
+
mem_used = mem.get("MemTotal", 0) - mem.get("MemAvailable", 0)
|
|
305
|
+
disk = shutil.disk_usage(Path.home())
|
|
306
|
+
load = os.getloadavg()
|
|
307
|
+
|
|
308
|
+
boxed = container_usage(apps)
|
|
309
|
+
rows = []
|
|
310
|
+
for name in sorted(apps):
|
|
311
|
+
app = apps[name]
|
|
312
|
+
unit = dict(props.get(service.unit_name(name), {}))
|
|
313
|
+
if name in boxed: # report the containers, not the docker client
|
|
314
|
+
unit["MemoryCurrent"] = str(boxed[name]["memory"])
|
|
315
|
+
unit["CPUUsageNSec"] = str(boxed[name]["cpu_ns"])
|
|
316
|
+
listening = detect.port_in_use(int(app["port"]))
|
|
317
|
+
active = unit.get("ActiveState", "inactive")
|
|
318
|
+
if app.get("adopted"):
|
|
319
|
+
status = "up" if listening else "down"
|
|
320
|
+
elif app.get("stopped"):
|
|
321
|
+
status = "stopped"
|
|
322
|
+
elif active == "active" and listening:
|
|
323
|
+
status = "up"
|
|
324
|
+
elif active in ("active", "activating"):
|
|
325
|
+
status = "starting"
|
|
326
|
+
elif active == "failed":
|
|
327
|
+
status = "failed"
|
|
328
|
+
else:
|
|
329
|
+
status = "down"
|
|
330
|
+
|
|
331
|
+
memory = _int(unit.get("MemoryCurrent"))
|
|
332
|
+
cpu_ns = _int(unit.get("CPUUsageNSec"))
|
|
333
|
+
app_cpu = None
|
|
334
|
+
if cpu_ns is not None:
|
|
335
|
+
last = self._last_unit_cpu.get(name)
|
|
336
|
+
if last and now > last[1] and cpu_ns >= last[0]:
|
|
337
|
+
app_cpu = 100.0 * (cpu_ns - last[0]) / ((now - last[1]) * 1e9)
|
|
338
|
+
self._last_unit_cpu[name] = (cpu_ns, now)
|
|
339
|
+
series = self.app_series.setdefault(name, deque(maxlen=KEEP_SAMPLES))
|
|
340
|
+
series.append((round(now), round(app_cpu, 2) if app_cpu is not None else None,
|
|
341
|
+
memory))
|
|
342
|
+
|
|
343
|
+
repo = app.get("repo")
|
|
344
|
+
timer = props.get(f"{source.sync_unit_base(name)}.timer", {}) if repo else {}
|
|
345
|
+
if app.get("kind") == "db":
|
|
346
|
+
url = f"127.0.0.1:{app['port']}"
|
|
347
|
+
else:
|
|
348
|
+
url = (f"https://{app['hostname']}" if app.get("tunnel") and app.get("hostname")
|
|
349
|
+
else f"http://{app.get('lan_ip', '127.0.0.1')}:{app['port']}")
|
|
350
|
+
probes = list(self.probes.get(name, []))
|
|
351
|
+
rows.append({
|
|
352
|
+
"name": name,
|
|
353
|
+
"status": status,
|
|
354
|
+
"port": app["port"],
|
|
355
|
+
"url": url,
|
|
356
|
+
"public": bool(app.get("tunnel")),
|
|
357
|
+
"hostname": app.get("hostname"),
|
|
358
|
+
"adopted": bool(app.get("adopted")),
|
|
359
|
+
"stopped": bool(app.get("stopped")),
|
|
360
|
+
"workdir": app.get("workdir"),
|
|
361
|
+
"cmd": app.get("cmd"),
|
|
362
|
+
"env_keys": sorted((app.get("env") or {}).keys()),
|
|
363
|
+
"limits": app.get("limits") or {},
|
|
364
|
+
"kind": app.get("kind") or "app",
|
|
365
|
+
"stage": app.get("stage") or ("preview" if app.get("preview_of") else "production"),
|
|
366
|
+
"containers": boxed.get(name, {}).get("containers"),
|
|
367
|
+
"preview_of": app.get("preview_of"),
|
|
368
|
+
"dockerfile": bool(app.get("dockerfile")),
|
|
369
|
+
"template": app.get("template"),
|
|
370
|
+
"db": ({k: app["db"].get(k) for k in ("engine", "url", "env_var", "volume", "image", "database",
|
|
371
|
+
"offsite", "daily_backup")}
|
|
372
|
+
if app.get("kind") == "db" else None),
|
|
373
|
+
"created": app.get("created"),
|
|
374
|
+
"updated": app.get("updated"),
|
|
375
|
+
"pid": _int(unit.get("MainPID")) or None,
|
|
376
|
+
"restarts": _int(unit.get("NRestarts")) or 0,
|
|
377
|
+
"since": _since(unit.get("ActiveEnterTimestamp", "")) if active == "active" else None,
|
|
378
|
+
"memory": memory,
|
|
379
|
+
"cpu": round(app_cpu, 1) if app_cpu is not None else None,
|
|
380
|
+
"repo": _repo_summary(repo, timer) if repo else None,
|
|
381
|
+
"probe": probes[-1] if probes else None,
|
|
382
|
+
"uptime": _uptime(probes),
|
|
383
|
+
})
|
|
384
|
+
|
|
385
|
+
snapshot = {
|
|
386
|
+
"t": now,
|
|
387
|
+
"host": socket.gethostname(),
|
|
388
|
+
"domain": state.get("domain"),
|
|
389
|
+
"apps": rows,
|
|
390
|
+
"system": {
|
|
391
|
+
"cpu": round(cpu, 1) if cpu is not None else None,
|
|
392
|
+
"cores": os.cpu_count() or 1,
|
|
393
|
+
"load": [round(x, 2) for x in load],
|
|
394
|
+
"mem_total": mem.get("MemTotal", 0),
|
|
395
|
+
"mem_used": mem_used,
|
|
396
|
+
"swap_total": mem.get("SwapTotal", 0),
|
|
397
|
+
"swap_used": mem.get("SwapTotal", 0) - mem.get("SwapFree", 0),
|
|
398
|
+
"disk_total": disk.total,
|
|
399
|
+
"disk_used": disk.used,
|
|
400
|
+
"uptime": uptime_seconds(),
|
|
401
|
+
"cloudflared": bool(tunnel.daemon_unit()),
|
|
402
|
+
"net_rx": round(rx_rate) if rx_rate is not None else None,
|
|
403
|
+
"net_tx": round(tx_rate) if tx_rate is not None else None,
|
|
404
|
+
},
|
|
405
|
+
"routes": routes(),
|
|
406
|
+
}
|
|
407
|
+
self.system_series.append((round(now), snapshot["system"]["cpu"], mem_used,
|
|
408
|
+
snapshot["system"]["net_rx"], snapshot["system"]["net_tx"]))
|
|
409
|
+
with self.lock:
|
|
410
|
+
self.snapshot = snapshot
|
|
411
|
+
|
|
412
|
+
if now - self._last_probe >= PROBE_EVERY and self.probes_wanted():
|
|
413
|
+
self._last_probe = now
|
|
414
|
+
threading.Thread(target=self._probe_all, args=(rows, snapshot["routes"]),
|
|
415
|
+
daemon=True).start()
|
|
416
|
+
|
|
417
|
+
def _probe_all(self, rows: list[dict], route_info: dict) -> None:
|
|
418
|
+
targets = {row["name"]: row["url"] for row in rows if row["public"]}
|
|
419
|
+
# Hand-written routes (n8n, an API...) are watched too, keyed by hostname.
|
|
420
|
+
for rule in route_info.get("rules", []):
|
|
421
|
+
host = rule.get("hostname")
|
|
422
|
+
if host and not rule.get("managed"):
|
|
423
|
+
targets[f"route:{host}"] = f"https://{host}"
|
|
424
|
+
stopped = {row["name"] for row in rows if row["stopped"]}
|
|
425
|
+
for key, url in targets.items():
|
|
426
|
+
result = probe(url)
|
|
427
|
+
with self.lock:
|
|
428
|
+
self.probes.setdefault(key, deque(maxlen=KEEP_PROBES)).append(result)
|
|
429
|
+
if key.startswith("route:") or key in stopped:
|
|
430
|
+
continue
|
|
431
|
+
self._watch_downtime(key, url, result)
|
|
432
|
+
|
|
433
|
+
def _watch_downtime(self, name: str, url: str, result: dict) -> None:
|
|
434
|
+
"""Alert after two failed checks in a row (one minute), and on recovery."""
|
|
435
|
+
if result["up"]:
|
|
436
|
+
self._down[name] = 0
|
|
437
|
+
if name in self._alerted:
|
|
438
|
+
self._alerted.discard(name)
|
|
439
|
+
notify.send(f"{name} is answering again", f"{url} — {result['code']} in {result['ms']} ms",
|
|
440
|
+
level="ok", app=name, wait=False)
|
|
441
|
+
return
|
|
442
|
+
self._down[name] = self._down.get(name, 0) + 1
|
|
443
|
+
if self._down[name] == 2 and name not in self._alerted:
|
|
444
|
+
self._alerted.add(name)
|
|
445
|
+
why = f"answers {result['code']}" if result["code"] else (result.get("error") or "unreachable")
|
|
446
|
+
notify.send(f"{name} stopped answering", f"{url} {why} for the last minute.",
|
|
447
|
+
level="bad", app=name, wait=False)
|
|
448
|
+
|
|
449
|
+
def probes_for(self, key: str) -> list[dict]:
|
|
450
|
+
with self.lock:
|
|
451
|
+
return list(self.probes.get(key, []))
|
|
452
|
+
|
|
453
|
+
def get(self) -> dict:
|
|
454
|
+
with self.lock:
|
|
455
|
+
snap = dict(self.snapshot)
|
|
456
|
+
# attach the latest probe of hand-written routes
|
|
457
|
+
rules = []
|
|
458
|
+
for rule in snap.get("routes", {}).get("rules", []):
|
|
459
|
+
rule = dict(rule)
|
|
460
|
+
if rule.get("hostname") and not rule.get("managed"):
|
|
461
|
+
history = self.probes_for(f"route:{rule['hostname']}")
|
|
462
|
+
rule["probe"] = history[-1] if history else None
|
|
463
|
+
rule["uptime"] = _uptime(history)
|
|
464
|
+
rules.append(rule)
|
|
465
|
+
if "routes" in snap:
|
|
466
|
+
snap["routes"] = dict(snap["routes"], rules=rules)
|
|
467
|
+
return snap
|
|
468
|
+
|
|
469
|
+
def series(self, name: str | None = None) -> list:
|
|
470
|
+
with self.lock:
|
|
471
|
+
if name is None:
|
|
472
|
+
return list(self.system_series)
|
|
473
|
+
return list(self.app_series.get(name, []))
|
|
474
|
+
|
|
475
|
+
|
|
476
|
+
def _uptime(probes: list[dict]) -> float | None:
|
|
477
|
+
if not probes:
|
|
478
|
+
return None
|
|
479
|
+
return round(100.0 * sum(1 for p in probes if p["up"]) / len(probes), 1)
|
|
480
|
+
|
|
481
|
+
|
|
482
|
+
def _repo_summary(repo: dict, timer: dict) -> dict:
|
|
483
|
+
return {
|
|
484
|
+
"url": repo.get("url"),
|
|
485
|
+
"branch": repo.get("branch"),
|
|
486
|
+
"sha": repo.get("sha"),
|
|
487
|
+
"commit": repo.get("commit"),
|
|
488
|
+
"deployed": repo.get("deployed"),
|
|
489
|
+
"failed_sha": repo.get("failed_sha"),
|
|
490
|
+
"paused": bool(repo.get("paused")),
|
|
491
|
+
"previews": bool(repo.get("previews")),
|
|
492
|
+
"webhook": bool(repo.get("webhook_secret")),
|
|
493
|
+
"preview_failed": repo.get("preview_failed") or {},
|
|
494
|
+
"build": repo.get("build", []),
|
|
495
|
+
"watching": timer.get("ActiveState") == "active",
|
|
496
|
+
"last": (repo.get("history") or [None])[0],
|
|
497
|
+
}
|
hoist/dash/fleet.py
ADDED
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
"""Several machines from one dashboard.
|
|
2
|
+
|
|
3
|
+
Every machine runs its own hoist and its own `hoist dash`; this dashboard
|
|
4
|
+
keeps a list of the others (URL plus a token, stored 0600) and forwards API
|
|
5
|
+
calls to them server-side. Nothing new runs on the other machines, and no
|
|
6
|
+
token ever reaches the browser.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import json
|
|
12
|
+
import os
|
|
13
|
+
import re
|
|
14
|
+
import secrets
|
|
15
|
+
import threading
|
|
16
|
+
import time
|
|
17
|
+
import urllib.error
|
|
18
|
+
import urllib.request
|
|
19
|
+
from pathlib import Path
|
|
20
|
+
|
|
21
|
+
from .. import config
|
|
22
|
+
|
|
23
|
+
ID = re.compile(r"^[a-z0-9][a-z0-9-]{0,30}$")
|
|
24
|
+
_lock = threading.Lock()
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class FleetError(Exception):
|
|
28
|
+
pass
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def path() -> Path:
|
|
32
|
+
return config.STATE_DIR / "servers.json"
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def servers() -> dict[str, dict]:
|
|
36
|
+
try:
|
|
37
|
+
return json.loads(path().read_text()).get("servers", {})
|
|
38
|
+
except (OSError, json.JSONDecodeError):
|
|
39
|
+
return {}
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def _save(data: dict) -> None:
|
|
43
|
+
path().parent.mkdir(parents=True, exist_ok=True)
|
|
44
|
+
tmp = path().with_name(f".servers.{os.getpid()}.tmp")
|
|
45
|
+
fd = os.open(tmp, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
|
|
46
|
+
with os.fdopen(fd, "w") as fh:
|
|
47
|
+
fh.write(json.dumps({"servers": data}, indent=2) + "\n")
|
|
48
|
+
os.replace(tmp, path())
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def _base(url: str) -> str:
|
|
52
|
+
url = url.strip().rstrip("/")
|
|
53
|
+
if not re.match(r"^https?://[^/\s]+$", url):
|
|
54
|
+
raise FleetError("use the dashboard's address, like https://dash.example.com or http://10.0.0.5:7457")
|
|
55
|
+
return url
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def add(name: str, url: str, token: str) -> dict:
|
|
59
|
+
"""Link a machine after checking its dashboard answers to the token."""
|
|
60
|
+
sid = re.sub(r"[^a-z0-9-]+", "-", name.strip().lower()).strip("-")[:31]
|
|
61
|
+
if not ID.match(sid or ""):
|
|
62
|
+
raise FleetError("give the machine a short name")
|
|
63
|
+
base = _base(url)
|
|
64
|
+
token = token.strip()
|
|
65
|
+
if "token=" in token: # a whole sign-in link was pasted
|
|
66
|
+
token = token.split("token=", 1)[1].split("&", 1)[0]
|
|
67
|
+
info = probe(base, token)
|
|
68
|
+
if info.get("error"):
|
|
69
|
+
raise FleetError(f"could not reach that dashboard: {info['error']}")
|
|
70
|
+
if info.get("role") != "owner":
|
|
71
|
+
raise FleetError("link the other machine with its owner's link, so it can be managed from here")
|
|
72
|
+
with _lock:
|
|
73
|
+
everyone = servers()
|
|
74
|
+
everyone[sid] = {"name": name.strip()[:40], "url": base, "token": token,
|
|
75
|
+
"added": config.now(), "host": info.get("host")}
|
|
76
|
+
_save(everyone)
|
|
77
|
+
return {"id": sid, "name": name.strip(), "url": base, "host": info.get("host")}
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def remove(sid: str) -> bool:
|
|
81
|
+
with _lock:
|
|
82
|
+
everyone = servers()
|
|
83
|
+
if everyone.pop(sid, None) is None:
|
|
84
|
+
return False
|
|
85
|
+
_save(everyone)
|
|
86
|
+
return True
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def request(server: dict, method: str, rest: str, body: bytes | None = None,
|
|
90
|
+
stream: bool = False, timeout: float = 15.0):
|
|
91
|
+
"""Forward one API call. Returns the open response (caller reads/closes)."""
|
|
92
|
+
headers = {"Authorization": f"Bearer {server['token']}", "User-Agent": "hoist-fleet"}
|
|
93
|
+
if method != "GET":
|
|
94
|
+
headers["X-Hoist"] = "1"
|
|
95
|
+
headers["Content-Type"] = "application/json"
|
|
96
|
+
req = urllib.request.Request(f"{server['url']}/api/{rest}", data=body, method=method, headers=headers)
|
|
97
|
+
return urllib.request.urlopen(req, timeout=None if stream else timeout)
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def probe(base: str, token: str) -> dict:
|
|
101
|
+
started = time.monotonic()
|
|
102
|
+
try:
|
|
103
|
+
with request({"url": base, "token": token}, "GET", "state", timeout=8) as r:
|
|
104
|
+
data = json.loads(r.read())
|
|
105
|
+
except urllib.error.HTTPError as exc:
|
|
106
|
+
return {"error": "the token was refused" if exc.code == 401 else f"HTTP {exc.code}"}
|
|
107
|
+
except Exception as exc: # noqa: BLE001
|
|
108
|
+
return {"error": str(getattr(exc, "reason", exc))[:160]}
|
|
109
|
+
data["latency_ms"] = round((time.monotonic() - started) * 1000)
|
|
110
|
+
return data
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def summary() -> list[dict]:
|
|
114
|
+
"""Every linked machine's health, fetched in parallel."""
|
|
115
|
+
out: list[dict] = []
|
|
116
|
+
threads = []
|
|
117
|
+
for sid, srv in servers().items():
|
|
118
|
+
row = {"id": sid, "name": srv["name"], "url": srv["url"]}
|
|
119
|
+
out.append(row)
|
|
120
|
+
|
|
121
|
+
def fill(row=row, srv=srv):
|
|
122
|
+
info = probe(srv["url"], srv["token"])
|
|
123
|
+
if info.get("error"):
|
|
124
|
+
row.update(ok=False, error=info["error"])
|
|
125
|
+
return
|
|
126
|
+
apps = [a for a in info.get("apps", []) if a.get("kind") != "db"]
|
|
127
|
+
broken = [a["name"] for a in apps if a.get("status") in ("down", "failed")
|
|
128
|
+
or (a.get("public") and a.get("probe") and not a["probe"].get("up"))]
|
|
129
|
+
row.update(ok=True, host=info.get("host"), latency_ms=info.get("latency_ms"),
|
|
130
|
+
apps=len(apps), dbs=len(info.get("apps", [])) - len(apps), broken=broken,
|
|
131
|
+
system=info.get("system"), version=info.get("version"))
|
|
132
|
+
thread = threading.Thread(target=fill, daemon=True)
|
|
133
|
+
thread.start()
|
|
134
|
+
threads.append(thread)
|
|
135
|
+
for thread in threads:
|
|
136
|
+
thread.join(12)
|
|
137
|
+
return out
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
__all__ = ["FleetError", "add", "remove", "request", "servers", "summary", "probe", "secrets"]
|