stack-guardian 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- stack_guardian/__init__.py +13 -0
- stack_guardian/__main__.py +6 -0
- stack_guardian/checks.d/10_core.py +174 -0
- stack_guardian/cli.py +295 -0
- stack_guardian/collect.py +206 -0
- stack_guardian/core.py +264 -0
- stack_guardian/finding.py +491 -0
- stack_guardian/fixes.py +287 -0
- stack_guardian/mcp_server.py +340 -0
- stack_guardian/multinode.py +152 -0
- stack_guardian/parsers.py +658 -0
- stack_guardian/serve.py +198 -0
- stack_guardian/state.py +266 -0
- stack_guardian-0.1.0.dist-info/METADATA +164 -0
- stack_guardian-0.1.0.dist-info/RECORD +19 -0
- stack_guardian-0.1.0.dist-info/WHEEL +5 -0
- stack_guardian-0.1.0.dist-info/entry_points.txt +4 -0
- stack_guardian-0.1.0.dist-info/licenses/LICENSE +21 -0
- stack_guardian-0.1.0.dist-info/top_level.txt +1 -0
stack_guardian/serve.py
ADDED
|
@@ -0,0 +1,198 @@
|
|
|
1
|
+
"""Tiny read-only HTTP endpoint for a stack_guardian health snapshot.
|
|
2
|
+
|
|
3
|
+
One machine asks another for its *current* health by GETting /snapshot; the
|
|
4
|
+
server collects a fresh snapshot on every request, never caches, and refuses
|
|
5
|
+
every write verb. Binds to loopback unless the caller explicitly opts in to a
|
|
6
|
+
remote bind with allow_remote=True.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
import http.server
|
|
10
|
+
import json
|
|
11
|
+
import sys
|
|
12
|
+
|
|
13
|
+
LOOPBACK = ("127.0.0.1", "::1", "localhost")
|
|
14
|
+
|
|
15
|
+
_READ_ONLY = {"error": "read-only: GET /snapshot or /health"}
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def make_handler(get_snapshot):
|
|
19
|
+
"""Return a BaseHTTPRequestHandler subclass bound to get_snapshot."""
|
|
20
|
+
|
|
21
|
+
class Handler(http.server.BaseHTTPRequestHandler):
|
|
22
|
+
server_version = "stack-guardian-serve"
|
|
23
|
+
|
|
24
|
+
def _reply(self, code, obj):
|
|
25
|
+
body = json.dumps(obj, default=str).encode("utf-8")
|
|
26
|
+
self.send_response(code)
|
|
27
|
+
if code == 405:
|
|
28
|
+
self.send_header("Allow", "GET")
|
|
29
|
+
self.send_header("Content-Type", "application/json")
|
|
30
|
+
self.send_header("Content-Length", str(len(body)))
|
|
31
|
+
self.end_headers()
|
|
32
|
+
self.wfile.write(body)
|
|
33
|
+
|
|
34
|
+
def _get(self):
|
|
35
|
+
path = self.path.split("?", 1)[0]
|
|
36
|
+
if path == "/health":
|
|
37
|
+
self._reply(200, {"ok": True})
|
|
38
|
+
elif path == "/snapshot":
|
|
39
|
+
try:
|
|
40
|
+
snapshot = get_snapshot()
|
|
41
|
+
except Exception as exc: # noqa: BLE001 - report, never crash the server
|
|
42
|
+
self._reply(500, {"error": "%s: %s" % (type(exc).__name__, exc)})
|
|
43
|
+
else:
|
|
44
|
+
self._reply(200, snapshot)
|
|
45
|
+
else:
|
|
46
|
+
self._reply(404, {"error": "not found"})
|
|
47
|
+
|
|
48
|
+
def _refuse_write(self):
|
|
49
|
+
self._reply(405, dict(_READ_ONLY))
|
|
50
|
+
|
|
51
|
+
def do_GET(self): # noqa: N802 - name fixed by BaseHTTPRequestHandler
|
|
52
|
+
self._get()
|
|
53
|
+
|
|
54
|
+
def do_POST(self): # noqa: N802
|
|
55
|
+
self._refuse_write()
|
|
56
|
+
|
|
57
|
+
def do_PUT(self): # noqa: N802
|
|
58
|
+
self._refuse_write()
|
|
59
|
+
|
|
60
|
+
def do_DELETE(self): # noqa: N802
|
|
61
|
+
self._refuse_write()
|
|
62
|
+
|
|
63
|
+
def do_PATCH(self): # noqa: N802
|
|
64
|
+
self._refuse_write()
|
|
65
|
+
|
|
66
|
+
def log_message(self, *args):
|
|
67
|
+
pass
|
|
68
|
+
|
|
69
|
+
return Handler
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def make_server(get_snapshot, host="127.0.0.1", port=0, allow_remote=False):
|
|
73
|
+
"""Build (but do not start) a threaded server, refusing non-loopback binds."""
|
|
74
|
+
if host not in LOOPBACK and not allow_remote:
|
|
75
|
+
raise ValueError("refusing to bind %s without allow_remote=True" % host)
|
|
76
|
+
return http.server.ThreadingHTTPServer((host, port), make_handler(get_snapshot))
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def _opt(argv, name):
|
|
80
|
+
"""Return the token following `name` in argv, or None when there is none."""
|
|
81
|
+
if name in argv:
|
|
82
|
+
i = argv.index(name)
|
|
83
|
+
if i + 1 < len(argv):
|
|
84
|
+
return argv[i + 1]
|
|
85
|
+
return None
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def main(argv=None):
|
|
89
|
+
argv = list(sys.argv[1:] if argv is None else argv)
|
|
90
|
+
if "--selftest" in argv:
|
|
91
|
+
return _selftest()
|
|
92
|
+
nodes = _opt(argv, "--nodes")
|
|
93
|
+
if nodes is None:
|
|
94
|
+
print("stack-guardian-serve: usage: --nodes FILE [--port N] [--host H --allow-remote]",
|
|
95
|
+
file=sys.stderr)
|
|
96
|
+
return 3
|
|
97
|
+
from . import collect as sg_collect
|
|
98
|
+
|
|
99
|
+
def snap():
|
|
100
|
+
with open(nodes, encoding="utf-8") as fh:
|
|
101
|
+
return sg_collect.snapshot(json.load(fh))
|
|
102
|
+
|
|
103
|
+
server = make_server(snap, _opt(argv, "--host") or "127.0.0.1",
|
|
104
|
+
int(_opt(argv, "--port") or "8788"), "--allow-remote" in argv)
|
|
105
|
+
print("serving http://%s:%d/snapshot" % server.server_address[:2], flush=True)
|
|
106
|
+
server.serve_forever()
|
|
107
|
+
return 0
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def _selftest():
|
|
111
|
+
import threading
|
|
112
|
+
import urllib.error
|
|
113
|
+
import urllib.request
|
|
114
|
+
state = {"n": 0}
|
|
115
|
+
|
|
116
|
+
def snap():
|
|
117
|
+
state["n"] += 1
|
|
118
|
+
return {"schema": 1, "taken_at": "T", "nodes": {"a": {"reachable": True}}, "n": state["n"]}
|
|
119
|
+
|
|
120
|
+
def boom():
|
|
121
|
+
raise RuntimeError("collector died")
|
|
122
|
+
|
|
123
|
+
def start(fn):
|
|
124
|
+
s = make_server(fn)
|
|
125
|
+
threading.Thread(target=s.serve_forever, daemon=True).start()
|
|
126
|
+
return s, "http://127.0.0.1:%d" % s.server_address[1]
|
|
127
|
+
|
|
128
|
+
def get(url, method="GET"):
|
|
129
|
+
req = urllib.request.Request(url, method=method, data=b"{}" if method != "GET" else None)
|
|
130
|
+
try:
|
|
131
|
+
with urllib.request.urlopen(req, timeout=10) as r:
|
|
132
|
+
return r.status, r.headers.get("Content-Type"), json.loads(r.read().decode("utf-8"))
|
|
133
|
+
except urllib.error.HTTPError as e:
|
|
134
|
+
return e.code, e.headers.get("Content-Type"), json.loads(e.read().decode("utf-8"))
|
|
135
|
+
|
|
136
|
+
s, base = start(snap)
|
|
137
|
+
bad, bad_base = start(boom)
|
|
138
|
+
try:
|
|
139
|
+
def c_snapshot():
|
|
140
|
+
code, ctype, body = get(base + "/snapshot")
|
|
141
|
+
return code == 200 and ctype == "application/json" and body["nodes"] == {"a": {"reachable": True}}
|
|
142
|
+
|
|
143
|
+
def c_fresh_each_call():
|
|
144
|
+
a = get(base + "/snapshot")[2]["n"]
|
|
145
|
+
b = get(base + "/snapshot?x=1")[2]["n"]
|
|
146
|
+
return b == a + 1
|
|
147
|
+
|
|
148
|
+
def c_health():
|
|
149
|
+
return get(base + "/health")[:3:2] == (200, {"ok": True})
|
|
150
|
+
|
|
151
|
+
def c_not_found():
|
|
152
|
+
return get(base + "/etc/passwd")[0] == 404
|
|
153
|
+
|
|
154
|
+
def c_post_refused():
|
|
155
|
+
code, _, body = get(base + "/snapshot", "POST")
|
|
156
|
+
return code == 405 and body["error"].startswith("read-only")
|
|
157
|
+
|
|
158
|
+
def c_delete_refused():
|
|
159
|
+
return get(base + "/snapshot", "DELETE")[0] == 405
|
|
160
|
+
|
|
161
|
+
def c_collector_error_500():
|
|
162
|
+
code, _, body = get(bad_base + "/snapshot")
|
|
163
|
+
return code == 500 and body["error"] == "RuntimeError: collector died"
|
|
164
|
+
|
|
165
|
+
def c_loopback_default():
|
|
166
|
+
return s.server_address[0] == "127.0.0.1"
|
|
167
|
+
|
|
168
|
+
def c_remote_refused():
|
|
169
|
+
try:
|
|
170
|
+
make_server(snap, host="0.0.0.0") # nosec B104 - asserts the refusal; nothing binds
|
|
171
|
+
except ValueError as exc:
|
|
172
|
+
return "allow_remote" in str(exc)
|
|
173
|
+
return False
|
|
174
|
+
|
|
175
|
+
checks = [("snapshot", c_snapshot), ("fresh_each_call", c_fresh_each_call), ("health", c_health),
|
|
176
|
+
("not_found", c_not_found), ("post_refused", c_post_refused), ("delete_refused", c_delete_refused),
|
|
177
|
+
("collector_error_500", c_collector_error_500), ("loopback_default", c_loopback_default),
|
|
178
|
+
("remote_refused", c_remote_refused)]
|
|
179
|
+
passed = failed = 0
|
|
180
|
+
for name, fn in checks:
|
|
181
|
+
try:
|
|
182
|
+
ok = bool(fn())
|
|
183
|
+
except Exception as exc: # noqa: BLE001
|
|
184
|
+
ok = False
|
|
185
|
+
print("FAIL %s -- raised %s: %s" % (name, type(exc).__name__, exc))
|
|
186
|
+
else:
|
|
187
|
+
print(("ok %s" if ok else "FAIL %s") % name)
|
|
188
|
+
passed += ok
|
|
189
|
+
failed += not ok
|
|
190
|
+
print("serve selftest: %d checks, %d passed, %d failed" % (len(checks), passed, failed))
|
|
191
|
+
return 0 if failed == 0 and checks else 1
|
|
192
|
+
finally:
|
|
193
|
+
s.shutdown()
|
|
194
|
+
bad.shutdown()
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
if __name__ == "__main__":
|
|
198
|
+
sys.exit(main())
|
stack_guardian/state.py
ADDED
|
@@ -0,0 +1,266 @@
|
|
|
1
|
+
"""Run history for stack-guardian: the last runs per node, streaks, and confidence grading."""
|
|
2
|
+
import json
|
|
3
|
+
import os
|
|
4
|
+
import shutil
|
|
5
|
+
import sys
|
|
6
|
+
import tempfile
|
|
7
|
+
import time
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
|
|
10
|
+
STATE_ENV = "STACK_GUARDIAN_STATE"
|
|
11
|
+
KEEP = 5
|
|
12
|
+
BAD = ("WARN", "FAIL")
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def state_path() -> Path:
|
|
16
|
+
"""Path(os.environ[STATE_ENV]) if that variable is set and non-empty, else Path.home() / ".stack-guardian" / "state.json"."""
|
|
17
|
+
path = os.environ.get(STATE_ENV, "")
|
|
18
|
+
if path:
|
|
19
|
+
return Path(path)
|
|
20
|
+
return Path.home() / ".stack-guardian" / "state.json"
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def fresh() -> dict:
|
|
24
|
+
"""a NEW dict every call: {"schema": 1, "nodes": {}}"""
|
|
25
|
+
return {"schema": 1, "nodes": {}}
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def load(path=None) -> dict:
|
|
29
|
+
"""path or state_path(); json.loads of the file (utf-8). Missing file, unreadable file, invalid JSON, a non-dict,
|
|
30
|
+
"schema" != 1, or "nodes" not a dict -> fresh(). Never raises. Anything but a missing file prints one stderr line."""
|
|
31
|
+
if path is None:
|
|
32
|
+
path = state_path()
|
|
33
|
+
try:
|
|
34
|
+
with open(path, "r", encoding="utf-8") as fh:
|
|
35
|
+
data = json.load(fh)
|
|
36
|
+
except FileNotFoundError:
|
|
37
|
+
return fresh()
|
|
38
|
+
except (OSError, TypeError, ValueError) as exc:
|
|
39
|
+
return _discard(path, "%s: %s" % (type(exc).__name__, exc))
|
|
40
|
+
if not isinstance(data, dict) or data.get("schema") != 1 or not isinstance(data.get("nodes"), dict):
|
|
41
|
+
return _discard(path, "not a schema 1 state")
|
|
42
|
+
return data
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def _discard(path, why):
|
|
46
|
+
print("stack-guardian: state %s discarded (%s); starting fresh" % (path, why), file=sys.stderr)
|
|
47
|
+
return fresh()
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def save(state, path=None) -> None:
|
|
51
|
+
"""path or state_path(); create parent folders; write json.dumps(state, sort_keys=True) to a unique temp file
|
|
52
|
+
beside it, then os.replace(tmp, path) -- concurrent saves never share a temp file and none is left behind."""
|
|
53
|
+
if path is None:
|
|
54
|
+
path = state_path()
|
|
55
|
+
path = Path(path)
|
|
56
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
57
|
+
fd, tmp_path = tempfile.mkstemp(dir=path.parent, prefix=path.name + ".", suffix=".tmp")
|
|
58
|
+
try:
|
|
59
|
+
with os.fdopen(fd, "w", encoding="utf-8") as fh:
|
|
60
|
+
json.dump(state, fh, sort_keys=True)
|
|
61
|
+
for attempt in range(20):
|
|
62
|
+
try:
|
|
63
|
+
os.replace(tmp_path, path)
|
|
64
|
+
break
|
|
65
|
+
except PermissionError: # Windows: another save is replacing the same file this instant
|
|
66
|
+
if attempt == 19:
|
|
67
|
+
raise
|
|
68
|
+
time.sleep(0.01)
|
|
69
|
+
except Exception:
|
|
70
|
+
# Clean up the temporary file if something goes wrong
|
|
71
|
+
try:
|
|
72
|
+
os.remove(tmp_path)
|
|
73
|
+
except OSError:
|
|
74
|
+
pass
|
|
75
|
+
raise
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def record_run(state, node, findings, at) -> dict:
|
|
79
|
+
"""runs = state["nodes"][node]["runs"] (create the levels that are missing); append
|
|
80
|
+
{"at": at, "codes": {f["code"]: f["level"] for each finding f whose f.get("node", "") is node or ""}}
|
|
81
|
+
then keep only the last KEEP runs (oldest dropped). Mutates and returns `state`."""
|
|
82
|
+
if node not in state["nodes"]:
|
|
83
|
+
state["nodes"][node] = {"runs": []}
|
|
84
|
+
runs = state["nodes"][node]["runs"]
|
|
85
|
+
|
|
86
|
+
# Filter findings for this node
|
|
87
|
+
node_findings = [f for f in findings if f.get("node", "") == node or f.get("node", "") == ""]
|
|
88
|
+
|
|
89
|
+
# Create the run entry
|
|
90
|
+
run_entry = {
|
|
91
|
+
"at": at,
|
|
92
|
+
"codes": {f["code"]: f["level"] for f in node_findings}
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
runs.append(run_entry)
|
|
96
|
+
|
|
97
|
+
# Keep only the last KEEP runs
|
|
98
|
+
if len(runs) > KEEP:
|
|
99
|
+
runs[:] = runs[-KEEP:]
|
|
100
|
+
|
|
101
|
+
return state
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def streak(state, node, code) -> int:
|
|
105
|
+
"""how many of the MOST RECENT runs of `node`, counting back from the newest, have codes[code] in BAD; stop at the first
|
|
106
|
+
run that does not. Unknown node -> 0."""
|
|
107
|
+
if node not in state["nodes"]:
|
|
108
|
+
return 0
|
|
109
|
+
|
|
110
|
+
runs = state["nodes"][node]["runs"]
|
|
111
|
+
count = 0
|
|
112
|
+
|
|
113
|
+
# Iterate from newest to oldest
|
|
114
|
+
for run in reversed(runs):
|
|
115
|
+
if code in run["codes"] and run["codes"][code] in BAD:
|
|
116
|
+
count += 1
|
|
117
|
+
else:
|
|
118
|
+
break
|
|
119
|
+
|
|
120
|
+
return count
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def clear_streak(state, node, code) -> int:
|
|
124
|
+
"""how many of the most recent runs, counting back from the newest, do NOT have codes[code] in BAD; stop at the first that does."""
|
|
125
|
+
if node not in state["nodes"]:
|
|
126
|
+
return 0
|
|
127
|
+
|
|
128
|
+
runs = state["nodes"][node]["runs"]
|
|
129
|
+
count = 0
|
|
130
|
+
|
|
131
|
+
# Iterate from newest to oldest
|
|
132
|
+
for run in reversed(runs):
|
|
133
|
+
if code in run["codes"] and run["codes"][code] in BAD:
|
|
134
|
+
break
|
|
135
|
+
else:
|
|
136
|
+
count += 1
|
|
137
|
+
|
|
138
|
+
return count
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def grade(state, node, finding) -> dict:
|
|
142
|
+
"""Returns a COPY (dict(finding)); never mutates the input. Only for level in BAD:
|
|
143
|
+
streak(state, node, finding["code"]) >= 2 -> confidence "VERIFIED";
|
|
144
|
+
otherwise a "VERIFIED" confidence becomes "LIKELY" (LIKELY and INSUFFICIENT stay).
|
|
145
|
+
Call it AFTER record_run for the current run (the current run counts as one sample). Other levels: copy unchanged."""
|
|
146
|
+
result = dict(finding)
|
|
147
|
+
|
|
148
|
+
if finding["level"] in BAD:
|
|
149
|
+
st = streak(state, node, finding["code"])
|
|
150
|
+
if st >= 2:
|
|
151
|
+
result["confidence"] = "VERIFIED"
|
|
152
|
+
elif result["confidence"] == "VERIFIED":
|
|
153
|
+
result["confidence"] = "LIKELY"
|
|
154
|
+
|
|
155
|
+
return result
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def due(state, node, code, fail_after=1) -> bool:
|
|
159
|
+
"""streak(state, node, code) >= fail_after"""
|
|
160
|
+
return streak(state, node, code) >= fail_after
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def _selftest():
|
|
164
|
+
tmp = tempfile.mkdtemp()
|
|
165
|
+
old_env = os.environ.get(STATE_ENV)
|
|
166
|
+
try:
|
|
167
|
+
def f(code, level="FAIL", node="gpu-box", confidence="VERIFIED"):
|
|
168
|
+
return {"code": code, "level": level, "node": node, "confidence": confidence}
|
|
169
|
+
|
|
170
|
+
def t_missing_file_fresh():
|
|
171
|
+
return load(os.path.join(tmp, "absent.json")) == {"schema": 1, "nodes": {}}
|
|
172
|
+
|
|
173
|
+
def t_corrupt_file_fresh():
|
|
174
|
+
p = os.path.join(tmp, "bad.json")
|
|
175
|
+
with open(p, "w", encoding="utf-8") as fh:
|
|
176
|
+
fh.write("{not json")
|
|
177
|
+
return load(p) == {"schema": 1, "nodes": {}}
|
|
178
|
+
|
|
179
|
+
def t_save_load_roundtrip():
|
|
180
|
+
p = os.path.join(tmp, "sub", "s.json")
|
|
181
|
+
s = record_run(fresh(), "gpu-box", [f("x.y")], "2026-10-03T18:00:00")
|
|
182
|
+
save(s, p)
|
|
183
|
+
return load(p) == s and not os.path.exists(p + ".tmp")
|
|
184
|
+
|
|
185
|
+
def t_keeps_last_5():
|
|
186
|
+
s = fresh()
|
|
187
|
+
for i in range(7):
|
|
188
|
+
record_run(s, "gpu-box", [], "t%d" % i)
|
|
189
|
+
runs = s["nodes"]["gpu-box"]["runs"]
|
|
190
|
+
return len(runs) == KEEP and runs[0]["at"] == "t2" and runs[-1]["at"] == "t6"
|
|
191
|
+
|
|
192
|
+
def t_other_node_ignored():
|
|
193
|
+
s = record_run(fresh(), "gpu-box", [f("a.b", node="gpu-box-2"), f("c.d")], "t0")
|
|
194
|
+
return s["nodes"]["gpu-box"]["runs"][0]["codes"] == {"c.d": "FAIL"}
|
|
195
|
+
|
|
196
|
+
def t_streak_counts_recent_bad_runs():
|
|
197
|
+
s = fresh()
|
|
198
|
+
record_run(s, "gpu-box", [f("x.y")], "t0")
|
|
199
|
+
record_run(s, "gpu-box", [], "t1")
|
|
200
|
+
record_run(s, "gpu-box", [f("x.y", "WARN")], "t2")
|
|
201
|
+
record_run(s, "gpu-box", [f("x.y")], "t3")
|
|
202
|
+
return streak(s, "gpu-box", "x.y") == 2 and clear_streak(s, "gpu-box", "x.y") == 0
|
|
203
|
+
|
|
204
|
+
def t_one_sample_is_likely_at_most():
|
|
205
|
+
s = record_run(fresh(), "gpu-box", [f("x.y")], "t0")
|
|
206
|
+
return grade(s, "gpu-box", f("x.y"))["confidence"] == "LIKELY"
|
|
207
|
+
|
|
208
|
+
def t_two_samples_verified():
|
|
209
|
+
s = fresh()
|
|
210
|
+
record_run(s, "gpu-box", [f("x.y", confidence="LIKELY")], "t0")
|
|
211
|
+
record_run(s, "gpu-box", [f("x.y", confidence="LIKELY")], "t1")
|
|
212
|
+
return grade(s, "gpu-box", f("x.y", confidence="LIKELY"))["confidence"] == "VERIFIED"
|
|
213
|
+
|
|
214
|
+
def t_ok_finding_untouched():
|
|
215
|
+
s = record_run(fresh(), "gpu-box", [f("x.ok", "OK")], "t0")
|
|
216
|
+
return grade(s, "gpu-box", f("x.ok", "OK"))["confidence"] == "VERIFIED"
|
|
217
|
+
|
|
218
|
+
def t_due_respects_fail_after():
|
|
219
|
+
s = fresh()
|
|
220
|
+
record_run(s, "gpu-box", [f("x.y")], "t0")
|
|
221
|
+
first = due(s, "gpu-box", "x.y", fail_after=2)
|
|
222
|
+
record_run(s, "gpu-box", [f("x.y")], "t1")
|
|
223
|
+
return first is False and due(s, "gpu-box", "x.y", fail_after=2) is True
|
|
224
|
+
|
|
225
|
+
def t_env_path():
|
|
226
|
+
os.environ[STATE_ENV] = os.path.join(tmp, "env.json")
|
|
227
|
+
return state_path() == Path(tmp) / "env.json"
|
|
228
|
+
|
|
229
|
+
checks = [
|
|
230
|
+
("missing_file_fresh", t_missing_file_fresh),
|
|
231
|
+
("corrupt_file_fresh", t_corrupt_file_fresh),
|
|
232
|
+
("save_load_roundtrip", t_save_load_roundtrip),
|
|
233
|
+
("keeps_last_5", t_keeps_last_5),
|
|
234
|
+
("other_node_ignored", t_other_node_ignored),
|
|
235
|
+
("streak_counts_recent_bad_runs", t_streak_counts_recent_bad_runs),
|
|
236
|
+
("one_sample_is_likely_at_most", t_one_sample_is_likely_at_most),
|
|
237
|
+
("two_samples_verified", t_two_samples_verified),
|
|
238
|
+
("ok_finding_untouched", t_ok_finding_untouched),
|
|
239
|
+
("due_respects_fail_after", t_due_respects_fail_after),
|
|
240
|
+
("env_path", t_env_path),
|
|
241
|
+
]
|
|
242
|
+
passed = 0
|
|
243
|
+
for name, fn in checks:
|
|
244
|
+
try:
|
|
245
|
+
ok = bool(fn())
|
|
246
|
+
except Exception as exc: # noqa: BLE001 - a crashed check is a failed check
|
|
247
|
+
ok = False
|
|
248
|
+
name = "%s (raised %s: %s)" % (name, type(exc).__name__, exc)
|
|
249
|
+
passed += ok
|
|
250
|
+
print(("ok " if ok else "FAIL ") + name)
|
|
251
|
+
failed = len(checks) - passed
|
|
252
|
+
print("sg_state selftest: %d checks, %d passed, %d failed" % (len(checks), passed, failed))
|
|
253
|
+
return 0 if failed == 0 and checks else 1
|
|
254
|
+
finally:
|
|
255
|
+
if old_env is None:
|
|
256
|
+
os.environ.pop(STATE_ENV, None)
|
|
257
|
+
else:
|
|
258
|
+
os.environ[STATE_ENV] = old_env
|
|
259
|
+
|
|
260
|
+
shutil.rmtree(tmp, ignore_errors=True)
|
|
261
|
+
|
|
262
|
+
|
|
263
|
+
if __name__ == "__main__":
|
|
264
|
+
if "--selftest" in sys.argv:
|
|
265
|
+
sys.exit(_selftest())
|
|
266
|
+
print(json.dumps(load(), indent=1))
|
|
@@ -0,0 +1,164 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: stack-guardian
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: A health doctor for a home AI stack: Ollama nodes, GPUs and tailnet, with graded confidence.
|
|
5
|
+
Author: LuminariSoftwares
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/LuminariSoftwares/stack-guardian
|
|
8
|
+
Project-URL: Issues, https://github.com/LuminariSoftwares/stack-guardian/issues
|
|
9
|
+
Project-URL: Changelog, https://github.com/LuminariSoftwares/stack-guardian/blob/main/CHANGELOG.md
|
|
10
|
+
Keywords: ollama,gpu,vram,tailscale,health-check,local-llm,mcp
|
|
11
|
+
Classifier: Programming Language :: Python :: 3
|
|
12
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
13
|
+
Classifier: Operating System :: OS Independent
|
|
14
|
+
Classifier: Environment :: Console
|
|
15
|
+
Classifier: Topic :: System :: Monitoring
|
|
16
|
+
Requires-Python: >=3.11
|
|
17
|
+
Description-Content-Type: text/markdown
|
|
18
|
+
License-File: LICENSE
|
|
19
|
+
Dynamic: license-file
|
|
20
|
+
|
|
21
|
+
# stack-guardian
|
|
22
|
+
|
|
23
|
+
[](https://github.com/LuminariSoftwares/stack-guardian/actions/workflows/ci.yml)
|
|
24
|
+
[](https://github.com/LuminariSoftwares/stack-guardian/actions/workflows/privacy.yml)
|
|
25
|
+
|
|
26
|
+
A health doctor for a home AI stack. It collects a snapshot of your nodes (Ollama endpoints, loaded models, NVIDIA
|
|
27
|
+
GPUs, tailnet peers), runs small Python rules against it, remembers the last runs, and reports findings with a
|
|
28
|
+
confidence grade: one sample is at most `LIKELY`, the same finding twice in a row is `VERIFIED`. A finding can carry a
|
|
29
|
+
typed fix that you run on purpose, never automatically.
|
|
30
|
+
|
|
31
|
+
Status: **0.1.0, first release.** Standard library only, Python 3.11+.
|
|
32
|
+
|
|
33
|
+
## Install
|
|
34
|
+
|
|
35
|
+
```
|
|
36
|
+
pip install stack-guardian
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
This gives you three commands: `stack-guardian` (the CLI), `stack-guardian-serve` and `stack-guardian-mcp`.
|
|
40
|
+
|
|
41
|
+
Try it offline first: the example snapshot makes every shipped rule fire once.
|
|
42
|
+
|
|
43
|
+
```
|
|
44
|
+
curl -LO https://raw.githubusercontent.com/LuminariSoftwares/stack-guardian/main/examples/snapshot.json
|
|
45
|
+
curl -LO https://raw.githubusercontent.com/LuminariSoftwares/stack-guardian/main/examples/ctx.json
|
|
46
|
+
stack-guardian doctor --snapshot snapshot.json --ctx ctx.json
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
From a clone instead (for writing rules or running the tests):
|
|
50
|
+
|
|
51
|
+
```
|
|
52
|
+
git clone https://github.com/LuminariSoftwares/stack-guardian.git
|
|
53
|
+
cd stack-guardian
|
|
54
|
+
python -m venv .venv
|
|
55
|
+
.venv\Scripts\activate # Windows; macOS/Linux: source .venv/bin/activate
|
|
56
|
+
pip install -e .
|
|
57
|
+
stack-guardian doctor --snapshot examples/snapshot.json --ctx examples/ctx.json
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
## For coding agents
|
|
61
|
+
|
|
62
|
+
- **MCP server** (any MCP client): `stack-guardian-mcp` speaks MCP over stdio with three read-only tools,
|
|
63
|
+
`stack_health`, `stack_snapshot` and `explain`. In Claude Code:
|
|
64
|
+
|
|
65
|
+
```
|
|
66
|
+
claude mcp add stack-guardian -e STACK_GUARDIAN_NODES=/path/to/nodes.json -- stack-guardian-mcp
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
Other clients: command `stack-guardian-mcp`, args `["--nodes", "/path/to/nodes.json"]`. `stack_health` grades
|
|
70
|
+
against the saved run history without adding to it, and reuses one snapshot for 10 s (`--ttl SECONDS`).
|
|
71
|
+
- **DSH plugin** (`npm install dsh-stack-guardian`, source in `dsh/`, Node 20+): adds one line at the start of a DeepSeek Harness session when the doctor reports
|
|
72
|
+
WARN or FAIL, and nothing otherwise. It runs `stack-guardian doctor --json --quiet`, so set `STACK_GUARDIAN_NODES`
|
|
73
|
+
to your nodes file in the harness environment.
|
|
74
|
+
|
|
75
|
+
## What it does
|
|
76
|
+
|
|
77
|
+
### 1. `doctor` - run the checks
|
|
78
|
+
|
|
79
|
+
Describe your nodes in a JSON file (see [examples/nodes.json](examples/nodes.json)) and check them live:
|
|
80
|
+
|
|
81
|
+
```
|
|
82
|
+
stack-guardian doctor --nodes examples/nodes.json --ctx examples/ctx.json
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
With neither `--snapshot` nor `--nodes`, `doctor` reads the nodes file named by `$STACK_GUARDIAN_NODES`.
|
|
86
|
+
|
|
87
|
+
Options: `--checks-dir DIR` (your own rules, default is the shipped `checks.d`), `--ctx FILE` (rule settings such as
|
|
88
|
+
the model each node is expected to hold), `--state FILE` (default `$STACK_GUARDIAN_STATE` or
|
|
89
|
+
`~/.stack-guardian/state.json`), `--json`, `--quiet` (print nothing when all is OK).
|
|
90
|
+
|
|
91
|
+
Exit codes: `0` all OK/INFO, `1` a WARN, `2` a FAIL, `3` usage, `4` the doctor itself crashed.
|
|
92
|
+
|
|
93
|
+
Shipped rules: `node.unreachable`, `ollama.unreachable`, `model.not_loaded`, `vram.unattributed`, `gpu.unavailable`,
|
|
94
|
+
`tailnet.unavailable`, `tailnet.peer_offline`, `tailnet.relayed`. Each raises one finding per affected node. A rule that
|
|
95
|
+
raises becomes a `check.crashed` WARN - a buggy rule never reports a broken stack.
|
|
96
|
+
|
|
97
|
+
### 2. `doctor --fix ID` - run a finding's typed fix
|
|
98
|
+
|
|
99
|
+
A rule may attach `fix={"action": ..., "args": {...}}` to a finding. There are exactly three actions and never a raw
|
|
100
|
+
shell string:
|
|
101
|
+
|
|
102
|
+
| action | what it does | needs |
|
|
103
|
+
|---|---|---|
|
|
104
|
+
| `unload_model` | POST `keep_alive: 0` to an Ollama `/api/generate` | `node`, `url`, `model` |
|
|
105
|
+
| `restart_lane` | runs the argv list you declared for that lane | `lane` declared in `--config` |
|
|
106
|
+
| `set_env` | writes `NAME=value` to `<state folder>/env` and sets it for the process | `name`, `value` |
|
|
107
|
+
|
|
108
|
+
```
|
|
109
|
+
stack-guardian doctor --checks-dir examples/checks.d --snapshot examples/snapshot.json --fix ollama.stuck --config examples/lanes.json
|
|
110
|
+
```
|
|
111
|
+
|
|
112
|
+
That is a **dry run**: it prints the plan and changes nothing. Add `--apply` to perform it. An applied fix first saves
|
|
113
|
+
the snapshot under `<state folder>/snapshots/`, then runs, then appends one JSON line to `<state folder>/fixes.jsonl`
|
|
114
|
+
(`--audit FILE` to move it). A `set_env` value is never written to the audit, only its length. `ID` is the finding
|
|
115
|
+
code, or `code@node` when several nodes raise the same code. With `--fix`, the exit code is the fix's: `0` plan or
|
|
116
|
+
success, `1` refused or failed, `3` usage.
|
|
117
|
+
|
|
118
|
+
### 3. Other commands
|
|
119
|
+
|
|
120
|
+
- `stack-guardian snapshot --nodes FILE` prints the raw snapshot the rules read.
|
|
121
|
+
- `stack-guardian-serve --nodes FILE [--port 8788]` serves a fresh snapshot at `GET /snapshot` (read-only, loopback
|
|
122
|
+
unless you pass `--host H --allow-remote`), so one machine can ask another for its health.
|
|
123
|
+
no auth: bind to localhost or a tailnet IP only.
|
|
124
|
+
- `stack-guardian-mcp --nodes FILE [--ctx FILE] [--state FILE] [--ttl SECONDS]`: see [For coding agents](#for-coding-agents).
|
|
125
|
+
|
|
126
|
+
## Writing a rule
|
|
127
|
+
|
|
128
|
+
```python
|
|
129
|
+
from stack_guardian import finding as F
|
|
130
|
+
|
|
131
|
+
def ollama_up(snapshot, ctx):
|
|
132
|
+
if not snapshot["nodes"]["gpu-box"].get("reachable"):
|
|
133
|
+
return F.make_finding("ollama.unreachable", "missing", "FAIL", "gpu-box", "Ollama does not answer")
|
|
134
|
+
return None
|
|
135
|
+
|
|
136
|
+
CHECKS = [("ollama.up", ollama_up)]
|
|
137
|
+
```
|
|
138
|
+
|
|
139
|
+
Drop the file in a folder and point `--checks-dir` at it. A full example with a fix is in
|
|
140
|
+
[examples/checks.d/20_ollama_fix.py](examples/checks.d/20_ollama_fix.py). A rule may also return a list of findings,
|
|
141
|
+
one per node.
|
|
142
|
+
|
|
143
|
+
## Examples
|
|
144
|
+
|
|
145
|
+
| file | what it is |
|
|
146
|
+
|---|---|
|
|
147
|
+
| `examples/nodes.json` | three nodes: a local one with GPUs, two remote ones over the tailnet |
|
|
148
|
+
| `examples/ctx.json` | rule settings: which model gpu-box-2 should hold |
|
|
149
|
+
| `examples/snapshot.json` | an offline snapshot on which every shipped rule fires once, for trying `doctor` and `--fix` |
|
|
150
|
+
| `examples/lanes.json` | the lanes `restart_lane` is allowed to run |
|
|
151
|
+
| `examples/checks.d/20_ollama_fix.py` | a custom rule whose finding carries a fix |
|
|
152
|
+
|
|
153
|
+
## Test
|
|
154
|
+
|
|
155
|
+
```
|
|
156
|
+
pip install pytest
|
|
157
|
+
python -m pytest -q
|
|
158
|
+
```
|
|
159
|
+
|
|
160
|
+
CI runs this on Ubuntu, Windows and macOS with Python 3.11, plus a privacy scan of every tracked file.
|
|
161
|
+
|
|
162
|
+
## License
|
|
163
|
+
|
|
164
|
+
MIT
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
stack_guardian/__init__.py,sha256=T3q-qBicefSvI0yTpw112QZ33IqkIjPVF4TodgQWO9g,496
|
|
2
|
+
stack_guardian/__main__.py,sha256=ibl4GU_04RgSpLteRQsvqE5RZw4s2n1_Cph0hHqkmGw,94
|
|
3
|
+
stack_guardian/cli.py,sha256=3JxijTGy_pWq_yWTzv-Fh0j_RjkYScGqMSk9m-bN6Ws,13462
|
|
4
|
+
stack_guardian/collect.py,sha256=56SfzV0RwiLsJTFhrkw2gzdxMweacVbOdLYtqjwDHzE,8799
|
|
5
|
+
stack_guardian/core.py,sha256=eRWpjWWRpSIn3inEz3oDkCNhwTrxV7yW53lQLVEPeSM,8841
|
|
6
|
+
stack_guardian/finding.py,sha256=QdnHq0ecNGMxrhRgpWgiy-wf_4Ho-Tzut5A3ecBmilw,16032
|
|
7
|
+
stack_guardian/fixes.py,sha256=okQPfm4Yd3bVm57WA4lUUGoXRHbLwWcAP_L8AFk2WW0,12912
|
|
8
|
+
stack_guardian/mcp_server.py,sha256=i4WhO451NJcy237fqfIVj1sIUh1VYxueB94hOPPAzIY,15701
|
|
9
|
+
stack_guardian/multinode.py,sha256=t8GWoQP6Tnu6mv20A0jSkHi1jnrtLjvKISKAM3eSNig,6175
|
|
10
|
+
stack_guardian/parsers.py,sha256=wUmfQrrDDNrmH0jXLQeM6hiOvDXKec2x_8odRQHs8vs,26281
|
|
11
|
+
stack_guardian/serve.py,sha256=2-KnIdHC0e8to6tTVEmk-pVQFXdtbgPorWBTuiT_npk,7026
|
|
12
|
+
stack_guardian/state.py,sha256=kh_9cgxqk2EMSsC_uFiJ7e6585Ij54PL3q2owOysnmA,9965
|
|
13
|
+
stack_guardian/checks.d/10_core.py,sha256=gZ0PAz6ZPzRhayO1q58WouBmQtivAxvU7rBPCYioQ9s,8428
|
|
14
|
+
stack_guardian-0.1.0.dist-info/licenses/LICENSE,sha256=fZftfP7aXYDvruO-a2SUD2UxszjndCeOA8u0pXRjQS8,1074
|
|
15
|
+
stack_guardian-0.1.0.dist-info/METADATA,sha256=QzhI-i-f47Xv-7ju-TC4MYIEzqGuLAkHLFU-Xuefo4s,7388
|
|
16
|
+
stack_guardian-0.1.0.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
|
|
17
|
+
stack_guardian-0.1.0.dist-info/entry_points.txt,sha256=0RxeHy2S0emvaG_zUmYE0DwKeCq6Q5V7zxUf3lo9zx4,160
|
|
18
|
+
stack_guardian-0.1.0.dist-info/top_level.txt,sha256=7J59zP8x8_DwTUFQiW-Grx_r2Cn_j-jDZYtn1q5lHgk,15
|
|
19
|
+
stack_guardian-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 LuminariSoftwares
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|