stack-guardian 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- stack_guardian-0.1.0/LICENSE +21 -0
- stack_guardian-0.1.0/PKG-INFO +164 -0
- stack_guardian-0.1.0/README.md +144 -0
- stack_guardian-0.1.0/pyproject.toml +38 -0
- stack_guardian-0.1.0/setup.cfg +4 -0
- stack_guardian-0.1.0/stack_guardian/__init__.py +13 -0
- stack_guardian-0.1.0/stack_guardian/__main__.py +6 -0
- stack_guardian-0.1.0/stack_guardian/checks.d/10_core.py +174 -0
- stack_guardian-0.1.0/stack_guardian/cli.py +295 -0
- stack_guardian-0.1.0/stack_guardian/collect.py +206 -0
- stack_guardian-0.1.0/stack_guardian/core.py +264 -0
- stack_guardian-0.1.0/stack_guardian/finding.py +491 -0
- stack_guardian-0.1.0/stack_guardian/fixes.py +287 -0
- stack_guardian-0.1.0/stack_guardian/mcp_server.py +340 -0
- stack_guardian-0.1.0/stack_guardian/multinode.py +152 -0
- stack_guardian-0.1.0/stack_guardian/parsers.py +658 -0
- stack_guardian-0.1.0/stack_guardian/serve.py +198 -0
- stack_guardian-0.1.0/stack_guardian/state.py +266 -0
- stack_guardian-0.1.0/stack_guardian.egg-info/PKG-INFO +164 -0
- stack_guardian-0.1.0/stack_guardian.egg-info/SOURCES.txt +25 -0
- stack_guardian-0.1.0/stack_guardian.egg-info/dependency_links.txt +1 -0
- stack_guardian-0.1.0/stack_guardian.egg-info/entry_points.txt +4 -0
- stack_guardian-0.1.0/stack_guardian.egg-info/top_level.txt +1 -0
- stack_guardian-0.1.0/tests/test_doctor_fix.py +137 -0
- stack_guardian-0.1.0/tests/test_examples.py +37 -0
- stack_guardian-0.1.0/tests/test_review_fixes.py +286 -0
- stack_guardian-0.1.0/tests/test_selftests.py +98 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 LuminariSoftwares
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,164 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: stack-guardian
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: A health doctor for a home AI stack: Ollama nodes, GPUs and tailnet, with graded confidence.
|
|
5
|
+
Author: LuminariSoftwares
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/LuminariSoftwares/stack-guardian
|
|
8
|
+
Project-URL: Issues, https://github.com/LuminariSoftwares/stack-guardian/issues
|
|
9
|
+
Project-URL: Changelog, https://github.com/LuminariSoftwares/stack-guardian/blob/main/CHANGELOG.md
|
|
10
|
+
Keywords: ollama,gpu,vram,tailscale,health-check,local-llm,mcp
|
|
11
|
+
Classifier: Programming Language :: Python :: 3
|
|
12
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
13
|
+
Classifier: Operating System :: OS Independent
|
|
14
|
+
Classifier: Environment :: Console
|
|
15
|
+
Classifier: Topic :: System :: Monitoring
|
|
16
|
+
Requires-Python: >=3.11
|
|
17
|
+
Description-Content-Type: text/markdown
|
|
18
|
+
License-File: LICENSE
|
|
19
|
+
Dynamic: license-file
|
|
20
|
+
|
|
21
|
+
# stack-guardian
|
|
22
|
+
|
|
23
|
+
[](https://github.com/LuminariSoftwares/stack-guardian/actions/workflows/ci.yml)
|
|
24
|
+
[](https://github.com/LuminariSoftwares/stack-guardian/actions/workflows/privacy.yml)
|
|
25
|
+
|
|
26
|
+
A health doctor for a home AI stack. It collects a snapshot of your nodes (Ollama endpoints, loaded models, NVIDIA
|
|
27
|
+
GPUs, tailnet peers), runs small Python rules against it, remembers the last runs, and reports findings with a
|
|
28
|
+
confidence grade: one sample is at most `LIKELY`, the same finding twice in a row is `VERIFIED`. A finding can carry a
|
|
29
|
+
typed fix that you run on purpose, never automatically.
|
|
30
|
+
|
|
31
|
+
Status: **0.1.0, first release.** Standard library only, Python 3.11+.
|
|
32
|
+
|
|
33
|
+
## Install
|
|
34
|
+
|
|
35
|
+
```
|
|
36
|
+
pip install stack-guardian
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
This gives you three commands: `stack-guardian` (the CLI), `stack-guardian-serve` and `stack-guardian-mcp`.
|
|
40
|
+
|
|
41
|
+
Try it offline first: the example snapshot makes every shipped rule fire once.
|
|
42
|
+
|
|
43
|
+
```
|
|
44
|
+
curl -LO https://raw.githubusercontent.com/LuminariSoftwares/stack-guardian/main/examples/snapshot.json
|
|
45
|
+
curl -LO https://raw.githubusercontent.com/LuminariSoftwares/stack-guardian/main/examples/ctx.json
|
|
46
|
+
stack-guardian doctor --snapshot snapshot.json --ctx ctx.json
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
From a clone instead (for writing rules or running the tests):
|
|
50
|
+
|
|
51
|
+
```
|
|
52
|
+
git clone https://github.com/LuminariSoftwares/stack-guardian.git
|
|
53
|
+
cd stack-guardian
|
|
54
|
+
python -m venv .venv
|
|
55
|
+
.venv\Scripts\activate # Windows; macOS/Linux: source .venv/bin/activate
|
|
56
|
+
pip install -e .
|
|
57
|
+
stack-guardian doctor --snapshot examples/snapshot.json --ctx examples/ctx.json
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
## For coding agents
|
|
61
|
+
|
|
62
|
+
- **MCP server** (any MCP client): `stack-guardian-mcp` speaks MCP over stdio with three read-only tools,
|
|
63
|
+
`stack_health`, `stack_snapshot` and `explain`. In Claude Code:
|
|
64
|
+
|
|
65
|
+
```
|
|
66
|
+
claude mcp add stack-guardian -e STACK_GUARDIAN_NODES=/path/to/nodes.json -- stack-guardian-mcp
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
Other clients: command `stack-guardian-mcp`, args `["--nodes", "/path/to/nodes.json"]`. `stack_health` grades
|
|
70
|
+
against the saved run history without adding to it, and reuses one snapshot for 10 s (`--ttl SECONDS`).
|
|
71
|
+
- **DSH plugin** (`npm install dsh-stack-guardian`, source in `dsh/`, Node 20+): adds one line at the start of a DeepSeek Harness session when the doctor reports
|
|
72
|
+
WARN or FAIL, and nothing otherwise. It runs `stack-guardian doctor --json --quiet`, so set `STACK_GUARDIAN_NODES`
|
|
73
|
+
to your nodes file in the harness environment.
|
|
74
|
+
|
|
75
|
+
## What it does
|
|
76
|
+
|
|
77
|
+
### 1. `doctor` - run the checks
|
|
78
|
+
|
|
79
|
+
Describe your nodes in a JSON file (see [examples/nodes.json](examples/nodes.json)) and check them live:
|
|
80
|
+
|
|
81
|
+
```
|
|
82
|
+
stack-guardian doctor --nodes examples/nodes.json --ctx examples/ctx.json
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
With neither `--snapshot` nor `--nodes`, `doctor` reads the nodes file named by `$STACK_GUARDIAN_NODES`.
|
|
86
|
+
|
|
87
|
+
Options: `--checks-dir DIR` (your own rules, default is the shipped `checks.d`), `--ctx FILE` (rule settings such as
|
|
88
|
+
the model each node is expected to hold), `--state FILE` (default `$STACK_GUARDIAN_STATE` or
|
|
89
|
+
`~/.stack-guardian/state.json`), `--json`, `--quiet` (print nothing when all is OK).
|
|
90
|
+
|
|
91
|
+
Exit codes: `0` all OK/INFO, `1` a WARN, `2` a FAIL, `3` usage, `4` the doctor itself crashed.
|
|
92
|
+
|
|
93
|
+
Shipped rules: `node.unreachable`, `ollama.unreachable`, `model.not_loaded`, `vram.unattributed`, `gpu.unavailable`,
|
|
94
|
+
`tailnet.unavailable`, `tailnet.peer_offline`, `tailnet.relayed`. Each raises one finding per affected node. A rule that
|
|
95
|
+
raises becomes a `check.crashed` WARN - a buggy rule never reports a broken stack.
|
|
96
|
+
|
|
97
|
+
### 2. `doctor --fix ID` - run a finding's typed fix
|
|
98
|
+
|
|
99
|
+
A rule may attach `fix={"action": ..., "args": {...}}` to a finding. There are exactly three actions and never a raw
|
|
100
|
+
shell string:
|
|
101
|
+
|
|
102
|
+
| action | what it does | needs |
|
|
103
|
+
|---|---|---|
|
|
104
|
+
| `unload_model` | POST `keep_alive: 0` to an Ollama `/api/generate` | `node`, `url`, `model` |
|
|
105
|
+
| `restart_lane` | runs the argv list you declared for that lane | `lane` declared in `--config` |
|
|
106
|
+
| `set_env` | writes `NAME=value` to `<state folder>/env` and sets it for the process | `name`, `value` |
|
|
107
|
+
|
|
108
|
+
```
|
|
109
|
+
stack-guardian doctor --checks-dir examples/checks.d --snapshot examples/snapshot.json --fix ollama.stuck --config examples/lanes.json
|
|
110
|
+
```
|
|
111
|
+
|
|
112
|
+
That is a **dry run**: it prints the plan and changes nothing. Add `--apply` to perform it. An applied fix first saves
|
|
113
|
+
the snapshot under `<state folder>/snapshots/`, then runs, then appends one JSON line to `<state folder>/fixes.jsonl`
|
|
114
|
+
(`--audit FILE` to move it). A `set_env` value is never written to the audit, only its length. `ID` is the finding
|
|
115
|
+
code, or `code@node` when several nodes raise the same code. With `--fix`, the exit code is the fix's: `0` plan or
|
|
116
|
+
success, `1` refused or failed, `3` usage.
|
|
117
|
+
|
|
118
|
+
### 3. Other commands
|
|
119
|
+
|
|
120
|
+
- `stack-guardian snapshot --nodes FILE` prints the raw snapshot the rules read.
|
|
121
|
+
- `stack-guardian-serve --nodes FILE [--port 8788]` serves a fresh snapshot at `GET /snapshot` (read-only, loopback
|
|
122
|
+
unless you pass `--host H --allow-remote`), so one machine can ask another for its health.
|
|
123
|
+
no auth: bind to localhost or a tailnet IP only.
|
|
124
|
+
- `stack-guardian-mcp --nodes FILE [--ctx FILE] [--state FILE] [--ttl SECONDS]`: see [For coding agents](#for-coding-agents).
|
|
125
|
+
|
|
126
|
+
## Writing a rule
|
|
127
|
+
|
|
128
|
+
```python
|
|
129
|
+
from stack_guardian import finding as F
|
|
130
|
+
|
|
131
|
+
def ollama_up(snapshot, ctx):
|
|
132
|
+
if not snapshot["nodes"]["gpu-box"].get("reachable"):
|
|
133
|
+
return F.make_finding("ollama.unreachable", "missing", "FAIL", "gpu-box", "Ollama does not answer")
|
|
134
|
+
return None
|
|
135
|
+
|
|
136
|
+
CHECKS = [("ollama.up", ollama_up)]
|
|
137
|
+
```
|
|
138
|
+
|
|
139
|
+
Drop the file in a folder and point `--checks-dir` at it. A full example with a fix is in
|
|
140
|
+
[examples/checks.d/20_ollama_fix.py](examples/checks.d/20_ollama_fix.py). A rule may also return a list of findings,
|
|
141
|
+
one per node.
|
|
142
|
+
|
|
143
|
+
## Examples
|
|
144
|
+
|
|
145
|
+
| file | what it is |
|
|
146
|
+
|---|---|
|
|
147
|
+
| `examples/nodes.json` | three nodes: a local one with GPUs, two remote ones over the tailnet |
|
|
148
|
+
| `examples/ctx.json` | rule settings: which model gpu-box-2 should hold |
|
|
149
|
+
| `examples/snapshot.json` | an offline snapshot on which every shipped rule fires once, for trying `doctor` and `--fix` |
|
|
150
|
+
| `examples/lanes.json` | the lanes `restart_lane` is allowed to run |
|
|
151
|
+
| `examples/checks.d/20_ollama_fix.py` | a custom rule whose finding carries a fix |
|
|
152
|
+
|
|
153
|
+
## Test
|
|
154
|
+
|
|
155
|
+
```
|
|
156
|
+
pip install pytest
|
|
157
|
+
python -m pytest -q
|
|
158
|
+
```
|
|
159
|
+
|
|
160
|
+
CI runs this on Ubuntu, Windows and macOS with Python 3.11, plus a privacy scan of every tracked file.
|
|
161
|
+
|
|
162
|
+
## License
|
|
163
|
+
|
|
164
|
+
MIT
|
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
# stack-guardian
|
|
2
|
+
|
|
3
|
+
[](https://github.com/LuminariSoftwares/stack-guardian/actions/workflows/ci.yml)
|
|
4
|
+
[](https://github.com/LuminariSoftwares/stack-guardian/actions/workflows/privacy.yml)
|
|
5
|
+
|
|
6
|
+
A health doctor for a home AI stack. It collects a snapshot of your nodes (Ollama endpoints, loaded models, NVIDIA
|
|
7
|
+
GPUs, tailnet peers), runs small Python rules against it, remembers the last runs, and reports findings with a
|
|
8
|
+
confidence grade: one sample is at most `LIKELY`, the same finding twice in a row is `VERIFIED`. A finding can carry a
|
|
9
|
+
typed fix that you run on purpose, never automatically.
|
|
10
|
+
|
|
11
|
+
Status: **0.1.0, first release.** Standard library only, Python 3.11+.
|
|
12
|
+
|
|
13
|
+
## Install
|
|
14
|
+
|
|
15
|
+
```
|
|
16
|
+
pip install stack-guardian
|
|
17
|
+
```
|
|
18
|
+
|
|
19
|
+
This gives you three commands: `stack-guardian` (the CLI), `stack-guardian-serve` and `stack-guardian-mcp`.
|
|
20
|
+
|
|
21
|
+
Try it offline first: the example snapshot makes every shipped rule fire once.
|
|
22
|
+
|
|
23
|
+
```
|
|
24
|
+
curl -LO https://raw.githubusercontent.com/LuminariSoftwares/stack-guardian/main/examples/snapshot.json
|
|
25
|
+
curl -LO https://raw.githubusercontent.com/LuminariSoftwares/stack-guardian/main/examples/ctx.json
|
|
26
|
+
stack-guardian doctor --snapshot snapshot.json --ctx ctx.json
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
From a clone instead (for writing rules or running the tests):
|
|
30
|
+
|
|
31
|
+
```
|
|
32
|
+
git clone https://github.com/LuminariSoftwares/stack-guardian.git
|
|
33
|
+
cd stack-guardian
|
|
34
|
+
python -m venv .venv
|
|
35
|
+
.venv\Scripts\activate # Windows; macOS/Linux: source .venv/bin/activate
|
|
36
|
+
pip install -e .
|
|
37
|
+
stack-guardian doctor --snapshot examples/snapshot.json --ctx examples/ctx.json
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
## For coding agents
|
|
41
|
+
|
|
42
|
+
- **MCP server** (any MCP client): `stack-guardian-mcp` speaks MCP over stdio with three read-only tools,
|
|
43
|
+
`stack_health`, `stack_snapshot` and `explain`. In Claude Code:
|
|
44
|
+
|
|
45
|
+
```
|
|
46
|
+
claude mcp add stack-guardian -e STACK_GUARDIAN_NODES=/path/to/nodes.json -- stack-guardian-mcp
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
Other clients: command `stack-guardian-mcp`, args `["--nodes", "/path/to/nodes.json"]`. `stack_health` grades
|
|
50
|
+
against the saved run history without adding to it, and reuses one snapshot for 10 s (`--ttl SECONDS`).
|
|
51
|
+
- **DSH plugin** (`npm install dsh-stack-guardian`, source in `dsh/`, Node 20+): adds one line at the start of a DeepSeek Harness session when the doctor reports
|
|
52
|
+
WARN or FAIL, and nothing otherwise. It runs `stack-guardian doctor --json --quiet`, so set `STACK_GUARDIAN_NODES`
|
|
53
|
+
to your nodes file in the harness environment.
|
|
54
|
+
|
|
55
|
+
## What it does
|
|
56
|
+
|
|
57
|
+
### 1. `doctor` - run the checks
|
|
58
|
+
|
|
59
|
+
Describe your nodes in a JSON file (see [examples/nodes.json](examples/nodes.json)) and check them live:
|
|
60
|
+
|
|
61
|
+
```
|
|
62
|
+
stack-guardian doctor --nodes examples/nodes.json --ctx examples/ctx.json
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
With neither `--snapshot` nor `--nodes`, `doctor` reads the nodes file named by `$STACK_GUARDIAN_NODES`.
|
|
66
|
+
|
|
67
|
+
Options: `--checks-dir DIR` (your own rules, default is the shipped `checks.d`), `--ctx FILE` (rule settings such as
|
|
68
|
+
the model each node is expected to hold), `--state FILE` (default `$STACK_GUARDIAN_STATE` or
|
|
69
|
+
`~/.stack-guardian/state.json`), `--json`, `--quiet` (print nothing when all is OK).
|
|
70
|
+
|
|
71
|
+
Exit codes: `0` all OK/INFO, `1` a WARN, `2` a FAIL, `3` usage, `4` the doctor itself crashed.
|
|
72
|
+
|
|
73
|
+
Shipped rules: `node.unreachable`, `ollama.unreachable`, `model.not_loaded`, `vram.unattributed`, `gpu.unavailable`,
|
|
74
|
+
`tailnet.unavailable`, `tailnet.peer_offline`, `tailnet.relayed`. Each raises one finding per affected node. A rule that
|
|
75
|
+
raises becomes a `check.crashed` WARN - a buggy rule never reports a broken stack.
|
|
76
|
+
|
|
77
|
+
### 2. `doctor --fix ID` - run a finding's typed fix
|
|
78
|
+
|
|
79
|
+
A rule may attach `fix={"action": ..., "args": {...}}` to a finding. There are exactly three actions and never a raw
|
|
80
|
+
shell string:
|
|
81
|
+
|
|
82
|
+
| action | what it does | needs |
|
|
83
|
+
|---|---|---|
|
|
84
|
+
| `unload_model` | POST `keep_alive: 0` to an Ollama `/api/generate` | `node`, `url`, `model` |
|
|
85
|
+
| `restart_lane` | runs the argv list you declared for that lane | `lane` declared in `--config` |
|
|
86
|
+
| `set_env` | writes `NAME=value` to `<state folder>/env` and sets it for the process | `name`, `value` |
|
|
87
|
+
|
|
88
|
+
```
|
|
89
|
+
stack-guardian doctor --checks-dir examples/checks.d --snapshot examples/snapshot.json --fix ollama.stuck --config examples/lanes.json
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
That is a **dry run**: it prints the plan and changes nothing. Add `--apply` to perform it. An applied fix first saves
|
|
93
|
+
the snapshot under `<state folder>/snapshots/`, then runs, then appends one JSON line to `<state folder>/fixes.jsonl`
|
|
94
|
+
(`--audit FILE` to move it). A `set_env` value is never written to the audit, only its length. `ID` is the finding
|
|
95
|
+
code, or `code@node` when several nodes raise the same code. With `--fix`, the exit code is the fix's: `0` plan or
|
|
96
|
+
success, `1` refused or failed, `3` usage.
|
|
97
|
+
|
|
98
|
+
### 3. Other commands
|
|
99
|
+
|
|
100
|
+
- `stack-guardian snapshot --nodes FILE` prints the raw snapshot the rules read.
|
|
101
|
+
- `stack-guardian-serve --nodes FILE [--port 8788]` serves a fresh snapshot at `GET /snapshot` (read-only, loopback
|
|
102
|
+
unless you pass `--host H --allow-remote`), so one machine can ask another for its health.
|
|
103
|
+
no auth: bind to localhost or a tailnet IP only.
|
|
104
|
+
- `stack-guardian-mcp --nodes FILE [--ctx FILE] [--state FILE] [--ttl SECONDS]`: see [For coding agents](#for-coding-agents).
|
|
105
|
+
|
|
106
|
+
## Writing a rule
|
|
107
|
+
|
|
108
|
+
```python
|
|
109
|
+
from stack_guardian import finding as F
|
|
110
|
+
|
|
111
|
+
def ollama_up(snapshot, ctx):
|
|
112
|
+
if not snapshot["nodes"]["gpu-box"].get("reachable"):
|
|
113
|
+
return F.make_finding("ollama.unreachable", "missing", "FAIL", "gpu-box", "Ollama does not answer")
|
|
114
|
+
return None
|
|
115
|
+
|
|
116
|
+
CHECKS = [("ollama.up", ollama_up)]
|
|
117
|
+
```
|
|
118
|
+
|
|
119
|
+
Drop the file in a folder and point `--checks-dir` at it. A full example with a fix is in
|
|
120
|
+
[examples/checks.d/20_ollama_fix.py](examples/checks.d/20_ollama_fix.py). A rule may also return a list of findings,
|
|
121
|
+
one per node.
|
|
122
|
+
|
|
123
|
+
## Examples
|
|
124
|
+
|
|
125
|
+
| file | what it is |
|
|
126
|
+
|---|---|
|
|
127
|
+
| `examples/nodes.json` | three nodes: a local one with GPUs, two remote ones over the tailnet |
|
|
128
|
+
| `examples/ctx.json` | rule settings: which model gpu-box-2 should hold |
|
|
129
|
+
| `examples/snapshot.json` | an offline snapshot on which every shipped rule fires once, for trying `doctor` and `--fix` |
|
|
130
|
+
| `examples/lanes.json` | the lanes `restart_lane` is allowed to run |
|
|
131
|
+
| `examples/checks.d/20_ollama_fix.py` | a custom rule whose finding carries a fix |
|
|
132
|
+
|
|
133
|
+
## Test
|
|
134
|
+
|
|
135
|
+
```
|
|
136
|
+
pip install pytest
|
|
137
|
+
python -m pytest -q
|
|
138
|
+
```
|
|
139
|
+
|
|
140
|
+
CI runs this on Ubuntu, Windows and macOS with Python 3.11, plus a privacy scan of every tracked file.
|
|
141
|
+
|
|
142
|
+
## License
|
|
143
|
+
|
|
144
|
+
MIT
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=68"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "stack-guardian"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "A health doctor for a home AI stack: Ollama nodes, GPUs and tailnet, with graded confidence."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.11"
|
|
11
|
+
license = { text = "MIT" }
|
|
12
|
+
authors = [{ name = "LuminariSoftwares" }]
|
|
13
|
+
keywords = ["ollama", "gpu", "vram", "tailscale", "health-check", "local-llm", "mcp"]
|
|
14
|
+
classifiers = [
|
|
15
|
+
"Programming Language :: Python :: 3",
|
|
16
|
+
"License :: OSI Approved :: MIT License",
|
|
17
|
+
"Operating System :: OS Independent",
|
|
18
|
+
"Environment :: Console",
|
|
19
|
+
"Topic :: System :: Monitoring",
|
|
20
|
+
]
|
|
21
|
+
dependencies = []
|
|
22
|
+
|
|
23
|
+
[project.urls]
|
|
24
|
+
Homepage = "https://github.com/LuminariSoftwares/stack-guardian"
|
|
25
|
+
Issues = "https://github.com/LuminariSoftwares/stack-guardian/issues"
|
|
26
|
+
Changelog = "https://github.com/LuminariSoftwares/stack-guardian/blob/main/CHANGELOG.md"
|
|
27
|
+
|
|
28
|
+
[project.scripts]
|
|
29
|
+
stack-guardian = "stack_guardian.cli:main"
|
|
30
|
+
stack-guardian-mcp = "stack_guardian.mcp_server:main"
|
|
31
|
+
stack-guardian-serve = "stack_guardian.serve:main"
|
|
32
|
+
|
|
33
|
+
[tool.setuptools]
|
|
34
|
+
packages = ["stack_guardian"]
|
|
35
|
+
|
|
36
|
+
[tool.setuptools.package-data]
|
|
37
|
+
# the shipped rules are plain .py files in a non-package folder; without this a wheel installs with no rules
|
|
38
|
+
stack_guardian = ["checks.d/*.py"]
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
"""stack-guardian: a health doctor for a home AI stack (Ollama nodes, GPUs, tailnet).
|
|
2
|
+
|
|
3
|
+
Check files (checks.d/*.py) export ``CHECKS = [(check_id, fn(snapshot, ctx) -> finding | None)]`` and build findings with
|
|
4
|
+
``stack_guardian.finding.make_finding``. ``import sg_finding`` keeps working inside a check file: the package registers
|
|
5
|
+
that name for checks written against the flat module.
|
|
6
|
+
"""
|
|
7
|
+
import sys
|
|
8
|
+
|
|
9
|
+
from . import finding
|
|
10
|
+
|
|
11
|
+
__version__ = "0.1.0.dev0"
|
|
12
|
+
|
|
13
|
+
sys.modules.setdefault("sg_finding", finding)
|
|
@@ -0,0 +1,174 @@
|
|
|
1
|
+
"""Core 0.1 rules for stack-guardian, a health doctor for a home AI stack.
|
|
2
|
+
|
|
3
|
+
Each rule inspects a whole snapshot and returns a list with ONE finding per
|
|
4
|
+
failing node (empty when the stack is healthy along that axis), so every node
|
|
5
|
+
keeps its own streak, matches a node= filter and can be named as code@node.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import sys
|
|
9
|
+
|
|
10
|
+
import sg_finding as F
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def _nodes(snapshot):
|
|
14
|
+
"""Return the snapshot's nodes as a list of (name, node) pairs, node None -> {}."""
|
|
15
|
+
nodes = (snapshot or {}).get("nodes") or {}
|
|
16
|
+
return [(name, node or {}) for name, node in nodes.items()]
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def node_unreachable(snapshot, ctx):
|
|
20
|
+
return [F.make_finding("node.unreachable", "missing", "FAIL", name,
|
|
21
|
+
"node did not answer the collector: %s" % node["error"],
|
|
22
|
+
evidence=["%s: %s" % (name, node["error"])],
|
|
23
|
+
try_=["check the machine is on and reachable", "raise timeout_s if the node is just slow"])
|
|
24
|
+
for name, node in _nodes(snapshot) if node.get("error")]
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def ollama_unreachable(snapshot, ctx):
|
|
28
|
+
out = []
|
|
29
|
+
for name, node in _nodes(snapshot):
|
|
30
|
+
bad = [(ep.get("url"), ep.get("error")) for ep in (node.get("ollama") or []) if (ep or {}).get("error")]
|
|
31
|
+
if bad:
|
|
32
|
+
out.append(F.make_finding(
|
|
33
|
+
"ollama.unreachable", "missing", "FAIL", name,
|
|
34
|
+
"%d Ollama endpoint(s) do not answer: %s" % (len(bad), ", ".join(url for url, _ in bad)),
|
|
35
|
+
evidence=["%s %s: %s" % (name, url, error) for url, error in bad],
|
|
36
|
+
try_=["check the Ollama service on that node"]))
|
|
37
|
+
return out
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def model_not_loaded(snapshot, ctx):
|
|
41
|
+
expect = (ctx or {}).get("expect") or {}
|
|
42
|
+
out = []
|
|
43
|
+
for name, node in _nodes(snapshot):
|
|
44
|
+
want = (expect.get(name) or {}).get("model")
|
|
45
|
+
if not want or not node.get("reachable"):
|
|
46
|
+
continue
|
|
47
|
+
running = [(m or {}).get("name") for ep in (node.get("ollama") or []) for m in ((ep or {}).get("running") or [])]
|
|
48
|
+
if want not in running:
|
|
49
|
+
out.append(F.make_finding("model.not_loaded", "partial", "WARN", name,
|
|
50
|
+
"expected model not loaded: %s wants %s" % (name, want),
|
|
51
|
+
evidence=["%s running: %s" % (name, ", ".join(running) or "nothing")]))
|
|
52
|
+
return out
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def vram_unattributed(snapshot, ctx):
|
|
56
|
+
limit = int((ctx or {}).get("vram_unattributed_mib", 2048))
|
|
57
|
+
out = []
|
|
58
|
+
for name, node in _nodes(snapshot):
|
|
59
|
+
bad = [((gpu or {}).get("name"), int((gpu or {}).get("unattributed_mib") or 0)) for gpu in (node.get("gpus") or [])]
|
|
60
|
+
bad = [(gpu, mib) for gpu, mib in bad if mib >= limit]
|
|
61
|
+
if bad:
|
|
62
|
+
out.append(F.make_finding("vram.unattributed", "partial", "WARN", name,
|
|
63
|
+
"%d GPU(s) hold VRAM no listed process explains" % len(bad),
|
|
64
|
+
evidence=["%s %s: %d MiB unattributed" % (name, gpu, mib) for gpu, mib in bad]))
|
|
65
|
+
return out
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def gpu_unavailable(snapshot, ctx):
|
|
69
|
+
return [F.make_finding("gpu.unavailable", "partial", "INFO", name, "GPU status unavailable",
|
|
70
|
+
evidence=["%s: %s" % (name, node["gpu_error"])], try_=["check nvidia-smi runs on that node"])
|
|
71
|
+
for name, node in _nodes(snapshot) if node.get("gpu_error")]
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def _tailnet(node):
|
|
75
|
+
return (node.get("tailnet") or {}) if "tailnet" in node else None
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def tailnet_unavailable(snapshot, ctx):
|
|
79
|
+
return [F.make_finding("tailnet.unavailable", "partial", "INFO", name, "tailnet status unavailable",
|
|
80
|
+
evidence=["%s: %s" % (name, _tailnet(node).get("error") or "tailscale gave no status")],
|
|
81
|
+
try_=["check tailscale is installed and tailscaled is running"])
|
|
82
|
+
for name, node in _nodes(snapshot) if _tailnet(node) is not None and _tailnet(node).get("peer") is None]
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def tailnet_peer_offline(snapshot, ctx):
|
|
86
|
+
return [F.make_finding("tailnet.peer_offline", "missing", "FAIL", name, "tailnet peer offline: " + name)
|
|
87
|
+
for name, node in _nodes(snapshot) if _tailnet(node) is not None and _tailnet(node).get("peer") is False]
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def tailnet_relayed(snapshot, ctx):
|
|
91
|
+
out = []
|
|
92
|
+
for name, node in _nodes(snapshot):
|
|
93
|
+
tailnet = node.get("tailnet") or {}
|
|
94
|
+
if tailnet.get("peer") and tailnet.get("direct") is False:
|
|
95
|
+
out.append(F.make_finding("tailnet.relayed", "partial", "INFO", name,
|
|
96
|
+
"tailnet traffic is relayed, not direct: %s via %s" % (name, tailnet.get("derp"))))
|
|
97
|
+
return out
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
CHECKS = [
|
|
101
|
+
("node.unreachable", node_unreachable),
|
|
102
|
+
("ollama.unreachable", ollama_unreachable),
|
|
103
|
+
("model.not_loaded", model_not_loaded),
|
|
104
|
+
("vram.unattributed", vram_unattributed),
|
|
105
|
+
("gpu.unavailable", gpu_unavailable),
|
|
106
|
+
("tailnet.unavailable", tailnet_unavailable),
|
|
107
|
+
("tailnet.peer_offline", tailnet_peer_offline),
|
|
108
|
+
("tailnet.relayed", tailnet_relayed),
|
|
109
|
+
]
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def _selftest():
|
|
113
|
+
def snap(**over):
|
|
114
|
+
box = {"reachable": True, "latency_ms": 5, "ollama": [{"url": "http://a:11434", "error": None, "running": [{"name": "qwen3:30b"}]}],
|
|
115
|
+
"tailnet": {"peer": True, "direct": True, "derp": None, "ips": []}}
|
|
116
|
+
box2 = {"reachable": True, "latency_ms": 1, "ollama": [{"url": "http://n:11435", "error": None, "running": []}],
|
|
117
|
+
"gpus": [{"name": "RTX 3060", "used_mib": 300, "unattributed_mib": 300}]}
|
|
118
|
+
box.update(over.get("gpu-box", {}))
|
|
119
|
+
box2.update(over.get("gpu-box-2", {}))
|
|
120
|
+
return {"schema": 1, "taken_at": "T", "nodes": {"gpu-box": box, "gpu-box-2": box2}}
|
|
121
|
+
|
|
122
|
+
ctx = {"expect": {"gpu-box": {"model": "qwen3:30b"}}, "vram_unattributed_mib": 2048}
|
|
123
|
+
|
|
124
|
+
def t_clean_snapshot_no_findings():
|
|
125
|
+
return all(fn(snap(), ctx) == [] for _, fn in CHECKS)
|
|
126
|
+
|
|
127
|
+
def t_dead_endpoint_fail():
|
|
128
|
+
[f] = ollama_unreachable(snap(**{"gpu-box-2": {"ollama": [{"url": "http://n:11435", "error": "OSError: refused", "running": []}]}}), ctx)
|
|
129
|
+
return f["code"] == "ollama.unreachable" and f["level"] == "FAIL" and f["node"] == "gpu-box-2" and "refused" in f["evidence"][0]
|
|
130
|
+
|
|
131
|
+
def t_model_not_loaded_warn():
|
|
132
|
+
[f] = model_not_loaded(snap(**{"gpu-box": {"ollama": [{"url": "http://a:11434", "error": None, "running": [{"name": "qwen3-coder:30b"}]}]}}), ctx)
|
|
133
|
+
return f["level"] == "WARN" and f["node"] == "gpu-box" and "qwen3-coder:30b" in f["evidence"][0]
|
|
134
|
+
|
|
135
|
+
def t_unreachable_node_not_blamed_for_model():
|
|
136
|
+
return model_not_loaded(snap(**{"gpu-box": {"reachable": False}}), ctx) == []
|
|
137
|
+
|
|
138
|
+
def t_vram_threshold():
|
|
139
|
+
[f] = vram_unattributed(snap(**{"gpu-box-2": {"gpus": [{"name": "RTX 4070", "used_mib": 9000, "unattributed_mib": 8000}]}}), ctx)
|
|
140
|
+
return f["level"] == "WARN" and "8000 MiB" in f["evidence"][0]
|
|
141
|
+
|
|
142
|
+
def t_tailnet_offline_and_relayed():
|
|
143
|
+
[off] = tailnet_peer_offline(snap(**{"gpu-box": {"tailnet": {"peer": False, "direct": None, "derp": None, "ips": []}}}), ctx)
|
|
144
|
+
[rel] = tailnet_relayed(snap(**{"gpu-box": {"tailnet": {"peer": True, "direct": False, "derp": "ord", "ips": []}}}), ctx)
|
|
145
|
+
return off["level"] == "FAIL" and rel["level"] == "INFO" and "ord" in rel["summary"]
|
|
146
|
+
|
|
147
|
+
def t_empty_snapshot():
|
|
148
|
+
return all(fn({}, {}) == [] for _, fn in CHECKS) and all(fn(None, None) == [] for _, fn in CHECKS)
|
|
149
|
+
|
|
150
|
+
checks = [
|
|
151
|
+
("clean_snapshot_no_findings", t_clean_snapshot_no_findings),
|
|
152
|
+
("dead_endpoint_fail", t_dead_endpoint_fail),
|
|
153
|
+
("model_not_loaded_warn", t_model_not_loaded_warn),
|
|
154
|
+
("unreachable_node_not_blamed_for_model", t_unreachable_node_not_blamed_for_model),
|
|
155
|
+
("vram_threshold", t_vram_threshold),
|
|
156
|
+
("tailnet_offline_and_relayed", t_tailnet_offline_and_relayed),
|
|
157
|
+
("empty_snapshot", t_empty_snapshot),
|
|
158
|
+
]
|
|
159
|
+
passed = 0
|
|
160
|
+
for name, fn in checks:
|
|
161
|
+
try:
|
|
162
|
+
ok = bool(fn())
|
|
163
|
+
except Exception as exc: # noqa: BLE001 - a crashed check is a failed check
|
|
164
|
+
ok = False
|
|
165
|
+
name = "%s (raised %s: %s)" % (name, type(exc).__name__, exc)
|
|
166
|
+
passed += ok
|
|
167
|
+
print(("ok " if ok else "FAIL ") + name)
|
|
168
|
+
failed = len(checks) - passed
|
|
169
|
+
print("sg_checks_core selftest: %d checks, %d passed, %d failed" % (len(checks), passed, failed))
|
|
170
|
+
return 0 if failed == 0 and checks else 1
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
if __name__ == "__main__":
|
|
174
|
+
sys.exit(_selftest() if "--selftest" in sys.argv else 2)
|