stack-guardian 0.1.0__py3-none-any.whl

stack_guardian/collect.py

@@ -0,0 +1,206 @@
+"""sg_collect -- snapshot collectors for the stack-guardian health doctor.
+
+Each collector probes one surface of the home AI stack: an Ollama endpoint over
+HTTP, the local NVIDIA GPUs through nvidia-smi, and the tailnet through
+`tailscale status --json`.  Every command runner and HTTP getter is injectable so
+tests never touch the network or the GPU.  A dead endpoint is data, never a crash.
+"""
+
+import json
+import subprocess
+import sys
+import time
+import urllib.request
+
+try:
+    from . import parsers as sg_parsers
+except ImportError:  # run as a plain file next to sg_parsers.py
+    import sg_parsers
+
+GPU_QUERY = ["nvidia-smi", "--query-gpu=index,uuid,name,memory.total,memory.used", "--format=csv,noheader,nounits"]
+APPS_QUERY = ["nvidia-smi", "--query-compute-apps=gpu_uuid,pid,process_name,used_memory", "--format=csv,noheader,nounits"]
+TAILSCALE_STATUS = ["tailscale", "status", "--json"]
+
+
+def default_run(argv, timeout=15):
+    """Run a command and return its stdout, or None if it failed or is missing."""
+    try:
+        proc = subprocess.run(argv, capture_output=True, text=True, timeout=timeout, check=False)
+    except (OSError, subprocess.SubprocessError):
+        return None
+    return proc.stdout if proc.returncode == 0 else None
+
+
+def default_http_get(url, timeout=5):
+    """GET a URL and return (status, body text)."""
+    with urllib.request.urlopen(url, timeout=timeout) as resp:
+        return resp.status, resp.read().decode("utf-8", errors="replace")
+
+
+def collect_ollama(url, http_get=None, clock=None):
+    """Probe one Ollama endpoint: version, installed models, loaded models."""
+    http_get = http_get or default_http_get
+    clock = clock or time.monotonic
+    base = url.rstrip("/")
+    out = {"url": base, "kind": "ollama", "version": None, "models": [], "running": [], "latency_ms": None, "error": None}
+    try:
+        t0 = clock()
+        status, body = http_get(base + "/api/version")
+        out["latency_ms"] = int(round((clock() - t0) * 1000))
+        if status != 200:
+            raise ValueError("HTTP %s on /api/version" % status)
+        out["version"] = json.loads(body).get("version")
+        status, body = http_get(base + "/api/tags")
+        if status == 200:
+            out["models"] = [m["name"] for m in (json.loads(body).get("models") or []) if isinstance(m, dict) and m.get("name")]
+        status, body = http_get(base + "/api/ps")
+        if status == 200:
+            out["running"] = sg_parsers.parse_ollama_ps(body)
+    except Exception as exc:
+        out["error"] = ("%s: %s" % (type(exc).__name__, exc))[:200]
+    return out
+
+
+def collect_gpus(run=None):
+    """Describe the local GPUs and their attributed compute processes."""
+    return collect_gpus_status(run)[0]
+
+
+def collect_gpus_status(run=None):
+    """(gpus, error): error is set when nvidia-smi is missing or fails, so "no GPUs" and "cannot tell" differ."""
+    run = run or default_run
+    gpu = run(GPU_QUERY)
+    if gpu is None:
+        return [], "nvidia-smi unavailable (not installed, or it failed)"
+    if not gpu.strip():
+        return [], None
+    return sg_parsers.parse_nvsmi(gpu, run(APPS_QUERY) or ""), None
+
+
+def collect_tailnet(peer_host, run=None):
+    """Report whether peer_host is visible on the tailnet and how it is reached; peer None when tailscale cannot say."""
+    run = run or default_run
+    out = run(TAILSCALE_STATUS)
+    if out is None:
+        return dict(sg_parsers.UNAVAILABLE, error="tailscale status failed (tailscale missing or tailscaled down)")
+    return sg_parsers.parse_tailscale(out, peer_host)
+
+
+def snapshot(nodes, run=None, http_get=None, now=None, clock=None, timeout_s=10.0):
+    """Build the one snapshot dict the checks read, in node input order. Every node is collected at
+    the same time and a node that has not answered within timeout_s is reported down (multinode.pull)."""
+    try:
+        from . import multinode as sg_multinode
+    except ImportError:  # run as a plain file next to the sg_* modules, like cli.py
+        import multinode as sg_multinode
+
+    def one(name, cfg):
+        ollama = [collect_ollama(u, http_get=http_get, clock=clock) for u in (cfg.get("ollama") or [])]
+        ok = [o for o in ollama if o["error"] is None]
+        node = {"reachable": bool(ok), "latency_ms": ok[0]["latency_ms"] if ok else None, "ollama": ollama}
+        if cfg.get("local"):
+            node["gpus"], gpu_error = collect_gpus_status(run=run)
+            if gpu_error:
+                node["gpu_error"] = gpu_error
+        if cfg.get("tailnet_host"):
+            node["tailnet"] = collect_tailnet(cfg["tailnet_host"], run=run)
+        return node
+    return sg_multinode.pull(nodes, one, timeout_s=timeout_s, now=now)
+
+
+def _selftest():
+    TAGS = '{"models": [{"name": "qwen3:30b"}, {"name": "nomic-embed-text"}]}'
+    PS = '{"models": [{"name": "qwen3:30b", "size_vram": 19000000000, "expires_at": "2026-10-04T00:00:00Z"}]}'
+
+    def http_ok(url, timeout=5):
+        if url.endswith("/api/version"):
+            return 200, '{"version": "0.34.2"}'
+        if url.endswith("/api/tags"):
+            return 200, TAGS
+        return 200, PS
+
+    def http_dead(url, timeout=5):
+        raise OSError("connection refused")
+
+    ticks = iter([10.0, 10.042, 20.0, 20.5])
+
+    def clock():
+        return next(ticks)
+
+    def run_ok(argv, timeout=15):
+        if argv[0] == "tailscale":
+            return '{"Peer": {"k": {"HostName": "gpu-box", "Online": true, "CurAddr": "1.2.3.4:41641", "TailscaleIPs": ["100.64.0.10"]}}}'
+        if "--query-gpu=index,uuid,name,memory.total,memory.used" in argv:
+            return "0, GPU-a, RTX 3060, 12288, 9000\n1, GPU-b, RTX 4070 Ti SUPER, 16376, 300\n"
+        return "GPU-a, 4242, C:\\ollama\\ollama.exe, 8800\n"
+
+    def run_none(argv, timeout=15):
+        return None
+
+    def t_ollama_ok():
+        o = collect_ollama("http://gpu-box:11434/", http_get=http_ok, clock=clock)
+        return (o["url"] == "http://gpu-box:11434" and o["version"] == "0.34.2" and o["models"] == ["qwen3:30b", "nomic-embed-text"]
+                and o["running"][0]["name"] == "qwen3:30b" and o["latency_ms"] == 42 and o["error"] is None)
+
+    def t_ollama_dead():
+        o = collect_ollama("http://x:11434", http_get=http_dead)
+        return o["version"] is None and o["models"] == [] and o["error"].startswith("OSError: connection refused")
+
+    def t_gpus():
+        g = collect_gpus(run=run_ok)
+        return len(g) == 2 and g[0]["attributed_mib"] == 8800 and g[0]["processes"][0]["name"] == "ollama.exe"
+
+    def t_no_nvidia_smi():
+        return collect_gpus(run=run_none) == []
+
+    def t_tailnet():
+        t = collect_tailnet("GPU-BOX", run=run_ok)
+        return t["peer"] is True and t["direct"] is True and t["ips"] == ["100.64.0.10"]
+
+    def t_tailnet_missing():
+        return collect_tailnet("gpu-box", run=run_none)["peer"] is None
+
+    def t_snapshot_shape():
+        s = snapshot({"gpu-box": {"ollama": ["http://gpu-box:11434"], "tailnet_host": "gpu-box"},
+                      "gpu-box-2": {"ollama": ["http://n1:11434"], "local": True}},
+                     run=run_ok, http_get=lambda u, timeout=5: http_dead(u) if "n1" in u else http_ok(u), now="T")
+        a, n = s["nodes"]["gpu-box"], s["nodes"]["gpu-box-2"]
+        return (s["schema"] == 1 and s["taken_at"] == "T" and a["reachable"] is True and "gpus" not in a
+                and a["tailnet"]["peer"] is True and n["reachable"] is False and n["latency_ms"] is None and len(n["gpus"]) == 2)
+
+    def t_empty():
+        return snapshot({}, now="T") == {"schema": 1, "taken_at": "T", "nodes": {}}
+
+    checks = [
+        ("ollama_ok", t_ollama_ok),
+        ("ollama_dead", t_ollama_dead),
+        ("gpus", t_gpus),
+        ("no_nvidia_smi", t_no_nvidia_smi),
+        ("tailnet", t_tailnet),
+        ("tailnet_missing", t_tailnet_missing),
+        ("snapshot_shape", t_snapshot_shape),
+        ("empty", t_empty),
+    ]
+    passed = 0
+    for name, fn in checks:
+        try:
+            ok = bool(fn())
+        except Exception as exc:  # noqa: BLE001 - a crashed check is a failed check
+            ok = False
+            name = "%s (raised %s: %s)" % (name, type(exc).__name__, exc)
+        passed += ok
+        print(("ok   " if ok else "FAIL ") + name)
+    failed = len(checks) - passed
+    print("sg_collect selftest: %d checks, %d passed, %d failed" % (len(checks), passed, failed))
+    return 0 if failed == 0 and checks else 1
+
+
+if __name__ == "__main__":
+    if "--selftest" in sys.argv:
+        sys.exit(_selftest())
+    if "--nodes" in sys.argv:
+        with open(sys.argv[sys.argv.index("--nodes") + 1], encoding="utf-8") as fh:
+            print(json.dumps(snapshot(json.load(fh)), indent=1))
+        sys.exit(0)
+    print("usage: sg_collect.py --nodes NODES.json | --selftest")
+    sys.exit(2)

stack_guardian/core.py

@@ -0,0 +1,264 @@
+"""stack-guardian core runner.
+
+Loads the Python checks of a stack-guardian folder, runs them concurrently
+against a snapshot dict and reports the findings it collects.
+"""
+
+import argparse
+import contextlib
+import importlib.util
+import json
+import os
+import shutil
+import sys
+import tempfile
+import time
+from concurrent.futures import ThreadPoolExecutor
+
+try:
+    from . import finding as sg_finding
+except ImportError:  # run as a plain file next to sg_finding.py
+    import sg_finding
+
+_LEVEL_RANK = {"FAIL": 0, "WARN": 1, "INFO": 2, "OK": 3}
+
+
+def _broken_check(message):
+    """Build a check function that raises RuntimeError(message) when run."""
+
+    def _check(*_args, **_kwargs):
+        raise RuntimeError(message)
+
+    _check.__name__ = "broken_check"
+    return _check
+
+
+def load_checks(checks_dir):
+    """Return the [(check_id, fn), ...] list published by checks_dir.
+
+    Files are visited in file-name order; names starting with "_" are
+    skipped. A file that cannot be imported, or that publishes no CHECKS
+    list, contributes a single ("load.<stem>", fn) entry whose fn raises,
+    so the doctor reports it as a crashed check instead of dying itself.
+    """
+    if not checks_dir or not os.path.isdir(checks_dir):
+        return []
+    try:
+        names = sorted(os.listdir(checks_dir))
+    except OSError:
+        return []
+
+    checks = []
+    for position, name in enumerate(names):
+        if name.startswith("_") or not name.endswith(".py"):
+            continue
+        path = os.path.join(checks_dir, name)
+        if not os.path.isfile(path):
+            continue
+        try:
+            mod_name = "sg_check_%d_%s" % (position, name[:-3])
+            spec = importlib.util.spec_from_file_location(mod_name, path)
+            module = importlib.util.module_from_spec(spec)
+            spec.loader.exec_module(module)
+            entries = getattr(module, "CHECKS", None)
+            if not isinstance(entries, list):
+                raise RuntimeError("no CHECKS list")
+            for check_id, fn in entries:
+                checks.append((check_id, fn))
+        except Exception as exc:
+            checks.append(("load." + name[:-3],
+                           _broken_check("%s: %s" % (name, exc))))
+    return checks
+
+
+def run_all(checks, snapshot, ctx=None, max_workers=8):
+    """Run every check through sg_finding.run_check, concurrently.
+
+    Findings come back in the order of `checks`; a check may return one finding
+    or a list of them (one per node); None results are dropped.
+    """
+    checks = list(checks)
+    if not checks:
+        return []
+    ctx = ctx or {}
+    results = [None] * len(checks)
+
+    def _work(index, check_id, fn):
+        results[index] = sg_finding.run_check(check_id, fn, snapshot, ctx)
+
+    with ThreadPoolExecutor(max_workers=max_workers) as pool:
+        futures = [pool.submit(_work, index, check_id, fn)
+                   for index, (check_id, fn) in enumerate(checks)]
+        for future in futures:
+            future.result()
+    flat = []
+    for result in results:
+        flat.extend(result if isinstance(result, list) else [result])
+    return [finding for finding in flat if finding is not None]
+
+
+def render(findings):
+    """Render findings one per line, worst level first, plus a verdict."""
+    findings = list(findings)
+    if not findings:
+        return "worst: OK"
+    ordered = sorted(findings,
+                     key=lambda f: _LEVEL_RANK.get(f.get("level", "OK"), 3))
+    lines = []
+    for finding in ordered:
+        node = finding.get("node") or ""
+        lines.append("%-4s %s%s: %s" % (finding.get("level", "OK"),
+                                       finding.get("code", ""),
+                                       " [" + node + "]" if node else "",
+                                       finding.get("summary", "")))
+    lines.append("worst: " + sg_finding.worst(findings))
+    return "\n".join(lines)
+
+
+@contextlib.contextmanager
+def _scratch_dir():
+    """A temporary directory that is always removed again."""
+    path = tempfile.mkdtemp()
+    try:
+        yield path
+    finally:
+        shutil.rmtree(path, ignore_errors=True)
+
+
+def _write_check_file(directory, name, body):
+    with open(os.path.join(directory, name), "w", encoding="utf-8") as handle:
+        handle.write(body)
+
+
+def _check_file(check_id):
+    """Source of a check file that publishes a single no-op check."""
+    return ('def _noop(*_args, **_kwargs):\n    return None\n\n'
+            'CHECKS = [("%s", _noop)]\n' % check_id)
+
+
+def _check_loads_in_file_order():
+    with _scratch_dir() as scratch:
+        _write_check_file(scratch, "b.py", _check_file("b1"))
+        _write_check_file(scratch, "a.py", _check_file("a1"))
+        return [check_id for check_id, _fn in load_checks(scratch)] == ["a1", "b1"]
+
+
+def _check_underscore_skipped():
+    with _scratch_dir() as scratch:
+        _write_check_file(scratch, "_helper.py", _check_file("helper"))
+        return load_checks(scratch) == []
+
+
+def _check_broken_file_is_a_check():
+    with _scratch_dir() as scratch:
+        _write_check_file(scratch, "bad.py", "this is not python\n")
+        checks = load_checks(scratch)
+        if [check_id for check_id, _fn in checks] != ["load.bad"]:
+            return False
+        findings = run_all(checks, {})
+        if len(findings) != 1:
+            return False
+        return (findings[0].get("code") == "check.crashed"
+                and findings[0].get("level") == "WARN")
+
+
+def _check_none_dropped():
+    with _scratch_dir() as scratch:
+        _write_check_file(scratch, "quiet.py", _check_file("quiet"))
+        return run_all(load_checks(scratch), {}) == []
+
+
+def _check_order_kept():
+    def _ok(code):
+        def _check(*_args, **_kwargs):
+            return sg_finding.make_finding(code, "ok", "OK", "", code)
+        return _check
+
+    def _slow(*_args, **_kwargs):
+        time.sleep(0.2)
+        return sg_finding.make_finding("x", "ok", "OK", "", "x")
+
+    checks = [("x", _slow), ("y", _ok("y")), ("z", _ok("z"))]
+    findings = run_all(checks, {})
+    return [finding.get("code") for finding in findings] == ["x", "y", "z"]
+
+
+def _check_render_sorted():
+    findings = [sg_finding.make_finding("ok_code", "ok", "OK", "", "ok summary"),
+                sg_finding.make_finding("bad_code", "missing", "FAIL", "web",
+                                        "fail summary")]
+    lines = render(findings).split("\n")
+    return lines[0].startswith("FAIL")
+
+
+def _check_render_empty():
+    return render([]) == "worst: OK"
+
+
+def _check_missing_dir():
+    with _scratch_dir() as scratch:
+        return load_checks(os.path.join(scratch, "absent")) == []
+
+
+_SELFTESTS = [
+    ("loads_in_file_order", _check_loads_in_file_order),
+    ("underscore_skipped", _check_underscore_skipped),
+    ("broken_file_is_a_check", _check_broken_file_is_a_check),
+    ("none_dropped", _check_none_dropped),
+    ("order_kept", _check_order_kept),
+    ("render_sorted", _check_render_sorted),
+    ("render_empty", _check_render_empty),
+    ("missing_dir", _check_missing_dir),
+]
+
+
+def run_selftest():
+    """Run the built-in checks, print the tally and return an exit code."""
+    total = len(_SELFTESTS)
+    passed = 0
+    failed = 0
+    for name, check in _SELFTESTS:
+        detail = ""
+        try:
+            ok = bool(check())
+        except Exception as exc:
+            ok = False
+            detail = "%s: %s" % (type(exc).__name__, exc)
+        if ok:
+            passed += 1
+        else:
+            failed += 1
+            print("FAIL %s%s" % (name, ": " + detail if detail else ""))
+    print("sg_core selftest: %d checks, %d passed, %d failed"
+          % (total, passed, failed))
+    return 0 if failed == 0 and total > 0 else 1
+
+
+def main(argv=None):
+    parser = argparse.ArgumentParser(prog="sg_core",
+                                     description="stack-guardian health doctor")
+    parser.add_argument("--checks-dir", help="folder holding the check modules")
+    parser.add_argument("--snapshot", help="JSON file holding the snapshot")
+    parser.add_argument("--json", action="store_true",
+                        help="print the findings as one JSON line")
+    parser.add_argument("--selftest", action="store_true",
+                        help="run the built-in checks and exit")
+    args = parser.parse_args(argv)
+
+    if args.selftest:
+        return run_selftest()
+    if not args.checks_dir or not args.snapshot:
+        parser.error("--checks-dir and --snapshot are required")
+
+    with open(args.snapshot, "r", encoding="utf-8") as handle:
+        snapshot = json.load(handle)
+    findings = run_all(load_checks(args.checks_dir), snapshot)
+    if args.json:
+        print(json.dumps(findings))
+    else:
+        print(render(findings))
+    return sg_finding.exit_code(findings)
+
+
+if __name__ == "__main__":
+    sys.exit(main())