hugpy-core 0.1.0__tar.gz

hugpy_core-0.1.0/PKG-INFO

@@ -0,0 +1,32 @@
+Metadata-Version: 2.4
+Name: hugpy-core
+Version: 0.1.0
+Summary: hugpy central, minimal: live worker registry (register/heartbeat), model->worker pick, and the OpenAI /v1 relay to workers. Lifted from legacy hugpy central without evict-to-fit, discovery, admission or DB state.
+Author-email: putkoff <support@hugpy.ai>
+License: Proprietary
+Project-URL: Homepage, https://hugpy.ai
+Classifier: Development Status :: 3 - Alpha
+Classifier: Environment :: Web Environment
+Classifier: Framework :: Flask
+Classifier: Intended Audience :: Developers
+Classifier: Operating System :: POSIX :: Linux
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+Requires-Dist: flask>=2
+Requires-Dist: abstract_flask
+Requires-Dist: psycopg[binary]>=3
+Provides-Extra: test
+Requires-Dist: pytest; extra == "test"
+
+# hugpy_core
+
+hugpy central, minimal. Workers (`hugpy_worker`, one per box, over fitevict)
+register and heartbeat here; the registry is live in memory. `/v1/chat/completions`
+picks an online worker that has the model (resident first; `alloc.worker` pins one)
+and relays to its `/infer/stream`. No evict-to-fit, discovery, admission or DB
+state — each box's fitevict owns placement and the call log.
+
+    hugpy-core --host 127.0.0.1 --port 7020

hugpy_core-0.1.0/README.md

@@ -0,0 +1,9 @@
+# hugpy_core
+
+hugpy central, minimal. Workers (`hugpy_worker`, one per box, over fitevict)
+register and heartbeat here; the registry is live in memory. `/v1/chat/completions`
+picks an online worker that has the model (resident first; `alloc.worker` pins one)
+and relays to its `/infer/stream`. No evict-to-fit, discovery, admission or DB
+state — each box's fitevict owns placement and the call log.
+
+    hugpy-core --host 127.0.0.1 --port 7020

hugpy_core-0.1.0/hugpy_core/__init__.py

@@ -0,0 +1,3 @@
+"""hugpy_core — hugpy central, minimal: live worker registry, model->worker
+pick, and the OpenAI /v1 relay to workers (hugpy_worker over fitevict)."""
+__version__ = "0.1.0"

hugpy_core-0.1.0/hugpy_core/app.py

@@ -0,0 +1,30 @@
+"""hugpy_core router app: abstract_flask app over the routes module."""
+from __future__ import annotations
+
+import argparse
+import logging
+
+from abstract_flask import get_Flask_app
+
+from . import routes
+
+logger = logging.getLogger("hugpy_core")
+
+
+def build_app():
+    return get_Flask_app(name="hugpy_core", routes=routes)
+
+
+def main(argv=None) -> int:
+    logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s")
+    ap = argparse.ArgumentParser(prog="hugpy_core")
+    ap.add_argument("--host", default="127.0.0.1")
+    ap.add_argument("--port", type=int, default=7020)
+    a = ap.parse_args(argv)
+    logger.info("hugpy_core router on %s:%s", a.host, a.port)
+    build_app().run(host=a.host, port=a.port, threaded=True)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

hugpy_core-0.1.0/hugpy_core/console_routes.py

@@ -0,0 +1,155 @@
+"""The console's remaining routes, served by hugpy_core itself (no fallback to
+the hugpy_next central). Live values come from the DB / the registry at the
+moment of the call; /llm/events pushes on every registry change."""
+from __future__ import annotations
+
+import json
+import time
+import urllib.request
+
+from flask import Blueprint, Response, jsonify, request, stream_with_context
+
+from . import registry
+
+IE_DSN = "dbname=inference_engine"
+NOTE_FLAGS = ["broken", "trash", "archive", "unknown", "needs-env", "experimental", "keep"]
+
+
+def _ie(sql: str, params: tuple = ()) -> list:
+    import psycopg
+    with psycopg.connect(IE_DSN, autocommit=True, connect_timeout=5) as c:
+        return c.execute(sql, params).fetchall()
+
+
+def worker_activity(worker: dict) -> list:
+    """One host view of the worker's GPU: every process on its card from
+    inference_engine.gpu_state (published by its fitevict front door the moment
+    the card changes), joined to the front door's residents."""
+    name = worker.get("name")
+    cards = _ie("SELECT device, processes, hugpy_pids, probe_pids FROM gpu_state WHERE worker = %s", (name,))
+    res = {int(pid): key for key, pid in _ie("SELECT key, pid FROM residents WHERE worker = %s AND pid IS NOT NULL", (name,))}
+    rows = []
+    for device, procs, ours, probes in cards:
+        for pid_s, nbytes in sorted((procs or {}).items()):
+            pid = int(pid_s)
+            key = res.get(pid)
+            kind = "hugpy" if key or pid in (ours or []) else ("probe" if pid in (probes or []) else "observed")
+            rows.append({"kind": kind, "model_key": key, "service": "fitevict" if kind == "hugpy" else
+                         ("llama.cpp dry run" if kind == "probe" else "gpu process"),
+                         "pids": [pid], "gpu_indices": [device], "vram_bytes": int(nbytes),
+                         "vram_mib": int(nbytes) // 1048576, "state": "running",
+                         "evictable": kind == "hugpy", "immutable": kind == "observed",
+                         "api_available": bool(key), "activity": {},
+                         "note": ("Hugpy managed (fitevict)" if kind == "hugpy" else
+                                  "transient sizing probe" if kind == "probe" else
+                                  "GPU process observed; not managed by hugpy")})
+    return rows
+
+
+def attach_console(bp: Blueprint, snapshot_payload) -> None:
+
+    @bp.get("/llm/workers/<worker_id>/activity")
+    def workers_activity(worker_id):
+        w = registry.lookup_worker(worker_id)
+        if w is None:
+            return jsonify({"ok": False, "error": f"no worker {worker_id!r}", "activity": []}), 404
+        try:
+            return jsonify({"ok": True, "activity": worker_activity(w)})
+        except Exception as exc:  # noqa: BLE001
+            return jsonify({"ok": False, "error": f"{type(exc).__name__}: {exc}", "activity": []}), 503
+
+    @bp.get("/llm/workers/<worker_id>/health")
+    def workers_health(worker_id):
+        w = registry.lookup_worker(worker_id)
+        if w is None:
+            return jsonify({"reachable": False, "error": f"no worker {worker_id!r}"}), 404
+        url = (w.get("url") or "").rstrip("/") + "/health"
+        try:
+            with urllib.request.urlopen(url, timeout=4) as r:
+                return jsonify({"reachable": True, "url": url, "health": json.loads(r.read() or b"{}")})
+        except Exception as exc:  # noqa: BLE001
+            return jsonify({"reachable": False, "url": url, "error": f"{type(exc).__name__}: {exc}"})
+
+    @bp.get("/llm/models/status")
+    def models_status():
+        from .routes import catalog_rows
+        rows = catalog_rows()
+        models = [{"model_key": r["model_key"], "framework": r.get("framework"), "status": r.get("status") or "known",
+                   "blocked": False, "archived": {"marked": False, "at": None, "by": None, "reason": None},
+                   "admission": {"status": "unknown"}, "task": r.get("task")} for r in rows]
+        return jsonify({"models": models, "count": len(models), "buckets": {}, "counts": {},
+                        "generated_at": time.time(), "live": True})
+
+    @bp.get("/llm/events")
+    def llm_events():
+        """SSE: the full snapshot on connect, then a fresh one the moment the
+        registry changes (a worker registers or heartbeats). No polling."""
+        def gen():
+            seen = registry.version()
+            yield "event: snapshot\ndata: " + json.dumps({"ts": time.time(), "versions": {"workers": seen},
+                                                          "feeds": snapshot_payload()}, default=str) + "\n\n"
+            started = time.time()
+            while time.time() - started < 3600:
+                v = registry.wait_change(seen, 15.0)
+                if v == seen:
+                    yield ": keepalive\n\n"
+                    continue
+                seen = v
+                feeds = snapshot_payload()
+                for name, payload in feeds.items():
+                    yield "event: feed\ndata: " + json.dumps({"feed": name, "version": v, "updated": time.time(),
+                                                              "payload": payload}, default=str) + "\n\n"
+        return Response(stream_with_context(gen()), mimetype="text/event-stream",
+                        headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"})
+
+    @bp.get("/readiness")
+    def readiness():
+        """The console's landing / health probe: hugpy_core is up; serving and
+        storage are read live (registry + the model drive)."""
+        import os
+        from .routes import catalog_rows
+        root = os.environ.get("HUGPY_CORE_STORAGE_ROOT", "/mnt/16T_toshiba/llm_storage")
+        st = os.statvfs(root) if os.path.isdir(root) else None
+        online = [w for w in registry.all_workers() if registry.is_online(w)]
+        serving = any((w.get("loaded_models") or []) for w in online)
+        return jsonify({
+            "central": {"auth_mode": "open", "version": "hugpy_core"},
+            "connect": {"base_url": request.host_url.rstrip("/")},
+            "console": {"configured": True, "url": None},
+            "serving": {"any_serving": serving, "enabled": bool(online), "slots": [],
+                        "workers_online": len(online)},
+            "storage": {"exists": st is not None, "root": root,
+                        "free_bytes": (st.f_bavail * st.f_frsize) if st else None,
+                        "free_gb": round(st.f_bavail * st.f_frsize / 1e9, 1) if st else None,
+                        "writable": os.access(root, os.W_OK), "model_count": len(catalog_rows())},
+        })
+
+    @bp.get("/llm/env-profiles")
+    def env_profiles():
+        return jsonify({"bases": ["worker"], "profiles": []})
+
+    @bp.get("/llm/model-groups")
+    def model_groups():
+        return jsonify({"groups": []})
+
+    @bp.get("/models/notes")
+    def models_notes():
+        return jsonify({"flags": NOTE_FLAGS, "notes": {}})
+
+    @bp.get("/llm/fleet/distribution")
+    def fleet_distribution():
+        return jsonify({"env_override": False, "mode": "feasible", "source": "default", "stored": None})
+
+    @bp.get("/llm/enroll-tokens")
+    def enroll_tokens():
+        return jsonify([])
+
+    @bp.get("/llm/help/tickets")
+    def help_tickets():
+        return jsonify({"tickets": [], "count": 0})
+
+    @bp.post("/llm/sessions/<session_id>/lease")
+    def session_lease(session_id):
+        body = request.get_json(silent=True) or {}
+        return jsonify({"ok": True, "session_id": session_id, "state": body.get("state") or "active",
+                        "lease_fresh": True})

hugpy_core-0.1.0/hugpy_core/keys.py

@@ -0,0 +1,62 @@
+# LIFTED from legacy hugpy (read-only reference):
+#   model_key_forms   <- py/foundation/hugpy_platform/src/hugpy_platform/model_keys.py:8-22 (verbatim)
+#   canonical_key     <- py/inference/hugpy_engine/src/hugpy_engine/fit/types.py:53-60 (verbatim)
+#   _canon_forms, serveable_match <- py/fleet/hugpy_fleet/src/hugpy_fleet/central/workers.py:3724,3762
+# DROPPED: the blocked-sibling guard in serveable_match (no blocklist here).
+"""Model-key identity: alias forms and the one membership predicate routing uses."""
+from __future__ import annotations
+
+import re
+from typing import Any
+
+
+def model_key_forms(model_key: Any) -> set[str]:
+    """Return the raw key, case variants, and its ``/`` and ``~`` tails."""
+    if not model_key:
+        return set()
+    raw = str(model_key).strip()
+    forms = {raw, raw.lower()}
+    tail = raw.split("/")[-1]
+    forms.add(tail)
+    forms.add(tail.lower())
+    if "~" in raw:
+        base = raw.split("~", 1)[1]
+        if base:
+            forms.add(base)
+            forms.add(base.lower())
+    return forms
+
+
+# Case is not a name token (the routing alias union already folds it); the
+# separator before the suffix may be ``-`` or ``_`` (``Qwen3.8_4B_Distilled_GGUF``
+# is a live key) — nothing else is folded.
+_FORMAT_SUFFIX_RE = re.compile(r"[-_]gguf$", re.IGNORECASE)
+
+
+def canonical_key(model_key: Any) -> str:
+    """``model_key`` with ONLY the trailing format suffix (``-GGUF``) removed.
+    Every other token is identity and is kept verbatim."""
+    s = str(model_key or "").strip()
+    return _FORMAT_SUFFIX_RE.sub("", s, count=1)
+
+
+def _canon_forms(forms) -> set:
+    """The alias forms of a key reduced to their canonical identity (lower-cased,
+    ``-GGUF`` stripped) — the set two keys must intersect on to be ONE model."""
+    return {canonical_key(f).lower() for f in (forms or ()) if f}
+
+
+def serveable_match(model_key: str, wanted: set, serveable) -> bool:
+    """True iff one of ``serveable``'s advertised keys names ``model_key``."""
+    want_c = _canon_forms(wanted)
+    for m in serveable:
+        if m == model_key:
+            return True
+        # Alias match = the "~"/"/"-tail union (k67) OR the ``-GGUF`` format
+        # equivalence (operator rule 2026-09-29). Both are IDENTITY: nothing
+        # here ever matches a key carrying an extra name token (``-Distill``,
+        # a quant marker…), so a worker holding ``X-Distill-GGUF`` is never
+        # credited with ``X-GGUF`` — not as resident, allocated, or on-disk.
+        if (wanted & model_key_forms(m)) or (want_c & _canon_forms(model_key_forms(m))):
+            return True
+    return False

hugpy_core-0.1.0/hugpy_core/live.py

@@ -0,0 +1,127 @@
+"""LIVE window: what is loaded / loading / failed / free on each box, pushed the
+instant it changes. inference_engine triggers notify 'live' (residents,
+call_log) and 'gpu_state' (the card); one LISTEN thread re-reads and wakes every
+open page. No polling anywhere."""
+from __future__ import annotations
+
+import json
+import threading
+import time
+
+from flask import Blueprint, Response, stream_with_context
+
+IE_DSN = "dbname=inference_engine"
+_cond = threading.Condition()
+_state = {"version": 0, "data": {}}
+_started = [False]
+_lock = threading.Lock()
+
+
+def _read(c) -> dict:
+    res = [dict(zip(("worker", "key", "state", "pid", "ctx", "ngl", "total_layers", "vram", "ram", "error",
+                     "engine", "since", "last_used", "calls", "busy", "loading"), r))
+           for r in c.execute("SELECT worker, key, state, pid, ctx, n_gpu_layers, total_layers, vram_bytes, ram_bytes, "
+                              "load_error, engine, extract(epoch from resident_since), extract(epoch from last_used), "
+                              "calls, busy, loading FROM residents ORDER BY worker, key").fetchall()]
+    gpu = [dict(zip(("worker", "device", "name", "total", "usable", "used", "free", "reason", "at"), r))
+           for r in c.execute("SELECT worker, device, name, total, usable, used, free, reason, extract(epoch from at) "
+                              "FROM gpu_state ORDER BY worker, device").fetchall()]
+    host = [dict(zip(("worker", "mem_total", "mem_available"), r))
+            for r in c.execute("SELECT worker, mem_total, mem_available FROM host_state").fetchall()]
+    calls = [dict(zip(("t", "worker", "model", "task", "status", "secs", "ctx", "error", "mode"), r))
+             for r in c.execute("SELECT extract(epoch from received_at), worker, model_name, task, http_status, "
+                                "extract(epoch from (coalesce(responded_at, now()) - received_at)), context_length, "
+                                "left(error, 160), response_meta->'decision'->>'mode' "
+                                "FROM call_log WHERE http_status IS NOT NULL OR dispatched_at IS NOT NULL "
+                                "ORDER BY received_at DESC LIMIT 25").fetchall()]   # front-door rows (central-only stubs excluded)
+    return {"residents": res, "gpu": gpu, "host": host, "calls": calls, "at": time.time()}
+
+
+def _loop():
+    import psycopg
+    while True:
+        try:
+            with psycopg.connect(IE_DSN, autocommit=True) as conn, psycopg.connect(IE_DSN, autocommit=True) as rc:
+                conn.execute("LISTEN live")
+                conn.execute("LISTEN gpu_state")
+                while True:
+                    d = _read(rc)
+                    with _cond:
+                        _state["data"] = d
+                        _state["version"] += 1
+                        _cond.notify_all()
+                    list(conn.notifies(stop_after=1))
+                    list(conn.notifies(timeout=0))      # coalesce a burst into one re-read
+        except Exception as exc:  # noqa: BLE001
+            print(f"[live] reconnect: {exc}")
+            time.sleep(2)
+
+
+def _ensure():
+    with _lock:
+        if not _started[0]:
+            _started[0] = True
+            threading.Thread(target=_loop, name="live-listen", daemon=True).start()
+
+
+PAGE = r"""<!doctype html><html><head><meta charset="utf-8"><title>hugpy live</title>
+<style>
+body{font:13px/1.4 ui-monospace,Menlo,monospace;background:#0f1115;color:#d6d9e0;margin:16px}
+h2{font-size:14px;margin:18px 0 6px;color:#9aa4b2}
+table{border-collapse:collapse;width:100%}td,th{padding:3px 8px;border-bottom:1px solid #222733;text-align:left;white-space:nowrap}
+th{color:#7d8696;font-weight:normal}.n{text-align:right}
+.loading{color:#f5b942}.serving{color:#5aa9ff}.idle{color:#4cc38a}.error,.dead{color:#ff6b6b}.fail{color:#ff6b6b}.ok{color:#4cc38a}
+.bar{height:16px;background:#222733;position:relative;margin:4px 0}.bar i{position:absolute;top:0;bottom:0;left:0;background:#5aa9ff}
+#st{color:#7d8696}
+</style></head><body>
+<div id=st>connecting…</div>
+<h2>GPU</h2><div id=gpu></div>
+<h2>Models on the boxes (fitevict front door)</h2><table id=res></table>
+<h2>Recent calls</h2><table id=calls></table>
+<script>
+const B=n=>n==null?'—':Number(n).toLocaleString('en-US')+' B';
+const T=e=>e?new Date(e*1000).toLocaleTimeString():'—';
+function draw(d){
+  document.getElementById('st').textContent='live · updated '+T(d.at);
+  document.getElementById('gpu').innerHTML=d.gpu.map(g=>{
+    const h=(d.host||[]).find(x=>x.worker===g.worker)||{};
+    const pct=g.usable?Math.min(100,100*g.used/g.usable):0;
+    return `<div>${g.worker} · ${g.name} · used ${B(g.used)} · free ${B(g.free)} of usable ${B(g.usable)} · RAM available ${B(h.mem_available)} · last change: ${g.reason} ${T(g.at)}</div><div class=bar><i style="width:${pct}%"></i></div>`}).join('');
+  document.getElementById('res').innerHTML='<tr><th>box</th><th>model</th><th>state</th><th>engine</th><th class=n>ctx</th><th class=n>gpu layers</th><th class=n>VRAM</th><th class=n>RAM</th><th>since</th><th class=n>calls</th><th>error</th></tr>'+
+    (d.residents.length?d.residents.map(r=>{const st=r.busy?'serving':(r.loading?'loading':r.state);
+      return `<tr><td>${r.worker}</td><td>${r.key}</td><td class="${st}">● ${st}</td><td>${r.engine||''}</td><td class=n>${r.ctx??'—'}</td><td class=n>${r.ngl==null?'—':(r.ngl<0?'all':r.ngl)}${r.total_layers?'/'+r.total_layers:''}</td><td class=n>${B(r.vram)}</td><td class=n>${B(r.ram)}</td><td>${T(r.since)}</td><td class=n>${r.calls??0}</td><td class=error>${r.error||''}</td></tr>`}).join(''):'<tr><td colspan=11>nothing loaded</td></tr>');
+  document.getElementById('calls').innerHTML='<tr><th>time</th><th>model</th><th>task</th><th>status</th><th class=n>secs</th><th class=n>ctx</th><th>mode</th><th>error</th></tr>'+
+    d.calls.map(c=>{const ok=c.status==200, run=c.status==null;
+      return `<tr><td>${T(c.t)}</td><td>${c.model}</td><td>${c.task||''}</td><td class="${run?'loading':(ok?'ok':'fail')}">${run?'running':c.status}</td><td class=n>${c.secs==null?'':Number(c.secs).toFixed(1)}</td><td class=n>${c.ctx??''}</td><td>${c.mode||''}</td><td class=fail>${c.error||''}</td></tr>`}).join('');
+}
+function go(){const es=new EventSource('live/stream');es.onmessage=e=>draw(JSON.parse(e.data));
+  es.onerror=()=>{document.getElementById('st').textContent='reconnecting…'}}
+go();
+</script></body></html>"""
+
+
+def attach_live(bp: Blueprint) -> None:
+    @bp.get("/live")
+    def live_page():
+        _ensure()
+        return Response(PAGE, mimetype="text/html")
+
+    @bp.get("/live/stream")
+    def live_stream():
+        _ensure()
+
+        def gen():
+            seen = -1
+            started = time.time()
+            while time.time() - started < 3600:
+                with _cond:
+                    if _state["version"] == seen:
+                        _cond.wait(timeout=15)
+                    v, d = _state["version"], _state["data"]
+                if v == seen:
+                    yield ": keepalive\n\n"
+                    continue
+                seen = v
+                yield "data: " + json.dumps(d, default=str) + "\n\n"
+        return Response(stream_with_context(gen()), mimetype="text/event-stream",
+                        headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"})