hugpy-engine 0.2.5__tar.gz → 0.2.6__tar.gz

{hugpy_engine-0.2.5/src/hugpy_engine.egg-info → hugpy_engine-0.2.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hugpy-engine
-Version: 0.2.5
+Version: 0.2.6
 Summary: Hugpy engine: model discovery and classification, allocation and eviction, native llama.cpp and GGUF runners, and the prompt-to-reply inference path with task and placement seams
 Author-email: putkoff <support@hugpy.ai>
 License-Expression: LicenseRef-Proprietary

{hugpy_engine-0.2.5 → hugpy_engine-0.2.6}/src/hugpy_engine/config/models/models_config.py

@@ -1618,14 +1618,34 @@ def record_worker_models(worker_id: str, rows: dict) -> list[str]:
                 clean[key]["worker_location"] = location
         old = (data.get(worker_id) or {}).get("models")
         changed = old != clean
-        if changed:
+        # DETACH STEP 2 (operator 2026-10-02): the catalog's home is the DB
+        # (model_worker_presence, source 'discovered'); the JSON is written only
+        # with HUGPY_WORKER_CATALOG_JSON=1 — or when the DB write did not land
+        # (no DB / DB down): the JSON is the no-DB fallback the reader uses, so
+        # a no-DB central must still persist its scan there (2026-10-02).
+        db_changed = False
+        db_ok = False
+        try:
+            from hugpy_engine.model_index import record_worker_presence
+            rows = []
+            for key, row in clean.items():
+                r = dict(row); r["model_key"] = key
+                if r.get("worker_location"):
+                    r["external_location"] = r["worker_location"]
+                rows.append(r)
+            st, det = record_worker_presence(worker_id, rows, source="discovered", complete=True)
+            db_ok = st == "ok"
+            db_changed = bool(db_ok and (det or {}).get("changed"))
+        except Exception:  # noqa: BLE001 — a DB miss never breaks a heartbeat
+            db_changed = False
+        if changed and (not db_ok or os.environ.get("HUGPY_WORKER_CATALOG_JSON", "0") == "1"):
             data[worker_id] = {"at": time.time(), "models": clean}
             tmp = path + f".{os.getpid()}.tmp"
             with open(tmp, "w", encoding="utf-8") as fh:
                 json.dump(data, fh)
             os.replace(tmp, path)
         fcntl.flock(lock, fcntl.LOCK_UN)
-    _apply_worker_catalog(force=changed)
+    _apply_worker_catalog(force=(changed or db_changed))
     # The canonical keys may differ from worker-local scan keys when central
     # already has a same-named model from another hub. Return those identities
     # so heartbeat placement uses the same keys as the shared model catalog.
@@ -1639,14 +1659,27 @@ def _apply_worker_catalog(force=False) -> None:
     registry_dict = globals().get("MODEL_REGISTRY_DICT")
     if registry is None or registry_dict is None:
         return  # import-time base build
+    data = None
+    # DB first (detach step 2): the discovered catalog lives in
+    # model_worker_presence; the JSON file is the no-DB fallback only.
     try:
-        stamp = os.stat(_WORKER_CATALOG_PATH).st_mtime_ns
+        from hugpy_engine.model_index import fetch_presence_catalog
+        data = fetch_presence_catalog()
+    except Exception:  # noqa: BLE001
+        data = None
+    if data is not None:
+        stamp = hash(json.dumps(data, sort_keys=True, default=str))
         if not force and stamp == _WORKER_CATALOG_SEEN:
             return
-        with open(_WORKER_CATALOG_PATH, encoding="utf-8") as fh:
-            data = json.load(fh)
-    except (OSError, ValueError):
-        return
+    else:
+        try:
+            stamp = os.stat(_WORKER_CATALOG_PATH).st_mtime_ns
+            if not force and stamp == _WORKER_CATALOG_SEEN:
+                return
+            with open(_WORKER_CATALOG_PATH, encoding="utf-8") as fh:
+                data = json.load(fh)
+        except (OSError, ValueError):
+            return
     locations: dict[str, dict[str, str]] = {}
     external_apis: dict[str, dict[str, dict]] = {}
     candidates: dict[str, dict] = {}

{hugpy_engine-0.2.5 → hugpy_engine-0.2.6}/src/hugpy_engine/dispatch/dispatch.py

@@ -1147,7 +1147,26 @@ def evict(model_key: str, task: Optional[str] = None) -> bool:
         heavy = evict_llama_runner(model_key)
     except Exception:
         heavy = False
-    return dropped or heavy
+    # TRANSFORMERS weights live in generate.coder.REGISTRY, not in the runner
+    # wrapper: drop them too, then reclaim so the freed VRAM is visible to the
+    # very next fit check (2026-10-02: an in-process eviction "freed 16 MB" of
+    # a 17 GB model and stranded the rest on the card).
+    tf = 0
+    try:
+        from hugpy_engine.generate.coder import REGISTRY as _TF_REGISTRY
+        tf = _TF_REGISTRY.evict_model(model_key)
+    except Exception:
+        tf = 0
+    if tf:
+        try:
+            import gc
+            gc.collect()
+            import torch
+            if torch.cuda.is_available():
+                torch.cuda.empty_cache()
+        except Exception:
+            pass
+    return dropped or heavy or bool(tf)
 
 
 def clear() -> None:
@@ -1159,6 +1178,14 @@ def clear() -> None:
         clear_llama_runners()
     except Exception:
         pass
+    try:
+        from hugpy_engine.generate.coder import REGISTRY as _TF_REGISTRY
+        for _k in {getattr(getattr(i, "cfg", None), "model_key", None) for i in list(_TF_REGISTRY._instances.values())}:
+            if _k:
+                _TF_REGISTRY.evict_model(_k)
+        _TF_REGISTRY.clear()
+    except Exception:
+        pass
 
 
 def supported_task_keys() -> List[Tuple[str, str]]:

{hugpy_engine-0.2.5 → hugpy_engine-0.2.6}/src/hugpy_engine/fit/flex.py

@@ -127,7 +127,8 @@ def band_ceiling(target: float, deviation_pct: Optional[float],
 
 # ── band math (ctx%: the 1..100 point scale) ─────────────────────────────────
 def ctx_band_bounds(ctx_pct: Optional[int],
-                    deviation_pct: Optional[float]) -> "tuple[int, int] | None":
+                    deviation_pct: Optional[float],
+                    floor_pct: Optional[int] = None) -> "tuple[int, int] | None":
     """The hard ctx band ``(floor_pct, ceil_pct)`` in whole percent points, or
     None when there is no ctx target (ctx band is opt-in — no target == today's
     default ctx, byte-identical).
@@ -143,6 +144,15 @@ def ctx_band_bounds(ctx_pct: Optional[int],
     except (TypeError, ValueError):
         return None
     c = max(1, min(100, c))
+    # CONTEXT RANGE (operator 2026-10-02): an explicit minimum makes the band
+    # [min, target] — a polite window below the target, never above it.
+    if floor_pct is not None:
+        try:
+            f = int(floor_pct)
+        except (TypeError, ValueError):
+            f = None
+        if f is not None:
+            return max(1, min(c, f)), c
     dev = float(deviation_pct or 0.0)
     if dev <= 0:
         return c, c
@@ -610,17 +620,27 @@ def plan_flex(subject: dict, residents: list, deficit_bytes: int) -> FlexPlan:
     subj_ctx = subject.get("ctx_pct")
     subj_dev = subject.get("ctx_deviation_pct")
     subj_kv = int(subject.get("kv_bytes") or 0)
-    band = ctx_band_bounds(subj_ctx, subj_dev)
+    band = ctx_band_bounds(subj_ctx, subj_dev, subject.get("ctx_floor_pct"))
     if band is not None and subj_kv > 0:
         floor_pct, _ceil = band
         if floor_pct < int(subj_ctx):
-            kv_floor = kv_at_ctx_pct(subj_kv, int(subj_ctx), floor_pct)
-            saved = max(0, subj_kv - kv_floor)
+            c = int(subj_ctx)
+            to_pct = floor_pct
+            if subject.get("ctx_floor_pct") is not None:
+                # An explicit RANGE: the LARGEST ctx in [min, target) whose KV
+                # saving covers the deficit ("adjusting to fit where possible",
+                # 2026-10-02) — never deeper than needed; the minimum when even
+                # it is not enough. A ± deviation band keeps its floor jump.
+                keep = subj_kv - remaining
+                fit_pct = int(c * keep // subj_kv) if keep > 0 else 0
+                to_pct = max(floor_pct, min(c - 1, fit_pct))
+            kv_to = kv_at_ctx_pct(subj_kv, c, to_pct)
+            saved = max(0, subj_kv - kv_to)
             if saved > 0:
-                self_ctx_pct = floor_pct
+                self_ctx_pct = to_pct
                 remaining -= saved
                 note_parts.append(
-                    f"self ctx {subj_ctx}%->{floor_pct}% freed {saved}B")
+                    f"self ctx {subj_ctx}%->{to_pct}% freed {saved}B")
 
     if remaining <= 0:
         return FlexPlan("flex", self_ctx_pct=self_ctx_pct,
@@ -635,7 +655,7 @@ def plan_flex(subject: dict, residents: list, deficit_bytes: int) -> FlexPlan:
     for r in residents:
         if r.get("protected"):
             continue                       # protection outranks priority, always
-        rb = ctx_band_bounds(r.get("ctx_pct"), r.get("ctx_deviation_pct"))
+        rb = ctx_band_bounds(r.get("ctx_pct"), r.get("ctx_deviation_pct"), r.get("ctx_floor_pct"))
         rkv = int(r.get("kv_bytes") or 0)
         if rb is None or rkv <= 0:
             continue                       # nothing to compress

hugpy_engine-0.2.6/src/hugpy_engine/fit/gguf_need.py

@@ -0,0 +1,203 @@
+"""THE GGUF need function (2026-09-30): what a llama-server load of a GGUF puts
+on the GPU and in host RAM, priced from the file's typed structure
+(``hugpy_storage.gguf_inspect.gguf_structure``) — never a flat file size for a
+MoE, never a ``layers x dense`` KV for a hybrid.
+
+One source of truth: the worker's admission (``_incoming_need_detail``), the
+allocation rows, central's placement preflight (``_worker_fit``), the planned
+split on the worker row and the Ctx slider preview all call :func:`gguf_need`.
+
+Placement follows llama.cpp exactly:
+  * ``--n-gpu-layers L`` puts blocks ``i >= n_block - L`` on the GPU; the
+    output tensor goes to the GPU only when ``L > n_block`` (``-1``/all does);
+    ``token_embd`` is ALWAYS on the CPU (llama.cpp's input device); a tied
+    output (no ``output.weight``) duplicates ``token_embd`` onto the output
+    device.
+  * ``--n-cpu-moe N`` keeps the expert tensors (``ffn_{up,down,gate,gate_up}_exps``)
+    of blocks ``i < N`` on the CPU; every always-on tensor of a GPU block stays
+    on the GPU.
+  * KV / recurrent state lives with its block (offload_kqv default): full
+    attention = ``ctx x n_head_kv x (k_len + v_len) x dtype``; sliding-window
+    blocks cap the cells at ``window + ubatch``; linear-attention (Gated
+    DeltaNet / Mamba) blocks hold a fixed f32 state per sequence, independent
+    of ctx.
+
+The weights margin applies to the WEIGHTS of each side separately (measured
+per side when a record exists, else the prior); KV, state, the compute
+cushion and an mmproj projector are added unmultiplied. A dense model keeps
+the historical pricing (file bytes x margin on the GPU when fully offloaded),
+so dense admission is byte-for-byte unchanged.
+"""
+from __future__ import annotations
+
+from typing import Any, Optional
+
+COMPUTE_CUSHION_BYTES = 512 * 2 ** 20            # spill._CTX_COMPUTE_RESERVE_BYTES
+SWA_UBATCH = 512                                  # llama.cpp default n_ubatch
+
+
+def structure_for(path: Optional[str]) -> dict:
+    """``gguf_structure`` of the served file, ``{}`` when unreadable."""
+    if not path:
+        return {}
+    try:
+        from hugpy_storage.gguf_inspect import gguf_structure
+        return gguf_structure(path) or {}
+    except Exception:  # noqa: BLE001 — a structure gap degrades to opaque pricing
+        return {}
+
+
+def _layers(struct: dict) -> dict:
+    return {int(k): v for k, v in (struct.get("layers") or {}).items()}
+
+
+def kv_breakdown(struct: dict, ctx: Optional[int], *, dtype_bytes: float = 2.0,
+                 n_seq: int = 1, gpu_layers: Optional[set] = None) -> dict:
+    """Context cost by attention kind: ``{kv_bytes, kv_full_bytes, kv_swa_bytes,
+    state_bytes, layers: {full, swa, linear}, per_token_bytes}``; with
+    ``gpu_layers`` also the GPU-side share (``gpu_bytes``)."""
+    ctx = max(0, int(ctx or 0))
+    win = struct.get("swa_window")
+    swa_cells = min(ctx, int(win) + SWA_UBATCH) if win else ctx
+    full = swa = state = gpu = 0
+    per_tok = 0
+    counts = {"full": 0, "swa": 0, "linear": 0}
+    for i, L in _layers(struct).items():
+        kind = L.get("attn")
+        e = int(L.get("kv_elems_per_token") or 0)
+        b = 0
+        if kind == "full":
+            b = int(ctx * e * dtype_bytes)
+            full += b
+            per_tok += int(e * dtype_bytes)
+        elif kind == "swa":
+            b = int(swa_cells * e * dtype_bytes)
+            swa += b
+        elif kind == "linear":
+            b = int(L.get("state_bytes") or 0) * max(1, int(n_seq))
+            state += b
+        if kind in counts:
+            counts[kind] += 1
+        if gpu_layers is not None and i in gpu_layers:
+            gpu += b
+    out = {"ctx": ctx, "kv_bytes": full + swa + state, "kv_full_bytes": full,
+           "kv_swa_bytes": swa, "state_bytes": state, "layers": counts,
+           "per_token_bytes": per_tok, "swa_window": win}
+    if gpu_layers is not None:
+        out["gpu_bytes"] = gpu
+    return out
+
+
+def gguf_need(struct: dict, *, ctx: Optional[int], n_cpu_moe: Optional[int] = 0,
+              n_gpu_layers: int = -1, gpu_margin: float = 1.15,
+              ram_margin: float = 1.15, dtype_bytes: float = 2.0,
+              cushion_bytes: int = COMPUTE_CUSHION_BYTES, mmproj_bytes: int = 0,
+              n_seq: int = 1) -> dict:
+    """GPU / RAM bytes of a llama-server load at ``ctx`` (see module doc)."""
+    layers = _layers(struct)
+    n_block = int(struct.get("block_count") or len(layers))
+    ngl = int(n_gpu_layers if n_gpu_layers is not None else -1)
+    if ngl < 0 or ngl > n_block:
+        start, out_on_gpu = 0, True
+    else:
+        start, out_on_gpu = n_block - ngl, False
+    gpu_layers = {i for i in layers if i >= start} if ngl != 0 else set()
+    is_moe = bool(struct.get("is_moe"))
+    ncm = max(0, int(n_cpu_moe or 0)) if is_moe else 0
+    gw = rw = 0
+    experts_gpu = experts_cpu = 0
+    for i, L in layers.items():
+        a, e = int(L.get("always_bytes") or 0), int(L.get("expert_bytes") or 0)
+        if i in gpu_layers:
+            gw += a
+            if i < ncm:
+                rw += e
+                experts_cpu += e
+            else:
+                gw += e
+                experts_gpu += e
+        else:
+            rw += a + e
+            experts_cpu += e
+    embd = int(struct.get("token_embd_bytes") or 0)
+    outb = int(struct.get("output_bytes") or 0)
+    glob = int(struct.get("global_bytes") or 0)
+    rw += embd                                     # input layer: always CPU
+    if out_on_gpu:
+        gw += outb + glob + (embd if struct.get("output_tied") else 0)
+    else:
+        rw += outb + glob
+    file_bytes = int(struct.get("file_bytes") or 0)
+    if not is_moe and out_on_gpu and file_bytes:
+        # Dense, fully offloaded: the historical pricing (file x margin on the
+        # GPU) — dense admission stays byte-identical.
+        gw, rw = file_bytes, 0
+    kv = kv_breakdown(struct, ctx, dtype_bytes=dtype_bytes, n_seq=n_seq,
+                      gpu_layers=gpu_layers)
+    kv_gpu = int(kv.get("gpu_bytes") or 0)
+    kv_cpu = int(kv["kv_bytes"]) - kv_gpu
+    # Compute cushion on the GPU for a MoE split (dense pricing never carried one).
+    cushion = int(cushion_bytes) if (gpu_layers and is_moe) else 0
+    gpu_total = int(gw * float(gpu_margin)) + kv_gpu + cushion + int(mmproj_bytes or 0)
+    ram_total = int(rw * float(ram_margin)) + kv_cpu
+    return {"n_cpu_moe": (min(ncm, n_block) if is_moe else None),
+            "n_gpu_layers": ngl, "block_count": n_block, "is_moe": is_moe,
+            "ctx": kv["ctx"],
+            "gpu_bytes": gpu_total, "ram_bytes": ram_total,
+            "gpu_weights_bytes": int(gw), "ram_weights_bytes": int(rw),
+            "gpu_margin": float(gpu_margin), "ram_margin": float(ram_margin),
+            "experts_gpu_bytes": int(experts_gpu), "experts_cpu_bytes": int(experts_cpu),
+            "kv_bytes": int(kv["kv_bytes"]), "kv_gpu_bytes": kv_gpu, "kv_cpu_bytes": kv_cpu,
+            "kv_full_bytes": kv["kv_full_bytes"], "kv_swa_bytes": kv["kv_swa_bytes"],
+            "state_bytes": kv["state_bytes"], "kv_layers": kv["layers"],
+            "cushion_bytes": cushion, "mmproj_bytes": int(mmproj_bytes or 0),
+            # mmap: the whole file is mapped; pages of GPU-uploaded tensors stay
+            # in page cache (RSS) until reclaimed — reclaimable, not a need.
+            "mmap_bytes": file_bytes}
+
+
+def choose_n_cpu_moe(struct: dict, room_bytes: Optional[int], **kw) -> dict:
+    """The SMALLEST ``n_cpu_moe`` whose GPU side fits ``room_bytes`` whole
+    (GPU bytes fall monotonically as N grows). ``fits`` False -> every expert
+    on the CPU and the backbone still does not fit (N = block_count)."""
+    n_block = int(struct.get("block_count") or 0)
+    if room_bytes is None:
+        need = gguf_need(struct, n_cpu_moe=n_block, **kw)
+        return dict(need, fits=None, choice="unmeasurable card — all experts on CPU")
+    for n in range(0, n_block + 1):
+        need = gguf_need(struct, n_cpu_moe=n, **kw)
+        if need["gpu_bytes"] <= int(room_bytes):
+            return dict(need, fits=True, room_bytes=int(room_bytes),
+                        choice=f"smallest n_cpu_moe fitting {int(room_bytes)} B")
+    need = gguf_need(struct, n_cpu_moe=n_block, **kw)
+    return dict(need, fits=False, room_bytes=int(room_bytes),
+                choice="backbone does not fit even with every expert on CPU")
+
+
+def whole_seat_ctx(need_at, room: int, upper: int, floor: int = 4096) -> "tuple[int, bool]":
+    """Largest ctx (1024 multiple in [floor, upper]) with ``need_at(ctx) <= room``."""
+    upper = max(int(floor), int(upper))
+    top = upper if upper < 1024 else (upper // 1024) * 1024
+    if need_at(top) <= room:
+        return top, True
+    lo, hi, best = -(-int(floor) // 1024), top // 1024, None
+    while lo <= hi:
+        mid = (lo + hi) // 2
+        if need_at(mid * 1024) <= room:
+            best, lo = mid * 1024, mid + 1
+        else:
+            hi = mid - 1
+    if best is None:
+        return int(floor), need_at(int(floor)) <= room
+    return max(int(floor), best), True
+
+
+def verdict_text(need: dict) -> str:
+    """"MoE: 12/48 layers' experts on CPU; GPU 17.2 GiB, RAM 21.0 GiB" (+ KV)."""
+    g = 2 ** 30
+    if not need.get("is_moe"):
+        return (f"GPU {need['gpu_bytes'] / g:.1f} GiB, RAM {need['ram_bytes'] / g:.1f} GiB "
+                f"(KV {need['kv_bytes'] / g:.2f} GiB @ {need['ctx']})")
+    return (f"MoE: {need['n_cpu_moe']}/{need['block_count']} layers' experts on CPU; "
+            f"GPU {need['gpu_bytes'] / g:.1f} GiB, RAM {need['ram_bytes'] / g:.1f} GiB "
+            f"(KV {need['kv_bytes'] / g:.2f} GiB @ {need['ctx']} ctx)")