dflash-console 0.3.249__tar.gz → 0.3.253__tar.gz

{dflash_console-0.3.249 → dflash_console-0.3.253}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: dflash-console
-Version: 0.3.249
+Version: 0.3.253
 Summary: Local control panel and terminal CLI for DFlash stacks and local model runtimes
 Project-URL: Homepage, https://github.com/ilan4ever/Dflash-Console
 Project-URL: Repository, https://github.com/ilan4ever/Dflash-Console
@@ -674,7 +674,7 @@ from one UI, then talk to them through a single OpenAI-compatible port.
 > supported platform**; Linux and macOS are not supported or tested (see
 > [Platform support](#platform-support) below).
 
-**Developer:** ILAN AVIV · **UI:** [http://127.0.0.1:8900/](http://127.0.0.1:8900/) · **Version:** v0.3.249
+**Developer:** ILAN AVIV · **UI:** [http://127.0.0.1:8900/](http://127.0.0.1:8900/) · **Version:** v0.3.253
 
 ## Download (Windows)
 
@@ -753,7 +753,7 @@ Typical first session:
 3. For a DFlash GGUF, right-click and **Find and attach draft** if you want speculative decoding.
 4. Chat in the **Playground**, or point any OpenAI client at `http://127.0.0.1:8001/v1` with optional `X-DFlash-Client: YourApp` so **Engines** shows who is using each model.
 
-### Recent improvements (v0.3.249)
+### Recent improvements (v0.3.253)
 
 - **Model library** — loading clears when the model is on the GPU; Engine and GPU column headers stay visible
 - **Model catalog** — a pasted repo name with a trailing period still opens the real model, and shared shard names are no longer marked installed

{dflash_console-0.3.249 → dflash_console-0.3.253}/README.md

@@ -8,7 +8,7 @@ from one UI, then talk to them through a single OpenAI-compatible port.
 > supported platform**; Linux and macOS are not supported or tested (see
 > [Platform support](#platform-support) below).
 
-**Developer:** ILAN AVIV · **UI:** [http://127.0.0.1:8900/](http://127.0.0.1:8900/) · **Version:** v0.3.249
+**Developer:** ILAN AVIV · **UI:** [http://127.0.0.1:8900/](http://127.0.0.1:8900/) · **Version:** v0.3.253
 
 ## Download (Windows)
 
@@ -87,7 +87,7 @@ Typical first session:
 3. For a DFlash GGUF, right-click and **Find and attach draft** if you want speculative decoding.
 4. Chat in the **Playground**, or point any OpenAI client at `http://127.0.0.1:8001/v1` with optional `X-DFlash-Client: YourApp` so **Engines** shows who is using each model.
 
-### Recent improvements (v0.3.249)
+### Recent improvements (v0.3.253)
 
 - **Model library** — loading clears when the model is on the GPU; Engine and GPU column headers stay visible
 - **Model catalog** — a pasted repo name with a trailing period still opens the real model, and shared shard names are no longer marked installed

{dflash_console-0.3.249 → dflash_console-0.3.253}/api/app.py

@@ -2175,6 +2175,7 @@ async def servers_status(
         cached = _cached_status_payload(include_external)
         if cached is not None:
             cached = _merge_adapter_engine_rows(dict(cached))
+            cached['boot_id'] = _BOOT_ID
             cached['stale'] = True
             snapshot_at = float(cached.get('updated_at') or 0.0)
             if snapshot_at:
@@ -2206,6 +2207,7 @@ async def servers_status(
 
         _write_engine_status_log(trace, build_ms)
         payload = _merge_adapter_engine_rows(payload)
+        payload['boot_id'] = _BOOT_ID
         _store_status_payload(payload, include_external=include_external)
         return payload
 

{dflash_console-0.3.249 → dflash_console-0.3.253}/core/catalog_load.py

@@ -19,7 +19,7 @@ from core.engine_state import note_engine_loaded, note_engine_on
 from core.gpu_devices import validate_gpu_device
 from core.local_models import list_local_models
 from core.memory_guardrails import assess_load
-from core.server_boot import load_server_checkpoint
+from core.server_boot import checkpoint_already_loaded, find_target_loaded_elsewhere, load_server_checkpoint
 
 
 def _release_peer_llm_engines(keep: str) -> None:
@@ -257,11 +257,15 @@ def execute_catalog_load(
         if server is None:
             raise HTTPException(status_code=404, detail=f'unknown server_id: {server_id}')
     else:
-        candidates = [s for s in list_servers(config) if s.get('enabled', True)]
-        if modality == 'embedding':
-            server = next((s for s in candidates if is_embedding_server(s)), None)
-        else:
-            server = next((s for s in candidates if not is_embedding_server(s)), None)
+        catalog_server_id = str(target.get('server_id') or '').strip()
+        if catalog_server_id:
+            server = get_server(config, catalog_server_id)
+        if server is None:
+            candidates = [s for s in list_servers(config) if s.get('enabled', True)]
+            if modality == 'embedding':
+                server = next((s for s in candidates if is_embedding_server(s)), None)
+            else:
+                server = next((s for s in candidates if not is_embedding_server(s)), None)
     if server is None:
         raise HTTPException(status_code=409, detail=f'no enabled server can run a {modality} model — pass server_id')
 
@@ -277,6 +281,55 @@ def execute_catalog_load(
     if inference_settings:
         candidate['inference_settings'] = normalize_inference_settings(inference_settings)
 
+    server_id_text = str(server.get('id') or '')
+    already = checkpoint_already_loaded(
+        candidate,
+        cfg=config,
+        model_path=resolved_path,
+        model_id=model_id,
+    )
+    if already:
+        note_engine_loaded(server_id_text, loaded_by=loaded_by)
+        model_name = str(already.get('model') or target.get('label') or 'Model')
+        return {
+            'success': True,
+            'modality': modality,
+            'runtime_id': 'llama-server',
+            'server_id': server_id_text,
+            'loaded': True,
+            'already_loaded': True,
+            'path': resolved_path,
+            'message': f'{model_name} is already loaded on this engine.',
+            'how_to_use': f'POST /api/servers/{server_id_text}/v1/chat/completions',
+            **already,
+        }
+    elsewhere = find_target_loaded_elsewhere(
+        candidate,
+        cfg=config,
+        model_path=resolved_path,
+        exclude_server_id=server_id_text,
+    )
+    if elsewhere:
+        use_id = str(elsewhere.get('server_id') or '')
+        label = str(elsewhere.get('label') or use_id or 'another engine')
+        return {
+            'success': True,
+            'modality': modality,
+            'runtime_id': 'llama-server',
+            'server_id': use_id or server_id_text,
+            'loaded': True,
+            'already_loaded': True,
+            'already_loaded_elsewhere': True,
+            'path': resolved_path,
+            'message': f'This model is already loaded on {label}.',
+            'how_to_use': (
+                f'POST /api/servers/{use_id}/v1/chat/completions'
+                if use_id
+                else f'POST /api/servers/{server_id_text}/v1/chat/completions'
+            ),
+            **elsewhere,
+        }
+
     check = assess_load(candidate, cfg=config)
     if check.get('level') == 'block':
         raise HTTPException(status_code=400, detail=str(check.get('message') or 'insufficient VRAM'))

{dflash_console-0.3.249 → dflash_console-0.3.253}/core/embedding_server.py

@@ -17,13 +17,13 @@ from core.gpu_devices import resolve_role_gpu_launch_params
 from core.log_utils import rotate_log
 from core.server_boot import (
     LOG_DIR,
+    _acquire_port_boot,
     _spawn_detached,
     _tcp_port_open,
     clear_server_tracking,
     forget_started_process,
     get_started_process,
     note_boot_cycle_end,
-    port_lock_for,
     register_started_launch,
     terminate_process_tree,
     wait_for_port_closed,
@@ -241,9 +241,14 @@ def start_embedding_server(server: dict[str, Any], *, cfg: dict[str, Any] | None
     port = int(entry.get('port') or 0)
     if port <= 0 or not entry.get('id'):
         return {'success': False, 'error': 'invalid embedding server config'}
-    lock = port_lock_for(port)
-    if not lock.acquire(blocking=False):
-        return {'success': False, 'error': 'boot already in progress', 'port': port}
+    acquired = _acquire_port_boot(port)
+    if acquired is None:
+        return {
+            'success': False,
+            'error': 'timed out waiting for the model server to finish starting',
+            'port': port,
+        }
+    lock, _joined = acquired
     try:
         return _start_embedding_server_locked(entry, cfg=cfg)
     finally:

dflash_console-0.3.253/core/gpu_process_memory_windows.py

@@ -0,0 +1,262 @@
+"""Per-process GPU memory on Windows via WDDM performance counters."""
+
+from __future__ import annotations
+
+import json
+import re
+import subprocess
+import sys
+import threading
+import time
+from typing import Any
+
+_CACHE: dict[str, Any] = {'at': 0.0, 'map': {}}
+_CACHE_LOCK = threading.Lock()
+_CACHE_TTL_SECONDS = 20.0
+_MAX_PROCESS_BYTES = 64 * (1024 ** 3)
+
+_ROW_RE = re.compile(
+    r'pid_(\d+)_luid_((?:0x)?[0-9a-f]+)_((?:0x)?[0-9a-f]+)_phys_(\d+)',
+    re.I,
+)
+
+
+def _subprocess_no_window_kwargs() -> dict[str, Any]:
+    if sys.platform == 'win32':
+        startupinfo = subprocess.STARTUPINFO()
+        startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW
+        flags = getattr(subprocess, 'CREATE_NO_WINDOW', 0)
+        return {'startupinfo': startupinfo, 'creationflags': flags}
+    return {}
+
+
+def _fetch_windows_process_gpu_bytes() -> dict[tuple[int, str], int]:
+    # Keep the larger sample per process+adapter. Windows repeats the same
+    # allocation on several engine nodes (3D, copy, compute) and both boards
+    # report phys_0, so the adapter LUID is what separates the GPUs.
+    script = r"""
+$rows = Get-CimInstance -Namespace root/cimv2 -ClassName Win32_PerfFormattedData_GPUPerformanceCounters_GPUProcessMemory -ErrorAction SilentlyContinue
+$agg = @{}
+foreach ($r in $rows) {
+  if ($r.Name -match 'pid_(\d+)_luid_((?:0x)?[0-9a-fA-F]+)_((?:0x)?[0-9a-fA-F]+)_phys_(\d+)') {
+    $procId = [int]$matches[1]
+    $luid = ($matches[2] + '_' + $matches[3]).ToLower()
+    $key = "$procId|$luid"
+    $bytes = [long]($r.DedicatedUsage)
+    if ($bytes -le 0) { $bytes = [long]($r.TotalCommitted) }
+    if ($bytes -le 0) { continue }
+    if (-not $agg.ContainsKey($key)) { $agg[$key] = [long]0 }
+    $current = [long]$agg[$key]
+    if ($bytes -gt $current) { $agg[$key] = $bytes }
+  }
+}
+$out = @()
+foreach ($kv in $agg.GetEnumerator()) {
+  $parts = $kv.Key.Split('|')
+  $out += @{ pid = [int]$parts[0]; luid = [string]$parts[1]; bytes = [long]$kv.Value }
+}
+$out | ConvertTo-Json -Compress
+"""
+    try:
+        result = subprocess.run(
+            ['powershell', '-NoProfile', '-NonInteractive', '-WindowStyle', 'Hidden', '-Command', script],
+            capture_output=True,
+            text=True,
+            timeout=12,
+            check=False,
+            **_subprocess_no_window_kwargs(),
+        )
+    except Exception:
+        return {}
+    if result.returncode != 0 or not result.stdout.strip():
+        return {}
+    try:
+        payload = json.loads(result.stdout.strip())
+    except json.JSONDecodeError:
+        return {}
+    rows = payload if isinstance(payload, list) else [payload]
+    mapping: dict[tuple[int, str], int] = {}
+    for row in rows:
+        if not isinstance(row, dict):
+            continue
+        try:
+            pid = int(row.get('pid') or 0)
+            luid = str(row.get('luid') or '').strip().lower()
+            bytes_val = int(row.get('bytes') or 0)
+        except (TypeError, ValueError):
+            continue
+        if pid <= 0 or not luid or bytes_val <= 0 or bytes_val > _MAX_PROCESS_BYTES:
+            continue
+        key = (pid, luid)
+        prior = mapping.get(key, 0)
+        if bytes_val > prior:
+            mapping[key] = bytes_val
+    return mapping
+
+
+def match_luids_to_gpus(
+    per_luid_bytes: dict[str, int],
+    gpus: list[dict[str, Any]] | None,
+) -> dict[str, int]:
+    """Pair adapter LUIDs with nvidia-smi GPU indexes.
+
+    Both adapters report ``phys_0``. The LUID whose dedicated total is closest
+    to a board's used memory is that board.
+    """
+    totals = {
+        str(luid).lower(): int(nbytes)
+        for luid, nbytes in (per_luid_bytes or {}).items()
+        if int(nbytes or 0) >= 64 * (1024 ** 2)
+    }
+    devices: list[tuple[int, int | None]] = []
+    for gpu in gpus or []:
+        if not isinstance(gpu, dict) or gpu.get('index') is None:
+            continue
+        try:
+            index = int(gpu.get('index'))
+        except (TypeError, ValueError):
+            continue
+        used = gpu.get('vram_used_gb')
+        used_bytes: int | None
+        try:
+            used_bytes = int(float(used) * (1024 ** 3)) if used is not None else None
+        except (TypeError, ValueError):
+            used_bytes = None
+        devices.append((index, used_bytes))
+    if not totals:
+        return {}
+    if len(devices) == 1:
+        only = devices[0][0]
+        return {luid: only for luid in totals}
+    remaining = list(devices)
+    assigned: dict[str, int] = {}
+    for luid, nbytes in sorted(totals.items(), key=lambda item: item[1], reverse=True):
+        if not remaining:
+            break
+        best_at = 0
+        best_err: int | None = None
+        for at, (index, used_bytes) in enumerate(remaining):
+            if used_bytes is None:
+                continue
+            err = abs(int(used_bytes) - int(nbytes))
+            if best_err is None or err < best_err:
+                best_err = err
+                best_at = at
+        if best_err is None:
+            best_at = 0
+        index, _used = remaining.pop(best_at)
+        assigned[luid] = index
+    return assigned
+
+
+def _gpu_devices_for_luid_match() -> list[dict[str, Any]]:
+    try:
+        from core.gpu_devices import query_gpu_devices
+
+        devices = query_gpu_devices()
+    except Exception:
+        return []
+    return devices if isinstance(devices, list) else []
+
+
+def _mapping_from_luid_rows(
+    per_pid_luid: dict[tuple[int, str], int],
+    gpus: list[dict[str, Any]] | None = None,
+) -> dict[tuple[int, int], int]:
+    per_luid: dict[str, int] = {}
+    for (_pid, luid), nbytes in per_pid_luid.items():
+        per_luid[luid] = per_luid.get(luid, 0) + int(nbytes)
+    assigned = match_luids_to_gpus(per_luid, gpus if gpus is not None else _gpu_devices_for_luid_match())
+    mapping: dict[tuple[int, int], int] = {}
+    for (pid, luid), nbytes in per_pid_luid.items():
+        gpu_index = assigned.get(luid)
+        if gpu_index is None:
+            continue
+        key = (int(pid), int(gpu_index))
+        mapping[key] = mapping.get(key, 0) + int(nbytes)
+    return mapping
+
+
+def query_windows_process_gpu_bytes() -> dict[tuple[int, int], int]:
+    """Return {(pid, nvidia gpu index): dedicated_bytes}."""
+    if sys.platform != 'win32':
+        return {}
+    now = time.time()
+    cached_at = float(_CACHE.get('at') or 0.0)
+    cached_map = _CACHE.get('map')
+    if isinstance(cached_map, dict) and (now - cached_at) < _CACHE_TTL_SECONDS:
+        return dict(cached_map)
+    with _CACHE_LOCK:
+        now = time.time()
+        cached_at = float(_CACHE.get('at') or 0.0)
+        cached_map = _CACHE.get('map')
+        if isinstance(cached_map, dict) and (now - cached_at) < _CACHE_TTL_SECONDS:
+            return dict(cached_map)
+        per_pid_luid = _fetch_windows_process_gpu_bytes()
+        mapping = _mapping_from_luid_rows(per_pid_luid)
+        _CACHE['at'] = time.time()
+        _CACHE['map'] = dict(mapping)
+        return mapping
+
+
+def process_vram_by_gpu(pid: int, mapping: dict[tuple[int, int], int] | None = None) -> dict[int, float]:
+    """Gigabytes of one process on each GPU. Empty when this PID has none."""
+    if pid <= 0:
+        return {}
+    data = mapping if mapping is not None else query_windows_process_gpu_bytes()
+    found: dict[int, float] = {}
+    for (proc_id, gpu_index), nbytes in data.items():
+        if int(proc_id) != int(pid) or int(nbytes) <= 0:
+            continue
+        gb = round(int(nbytes) / (1024 ** 3), 3)
+        if gb <= 0:
+            continue
+        found[int(gpu_index)] = round(found.get(int(gpu_index), 0.0) + gb, 3)
+    return found
+
+
+def lookup_windows_process_vram_gb(
+    pid: int,
+    gpu_index: int,
+    mapping: dict[tuple[int, int], int] | None = None,
+) -> float | None:
+    """VRAM in GB for this process on this GPU only.
+
+    Memory on another board is not borrowed. That is what made the Titan's
+    usage show up as a 4090 number.
+    """
+    if pid <= 0:
+        return None
+    found = process_vram_by_gpu(pid, mapping)
+    gb = found.get(int(gpu_index))
+    return gb if gb and gb > 0 else None
+
+
+def apply_windows_process_vram(processes: list[dict[str, Any]]) -> bool:
+    """Fill missing per-process VRAM on Windows. Returns True when any row was enriched."""
+    if sys.platform != 'win32' or not processes:
+        return False
+    mapping = query_windows_process_gpu_bytes()
+    if not mapping:
+        return False
+    enriched = False
+    for proc in processes:
+        if not isinstance(proc, dict):
+            continue
+        try:
+            pid = int(proc.get('pid') or 0)
+            gpu_index = int(proc.get('gpu_index') or 0)
+        except (TypeError, ValueError):
+            continue
+        if pid <= 0:
+            continue
+        if proc.get('vram_gb') is not None and float(proc.get('vram_gb') or 0) > 0:
+            continue
+        gb = lookup_windows_process_vram_gb(pid, gpu_index, mapping)
+        if gb is None:
+            continue
+        proc['vram_gb'] = gb
+        proc['vram_mb'] = round(gb * 1024, 1)
+        proc['vram_source'] = 'windows'
+        enriched = True
+    return enriched