coderouter-cli 2.9.4__py3-none-any.whl → 2.11.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- coderouter/cli.py +125 -0
- coderouter/config/schemas.py +144 -10
- coderouter/ingress/launcher_routes.py +289 -22
- coderouter/launcher_devices.py +188 -4
- coderouter/launcher_speculative.py +5 -1
- coderouter/launcher_swap.py +11 -2
- coderouter/vscode_init.py +575 -0
- {coderouter_cli-2.9.4.dist-info → coderouter_cli-2.11.0.dist-info}/METADATA +5 -1
- {coderouter_cli-2.9.4.dist-info → coderouter_cli-2.11.0.dist-info}/RECORD +12 -11
- {coderouter_cli-2.9.4.dist-info → coderouter_cli-2.11.0.dist-info}/WHEEL +0 -0
- {coderouter_cli-2.9.4.dist-info → coderouter_cli-2.11.0.dist-info}/entry_points.txt +0 -0
- {coderouter_cli-2.9.4.dist-info → coderouter_cli-2.11.0.dist-info}/licenses/LICENSE +0 -0
|
@@ -47,6 +47,7 @@ from coderouter.launcher_devices import (
|
|
|
47
47
|
SweepPlan,
|
|
48
48
|
SweepState,
|
|
49
49
|
SweepStep,
|
|
50
|
+
base_backend,
|
|
50
51
|
build_auto_sweep_configs,
|
|
51
52
|
build_sweep_steps,
|
|
52
53
|
detect_llama_devices,
|
|
@@ -54,8 +55,11 @@ from coderouter.launcher_devices import (
|
|
|
54
55
|
is_port_free,
|
|
55
56
|
load_latest_results,
|
|
56
57
|
render_bench_command,
|
|
58
|
+
resolve_option_profiles,
|
|
57
59
|
selectable_devices,
|
|
58
60
|
suggest_tensor_split,
|
|
61
|
+
unknown_device_ids,
|
|
62
|
+
variant_of,
|
|
59
63
|
)
|
|
60
64
|
from coderouter.launcher_speculative import resolve_speculative
|
|
61
65
|
from coderouter.logging import get_logger
|
|
@@ -281,6 +285,9 @@ def _scan_models(model_dirs: list[str]) -> list[dict[str, Any]]:
|
|
|
281
285
|
# ---------------------------------------------------------------------------
|
|
282
286
|
|
|
283
287
|
|
|
288
|
+
# 基底バックエンド名 → 既定の実行ファイル名。キーは常に**基底名**であり、
|
|
289
|
+
# ``llama.cpp-cuda`` のようなバリアント名は入らない。参照側は必ず
|
|
290
|
+
# ``base_backend()`` を通してから引くこと(launcher_devices §2.0)。
|
|
284
291
|
_BACKEND_DEFAULTS: dict[str, str] = {
|
|
285
292
|
"llama.cpp": "llama-server",
|
|
286
293
|
"vllm": "python",
|
|
@@ -315,8 +322,15 @@ def _assert_no_model_override(backend: str, tokens: list[str]) -> None:
|
|
|
315
322
|
``tokens`` is the flat list of argument tokens sourced from ``options``
|
|
316
323
|
keys or ``shlex.split(extra_args)``. Matching is exact on the flag name;
|
|
317
324
|
``--model=foo`` style is caught by comparing the part before ``=``.
|
|
325
|
+
|
|
326
|
+
``backend`` may be a variant name (``llama.cpp-cuda``), so the lookup goes
|
|
327
|
+
through :func:`base_backend`. A missing entry must NOT silently disable the
|
|
328
|
+
guard — an unknown base falls back to the union of every banned set, i.e.
|
|
329
|
+
the strictest possible check (fail-closed).
|
|
318
330
|
"""
|
|
319
|
-
banned = _MODEL_FLAGS.get(backend
|
|
331
|
+
banned = _MODEL_FLAGS.get(base_backend(backend))
|
|
332
|
+
if banned is None:
|
|
333
|
+
banned = frozenset().union(*_MODEL_FLAGS.values())
|
|
320
334
|
for token in tokens:
|
|
321
335
|
name = token.split("=", 1)[0]
|
|
322
336
|
if name in banned:
|
|
@@ -328,11 +342,36 @@ def _assert_no_model_override(backend: str, tokens: list[str]) -> None:
|
|
|
328
342
|
|
|
329
343
|
|
|
330
344
|
def _resolve_binary(backend: str, configured: str | None) -> str:
|
|
331
|
-
"""Return the executable to use, expanding ~ and env vars.
|
|
332
|
-
|
|
345
|
+
"""Return the executable to use, expanding ~ and env vars.
|
|
346
|
+
|
|
347
|
+
``backend`` may be a variant name; the default-executable fallback is
|
|
348
|
+
keyed by :func:`base_backend` so ``llama.cpp-cuda`` without a configured
|
|
349
|
+
``binary`` would fall back to ``llama-server`` rather than to the literal
|
|
350
|
+
backend name. In practice variants require ``binary`` (validated at config
|
|
351
|
+
load — see ``LauncherConfig``), so that fallback should be unreachable.
|
|
352
|
+
"""
|
|
353
|
+
raw = configured or _BACKEND_DEFAULTS.get(base_backend(backend), backend)
|
|
333
354
|
return str(Path(raw).expanduser())
|
|
334
355
|
|
|
335
356
|
|
|
357
|
+
def _backend_names(backends_cfg: dict[str, Any] | None) -> list[str]:
|
|
358
|
+
"""Launcher が提示するバックエンド名の一覧 (順序が UI の並び順)。
|
|
359
|
+
|
|
360
|
+
基底 3 つを常に先頭に (従来どおりの順)、そのあとに ``launcher.backends``
|
|
361
|
+
に書かれたバリアントを記述順で並べる。設計 §6。
|
|
362
|
+
|
|
363
|
+
``launcher.backends`` にバリアントを書かない利用者には従来と同じ 3 要素が
|
|
364
|
+
返るので、UI の選択肢も API 応答も完全に不変 —— 特化ビルドは「書いた人に
|
|
365
|
+
だけ見える上級者向けオプション」であり、バッジや注記ではなく構造でそれを
|
|
366
|
+
担保する。
|
|
367
|
+
"""
|
|
368
|
+
names = list(_BACKEND_DEFAULTS)
|
|
369
|
+
for name in backends_cfg or {}:
|
|
370
|
+
if name not in names:
|
|
371
|
+
names.append(name)
|
|
372
|
+
return names
|
|
373
|
+
|
|
374
|
+
|
|
336
375
|
def _resolve_backends_sync(
|
|
337
376
|
backends_cfg: dict[str, Any] | None,
|
|
338
377
|
) -> dict[str, dict[str, Any]]:
|
|
@@ -340,9 +379,16 @@ def _resolve_backends_sync(
|
|
|
340
379
|
|
|
341
380
|
Performs blocking filesystem I/O (``is_file`` / ``shutil.which``),
|
|
342
381
|
so async-route callers must invoke it via ``asyncio.to_thread``.
|
|
382
|
+
|
|
383
|
+
Covers the base backends plus any variant declared in
|
|
384
|
+
``launcher.backends`` (see :func:`_backend_names`). Each entry gains
|
|
385
|
+
``base`` / ``variant`` so the UI can label a variant without re-parsing
|
|
386
|
+
the name client-side.
|
|
343
387
|
"""
|
|
344
388
|
result: dict[str, dict[str, Any]] = {}
|
|
345
|
-
for backend
|
|
389
|
+
for backend in _backend_names(backends_cfg):
|
|
390
|
+
base = base_backend(backend)
|
|
391
|
+
default_bin = _BACKEND_DEFAULTS.get(base, backend)
|
|
346
392
|
configured: str | None = None
|
|
347
393
|
if backends_cfg:
|
|
348
394
|
bc = backends_cfg.get(backend)
|
|
@@ -360,6 +406,9 @@ def _resolve_backends_sync(
|
|
|
360
406
|
"default": default_bin,
|
|
361
407
|
"is_custom": configured is not None,
|
|
362
408
|
"found": found,
|
|
409
|
+
# 以下 2 キーはバリアント機能で追加 (基底名では variant=None)。
|
|
410
|
+
"base": base,
|
|
411
|
+
"variant": variant_of(backend),
|
|
363
412
|
}
|
|
364
413
|
return result
|
|
365
414
|
|
|
@@ -436,6 +485,8 @@ def _suggest_launch_flags(backend: str, size_gb: float,
|
|
|
436
485
|
- mlx : 統合メモリ前提で起動時フラグ不要 (空文字)
|
|
437
486
|
あくまで目安。他プロセスのメモリ使用や量子化方式までは考慮しない。
|
|
438
487
|
"""
|
|
488
|
+
# バリアント名 (llama.cpp-cuda 等) でも同じフラグ体系なので基底名で判定。
|
|
489
|
+
backend = base_backend(backend)
|
|
439
490
|
if backend == "mlx":
|
|
440
491
|
# MLX は統合メモリ + Metal 前提。llama.cpp の -ngl に相当する
|
|
441
492
|
# レイヤーオフロードの概念がなく、mlx_lm.server は起動時の
|
|
@@ -552,20 +603,23 @@ def _build_cmd(
|
|
|
552
603
|
model-override guard, and never apply to vllm/mlx (device 非対応)。
|
|
553
604
|
"""
|
|
554
605
|
exe = _resolve_binary(backend, binary)
|
|
606
|
+
# argv の形はバリアントによらず基底バックエンドで決まる
|
|
607
|
+
# (llama.cpp-cuda も llama.cpp と同じフラグ体系)。
|
|
608
|
+
base = base_backend(backend)
|
|
555
609
|
|
|
556
|
-
if
|
|
610
|
+
if base == "llama.cpp":
|
|
557
611
|
cmd: list[str] = [exe, "-m", model_path, "--port", str(port)]
|
|
558
612
|
if device_args:
|
|
559
613
|
cmd.extend(device_args)
|
|
560
614
|
if spec_tokens:
|
|
561
615
|
cmd.extend(spec_tokens)
|
|
562
|
-
elif
|
|
616
|
+
elif base == "vllm":
|
|
563
617
|
cmd = [
|
|
564
618
|
exe, "-m", "vllm.entrypoints.openai.api_server",
|
|
565
619
|
"--model", model_path,
|
|
566
620
|
"--port", str(port),
|
|
567
621
|
]
|
|
568
|
-
elif
|
|
622
|
+
elif base == "mlx":
|
|
569
623
|
cmd = [
|
|
570
624
|
exe, "-m", "mlx_lm.server",
|
|
571
625
|
"--model", model_path,
|
|
@@ -574,7 +628,8 @@ def _build_cmd(
|
|
|
574
628
|
else:
|
|
575
629
|
raise ValueError(
|
|
576
630
|
f"Unknown backend: {backend!r}. "
|
|
577
|
-
"Expected 'llama.cpp', 'vllm' or 'mlx'
|
|
631
|
+
"Expected 'llama.cpp', 'vllm' or 'mlx' "
|
|
632
|
+
"(optionally with a '-<variant>' suffix)."
|
|
578
633
|
)
|
|
579
634
|
|
|
580
635
|
# H8: reject model-path re-specification via options keys.
|
|
@@ -653,8 +708,14 @@ async def _backend_ready(backend: str, port: int, *, probe_timeout_s: float) ->
|
|
|
653
708
|
documented health endpoint) fall back to a bare TCP connect: it can't
|
|
654
709
|
distinguish "loaded" from "listening", but it is a strict improvement
|
|
655
710
|
over registering the provider before the port is even open.
|
|
711
|
+
|
|
712
|
+
``backend`` may be a variant name (``llama.cpp-cuda``): the check goes
|
|
713
|
+
through :func:`base_backend` so a variant keeps the ``/health`` probe.
|
|
714
|
+
Falling through to the bare TCP connect here would silently re-introduce
|
|
715
|
+
the bug readiness gating was added to fix (provider registered before the
|
|
716
|
+
model finished loading).
|
|
656
717
|
"""
|
|
657
|
-
if backend in ("llama.cpp", "vllm"):
|
|
718
|
+
if base_backend(backend) in ("llama.cpp", "vllm"):
|
|
658
719
|
try:
|
|
659
720
|
async with httpx.AsyncClient(timeout=probe_timeout_s) as client:
|
|
660
721
|
resp = await client.get(f"http://localhost:{port}/health")
|
|
@@ -1032,15 +1093,30 @@ async def api_models(request: Request) -> dict[str, Any]:
|
|
|
1032
1093
|
|
|
1033
1094
|
@router.get("/api/launcher/option-profiles")
|
|
1034
1095
|
async def api_option_profiles(request: Request) -> dict[str, Any]:
|
|
1035
|
-
"""Return option_profiles from providers.yaml launcher config.
|
|
1096
|
+
"""Return option_profiles from providers.yaml launcher config.
|
|
1097
|
+
|
|
1098
|
+
A backend-variant key inherits the base backend's presets: the response
|
|
1099
|
+
for ``llama.cpp-cuda`` is the ``llama.cpp`` list followed by the
|
|
1100
|
+
variant-specific list, with same-``name`` presets replaced in place
|
|
1101
|
+
(:func:`resolve_option_profiles`). Every declared backend gets an entry
|
|
1102
|
+
so the UI can key on the selected backend name directly, and a config
|
|
1103
|
+
without variants yields exactly the pre-feature payload.
|
|
1104
|
+
"""
|
|
1036
1105
|
cfg = request.app.state.config
|
|
1037
1106
|
launcher_cfg = getattr(cfg, "launcher", None)
|
|
1038
1107
|
if not launcher_cfg:
|
|
1039
1108
|
return {"profiles": {}, "_note": "launcher: block not found in providers.yaml"}
|
|
1040
1109
|
if not launcher_cfg.option_profiles:
|
|
1041
1110
|
return {"profiles": {}, "_note": "option_profiles is empty — add option_profiles: under launcher: in providers.yaml"}
|
|
1111
|
+
names = list(launcher_cfg.option_profiles)
|
|
1112
|
+
for name in _backend_names(launcher_cfg.backends):
|
|
1113
|
+
if name not in names:
|
|
1114
|
+
names.append(name)
|
|
1042
1115
|
result: dict[str, list[dict]] = {}
|
|
1043
|
-
for backend
|
|
1116
|
+
for backend in names:
|
|
1117
|
+
profiles = resolve_option_profiles(launcher_cfg.option_profiles, backend)
|
|
1118
|
+
if not profiles and backend not in launcher_cfg.option_profiles:
|
|
1119
|
+
continue # 継承もローカル定義も無いバックエンドはキーを作らない
|
|
1044
1120
|
result[backend] = [{"name": p.name, "args": p.args} for p in profiles]
|
|
1045
1121
|
return {"profiles": result}
|
|
1046
1122
|
|
|
@@ -1323,10 +1399,12 @@ async def api_start(req: StartRequest, request: Request) -> dict[str, Any]:
|
|
|
1323
1399
|
_require_launcher_token(request)
|
|
1324
1400
|
cfg = request.app.state.config
|
|
1325
1401
|
launcher_cfg = getattr(cfg, "launcher", None)
|
|
1402
|
+
_assert_backend_declared(launcher_cfg, req.backend)
|
|
1326
1403
|
# デバイス選択 → CLI 断片(llama.cpp のみ・選択された場合のみ)。未指定なら
|
|
1327
1404
|
# None を渡し、既存の argv と完全一致(後方互換)。
|
|
1328
1405
|
device_args: list[str] | None = None
|
|
1329
|
-
if req.backend == "llama.cpp" and req.device_ids:
|
|
1406
|
+
if base_backend(req.backend) == "llama.cpp" and req.device_ids:
|
|
1407
|
+
await _assert_device_ids_known(launcher_cfg, req.backend, req.device_ids)
|
|
1330
1408
|
device_args = DeviceSelection(
|
|
1331
1409
|
device_ids=list(req.device_ids), tensor_split=list(req.tensor_split)
|
|
1332
1410
|
).to_cli_args()
|
|
@@ -1436,7 +1514,11 @@ async def api_suggest(request: Request, model_path: str = "",
|
|
|
1436
1514
|
|
|
1437
1515
|
|
|
1438
1516
|
def _configured_binary_for(launcher_cfg: Any, backend: str) -> str | None:
|
|
1439
|
-
"""launcher.backends[backend].binary を取り出す(未設定なら None)。
|
|
1517
|
+
"""launcher.backends[backend].binary を取り出す(未設定なら None)。
|
|
1518
|
+
|
|
1519
|
+
``backend`` はバリアント名 (``llama.cpp-cuda``) でもよい —— ``backends`` は
|
|
1520
|
+
バリアント名もキーに取るので、そのまま引ける。
|
|
1521
|
+
"""
|
|
1440
1522
|
if launcher_cfg and getattr(launcher_cfg, "backends", None):
|
|
1441
1523
|
bc = launcher_cfg.backends.get(backend)
|
|
1442
1524
|
if bc and bc.binary:
|
|
@@ -1444,6 +1526,61 @@ def _configured_binary_for(launcher_cfg: Any, backend: str) -> str | None:
|
|
|
1444
1526
|
return None
|
|
1445
1527
|
|
|
1446
1528
|
|
|
1529
|
+
def _assert_backend_declared(launcher_cfg: Any, backend: str) -> None:
|
|
1530
|
+
"""バリアント名は ``launcher.backends`` に宣言済みでなければ 400。
|
|
1531
|
+
|
|
1532
|
+
設計 §8。バリアントの実行ファイルパスは ``launcher.backends`` にしか無い
|
|
1533
|
+
ので、宣言されていない名前を受け取ったら**フォールバックせずに拒否**する。
|
|
1534
|
+
基底名 (``llama.cpp`` 等) は PATH 解決で動くため従来どおり宣言不要。
|
|
1535
|
+
|
|
1536
|
+
これは同時に「API はバックエンド名しか受け取らず、実行ファイルパスは常に
|
|
1537
|
+
オペレータの静的設定から来る」という不変則の実装でもある (§11)。
|
|
1538
|
+
"""
|
|
1539
|
+
if not variant_of(backend):
|
|
1540
|
+
return
|
|
1541
|
+
declared = getattr(launcher_cfg, "backends", None) or {}
|
|
1542
|
+
if backend not in declared:
|
|
1543
|
+
raise HTTPException(
|
|
1544
|
+
status_code=400,
|
|
1545
|
+
detail=(
|
|
1546
|
+
f"Unknown backend variant {backend!r}: not declared in "
|
|
1547
|
+
"launcher.backends. Add it (with a 'binary' path) to "
|
|
1548
|
+
"providers.yaml first."
|
|
1549
|
+
),
|
|
1550
|
+
)
|
|
1551
|
+
|
|
1552
|
+
|
|
1553
|
+
async def _assert_device_ids_known(
|
|
1554
|
+
launcher_cfg: Any, backend: str, device_ids: list[str]
|
|
1555
|
+
) -> None:
|
|
1556
|
+
"""選択デバイス ID がそのビルドに実在しなければ 400 (設計 §7.2)。
|
|
1557
|
+
|
|
1558
|
+
デバイス ID の名前空間はビルドごとに違う (``CUDA0`` と ``Vulkan0`` は同じ
|
|
1559
|
+
GPU を指さない)。CUDA ビルドで ``CUDA0`` を選んだまま Vulkan ビルドで起動
|
|
1560
|
+
すると ``--device CUDA0`` が渡って llama-server が起動失敗するので、spawn
|
|
1561
|
+
前に弾く。
|
|
1562
|
+
|
|
1563
|
+
``--list-devices`` 自体が失敗した環境では検証をスキップする
|
|
1564
|
+
(:func:`unknown_device_ids` の best-effort 契約)。
|
|
1565
|
+
"""
|
|
1566
|
+
if not device_ids:
|
|
1567
|
+
return
|
|
1568
|
+
configured = _configured_binary_for(launcher_cfg, backend)
|
|
1569
|
+
binary = _resolve_binary(backend, configured)
|
|
1570
|
+
probe = await asyncio.to_thread(detect_llama_devices, binary)
|
|
1571
|
+
unknown = unknown_device_ids(list(device_ids), probe)
|
|
1572
|
+
if unknown:
|
|
1573
|
+
known = [d.id for d in probe.devices]
|
|
1574
|
+
raise HTTPException(
|
|
1575
|
+
status_code=400,
|
|
1576
|
+
detail=(
|
|
1577
|
+
f"Device id(s) {unknown} do not exist in backend {backend!r} "
|
|
1578
|
+
f"(detected: {known}). Device ids are build-specific — "
|
|
1579
|
+
"re-select devices after switching build."
|
|
1580
|
+
),
|
|
1581
|
+
)
|
|
1582
|
+
|
|
1583
|
+
|
|
1447
1584
|
@router.get("/api/launcher/devices")
|
|
1448
1585
|
async def api_devices(
|
|
1449
1586
|
request: Request, backend: str = "llama.cpp", refresh: int = 0
|
|
@@ -1504,6 +1641,9 @@ class SweepConfigItem(BaseModel):
|
|
|
1504
1641
|
label: str
|
|
1505
1642
|
device_ids: list[str] = Field(default_factory=list)
|
|
1506
1643
|
tensor_split: list[float] = Field(default_factory=list)
|
|
1644
|
+
# バリアント横断スイープ: このステップだけ別ビルドで起動する。None なら
|
|
1645
|
+
# ``SweepRequest.backend`` を使う=従来と完全に同一の argv。
|
|
1646
|
+
backend: str | None = None
|
|
1507
1647
|
|
|
1508
1648
|
|
|
1509
1649
|
class SweepRequest(BaseModel):
|
|
@@ -1621,12 +1761,15 @@ class _SweepRunner:
|
|
|
1621
1761
|
step.state = SweepState.STARTING
|
|
1622
1762
|
self.log_tail.append(f"[sweep] {step.label}: starting")
|
|
1623
1763
|
device_args = step.selection.to_cli_args() or None
|
|
1764
|
+
# ステップ個別のバックエンド(バリアント横断スイープ)。未指定なら
|
|
1765
|
+
# プラン既定 = 従来の挙動。
|
|
1766
|
+
step_backend = step.backend or self.plan.backend
|
|
1624
1767
|
try:
|
|
1625
1768
|
mp = await spawn_process(
|
|
1626
1769
|
self.app,
|
|
1627
1770
|
self.launcher_cfg,
|
|
1628
1771
|
name=f"sweep-{step.label}",
|
|
1629
|
-
backend=
|
|
1772
|
+
backend=step_backend,
|
|
1630
1773
|
model_path=self.plan.model_path,
|
|
1631
1774
|
port=self.plan.port,
|
|
1632
1775
|
options=self.plan.options,
|
|
@@ -1768,6 +1911,13 @@ async def api_sweep_start(req: SweepRequest, request: Request) -> dict[str, Any]
|
|
|
1768
1911
|
status_code=400, detail=f"Port {req.port} is not free."
|
|
1769
1912
|
)
|
|
1770
1913
|
|
|
1914
|
+
# バリアント横断スイープ: 各構成が別ビルドを指せる。宣言されていない
|
|
1915
|
+
# バリアント名は通常起動と同じく 400 で弾く(フォールバックしない)。
|
|
1916
|
+
_assert_backend_declared(launcher_cfg, req.backend)
|
|
1917
|
+
for item in req.configs:
|
|
1918
|
+
if item.backend:
|
|
1919
|
+
_assert_backend_declared(launcher_cfg, item.backend)
|
|
1920
|
+
|
|
1771
1921
|
labeled = [
|
|
1772
1922
|
(
|
|
1773
1923
|
item.label,
|
|
@@ -1775,6 +1925,7 @@ async def api_sweep_start(req: SweepRequest, request: Request) -> dict[str, Any]
|
|
|
1775
1925
|
device_ids=list(item.device_ids),
|
|
1776
1926
|
tensor_split=list(item.tensor_split),
|
|
1777
1927
|
),
|
|
1928
|
+
item.backend,
|
|
1778
1929
|
)
|
|
1779
1930
|
for item in req.configs
|
|
1780
1931
|
]
|
|
@@ -1927,6 +2078,10 @@ _LAUNCHER_HTML = r"""<!doctype html>
|
|
|
1927
2078
|
|
|
1928
2079
|
<div>
|
|
1929
2080
|
<label class="block text-xs text-slate-400 mb-1">バックエンド</label>
|
|
2081
|
+
<!-- 選択肢は fetchBackends() が /api/launcher/backends の応答から
|
|
2082
|
+
動的生成する。providers.yaml に llama.cpp-cuda のようなバリアント
|
|
2083
|
+
を書くとここに増える。書かなければ基底 3 つのまま(従来と同一)。
|
|
2084
|
+
初期値は JS が届く前でも表示が崩れないための土台。 -->
|
|
1930
2085
|
<select id="f-backend" onchange="onBackendChange()">
|
|
1931
2086
|
<option value="llama.cpp">llama.cpp</option>
|
|
1932
2087
|
<option value="vllm">vllm</option>
|
|
@@ -2031,7 +2186,13 @@ _LAUNCHER_HTML = r"""<!doctype html>
|
|
|
2031
2186
|
</p>
|
|
2032
2187
|
<div class="grid grid-cols-1 md:grid-cols-2 gap-3">
|
|
2033
2188
|
<div>
|
|
2034
|
-
<
|
|
2189
|
+
<div class="flex items-center justify-between mb-1">
|
|
2190
|
+
<label class="block text-xs text-slate-400">構成候補</label>
|
|
2191
|
+
<!-- 宣言済みの llama.cpp バリアントを全部プローブして横断構成を作る。
|
|
2192
|
+
「CUDA ビルドと Vulkan ビルドどちらが速いか」を 1 回で回すための入口。 -->
|
|
2193
|
+
<button id="btn-cross-variant" onclick="buildCrossVariantConfigs()"
|
|
2194
|
+
class="btn-sm btn-slate hidden">⚙ ビルド横断</button>
|
|
2195
|
+
</div>
|
|
2035
2196
|
<div id="sweep-configs" class="text-xs text-slate-400 space-y-1 bg-slate-800/40 rounded p-2 max-h-40 overflow-y-auto">
|
|
2036
2197
|
<span class="text-slate-600">先に「🔍 検出」でデバイスを取得してください</span>
|
|
2037
2198
|
</div>
|
|
@@ -2254,12 +2415,30 @@ _LAUNCHER_HTML = r"""<!doctype html>
|
|
|
2254
2415
|
if (!r.ok) throw new Error(`HTTP ${r.status}`);
|
|
2255
2416
|
const d = await r.json();
|
|
2256
2417
|
allBackends = d.backends || {};
|
|
2418
|
+
populateBackendSelect();
|
|
2419
|
+
updateCrossVariantButton();
|
|
2257
2420
|
} catch (e) {
|
|
2258
2421
|
console.error("[Launcher] fetchBackends failed:", e);
|
|
2259
2422
|
}
|
|
2260
2423
|
renderBinaryHint(); // always call outside try-catch so errors surface
|
|
2261
2424
|
};
|
|
2262
2425
|
|
|
2426
|
+
// バックエンドセレクトをサーバ応答から作り直す。バリアント
|
|
2427
|
+
// (llama.cpp-cuda 等) には ⚙ を付けて「対応ランタイムを自分で入れる
|
|
2428
|
+
// 上級者向け」であることを示す。バリアントが 0 個なら従来と同じ 3 択。
|
|
2429
|
+
const populateBackendSelect = () => {
|
|
2430
|
+
const sel = document.getElementById("f-backend");
|
|
2431
|
+
const names = Object.keys(allBackends);
|
|
2432
|
+
if (!names.length) return;
|
|
2433
|
+
const prev = sel.value;
|
|
2434
|
+
sel.innerHTML = names.map((n) => {
|
|
2435
|
+
const info = allBackends[n] || {};
|
|
2436
|
+
const label = info.variant ? `${n} ⚙` : n;
|
|
2437
|
+
return `<option value="${esc(n)}">${esc(label)}</option>`;
|
|
2438
|
+
}).join("");
|
|
2439
|
+
sel.value = names.includes(prev) ? prev : names[0];
|
|
2440
|
+
};
|
|
2441
|
+
|
|
2263
2442
|
const renderBinaryHint = () => {
|
|
2264
2443
|
const backend = document.getElementById("f-backend").value;
|
|
2265
2444
|
const hint = document.getElementById("binary-hint");
|
|
@@ -2276,10 +2455,20 @@ _LAUNCHER_HTML = r"""<!doctype html>
|
|
|
2276
2455
|
const pathColor = info.found
|
|
2277
2456
|
? (info.is_custom ? "#818cf8" : "#4ade80") // indigo-400 / green-400
|
|
2278
2457
|
: "#f87171"; // red-400
|
|
2279
|
-
|
|
2458
|
+
// バリアント (特化ビルド) は対応 GPU ランタイムをユーザー自身が入れて
|
|
2459
|
+
// いる前提なので、選択中はその旨を一行添える。基底名では出さない
|
|
2460
|
+
// (通常利用者の画面を汚さない)。
|
|
2461
|
+
const runtimeNote = info.variant
|
|
2462
|
+
? `<div style="color:#fbbf24;margin-top:2px">⚙ ${esc(info.variant)} ビルド`
|
|
2463
|
+
+ ` — 対応ランタイム(CUDA / Vulkan / ROCm)が導入済みの環境でのみ動作します</div>`
|
|
2464
|
+
: "";
|
|
2465
|
+
hint.innerHTML = "<div style=\"display:flex;align-items:center;overflow:hidden\">"
|
|
2466
|
+
+ dot
|
|
2280
2467
|
+ `<span style="font-family:monospace;color:${pathColor};overflow:hidden;text-overflow:ellipsis;white-space:nowrap">${esc(info.resolved)}</span>`
|
|
2281
|
-
+ `<span style="color:#64748b;margin-left:6px;white-space:nowrap;flex-shrink:0">(${label} — ${statusText})</span
|
|
2282
|
-
|
|
2468
|
+
+ `<span style="color:#64748b;margin-left:6px;white-space:nowrap;flex-shrink:0">(${label} — ${statusText})</span>`
|
|
2469
|
+
+ "</div>"
|
|
2470
|
+
+ runtimeNote;
|
|
2471
|
+
hint.style.cssText = "overflow:hidden";
|
|
2283
2472
|
// Enable/disable launch button based on binary availability
|
|
2284
2473
|
if (!info.found) {
|
|
2285
2474
|
btn.disabled = true;
|
|
@@ -2320,6 +2509,20 @@ _LAUNCHER_HTML = r"""<!doctype html>
|
|
|
2320
2509
|
};
|
|
2321
2510
|
|
|
2322
2511
|
window.onBackendChange = () => {
|
|
2512
|
+
// ★ デバイス id の名前空間はビルドごとに違う ("CUDA0" と "Vulkan0" は
|
|
2513
|
+
// 同じ GPU ではない)。バックエンド/バリアントを切り替えたら選択を破棄
|
|
2514
|
+
// して検出やり直しにする。残したままだと --device CUDA0 が Vulkan
|
|
2515
|
+
// ビルドに渡って起動失敗する (サーバ側でも 400 で弾くが、UI 側で
|
|
2516
|
+
// 先に消しておくのが本筋)。
|
|
2517
|
+
_devices = [];
|
|
2518
|
+
_deviceSel = {};
|
|
2519
|
+
_deviceOk = false;
|
|
2520
|
+
_autoConfigs = [];
|
|
2521
|
+
const box = document.getElementById("device-list");
|
|
2522
|
+
if (box) {
|
|
2523
|
+
box.innerHTML =
|
|
2524
|
+
'<span class="text-slate-600">「検出」を押してデバイスを取得</span>';
|
|
2525
|
+
}
|
|
2323
2526
|
populateProfileSelect();
|
|
2324
2527
|
renderBinaryHint();
|
|
2325
2528
|
updateDeviceBlock();
|
|
@@ -2360,7 +2563,17 @@ _LAUNCHER_HTML = r"""<!doctype html>
|
|
|
2360
2563
|
let _deviceOk = false;
|
|
2361
2564
|
let _autoConfigs = []; // サーバ生成のスイープ構成候補
|
|
2362
2565
|
|
|
2363
|
-
|
|
2566
|
+
// バックエンド名は "llama.cpp" だけでなく "llama.cpp-cuda" のようなバリアント
|
|
2567
|
+
// 名も来る (docs/designs/launcher-multi-build.md)。素の === 比較にすると
|
|
2568
|
+
// バリアント選択時にデバイス欄が出なくなるので基底名で判定する。
|
|
2569
|
+
const baseBackend = (name) => {
|
|
2570
|
+
for (const b of ["llama.cpp", "vllm", "mlx"]) {
|
|
2571
|
+
if (name === b || name.startsWith(b + "-")) return b;
|
|
2572
|
+
}
|
|
2573
|
+
return name;
|
|
2574
|
+
};
|
|
2575
|
+
const isLlama = () =>
|
|
2576
|
+
baseBackend(document.getElementById("f-backend").value) === "llama.cpp";
|
|
2364
2577
|
|
|
2365
2578
|
// デバイス id → バックエンド接頭辞(末尾数字を除く)。launcher_devices の
|
|
2366
2579
|
// backend_of と同じ規則。"CUDA0"→"CUDA" / "Vulkan2"→"Vulkan" / "MTL0"→"MTL"。
|
|
@@ -2488,15 +2701,58 @@ _LAUNCHER_HTML = r"""<!doctype html>
|
|
|
2488
2701
|
// 0 MiB 除外・バックエンド単位のグループ化・跨バックエンド混成の除外は
|
|
2489
2702
|
// すべてサーバ側ロジックに一本化されている。
|
|
2490
2703
|
const buildSweepConfigs = () => {
|
|
2704
|
+
const backend = document.getElementById("f-backend").value;
|
|
2491
2705
|
_sweepConfigs = (_autoConfigs || []).map(c => ({
|
|
2492
2706
|
label: c.label,
|
|
2493
2707
|
device_ids: c.device_ids || [],
|
|
2494
2708
|
tensor_split: c.tensor_split || [],
|
|
2709
|
+
backend: backend,
|
|
2495
2710
|
checked: true,
|
|
2496
2711
|
}));
|
|
2497
2712
|
renderSweepConfigs();
|
|
2498
2713
|
};
|
|
2499
2714
|
|
|
2715
|
+
// 宣言済みの llama.cpp バリアントを全部プローブし、ビルド横断の構成を作る。
|
|
2716
|
+
// ラベルにビルド名を前置するので {config} 経由でベンチ結果も見分けられる。
|
|
2717
|
+
// ビルド間の混成構成は作らない(1 プロセスは 1 実行ファイルなので原理的に不可)。
|
|
2718
|
+
window.buildCrossVariantConfigs = async () => {
|
|
2719
|
+
const names = Object.keys(allBackends).filter(n => baseBackend(n) === "llama.cpp");
|
|
2720
|
+
if (names.length < 2) {
|
|
2721
|
+
showSweepErr("横断できるビルドが 1 つしかありません。providers.yaml の "
|
|
2722
|
+
+ "launcher.backends に llama.cpp-cuda などを追加してください");
|
|
2723
|
+
return;
|
|
2724
|
+
}
|
|
2725
|
+
showSweepErr("");
|
|
2726
|
+
const box = document.getElementById("sweep-configs");
|
|
2727
|
+
box.innerHTML = '<span class="text-slate-600">全ビルドを検出中…</span>';
|
|
2728
|
+
const collected = [];
|
|
2729
|
+
for (const name of names) {
|
|
2730
|
+
try {
|
|
2731
|
+
const r = await fetch("/api/launcher/devices?backend=" + encodeURIComponent(name));
|
|
2732
|
+
const d = await r.json();
|
|
2733
|
+
if (!r.ok || !d.ok) continue; // 検出できないビルドは黙って飛ばす
|
|
2734
|
+
const prefix = (allBackends[name] || {}).variant || name;
|
|
2735
|
+
for (const c of (d.auto_configs || [])) {
|
|
2736
|
+
collected.push({
|
|
2737
|
+
label: prefix + " / " + c.label,
|
|
2738
|
+
device_ids: c.device_ids || [],
|
|
2739
|
+
tensor_split: c.tensor_split || [],
|
|
2740
|
+
backend: name,
|
|
2741
|
+
checked: true,
|
|
2742
|
+
});
|
|
2743
|
+
}
|
|
2744
|
+
} catch (_) {}
|
|
2745
|
+
}
|
|
2746
|
+
if (!collected.length) {
|
|
2747
|
+
showSweepErr("どのビルドでもデバイスを検出できませんでした");
|
|
2748
|
+
renderSweepConfigs();
|
|
2749
|
+
return;
|
|
2750
|
+
}
|
|
2751
|
+
_sweepConfigs = collected;
|
|
2752
|
+
renderSweepConfigs();
|
|
2753
|
+
statusMsg(`ビルド横断構成 ${collected.length} 件を生成 (${names.length} ビルド)`);
|
|
2754
|
+
};
|
|
2755
|
+
|
|
2500
2756
|
const renderSweepConfigs = () => {
|
|
2501
2757
|
const box = document.getElementById("sweep-configs");
|
|
2502
2758
|
if (!_sweepConfigs.length) {
|
|
@@ -2512,6 +2768,15 @@ _LAUNCHER_HTML = r"""<!doctype html>
|
|
|
2512
2768
|
).join("");
|
|
2513
2769
|
};
|
|
2514
2770
|
|
|
2771
|
+
// 「⚙ ビルド横断」ボタンは llama.cpp のビルドが 2 つ以上宣言されている
|
|
2772
|
+
// ときだけ出す(バリアントを書いていない利用者の画面は従来どおり)。
|
|
2773
|
+
const updateCrossVariantButton = () => {
|
|
2774
|
+
const btn = document.getElementById("btn-cross-variant");
|
|
2775
|
+
if (!btn) return;
|
|
2776
|
+
const n = Object.keys(allBackends).filter(x => baseBackend(x) === "llama.cpp").length;
|
|
2777
|
+
btn.classList.toggle("hidden", n < 2);
|
|
2778
|
+
};
|
|
2779
|
+
|
|
2515
2780
|
window.onSweepCfgToggle = () => {
|
|
2516
2781
|
document.querySelectorAll('#sweep-configs input[data-cfg]').forEach(cb => {
|
|
2517
2782
|
_sweepConfigs[parseInt(cb.getAttribute("data-cfg"))].checked = cb.checked;
|
|
@@ -2529,7 +2794,8 @@ _LAUNCHER_HTML = r"""<!doctype html>
|
|
|
2529
2794
|
const model = document.getElementById("f-model").value.trim();
|
|
2530
2795
|
if (!model) { showSweepErr("起動フォームでモデルパスを選択してください"); return; }
|
|
2531
2796
|
const configs = _sweepConfigs.filter(c => c.checked)
|
|
2532
|
-
.map(c => ({label: c.label, device_ids: c.device_ids,
|
|
2797
|
+
.map(c => ({label: c.label, device_ids: c.device_ids,
|
|
2798
|
+
tensor_split: c.tensor_split, backend: c.backend || null}));
|
|
2533
2799
|
if (!configs.length) { showSweepErr("構成を 1 つ以上選択してください"); return; }
|
|
2534
2800
|
const port = parseInt(document.getElementById("sweep-port").value);
|
|
2535
2801
|
const cmd = document.getElementById("sweep-cmd").value.trim();
|
|
@@ -2597,7 +2863,8 @@ _LAUNCHER_HTML = r"""<!doctype html>
|
|
|
2597
2863
|
const tok = sm.tokens_per_sec != null ? sm.tokens_per_sec : "—";
|
|
2598
2864
|
const ttft = sm.ttft_ms != null ? sm.ttft_ms : "—";
|
|
2599
2865
|
const dev = (s.device_ids || []).join(",")
|
|
2600
|
-
+ ((s.tensor_split || []).length ? " · " + s.tensor_split.join(",") : "")
|
|
2866
|
+
+ ((s.tensor_split || []).length ? " · " + s.tensor_split.join(",") : "")
|
|
2867
|
+
+ (s.backend ? " [" + s.backend + "]" : "");
|
|
2601
2868
|
const exit = s.bench_exit_code != null ? s.bench_exit_code : "—";
|
|
2602
2869
|
return `<tr>
|
|
2603
2870
|
<td class="py-2 pr-3">${esc(s.label)}</td>
|
|
@@ -2628,7 +2895,7 @@ _LAUNCHER_HTML = r"""<!doctype html>
|
|
|
2628
2895
|
|
|
2629
2896
|
// デバイス選択 (llama.cpp のみ)。未選択なら空 → サーバ側で --device 不付与。
|
|
2630
2897
|
let deviceIds = [], tensorSplit = [];
|
|
2631
|
-
if (backend === "llama.cpp") {
|
|
2898
|
+
if (baseBackend(backend) === "llama.cpp") {
|
|
2632
2899
|
deviceIds = selectedDeviceIds();
|
|
2633
2900
|
const ts = document.getElementById("f-tsplit").value.trim();
|
|
2634
2901
|
if (ts && deviceIds.length > 1) {
|