coderouter-cli 2.7.5__py3-none-any.whl → 2.7.6__py3-none-any.whl

coderouter/gguf_introspect.py

@@ -136,6 +136,17 @@ class GGUFInfo:
     n_kv_heads: int | None
     file_type: int | None
     file_size_bytes: int
+    n_nextn: int | None = None
+
+    @property
+    def supports_mtp(self) -> bool:
+        """True when the GGUF embeds Multi-Token-Prediction (nextn) layers.
+
+        Derived from ``{arch}.nextn_predict_layers`` — a positive count means
+        the main model carries MTP/nextn tensors and can drive llama.cpp's
+        ``--spec-type draft-mtp`` without a separate draft gguf.
+        """
+        return bool(self.n_nextn and self.n_nextn > 0)
 
     @property
     def quant_name(self) -> str | None:
@@ -224,6 +235,10 @@ _KEY_BLOCK_COUNT = ".block_count"
 _KEY_EMBED_LEN = ".embedding_length"
 _KEY_HEAD_COUNT = ".attention.head_count"
 _KEY_HEAD_COUNT_KV = ".attention.head_count_kv"
+# Number of Multi-Token-Prediction (nextn) layers embedded in the main model
+# (e.g. ``glm4moe.nextn_predict_layers``). A positive value means the GGUF can
+# drive llama.cpp speculative decoding via ``--spec-type draft-mtp`` alone.
+_KEY_NEXTN = ".nextn_predict_layers"
 
 
 def read_gguf_metadata(path: str | Path) -> GGUFInfo:
@@ -245,6 +260,7 @@ def read_gguf_metadata(path: str | Path) -> GGUFInfo:
     n_heads: int | None = None
     n_kv_heads: int | None = None
     file_type: int | None = None
+    n_nextn: int | None = None
 
     with p.open("rb") as fh:
         magic = fh.read(4)
@@ -275,6 +291,8 @@ def read_gguf_metadata(path: str | Path) -> GGUFInfo:
                 n_kv_heads = value
             elif key.endswith(_KEY_HEAD_COUNT) and isinstance(value, int):
                 n_heads = value
+            elif key.endswith(_KEY_NEXTN) and isinstance(value, int):
+                n_nextn = value
 
     return GGUFInfo(
         architecture=arch,
@@ -284,6 +302,7 @@ def read_gguf_metadata(path: str | Path) -> GGUFInfo:
         n_kv_heads=n_kv_heads,
         file_type=file_type,
         file_size_bytes=file_size,
+        n_nextn=n_nextn,
     )
 
 

coderouter/ingress/launcher_routes.py

@@ -30,6 +30,7 @@ import secrets
 import shlex
 import shutil
 import subprocess
+import time
 import uuid
 from collections import deque
 from dataclasses import dataclass, field
@@ -40,6 +41,7 @@ from fastapi import APIRouter, HTTPException, Request
 from fastapi.responses import HTMLResponse
 from pydantic import BaseModel
 
+from coderouter.launcher_speculative import resolve_speculative
 from coderouter.logging import get_logger
 
 logger = get_logger(__name__)
@@ -115,9 +117,23 @@ class ManagedProcess:
     options: dict[str, Any]
     extra_args: str
     status: str = "starting"   # "starting" | "running" | "stopped" | "error"
+    # MTP / speculative-decoding controls (defaults keep existing call sites
+    # working). Recorded for introspection; the resolved flags live in the cmd.
+    draft_model_path: str | None = None
+    mtp_mode: str = "auto"
     pid: int | None = None
     returncode: int | None = None
     log_tail: deque = field(default_factory=lambda: deque(maxlen=200))
+    # MTP auto-fallback state. When the speculative flags were added by AUTO
+    # detection and the child dies during startup, the process is relaunched
+    # ONCE without them (some archs' draft-mtp support in llama.cpp is
+    # immature and crashes the context init). These fields make that possible
+    # and impossible to loop.
+    spec_tokens: list = field(default_factory=list)
+    spec_auto: bool = False
+    mtp_fallback_done: bool = False
+    fallback_cmd: list | None = None
+    started_at: float = 0.0
     # asyncio subprocess handle — not serialised
     _proc: Any = field(default=None, repr=False, compare=False)
 
@@ -208,8 +224,15 @@ _BACKEND_DEFAULTS: dict[str, str] = {
 #   - llama.cpp: ``-m`` / ``--model`` select the GGUF file.
 #   - vllm / mlx: ``--model`` selects the model; ``-m`` selects the python
 #     module that ``_build_cmd`` pins, so it must not be re-specified either.
+#   - llama.cpp: the draft/MTP model is likewise set only via the dedicated
+#     ``draft_model_path`` field, so its aliases (``-md`` / ``--model-draft``
+#     / ``--spec-draft-model``) are rejected in options / extra_args too. The
+#     remaining spec knobs (``--spec-type``, ``--spec-draft-n-max`` …) stay
+#     free-form.
 _MODEL_FLAGS: dict[str, frozenset[str]] = {
-    "llama.cpp": frozenset({"-m", "--model"}),
+    "llama.cpp": frozenset(
+        {"-m", "--model", "-md", "--model-draft", "--spec-draft-model"}
+    ),
     "vllm": frozenset({"-m", "--model"}),
     "mlx": frozenset({"-m", "--model"}),
 }
@@ -407,6 +430,24 @@ def _resolve_within_model_dirs(model_path: str, model_dirs: list[str]) -> Path:
     raise ValueError("model_path is not under any configured model_dirs.")
 
 
+def _option_tokens(options: dict[str, Any]) -> list[str]:
+    """Flatten an ``options`` dict into CLI tokens (``{flag: val}`` semantics).
+
+    Boolean values become a bare flag when true and vanish when false;
+    everything else becomes ``[flag, str(val)]``. Shared by ``_build_cmd`` and
+    ``api_start`` so the tokens fed to ``resolve_speculative`` match exactly
+    what lands on the command line (no double-parsing drift).
+    """
+    tokens: list[str] = []
+    for flag, val in options.items():
+        if isinstance(val, bool):
+            if val:
+                tokens.append(flag)
+        else:
+            tokens.extend([flag, str(val)])
+    return tokens
+
+
 def _build_cmd(
     backend: str,
     model_path: str,
@@ -414,17 +455,27 @@ def _build_cmd(
     options: dict[str, Any],
     extra_args: str,
     binary: str | None = None,
+    spec_tokens: list[str] | None = None,
 ) -> list[str]:
     """Build the CLI command list for the given backend and options.
 
     ``binary`` overrides the default executable (``llama-server`` /
     ``python``).  When None, the default is used and PATH resolution
     is left to the OS.
+
+    ``spec_tokens`` are pre-resolved speculative-decoding / MTP flags
+    (from :func:`coderouter.launcher_speculative.resolve_speculative`).
+    For llama.cpp they are appended right after the port args and BEFORE
+    the profile / extra args. They are trusted (the launcher, not the
+    caller, produced them) so they bypass the model-override guard even
+    though they may contain ``--model-draft``.
     """
     exe = _resolve_binary(backend, binary)
 
     if backend == "llama.cpp":
         cmd: list[str] = [exe, "-m", model_path, "--port", str(port)]
+        if spec_tokens:
+            cmd.extend(spec_tokens)
     elif backend == "vllm":
         cmd = [
             exe, "-m", "vllm.entrypoints.openai.api_server",
@@ -446,12 +497,7 @@ def _build_cmd(
     # H8: reject model-path re-specification via options keys.
     _assert_no_model_override(backend, list(options.keys()))
 
-    for flag, val in options.items():
-        if isinstance(val, bool):
-            if val:
-                cmd.append(flag)
-        else:
-            cmd.extend([flag, str(val)])
+    cmd.extend(_option_tokens(options))
 
     if extra_args.strip():
         extra_tokens = shlex.split(extra_args)
@@ -474,12 +520,37 @@ def _build_cmd(
 # recover by reading the buffered chunk and continuing so log tailing survives.
 _LOG_STREAM_LIMIT = 256 * 1024  # 256 KB
 
+# Window (seconds) after spawn in which a non-zero exit is treated as a
+# startup crash eligible for the MTP auto-fallback. A backend that ran fine
+# for longer and then died is a normal crash, not a load-time failure, so it
+# is never retried.
+_MTP_FALLBACK_WINDOW_SECS = 180.0
 
-async def _tail_logs(proc: ManagedProcess) -> None:
-    """Read stdout+stderr into proc.log_tail until the process exits."""
+
+def _should_mtp_fallback(proc: ManagedProcess) -> bool:
+    """True when a just-crashed process qualifies for the one-shot MTP retry.
+
+    Only auto-detected speculative launches that die within the startup
+    window are eligible, and only once (guarded by ``mtp_fallback_done``).
+    """
     p = proc._proc
-    if p is None:
-        return
+    rc = p.returncode if p is not None else proc.returncode
+    return (
+        proc.spec_auto
+        and not proc.mtp_fallback_done
+        and bool(proc.fallback_cmd)
+        and rc not in (0, None)
+        and (time.monotonic() - proc.started_at) <= _MTP_FALLBACK_WINDOW_SECS
+    )
+
+
+async def _tail_logs(proc: ManagedProcess) -> None:
+    """Read stdout+stderr into proc.log_tail until the process exits.
+
+    Loops so that a one-shot MTP fallback (relaunch without the auto-detected
+    speculative flags) can be tailed by the same task. The loop exits as soon
+    as no fallback applies.
+    """
 
     async def _drain(stream: asyncio.StreamReader | None) -> None:
         if stream is None:
@@ -505,14 +576,53 @@ async def _tail_logs(proc: ManagedProcess) -> None:
                 break
             proc.log_tail.append(line.decode(errors="replace").rstrip())
 
-    await asyncio.gather(_drain(p.stdout), _drain(p.stderr))
-    await p.wait()
-    proc.returncode = p.returncode
-    proc.pid = None
-    proc.status = "stopped" if (p.returncode or 0) == 0 else "error"
-    proc.log_tail.append(
-        f"[launcher] process exited with code {p.returncode}"
-    )
+    while True:
+        p = proc._proc
+        if p is None:
+            return
+
+        await asyncio.gather(_drain(p.stdout), _drain(p.stderr))
+        await p.wait()
+        proc.returncode = p.returncode
+        proc.pid = None
+        proc.status = "stopped" if (p.returncode or 0) == 0 else "error"
+        proc.log_tail.append(
+            f"[launcher] process exited with code {p.returncode}"
+        )
+
+        if not _should_mtp_fallback(proc):
+            return
+
+        # Auto-detected MTP crashed during startup — retry ONCE without the
+        # speculative flags. The port is unchanged, so the existing provider
+        # registration still stands (do not re-register).
+        rc = proc.returncode
+        proc.mtp_fallback_done = True
+        proc.log_tail.append(
+            f"[launcher] MTP startup failure detected (exit code {rc}); "
+            "retrying without speculative decoding"
+        )
+        proc.log_tail.append(
+            f"[launcher] cmd: {' '.join(proc.fallback_cmd)}"
+        )
+        try:
+            p = await asyncio.create_subprocess_exec(
+                *proc.fallback_cmd,
+                stdout=asyncio.subprocess.PIPE,
+                stderr=asyncio.subprocess.PIPE,
+                limit=_LOG_STREAM_LIMIT,
+            )
+        except (FileNotFoundError, OSError) as exc:
+            proc.log_tail.append(f"[launcher] fallback relaunch failed: {exc}")
+            proc.status = "error"
+            return
+        proc._proc = p
+        proc.pid = p.pid
+        proc.status = "running"
+        proc.returncode = None
+        proc.started_at = time.monotonic()
+        proc.log_tail.append(f"[launcher] started PID {p.pid}")
+        # Loop to tail the relaunched process.
 
 
 async def shutdown_launcher(app: Any) -> None:
@@ -555,6 +665,11 @@ class StartRequest(BaseModel):
     port: int
     options: dict[str, Any] = {}
     extra_args: str = ""
+    # MTP / speculative-decoding controls (llama.cpp only). ``draft_model_path``
+    # is an explicit companion draft/MTP gguf; ``mtp_mode`` is "auto" (detect)
+    # or "off" (never emit speculative flags).
+    draft_model_path: str | None = None
+    mtp_mode: str = "auto"
 
 
 # ---------------------------------------------------------------------------
@@ -690,15 +805,69 @@ async def api_start(req: StartRequest, request: Request) -> dict[str, Any]:
         if bc and bc.binary:
             configured_binary = bc.binary
 
+    # Validate the MTP mode up front (only "auto" / "off" are accepted).
+    if req.mtp_mode not in ("auto", "off"):
+        raise HTTPException(
+            status_code=400,
+            detail=f"mtp_mode must be 'auto' or 'off', got {req.mtp_mode!r}.",
+        )
+
+    # Build the user token list once (options flags + extra_args) so
+    # resolve_speculative sees exactly what _build_cmd will emit. shlex.split
+    # can raise on unbalanced quotes → surface as 400 like other bad input.
+    try:
+        user_tokens = _option_tokens(req.options)
+        if req.extra_args.strip():
+            user_tokens += shlex.split(req.extra_args)
+    except ValueError as exc:
+        raise HTTPException(status_code=400, detail=str(exc)) from exc
+
+    # Resolve MTP / speculative-decoding flags (llama.cpp only; no-op elsewhere).
+    try:
+        spec_tokens, spec_notes = resolve_speculative(
+            req.backend,
+            req.model_path,
+            req.draft_model_path,
+            req.mtp_mode,
+            user_tokens,
+        )
+    except ValueError as exc:
+        raise HTTPException(status_code=400, detail=str(exc)) from exc
+
     try:
         cmd = _build_cmd(
             req.backend, req.model_path, req.port,
             req.options, req.extra_args,
             binary=configured_binary,
+            spec_tokens=spec_tokens,
         )
     except ValueError as exc:
         raise HTTPException(status_code=400, detail=str(exc)) from exc
 
+    # Speculative flags qualify for the one-shot startup-crash fallback only
+    # when they came from AUTO detection (mtp_mode="auto", no explicit draft
+    # model, and detection actually emitted flags). Explicit draft paths and
+    # operator-supplied --spec-type are never auto-retried. The fallback cmd
+    # is rebuilt from scratch with spec_tokens=None (exact — never spliced).
+    spec_auto = (
+        req.mtp_mode == "auto"
+        and req.draft_model_path is None
+        and bool(spec_tokens)
+    )
+    fallback_cmd: list[str] | None = None
+    if spec_auto:
+        try:
+            fallback_cmd = _build_cmd(
+                req.backend, req.model_path, req.port,
+                req.options, req.extra_args,
+                binary=configured_binary,
+                spec_tokens=None,
+            )
+        except ValueError:
+            # A failure to rebuild the non-spec command simply disables the
+            # fallback; it must never block the primary start.
+            fallback_cmd = None
+
     proc_id = uuid.uuid4().hex[:8]
     proc = ManagedProcess(
         id=proc_id,
@@ -708,9 +877,16 @@ async def api_start(req: StartRequest, request: Request) -> dict[str, Any]:
         port=req.port,
         options=req.options,
         extra_args=req.extra_args,
+        draft_model_path=req.draft_model_path,
+        mtp_mode=req.mtp_mode,
         status="starting",
+        spec_tokens=spec_tokens,
+        spec_auto=spec_auto and fallback_cmd is not None,
+        fallback_cmd=fallback_cmd,
     )
     proc.log_tail.append(f"[launcher] cmd: {' '.join(cmd)}")
+    for note in spec_notes:
+        proc.log_tail.append(f"[launcher] {note}")
 
     try:
         p = await asyncio.create_subprocess_exec(
@@ -732,6 +908,7 @@ async def api_start(req: StartRequest, request: Request) -> dict[str, Any]:
     proc._proc = p
     proc.pid = p.pid
     proc.status = "running"
+    proc.started_at = time.monotonic()
     proc.log_tail.append(f"[launcher] started PID {p.pid}")
 
     _registry(request).add(proc)
@@ -767,6 +944,7 @@ async def api_start(req: StartRequest, request: Request) -> dict[str, Any]:
         "pid": p.pid,
         "command": cmd,
         "provider_sync": provider_sync,
+        "speculative": spec_tokens,
     }
 
 
@@ -971,6 +1149,20 @@ _LAUNCHER_HTML = r"""<!doctype html>
         <div id="profile-args" class="mt-1 text-xs font-mono text-slate-400 bg-slate-800/50 rounded p-2 hidden"></div>
       </div>
 
+      <div class="grid grid-cols-2 gap-2">
+        <div>
+          <label class="block text-xs text-slate-400 mb-1">MTP/draft gguf (空欄で自動検出)</label>
+          <input id="f-draft" type="text" placeholder="companion .gguf (任意)" />
+        </div>
+        <div>
+          <label class="block text-xs text-slate-400 mb-1">MTP</label>
+          <select id="f-mtp">
+            <option value="auto">auto</option>
+            <option value="off">off</option>
+          </select>
+        </div>
+      </div>
+
       <div>
         <div class="flex items-center justify-between mb-1">
           <label class="block text-xs text-slate-400">追加オプション(自由入力)</label>
@@ -1292,6 +1484,8 @@ _LAUNCHER_HTML = r"""<!doctype html>
     const backend = document.getElementById("f-backend").value;
     const model = document.getElementById("f-model").value.trim();
     const extra = document.getElementById("f-extra").value.trim();
+    const draft = document.getElementById("f-draft").value.trim();
+    const mtp = document.getElementById("f-mtp").value;
 
     if (!name) { showLaunchErr("名前を入力してください"); return; }
     if (!model) { showLaunchErr("モデルパスを入力してください"); return; }
@@ -1306,7 +1500,8 @@ _LAUNCHER_HTML = r"""<!doctype html>
         method: "POST",
         headers: authHeaders({"Content-Type": "application/json"}),
         body: JSON.stringify({name, backend, model_path: model, port,
-                              options: selectedProfileArgs(), extra_args: extra}),
+                              options: selectedProfileArgs(), extra_args: extra,
+                              draft_model_path: draft || null, mtp_mode: mtp}),
       });
       const d = await res.json();
       if (!res.ok) { showLaunchErr(d.detail || "起動失敗"); return; }

coderouter/launcher_speculative.py

@@ -0,0 +1,333 @@
+"""Speculative-decoding / MTP flag resolution for the llama.cpp launcher.
+
+Why this module exists
+======================
+
+llama.cpp's ``llama-server`` (2026) folds Multi-Token Prediction (MTP) into
+its speculative-decoding framework. Instead of asking the operator to hand-
+assemble the right ``--spec-type`` / ``--model-draft`` incantation, the
+launcher exposes two high-level knobs — ``draft_model_path`` and
+``mtp_mode`` — and derives the concrete CLI tokens here.
+
+Both launchers (the FastAPI web UI in
+:mod:`coderouter.ingress.launcher_routes` and the tkinter desktop app in
+``launcher_gui.py``) call :func:`resolve_speculative` so the decision logic
+lives in exactly one place.
+
+Decision summary (llama.cpp only)
+---------------------------------
+
+* ``mtp_mode="off"`` — never emit speculative flags (reproduces the historical
+  command exactly). Combining it with ``draft_model_path`` is a conflict.
+* Explicit ``draft_model_path`` — validate it, then pick ``draft-mtp`` when the
+  filename looks MTP-ish (``/mtp/i``) or ``draft-simple`` otherwise.
+* ``mtp_mode="auto"`` (the default) — inspect the main GGUF: if it embeds nextn
+  layers (``{arch}.nextn_predict_layers > 0``) use ``--spec-type draft-mtp``
+  with no separate draft model; otherwise scan the *same directory* for a
+  small companion draft/MTP gguf and wire it up if one is found.
+
+The functions here are pure stdlib + :mod:`coderouter.gguf_introspect`; they
+perform filesystem reads but never spawn processes and hold no FastAPI
+dependency, which keeps them trivially unit-testable.
+
+Security
+--------
+
+Every resolved path is placed into an argv list by the caller (never a shell
+string), and ``draft_model_path`` must resolve to an existing regular file —
+so this module adds no new shell-interpolation or path-probing surface beyond
+reading GGUF headers the operator already pointed us at.
+"""
+
+from __future__ import annotations
+
+import re
+from pathlib import Path
+
+from coderouter.gguf_introspect import try_read_gguf_metadata
+
+__all__ = ["find_draft_companion", "resolve_speculative"]
+
+_LLAMA_CPP = "llama.cpp"
+
+# Companion candidates must be meaningfully smaller than the main model — a
+# draft / MTP head is a fraction of the target's weights. Guards against
+# mistaking a second full model in the same folder for a draft.
+_COMPANION_MAX_SIZE_RATIO = 0.5
+
+# Minimum stripped-prefix length before a shared-prefix match is trusted
+# (avoids matching two unrelated models that merely share a short token).
+_MIN_PREFIX_LEN = 3
+
+# Trailing GGUF shard marker, e.g. "-00001-of-00003".
+_SHARD_RE = re.compile(r"[-._]\d{5}-of-\d{5}$", re.IGNORECASE)
+
+# Trailing quantisation token, e.g. "-Q4_K_M", ".IQ3_XXS", "-F16", "-BF16".
+_QUANT_RE = re.compile(
+    r"[-._]?(i?q\d[_a-z0-9]*|bf16|fp?16|fp?32)$",
+    re.IGNORECASE,
+)
+
+# Filename hint that the companion carries MTP tensors (→ draft-mtp).
+_MTP_NAME_RE = re.compile(r"mtp", re.IGNORECASE)
+
+
+def _strip_quant_suffix(stem: str) -> str:
+    """Return ``stem`` with a trailing shard marker and quant token removed.
+
+    Used to derive a model's "family prefix" so a companion sitting next to
+    ``Foo-7B-Q4_K_M.gguf`` (prefix ``Foo-7B``) can be matched even though its
+    own quant/shape suffix differs.
+    """
+    s = _SHARD_RE.sub("", stem)
+    s = _QUANT_RE.sub("", s)
+    return s
+
+
+def _has_spec_type(user_tokens: list[str]) -> bool:
+    """True if the operator already supplied ``--spec-type`` in extra args."""
+    return any(
+        tok == "--spec-type" or tok.startswith("--spec-type=")
+        for tok in user_tokens
+    )
+
+
+def _has_split_mode_tensor(user_tokens: list[str]) -> bool:
+    """True if the tokens request ``--split-mode tensor`` (either form).
+
+    llama.cpp issue #24309: a nextn-embedded model combined with tensor split
+    crashes, so callers warn (but do not block) when this pairs with emitted
+    speculative flags.
+    """
+    for i, tok in enumerate(user_tokens):
+        if tok == "--split-mode=tensor":
+            return True
+        if (
+            tok == "--split-mode"
+            and i + 1 < len(user_tokens)
+            and user_tokens[i + 1] == "tensor"
+        ):
+            return True
+    return False
+
+
+def _spec_type_for_name(path: Path) -> str:
+    """Pick ``draft-mtp`` for MTP-named files, ``draft-simple`` otherwise."""
+    return "draft-mtp" if _MTP_NAME_RE.search(path.name) else "draft-simple"
+
+
+def find_draft_companion(main_path: str | Path) -> Path | None:
+    """Scan the directory of ``main_path`` for a companion draft/MTP gguf.
+
+    A candidate qualifies when it is a ``*.gguf`` other than the main file,
+    is smaller than :data:`_COMPANION_MAX_SIZE_RATIO` of the main file, and
+    either:
+
+    * has an ``mtp`` or ``draft`` hint in its filename, or
+    * shares the main file's family prefix (main filename with its shard /
+      quant suffix stripped).
+
+    If a candidate's GGUF ``architecture`` is readable and differs from the
+    main model's, it is dropped (tokenizer/vocab must match for speculation);
+    unreadable metadata is kept on a best-effort basis.
+
+    Candidates are ranked by name hint (``mtp`` > ``draft`` > none), then by
+    whether they share the prefix, then by smallest size. Returns the best
+    match, or ``None`` when the directory holds no suitable companion.
+    """
+    main = Path(main_path).expanduser()
+    directory = main.parent
+    try:
+        main_size = main.stat().st_size
+    except OSError:
+        return None
+    if main_size <= 0:
+        return None
+
+    main_prefix = _strip_quant_suffix(main.stem)
+    main_meta = try_read_gguf_metadata(main)
+    main_arch = main_meta.architecture if main_meta else None
+
+    try:
+        entries = list(directory.iterdir())
+    except OSError:
+        return None
+
+    # Each ranked candidate: (name_hint, shares_prefix, -size) sorted so that
+    # the strongest hint / prefix / smallest file sorts first.
+    ranked: list[tuple[int, int, int, Path]] = []
+    for cand in entries:
+        if cand == main:
+            continue
+        if cand.suffix.lower() != ".gguf":
+            continue
+        try:
+            if not cand.is_file():
+                continue
+            size = cand.stat().st_size
+        except OSError:
+            continue
+        if size <= 0 or size >= main_size * _COMPANION_MAX_SIZE_RATIO:
+            continue
+
+        name_lower = cand.name.lower()
+        has_mtp = "mtp" in name_lower
+        has_draft = "draft" in name_lower
+        shares_prefix = (
+            len(main_prefix) >= _MIN_PREFIX_LEN
+            and cand.stem.lower().startswith(main_prefix.lower())
+        )
+        if not (has_mtp or has_draft or shares_prefix):
+            continue
+
+        # Reject a candidate whose architecture is readable and mismatches the
+        # main model — a speculator must share the target's vocabulary.
+        cand_meta = try_read_gguf_metadata(cand)
+        if (
+            cand_meta is not None
+            and cand_meta.architecture is not None
+            and main_arch is not None
+            and cand_meta.architecture != main_arch
+        ):
+            continue
+
+        name_hint = 2 if has_mtp else (1 if has_draft else 0)
+        ranked.append((name_hint, int(shares_prefix), size, cand))
+
+    if not ranked:
+        return None
+    # Highest hint, then prefix match, then smallest size.
+    ranked.sort(key=lambda t: (-t[0], -t[1], t[2]))
+    return ranked[0][3]
+
+
+def resolve_speculative(
+    backend: str,
+    model_path: str,
+    draft_model_path: str | None,
+    mtp_mode: str,
+    user_tokens: list[str],
+) -> tuple[list[str], list[str]]:
+    """Resolve speculative-decoding / MTP CLI tokens for a launch.
+
+    Parameters
+    ----------
+    backend:
+        Target backend id. Only ``"llama.cpp"`` supports speculation; other
+        backends must not receive ``draft_model_path`` / ``mtp_mode="off"``.
+    model_path:
+        The vetted main model path (already set via the launcher's dedicated
+        field). Inspected for embedded nextn layers / same-folder companions.
+    draft_model_path:
+        Optional explicit companion draft or MTP gguf. ``None`` means "not
+        supplied".
+    mtp_mode:
+        ``"auto"`` (default behaviour — detect) or ``"off"`` (never emit
+        speculative flags).
+    user_tokens:
+        The flat list of option/extra-args tokens already destined for the
+        command line. Used to defer to an operator-supplied ``--spec-type``
+        and to warn about the tensor-split crash.
+
+    Returns
+    -------
+    ``(spec_tokens, notes)`` where ``spec_tokens`` is the list of CLI tokens to
+    append (empty when none apply) and ``notes`` is a list of human-readable
+    strings describing what was decided (surfaced in the process log).
+
+    Raises
+    ------
+    ValueError
+        On misuse — the web API maps this to HTTP 400:
+
+        * ``draft_model_path`` / ``mtp_mode="off"`` on a non-llama.cpp backend,
+        * ``mtp_mode="off"`` combined with an explicit ``draft_model_path``,
+        * an explicit ``draft_model_path`` that does not resolve to a file.
+    """
+    notes: list[str] = []
+
+    # 1. Non-llama.cpp backends do not support speculation. Reject explicit
+    #    use; otherwise (auto default, no draft) stay a silent no-op.
+    if backend != _LLAMA_CPP:
+        if draft_model_path or mtp_mode == "off":
+            raise ValueError(
+                "draft_model_path / mtp_mode are only supported for llama.cpp"
+            )
+        return [], notes
+
+    # 2. Explicit opt-out. Never emit flags; conflicting draft path is an error.
+    if mtp_mode == "off":
+        if draft_model_path:
+            raise ValueError(
+                "mtp_mode='off' conflicts with an explicit draft_model_path"
+            )
+        return [], notes
+
+    # 3. Operator already drives speculation via extra args → defer entirely.
+    if _has_spec_type(user_tokens):
+        notes.append(
+            "spec flags supplied via extra args; auto MTP detection skipped"
+        )
+        return [], notes
+
+    spec_tokens: list[str] = []
+
+    # 4. Explicit companion draft / MTP gguf.
+    if draft_model_path:
+        draft = Path(draft_model_path).expanduser()
+        if not draft.is_file():
+            raise ValueError(
+                f"draft_model_path does not exist or is not a file: {draft}"
+            )
+        spec_type = _spec_type_for_name(draft)
+        spec_tokens = ["--spec-type", spec_type, "--model-draft", str(draft)]
+        notes.append(
+            f"MTP: explicit draft model {draft.name} → --spec-type {spec_type}"
+        )
+    # 5. Auto detection (only when the main path is an existing .gguf).
+    else:
+        main = Path(model_path).expanduser()
+        if main.suffix.lower() != ".gguf" or not main.is_file():
+            notes.append(
+                "MTP auto: main model is not an existing .gguf; "
+                "skipping speculative detection"
+            )
+        else:
+            info = try_read_gguf_metadata(main)
+            if info is not None and info.n_nextn and info.n_nextn > 0:
+                # 5a. Main gguf embeds nextn/MTP tensors — no draft model needed.
+                spec_tokens = ["--spec-type", "draft-mtp"]
+                notes.append(
+                    f"MTP: nextn layers ({info.n_nextn}) detected in main "
+                    "gguf → --spec-type draft-mtp"
+                )
+            else:
+                # 5b. Look for a companion gguf next to the main model.
+                companion = find_draft_companion(main)
+                if companion is not None:
+                    spec_type = _spec_type_for_name(companion)
+                    spec_tokens = [
+                        "--spec-type",
+                        spec_type,
+                        "--model-draft",
+                        str(companion),
+                    ]
+                    notes.append(
+                        f"MTP: companion gguf {companion.name} found next to "
+                        f"main → --spec-type {spec_type}"
+                    )
+                else:
+                    # 5c. Nothing found — start without speculative decoding.
+                    notes.append(
+                        f"MTP/draft gguf not found next to {main.name}; "
+                        "starting without speculative decoding"
+                    )
+
+    # 6. Warn (do not block) about the nextn + tensor-split crash.
+    if spec_tokens and _has_split_mode_tensor(user_tokens):
+        notes.append(
+            "WARNING: --split-mode tensor with speculative/nextn is known to "
+            "crash llama.cpp (issue #24309); consider --split-mode layer"
+        )
+
+    return spec_tokens, notes

{coderouter_cli-2.7.5.dist-info → coderouter_cli-2.7.6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: coderouter-cli
-Version: 2.7.5
+Version: 2.7.6
 Summary: Local-first, free-first, fallback-built-in LLM router. Claude Code / OpenAI compatible.
 Project-URL: Homepage, https://github.com/zephel01/CodeRouter
 Project-URL: Repository, https://github.com/zephel01/CodeRouter

{coderouter_cli-2.7.5.dist-info → coderouter_cli-2.7.6.dist-info}/RECORD

@@ -7,9 +7,10 @@ coderouter/doctor.py,sha256=2luNk6BHSRvpQStJnHcqzNvNi-SKdOuKV0WZdorZhVk,82854
 coderouter/doctor_apply.py,sha256=r_J6xbu5-HivofPNriw4_vjNYs_VRs7GsGTS0oMEX10,24209
 coderouter/env_security.py,sha256=FEBZnXfJ0xE39kmMMn39zk0W_DRRnmcB_REmP9f4xWo,14796
 coderouter/errors.py,sha256=Xmq67lheyw8iv3Ox39jh2c4tvNI5RcUR4QkoxVDN6l4,1130
-coderouter/gguf_introspect.py,sha256=FZO14STLSp94Rfo5AInGwYUOpfjiXOW6CH5RiczTWDE,9514
+coderouter/gguf_introspect.py,sha256=KeE00CfbYRa4gSTrzuwC6zVuHi20IzLC4nUQa98BEXI,10387
 coderouter/hardware.py,sha256=gn3_9qbVcGRR81yKMn1lJE_8-YDRau0LxIH_M-f7pxE,8356
 coderouter/language_tax.py,sha256=LTbE3tIfoJuV2O3T0NixRKhzq_dEOTUuPEerJv2q9uk,9360
+coderouter/launcher_speculative.py,sha256=kWaHNmhzBsbWuM_lRGDlbWkgJ1suuxBWQNTKhJOC-yg,12644
 coderouter/logging.py,sha256=91cveN8SCJEMNz9DGi6TrFSlc8cx0wdUkOdSLWLA-z8,56144
 coderouter/output_filters.py,sha256=0ry_rPiS_kC-FnHgaNVP6v7e6Al2djxzu9vBzZ8kEkE,25314
 coderouter/token_estimation.py,sha256=iz22vZEEW2P7uKLB2pYvPNpIbZGbgXRO5MtfkS_-9Sk,7531
@@ -41,7 +42,7 @@ coderouter/ingress/__init__.py,sha256=WQsCH2CGJCAhy0mS6GSEdeYZRkkQu2OHDsP4CJWTLu
 coderouter/ingress/anthropic_routes.py,sha256=1ThXzx8VgxRV7UCi3NF5ey7kbr0ADPKTXyQuYS5gK1w,27063
 coderouter/ingress/app.py,sha256=h6s-UWsbGP22rku4pPVlvRdrtbMj4KoEEUZoQQOrTTw,19370
 coderouter/ingress/dashboard_routes.py,sha256=1SqNNrVTfQiZcx5n2MvjAzWirittX0e61tCt-MKp4Ag,25617
-coderouter/ingress/launcher_routes.py,sha256=QzG0l0c2ZmAcQai3E-xs3cV5fUYf3ajjZauvI9pKDh4,57959
+coderouter/ingress/launcher_routes.py,sha256=8EY3a6O_vbA45h2uhp_wc37bP_DJFqVVjsd68d_bwtk,66179
 coderouter/ingress/metrics_routes.py,sha256=M22dwOGn24P05Ge4W3c7d7mYytSGWjIR-pPSPOAiHJY,3965
 coderouter/ingress/openai_routes.py,sha256=TrugsQNJfNPgKKLzfT1aXzrs4emWYGE4XebEBuaZoWk,12505
 coderouter/metrics/__init__.py,sha256=7Es351DPS7yLM0yVF_F0eesmiD83n7Zzhie44chht38,1465
@@ -67,8 +68,8 @@ coderouter/translation/__init__.py,sha256=PYXN7XVEwpG1uC8RLy6fvnGbzEZhhrEuUapH8I
 coderouter/translation/anthropic.py,sha256=aZkcYH4x82b0x7efJgJb9RWn9Hbyc9pEOthXe4vjUdU,11113
 coderouter/translation/convert.py,sha256=VV4kpu4dvLmGBmJfQnCLY5ryy_AnQcV2NuwjqRtaR8Q,52633
 coderouter/translation/tool_repair.py,sha256=DGOKArM53yfVbIfiAveoul1vGq9NEKmjagf43Qh4IkM,57630
-coderouter_cli-2.7.5.dist-info/METADATA,sha256=VsPe8Z7VLlfIRAm5OHEP8VP-gE6CZmWcQFDqWlxiXIE,15546
-coderouter_cli-2.7.5.dist-info/WHEEL,sha256=mffPy8wBnZQn2VnJUU5jE99KsxaSfiyMHV9Yt0aLVxs,87
-coderouter_cli-2.7.5.dist-info/entry_points.txt,sha256=-dnLfD1YZ2WjH2zSdNCvlO65wYltM9bsHt9Fhg3yGss,51
-coderouter_cli-2.7.5.dist-info/licenses/LICENSE,sha256=wkEzoR86jFw33jvfOHjULqmkGEfxTFMgMaJnpR8mPRw,1065
-coderouter_cli-2.7.5.dist-info/RECORD,,
+coderouter_cli-2.7.6.dist-info/METADATA,sha256=VNl2YqpfRQDN5Ww8ukg21mjL5JVvgAOtPj6HKTZKP6Y,15546
+coderouter_cli-2.7.6.dist-info/WHEEL,sha256=lCkmxWfQsSc9CfIClYeavTdQeEX2toPqufh9gI35EQA,87
+coderouter_cli-2.7.6.dist-info/entry_points.txt,sha256=-dnLfD1YZ2WjH2zSdNCvlO65wYltM9bsHt9Fhg3yGss,51
+coderouter_cli-2.7.6.dist-info/licenses/LICENSE,sha256=wkEzoR86jFw33jvfOHjULqmkGEfxTFMgMaJnpR8mPRw,1065
+coderouter_cli-2.7.6.dist-info/RECORD,,

{coderouter_cli-2.7.5.dist-info → coderouter_cli-2.7.6.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: hatchling 1.30.1
+Generator: hatchling 1.31.0
 Root-Is-Purelib: true
 Tag: py3-none-any