coderouter-cli 2.7.5__py3-none-any.whl → 2.7.7__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- coderouter/adapters/agent_cli.py +591 -0
- coderouter/adapters/registry.py +6 -0
- coderouter/config/schemas.py +162 -3
- coderouter/gguf_introspect.py +19 -0
- coderouter/ingress/launcher_routes.py +215 -20
- coderouter/launcher_speculative.py +333 -0
- {coderouter_cli-2.7.5.dist-info → coderouter_cli-2.7.7.dist-info}/METADATA +1 -1
- {coderouter_cli-2.7.5.dist-info → coderouter_cli-2.7.7.dist-info}/RECORD +11 -9
- {coderouter_cli-2.7.5.dist-info → coderouter_cli-2.7.7.dist-info}/WHEEL +1 -1
- {coderouter_cli-2.7.5.dist-info → coderouter_cli-2.7.7.dist-info}/entry_points.txt +0 -0
- {coderouter_cli-2.7.5.dist-info → coderouter_cli-2.7.7.dist-info}/licenses/LICENSE +0 -0
coderouter/config/schemas.py
CHANGED
|
@@ -163,6 +163,125 @@ class CostConfig(BaseModel):
|
|
|
163
163
|
)
|
|
164
164
|
|
|
165
165
|
|
|
166
|
+
class AgentCliConfig(BaseModel):
|
|
167
|
+
"""External coding-agent CLI settings for ``kind="agent_cli"`` providers.
|
|
168
|
+
|
|
169
|
+
Introduced by the external-agents-adapter design (Phase 1). One
|
|
170
|
+
``agent_cli`` sub-config drives the :class:`AgentCliAdapter`, which
|
|
171
|
+
invokes an external coding-agent CLI (codex / gemini / grok / claude)
|
|
172
|
+
in a single one-shot ``exec`` and returns the final answer as one
|
|
173
|
+
``prompt in → text out`` transformation. Phase 1a implements the
|
|
174
|
+
``claude`` (Claude Code CLI) target only; the other agents are declared
|
|
175
|
+
at the schema level so configs are forward-compatible, but the adapter
|
|
176
|
+
rejects them until their phase lands.
|
|
177
|
+
|
|
178
|
+
Follows the ``extra="forbid"`` convention used across this module so a
|
|
179
|
+
typo'd key fails at config-load rather than being silently ignored.
|
|
180
|
+
"""
|
|
181
|
+
|
|
182
|
+
model_config = ConfigDict(extra="forbid")
|
|
183
|
+
|
|
184
|
+
agent: Literal["codex", "gemini", "grok", "claude"] = Field(
|
|
185
|
+
...,
|
|
186
|
+
description="External coding-agent CLI to invoke. Phase 1a: 'claude' only.",
|
|
187
|
+
)
|
|
188
|
+
command: str | None = Field(
|
|
189
|
+
default=None,
|
|
190
|
+
description=(
|
|
191
|
+
"CLI executable name or absolute path (resolved via PATH). "
|
|
192
|
+
"When unset, defaults to the ``agent`` name."
|
|
193
|
+
),
|
|
194
|
+
)
|
|
195
|
+
workdir: str | None = Field(
|
|
196
|
+
default=None,
|
|
197
|
+
description=(
|
|
198
|
+
"Working directory for the one-shot exec. ``~`` / env-var "
|
|
199
|
+
"expansion is applied. When unset, a dedicated isolated "
|
|
200
|
+
"directory (``~/.coderouter/agents/<name>``) is used."
|
|
201
|
+
),
|
|
202
|
+
)
|
|
203
|
+
exec_timeout_s: float = Field(
|
|
204
|
+
default=600.0,
|
|
205
|
+
ge=1.0,
|
|
206
|
+
le=1800.0,
|
|
207
|
+
description=(
|
|
208
|
+
"Forced timeout (seconds) for the one-shot exec. Independent "
|
|
209
|
+
"of ``ProviderConfig.timeout_s`` — the CLI has no built-in "
|
|
210
|
+
"wall clock so this is enforced with an asyncio watchdog + "
|
|
211
|
+
"process-group SIGKILL."
|
|
212
|
+
),
|
|
213
|
+
)
|
|
214
|
+
allow_file_writes: bool = Field(
|
|
215
|
+
default=False,
|
|
216
|
+
description=(
|
|
217
|
+
"Allow the agent to write to the filesystem. Default False "
|
|
218
|
+
"(read-only). When False the sandbox mapping is clamped to "
|
|
219
|
+
"read-only regardless of ``sandbox_mode`` (defense in depth)."
|
|
220
|
+
),
|
|
221
|
+
)
|
|
222
|
+
sandbox_mode: Literal["read_only", "edit", "full_auto"] = Field(
|
|
223
|
+
default="read_only",
|
|
224
|
+
description=(
|
|
225
|
+
"Source mode mapped onto each CLI's sandbox / approval flags. "
|
|
226
|
+
"Default read_only maps to claude ``--permission-mode plan``."
|
|
227
|
+
),
|
|
228
|
+
)
|
|
229
|
+
model: str | None = Field(
|
|
230
|
+
default=None,
|
|
231
|
+
description=(
|
|
232
|
+
"Model name passed to the CLI's ``--model``. When unset, "
|
|
233
|
+
"``ProviderConfig.model`` is used."
|
|
234
|
+
),
|
|
235
|
+
)
|
|
236
|
+
max_turns: int | None = Field(
|
|
237
|
+
default=8,
|
|
238
|
+
ge=1,
|
|
239
|
+
le=50,
|
|
240
|
+
description=(
|
|
241
|
+
"Turn cap. Passed to the CLI's ``--max-turns`` where supported "
|
|
242
|
+
"(codex ignores it — the CLI has no such flag)."
|
|
243
|
+
),
|
|
244
|
+
)
|
|
245
|
+
passthrough_env: list[str] = Field(
|
|
246
|
+
default_factory=list,
|
|
247
|
+
description=(
|
|
248
|
+
"Allowlist of environment variable NAMES forwarded from the "
|
|
249
|
+
"parent process into the child. The child otherwise inherits "
|
|
250
|
+
"no parent environment (subscription-first auth policy). "
|
|
251
|
+
"``ANTHROPIC_API_KEY`` is NOT forwarded unless listed here."
|
|
252
|
+
),
|
|
253
|
+
)
|
|
254
|
+
agent_depth_limit: int = Field(
|
|
255
|
+
default=2,
|
|
256
|
+
ge=1,
|
|
257
|
+
le=4,
|
|
258
|
+
description=(
|
|
259
|
+
"Recursion nesting cap. ``CODEROUTER_AGENT_DEPTH`` is "
|
|
260
|
+
"propagated (incremented) into the child; ``generate()`` "
|
|
261
|
+
"refuses when the current depth is at or above this limit."
|
|
262
|
+
),
|
|
263
|
+
)
|
|
264
|
+
|
|
265
|
+
@model_validator(mode="after")
|
|
266
|
+
def _resolve_and_check(self) -> AgentCliConfig:
|
|
267
|
+
"""Default ``command`` to ``agent`` and reject contradictory sandbox.
|
|
268
|
+
|
|
269
|
+
Rule (design §5.2.1 #2): ``allow_file_writes=True`` together with
|
|
270
|
+
``sandbox_mode="read_only"`` is contradictory — the operator asked
|
|
271
|
+
for writes while pinning a read-only sandbox. Fail fast at load,
|
|
272
|
+
matching the module's other cross-field validators.
|
|
273
|
+
"""
|
|
274
|
+
if self.command is None:
|
|
275
|
+
self.command = self.agent
|
|
276
|
+
if self.allow_file_writes and self.sandbox_mode == "read_only":
|
|
277
|
+
raise ValueError(
|
|
278
|
+
"agent_cli: allow_file_writes=True conflicts with "
|
|
279
|
+
"sandbox_mode='read_only'. Set sandbox_mode to 'edit' or "
|
|
280
|
+
"'full_auto' to permit writes, or keep allow_file_writes=False."
|
|
281
|
+
)
|
|
282
|
+
return self
|
|
283
|
+
|
|
284
|
+
|
|
166
285
|
class ProviderConfig(BaseModel):
|
|
167
286
|
"""A single provider entry from providers.yaml.
|
|
168
287
|
|
|
@@ -170,20 +289,26 @@ class ProviderConfig(BaseModel):
|
|
|
170
289
|
- Local llama.cpp server: kind=openai_compat, base_url=http://localhost:8080/v1
|
|
171
290
|
- OpenRouter free: kind=openai_compat, base_url=https://openrouter.ai/api/v1
|
|
172
291
|
- (future) Anthropic: kind=anthropic, base_url=https://api.anthropic.com
|
|
292
|
+
- External agent CLI: kind=agent_cli, agent_cli={agent: claude, ...}
|
|
173
293
|
"""
|
|
174
294
|
|
|
175
295
|
model_config = ConfigDict(extra="forbid")
|
|
176
296
|
|
|
177
297
|
name: str = Field(..., description="Unique identifier used in profiles.yaml")
|
|
178
|
-
kind: Literal["openai_compat", "anthropic"] = Field(
|
|
298
|
+
kind: Literal["openai_compat", "anthropic", "agent_cli"] = Field(
|
|
179
299
|
default="openai_compat",
|
|
180
300
|
description=(
|
|
181
301
|
"Adapter type. 'openai_compat' covers llama.cpp / Ollama / "
|
|
182
302
|
"OpenRouter / LM Studio / Together / Groq. 'anthropic' is the "
|
|
183
|
-
"native Anthropic Messages API passthrough (v0.3.x)."
|
|
303
|
+
"native Anthropic Messages API passthrough (v0.3.x). 'agent_cli' "
|
|
304
|
+
"invokes an external coding-agent CLI one-shot (see AgentCliConfig)."
|
|
184
305
|
),
|
|
185
306
|
)
|
|
186
|
-
base_url
|
|
307
|
+
# base_url is required for HTTP-backed adapters (openai_compat / anthropic)
|
|
308
|
+
# but meaningless for agent_cli (which shells out to a local CLI rather
|
|
309
|
+
# than calling a URL). It is optional at the field level and enforced per
|
|
310
|
+
# kind by ``_check_kind_requirements`` below.
|
|
311
|
+
base_url: HttpUrl | None = None
|
|
187
312
|
model: str = Field(..., description="Upstream model id sent in the request body")
|
|
188
313
|
api_key_env: str | None = Field(
|
|
189
314
|
default=None,
|
|
@@ -243,6 +368,14 @@ class ProviderConfig(BaseModel):
|
|
|
243
368
|
|
|
244
369
|
capabilities: Capabilities = Field(default_factory=Capabilities)
|
|
245
370
|
|
|
371
|
+
# kind="agent_cli": required external coding-agent CLI settings. Opt-in
|
|
372
|
+
# (default None) exactly like ``restart_command`` — only meaningful when
|
|
373
|
+
# ``kind == "agent_cli"``, and enforced by ``_check_kind_requirements``.
|
|
374
|
+
agent_cli: AgentCliConfig | None = Field(
|
|
375
|
+
default=None,
|
|
376
|
+
description="Required when kind='agent_cli': external agent CLI settings.",
|
|
377
|
+
)
|
|
378
|
+
|
|
246
379
|
cost: CostConfig | None = Field(
|
|
247
380
|
default=None,
|
|
248
381
|
description=(
|
|
@@ -298,6 +431,32 @@ class ProviderConfig(BaseModel):
|
|
|
298
431
|
validate_output_filters(self.output_filters)
|
|
299
432
|
return self
|
|
300
433
|
|
|
434
|
+
@model_validator(mode="after")
|
|
435
|
+
def _check_kind_requirements(self) -> ProviderConfig:
|
|
436
|
+
"""Enforce per-``kind`` field requirements (external-agents design §5.2.2).
|
|
437
|
+
|
|
438
|
+
- HTTP-backed adapters (``openai_compat`` / ``anthropic``) require a
|
|
439
|
+
``base_url`` — they have nowhere to send the request otherwise.
|
|
440
|
+
``base_url`` was relaxed to Optional so ``agent_cli`` providers can
|
|
441
|
+
omit it; this validator restores the required-ness for the HTTP kinds.
|
|
442
|
+
- ``agent_cli`` requires the ``agent_cli`` sub-config (the adapter has
|
|
443
|
+
no CLI to invoke without it).
|
|
444
|
+
|
|
445
|
+
Same fast-fail philosophy as ``_check_output_filters_known`` — a
|
|
446
|
+
misconfigured provider surfaces at config-load, not at first request.
|
|
447
|
+
"""
|
|
448
|
+
if self.kind in ("openai_compat", "anthropic") and self.base_url is None:
|
|
449
|
+
raise ValueError(
|
|
450
|
+
f"provider {self.name!r}: base_url is required for "
|
|
451
|
+
f"kind={self.kind!r}."
|
|
452
|
+
)
|
|
453
|
+
if self.kind == "agent_cli" and self.agent_cli is None:
|
|
454
|
+
raise ValueError(
|
|
455
|
+
f"provider {self.name!r}: agent_cli sub-config is required "
|
|
456
|
+
f"for kind='agent_cli'."
|
|
457
|
+
)
|
|
458
|
+
return self
|
|
459
|
+
|
|
301
460
|
|
|
302
461
|
class FallbackChain(BaseModel):
|
|
303
462
|
"""An ordered list of provider names to try in sequence.
|
coderouter/gguf_introspect.py
CHANGED
|
@@ -136,6 +136,17 @@ class GGUFInfo:
|
|
|
136
136
|
n_kv_heads: int | None
|
|
137
137
|
file_type: int | None
|
|
138
138
|
file_size_bytes: int
|
|
139
|
+
n_nextn: int | None = None
|
|
140
|
+
|
|
141
|
+
@property
|
|
142
|
+
def supports_mtp(self) -> bool:
|
|
143
|
+
"""True when the GGUF embeds Multi-Token-Prediction (nextn) layers.
|
|
144
|
+
|
|
145
|
+
Derived from ``{arch}.nextn_predict_layers`` — a positive count means
|
|
146
|
+
the main model carries MTP/nextn tensors and can drive llama.cpp's
|
|
147
|
+
``--spec-type draft-mtp`` without a separate draft gguf.
|
|
148
|
+
"""
|
|
149
|
+
return bool(self.n_nextn and self.n_nextn > 0)
|
|
139
150
|
|
|
140
151
|
@property
|
|
141
152
|
def quant_name(self) -> str | None:
|
|
@@ -224,6 +235,10 @@ _KEY_BLOCK_COUNT = ".block_count"
|
|
|
224
235
|
_KEY_EMBED_LEN = ".embedding_length"
|
|
225
236
|
_KEY_HEAD_COUNT = ".attention.head_count"
|
|
226
237
|
_KEY_HEAD_COUNT_KV = ".attention.head_count_kv"
|
|
238
|
+
# Number of Multi-Token-Prediction (nextn) layers embedded in the main model
|
|
239
|
+
# (e.g. ``glm4moe.nextn_predict_layers``). A positive value means the GGUF can
|
|
240
|
+
# drive llama.cpp speculative decoding via ``--spec-type draft-mtp`` alone.
|
|
241
|
+
_KEY_NEXTN = ".nextn_predict_layers"
|
|
227
242
|
|
|
228
243
|
|
|
229
244
|
def read_gguf_metadata(path: str | Path) -> GGUFInfo:
|
|
@@ -245,6 +260,7 @@ def read_gguf_metadata(path: str | Path) -> GGUFInfo:
|
|
|
245
260
|
n_heads: int | None = None
|
|
246
261
|
n_kv_heads: int | None = None
|
|
247
262
|
file_type: int | None = None
|
|
263
|
+
n_nextn: int | None = None
|
|
248
264
|
|
|
249
265
|
with p.open("rb") as fh:
|
|
250
266
|
magic = fh.read(4)
|
|
@@ -275,6 +291,8 @@ def read_gguf_metadata(path: str | Path) -> GGUFInfo:
|
|
|
275
291
|
n_kv_heads = value
|
|
276
292
|
elif key.endswith(_KEY_HEAD_COUNT) and isinstance(value, int):
|
|
277
293
|
n_heads = value
|
|
294
|
+
elif key.endswith(_KEY_NEXTN) and isinstance(value, int):
|
|
295
|
+
n_nextn = value
|
|
278
296
|
|
|
279
297
|
return GGUFInfo(
|
|
280
298
|
architecture=arch,
|
|
@@ -284,6 +302,7 @@ def read_gguf_metadata(path: str | Path) -> GGUFInfo:
|
|
|
284
302
|
n_kv_heads=n_kv_heads,
|
|
285
303
|
file_type=file_type,
|
|
286
304
|
file_size_bytes=file_size,
|
|
305
|
+
n_nextn=n_nextn,
|
|
287
306
|
)
|
|
288
307
|
|
|
289
308
|
|
|
@@ -30,6 +30,7 @@ import secrets
|
|
|
30
30
|
import shlex
|
|
31
31
|
import shutil
|
|
32
32
|
import subprocess
|
|
33
|
+
import time
|
|
33
34
|
import uuid
|
|
34
35
|
from collections import deque
|
|
35
36
|
from dataclasses import dataclass, field
|
|
@@ -40,6 +41,7 @@ from fastapi import APIRouter, HTTPException, Request
|
|
|
40
41
|
from fastapi.responses import HTMLResponse
|
|
41
42
|
from pydantic import BaseModel
|
|
42
43
|
|
|
44
|
+
from coderouter.launcher_speculative import resolve_speculative
|
|
43
45
|
from coderouter.logging import get_logger
|
|
44
46
|
|
|
45
47
|
logger = get_logger(__name__)
|
|
@@ -115,9 +117,23 @@ class ManagedProcess:
|
|
|
115
117
|
options: dict[str, Any]
|
|
116
118
|
extra_args: str
|
|
117
119
|
status: str = "starting" # "starting" | "running" | "stopped" | "error"
|
|
120
|
+
# MTP / speculative-decoding controls (defaults keep existing call sites
|
|
121
|
+
# working). Recorded for introspection; the resolved flags live in the cmd.
|
|
122
|
+
draft_model_path: str | None = None
|
|
123
|
+
mtp_mode: str = "auto"
|
|
118
124
|
pid: int | None = None
|
|
119
125
|
returncode: int | None = None
|
|
120
126
|
log_tail: deque = field(default_factory=lambda: deque(maxlen=200))
|
|
127
|
+
# MTP auto-fallback state. When the speculative flags were added by AUTO
|
|
128
|
+
# detection and the child dies during startup, the process is relaunched
|
|
129
|
+
# ONCE without them (some archs' draft-mtp support in llama.cpp is
|
|
130
|
+
# immature and crashes the context init). These fields make that possible
|
|
131
|
+
# and impossible to loop.
|
|
132
|
+
spec_tokens: list = field(default_factory=list)
|
|
133
|
+
spec_auto: bool = False
|
|
134
|
+
mtp_fallback_done: bool = False
|
|
135
|
+
fallback_cmd: list | None = None
|
|
136
|
+
started_at: float = 0.0
|
|
121
137
|
# asyncio subprocess handle — not serialised
|
|
122
138
|
_proc: Any = field(default=None, repr=False, compare=False)
|
|
123
139
|
|
|
@@ -208,8 +224,15 @@ _BACKEND_DEFAULTS: dict[str, str] = {
|
|
|
208
224
|
# - llama.cpp: ``-m`` / ``--model`` select the GGUF file.
|
|
209
225
|
# - vllm / mlx: ``--model`` selects the model; ``-m`` selects the python
|
|
210
226
|
# module that ``_build_cmd`` pins, so it must not be re-specified either.
|
|
227
|
+
# - llama.cpp: the draft/MTP model is likewise set only via the dedicated
|
|
228
|
+
# ``draft_model_path`` field, so its aliases (``-md`` / ``--model-draft``
|
|
229
|
+
# / ``--spec-draft-model``) are rejected in options / extra_args too. The
|
|
230
|
+
# remaining spec knobs (``--spec-type``, ``--spec-draft-n-max`` …) stay
|
|
231
|
+
# free-form.
|
|
211
232
|
_MODEL_FLAGS: dict[str, frozenset[str]] = {
|
|
212
|
-
"llama.cpp": frozenset(
|
|
233
|
+
"llama.cpp": frozenset(
|
|
234
|
+
{"-m", "--model", "-md", "--model-draft", "--spec-draft-model"}
|
|
235
|
+
),
|
|
213
236
|
"vllm": frozenset({"-m", "--model"}),
|
|
214
237
|
"mlx": frozenset({"-m", "--model"}),
|
|
215
238
|
}
|
|
@@ -407,6 +430,24 @@ def _resolve_within_model_dirs(model_path: str, model_dirs: list[str]) -> Path:
|
|
|
407
430
|
raise ValueError("model_path is not under any configured model_dirs.")
|
|
408
431
|
|
|
409
432
|
|
|
433
|
+
def _option_tokens(options: dict[str, Any]) -> list[str]:
|
|
434
|
+
"""Flatten an ``options`` dict into CLI tokens (``{flag: val}`` semantics).
|
|
435
|
+
|
|
436
|
+
Boolean values become a bare flag when true and vanish when false;
|
|
437
|
+
everything else becomes ``[flag, str(val)]``. Shared by ``_build_cmd`` and
|
|
438
|
+
``api_start`` so the tokens fed to ``resolve_speculative`` match exactly
|
|
439
|
+
what lands on the command line (no double-parsing drift).
|
|
440
|
+
"""
|
|
441
|
+
tokens: list[str] = []
|
|
442
|
+
for flag, val in options.items():
|
|
443
|
+
if isinstance(val, bool):
|
|
444
|
+
if val:
|
|
445
|
+
tokens.append(flag)
|
|
446
|
+
else:
|
|
447
|
+
tokens.extend([flag, str(val)])
|
|
448
|
+
return tokens
|
|
449
|
+
|
|
450
|
+
|
|
410
451
|
def _build_cmd(
|
|
411
452
|
backend: str,
|
|
412
453
|
model_path: str,
|
|
@@ -414,17 +455,27 @@ def _build_cmd(
|
|
|
414
455
|
options: dict[str, Any],
|
|
415
456
|
extra_args: str,
|
|
416
457
|
binary: str | None = None,
|
|
458
|
+
spec_tokens: list[str] | None = None,
|
|
417
459
|
) -> list[str]:
|
|
418
460
|
"""Build the CLI command list for the given backend and options.
|
|
419
461
|
|
|
420
462
|
``binary`` overrides the default executable (``llama-server`` /
|
|
421
463
|
``python``). When None, the default is used and PATH resolution
|
|
422
464
|
is left to the OS.
|
|
465
|
+
|
|
466
|
+
``spec_tokens`` are pre-resolved speculative-decoding / MTP flags
|
|
467
|
+
(from :func:`coderouter.launcher_speculative.resolve_speculative`).
|
|
468
|
+
For llama.cpp they are appended right after the port args and BEFORE
|
|
469
|
+
the profile / extra args. They are trusted (the launcher, not the
|
|
470
|
+
caller, produced them) so they bypass the model-override guard even
|
|
471
|
+
though they may contain ``--model-draft``.
|
|
423
472
|
"""
|
|
424
473
|
exe = _resolve_binary(backend, binary)
|
|
425
474
|
|
|
426
475
|
if backend == "llama.cpp":
|
|
427
476
|
cmd: list[str] = [exe, "-m", model_path, "--port", str(port)]
|
|
477
|
+
if spec_tokens:
|
|
478
|
+
cmd.extend(spec_tokens)
|
|
428
479
|
elif backend == "vllm":
|
|
429
480
|
cmd = [
|
|
430
481
|
exe, "-m", "vllm.entrypoints.openai.api_server",
|
|
@@ -446,12 +497,7 @@ def _build_cmd(
|
|
|
446
497
|
# H8: reject model-path re-specification via options keys.
|
|
447
498
|
_assert_no_model_override(backend, list(options.keys()))
|
|
448
499
|
|
|
449
|
-
|
|
450
|
-
if isinstance(val, bool):
|
|
451
|
-
if val:
|
|
452
|
-
cmd.append(flag)
|
|
453
|
-
else:
|
|
454
|
-
cmd.extend([flag, str(val)])
|
|
500
|
+
cmd.extend(_option_tokens(options))
|
|
455
501
|
|
|
456
502
|
if extra_args.strip():
|
|
457
503
|
extra_tokens = shlex.split(extra_args)
|
|
@@ -474,12 +520,37 @@ def _build_cmd(
|
|
|
474
520
|
# recover by reading the buffered chunk and continuing so log tailing survives.
|
|
475
521
|
_LOG_STREAM_LIMIT = 256 * 1024 # 256 KB
|
|
476
522
|
|
|
523
|
+
# Window (seconds) after spawn in which a non-zero exit is treated as a
|
|
524
|
+
# startup crash eligible for the MTP auto-fallback. A backend that ran fine
|
|
525
|
+
# for longer and then died is a normal crash, not a load-time failure, so it
|
|
526
|
+
# is never retried.
|
|
527
|
+
_MTP_FALLBACK_WINDOW_SECS = 180.0
|
|
477
528
|
|
|
478
|
-
|
|
479
|
-
|
|
529
|
+
|
|
530
|
+
def _should_mtp_fallback(proc: ManagedProcess) -> bool:
|
|
531
|
+
"""True when a just-crashed process qualifies for the one-shot MTP retry.
|
|
532
|
+
|
|
533
|
+
Only auto-detected speculative launches that die within the startup
|
|
534
|
+
window are eligible, and only once (guarded by ``mtp_fallback_done``).
|
|
535
|
+
"""
|
|
480
536
|
p = proc._proc
|
|
481
|
-
if p is None
|
|
482
|
-
|
|
537
|
+
rc = p.returncode if p is not None else proc.returncode
|
|
538
|
+
return (
|
|
539
|
+
proc.spec_auto
|
|
540
|
+
and not proc.mtp_fallback_done
|
|
541
|
+
and bool(proc.fallback_cmd)
|
|
542
|
+
and rc not in (0, None)
|
|
543
|
+
and (time.monotonic() - proc.started_at) <= _MTP_FALLBACK_WINDOW_SECS
|
|
544
|
+
)
|
|
545
|
+
|
|
546
|
+
|
|
547
|
+
async def _tail_logs(proc: ManagedProcess) -> None:
|
|
548
|
+
"""Read stdout+stderr into proc.log_tail until the process exits.
|
|
549
|
+
|
|
550
|
+
Loops so that a one-shot MTP fallback (relaunch without the auto-detected
|
|
551
|
+
speculative flags) can be tailed by the same task. The loop exits as soon
|
|
552
|
+
as no fallback applies.
|
|
553
|
+
"""
|
|
483
554
|
|
|
484
555
|
async def _drain(stream: asyncio.StreamReader | None) -> None:
|
|
485
556
|
if stream is None:
|
|
@@ -505,14 +576,53 @@ async def _tail_logs(proc: ManagedProcess) -> None:
|
|
|
505
576
|
break
|
|
506
577
|
proc.log_tail.append(line.decode(errors="replace").rstrip())
|
|
507
578
|
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
579
|
+
while True:
|
|
580
|
+
p = proc._proc
|
|
581
|
+
if p is None:
|
|
582
|
+
return
|
|
583
|
+
|
|
584
|
+
await asyncio.gather(_drain(p.stdout), _drain(p.stderr))
|
|
585
|
+
await p.wait()
|
|
586
|
+
proc.returncode = p.returncode
|
|
587
|
+
proc.pid = None
|
|
588
|
+
proc.status = "stopped" if (p.returncode or 0) == 0 else "error"
|
|
589
|
+
proc.log_tail.append(
|
|
590
|
+
f"[launcher] process exited with code {p.returncode}"
|
|
591
|
+
)
|
|
592
|
+
|
|
593
|
+
if not _should_mtp_fallback(proc):
|
|
594
|
+
return
|
|
595
|
+
|
|
596
|
+
# Auto-detected MTP crashed during startup — retry ONCE without the
|
|
597
|
+
# speculative flags. The port is unchanged, so the existing provider
|
|
598
|
+
# registration still stands (do not re-register).
|
|
599
|
+
rc = proc.returncode
|
|
600
|
+
proc.mtp_fallback_done = True
|
|
601
|
+
proc.log_tail.append(
|
|
602
|
+
f"[launcher] MTP startup failure detected (exit code {rc}); "
|
|
603
|
+
"retrying without speculative decoding"
|
|
604
|
+
)
|
|
605
|
+
proc.log_tail.append(
|
|
606
|
+
f"[launcher] cmd: {' '.join(proc.fallback_cmd)}"
|
|
607
|
+
)
|
|
608
|
+
try:
|
|
609
|
+
p = await asyncio.create_subprocess_exec(
|
|
610
|
+
*proc.fallback_cmd,
|
|
611
|
+
stdout=asyncio.subprocess.PIPE,
|
|
612
|
+
stderr=asyncio.subprocess.PIPE,
|
|
613
|
+
limit=_LOG_STREAM_LIMIT,
|
|
614
|
+
)
|
|
615
|
+
except (FileNotFoundError, OSError) as exc:
|
|
616
|
+
proc.log_tail.append(f"[launcher] fallback relaunch failed: {exc}")
|
|
617
|
+
proc.status = "error"
|
|
618
|
+
return
|
|
619
|
+
proc._proc = p
|
|
620
|
+
proc.pid = p.pid
|
|
621
|
+
proc.status = "running"
|
|
622
|
+
proc.returncode = None
|
|
623
|
+
proc.started_at = time.monotonic()
|
|
624
|
+
proc.log_tail.append(f"[launcher] started PID {p.pid}")
|
|
625
|
+
# Loop to tail the relaunched process.
|
|
516
626
|
|
|
517
627
|
|
|
518
628
|
async def shutdown_launcher(app: Any) -> None:
|
|
@@ -555,6 +665,11 @@ class StartRequest(BaseModel):
|
|
|
555
665
|
port: int
|
|
556
666
|
options: dict[str, Any] = {}
|
|
557
667
|
extra_args: str = ""
|
|
668
|
+
# MTP / speculative-decoding controls (llama.cpp only). ``draft_model_path``
|
|
669
|
+
# is an explicit companion draft/MTP gguf; ``mtp_mode`` is "auto" (detect)
|
|
670
|
+
# or "off" (never emit speculative flags).
|
|
671
|
+
draft_model_path: str | None = None
|
|
672
|
+
mtp_mode: str = "auto"
|
|
558
673
|
|
|
559
674
|
|
|
560
675
|
# ---------------------------------------------------------------------------
|
|
@@ -690,15 +805,69 @@ async def api_start(req: StartRequest, request: Request) -> dict[str, Any]:
|
|
|
690
805
|
if bc and bc.binary:
|
|
691
806
|
configured_binary = bc.binary
|
|
692
807
|
|
|
808
|
+
# Validate the MTP mode up front (only "auto" / "off" are accepted).
|
|
809
|
+
if req.mtp_mode not in ("auto", "off"):
|
|
810
|
+
raise HTTPException(
|
|
811
|
+
status_code=400,
|
|
812
|
+
detail=f"mtp_mode must be 'auto' or 'off', got {req.mtp_mode!r}.",
|
|
813
|
+
)
|
|
814
|
+
|
|
815
|
+
# Build the user token list once (options flags + extra_args) so
|
|
816
|
+
# resolve_speculative sees exactly what _build_cmd will emit. shlex.split
|
|
817
|
+
# can raise on unbalanced quotes → surface as 400 like other bad input.
|
|
818
|
+
try:
|
|
819
|
+
user_tokens = _option_tokens(req.options)
|
|
820
|
+
if req.extra_args.strip():
|
|
821
|
+
user_tokens += shlex.split(req.extra_args)
|
|
822
|
+
except ValueError as exc:
|
|
823
|
+
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
|
824
|
+
|
|
825
|
+
# Resolve MTP / speculative-decoding flags (llama.cpp only; no-op elsewhere).
|
|
826
|
+
try:
|
|
827
|
+
spec_tokens, spec_notes = resolve_speculative(
|
|
828
|
+
req.backend,
|
|
829
|
+
req.model_path,
|
|
830
|
+
req.draft_model_path,
|
|
831
|
+
req.mtp_mode,
|
|
832
|
+
user_tokens,
|
|
833
|
+
)
|
|
834
|
+
except ValueError as exc:
|
|
835
|
+
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
|
836
|
+
|
|
693
837
|
try:
|
|
694
838
|
cmd = _build_cmd(
|
|
695
839
|
req.backend, req.model_path, req.port,
|
|
696
840
|
req.options, req.extra_args,
|
|
697
841
|
binary=configured_binary,
|
|
842
|
+
spec_tokens=spec_tokens,
|
|
698
843
|
)
|
|
699
844
|
except ValueError as exc:
|
|
700
845
|
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
|
701
846
|
|
|
847
|
+
# Speculative flags qualify for the one-shot startup-crash fallback only
|
|
848
|
+
# when they came from AUTO detection (mtp_mode="auto", no explicit draft
|
|
849
|
+
# model, and detection actually emitted flags). Explicit draft paths and
|
|
850
|
+
# operator-supplied --spec-type are never auto-retried. The fallback cmd
|
|
851
|
+
# is rebuilt from scratch with spec_tokens=None (exact — never spliced).
|
|
852
|
+
spec_auto = (
|
|
853
|
+
req.mtp_mode == "auto"
|
|
854
|
+
and req.draft_model_path is None
|
|
855
|
+
and bool(spec_tokens)
|
|
856
|
+
)
|
|
857
|
+
fallback_cmd: list[str] | None = None
|
|
858
|
+
if spec_auto:
|
|
859
|
+
try:
|
|
860
|
+
fallback_cmd = _build_cmd(
|
|
861
|
+
req.backend, req.model_path, req.port,
|
|
862
|
+
req.options, req.extra_args,
|
|
863
|
+
binary=configured_binary,
|
|
864
|
+
spec_tokens=None,
|
|
865
|
+
)
|
|
866
|
+
except ValueError:
|
|
867
|
+
# A failure to rebuild the non-spec command simply disables the
|
|
868
|
+
# fallback; it must never block the primary start.
|
|
869
|
+
fallback_cmd = None
|
|
870
|
+
|
|
702
871
|
proc_id = uuid.uuid4().hex[:8]
|
|
703
872
|
proc = ManagedProcess(
|
|
704
873
|
id=proc_id,
|
|
@@ -708,9 +877,16 @@ async def api_start(req: StartRequest, request: Request) -> dict[str, Any]:
|
|
|
708
877
|
port=req.port,
|
|
709
878
|
options=req.options,
|
|
710
879
|
extra_args=req.extra_args,
|
|
880
|
+
draft_model_path=req.draft_model_path,
|
|
881
|
+
mtp_mode=req.mtp_mode,
|
|
711
882
|
status="starting",
|
|
883
|
+
spec_tokens=spec_tokens,
|
|
884
|
+
spec_auto=spec_auto and fallback_cmd is not None,
|
|
885
|
+
fallback_cmd=fallback_cmd,
|
|
712
886
|
)
|
|
713
887
|
proc.log_tail.append(f"[launcher] cmd: {' '.join(cmd)}")
|
|
888
|
+
for note in spec_notes:
|
|
889
|
+
proc.log_tail.append(f"[launcher] {note}")
|
|
714
890
|
|
|
715
891
|
try:
|
|
716
892
|
p = await asyncio.create_subprocess_exec(
|
|
@@ -732,6 +908,7 @@ async def api_start(req: StartRequest, request: Request) -> dict[str, Any]:
|
|
|
732
908
|
proc._proc = p
|
|
733
909
|
proc.pid = p.pid
|
|
734
910
|
proc.status = "running"
|
|
911
|
+
proc.started_at = time.monotonic()
|
|
735
912
|
proc.log_tail.append(f"[launcher] started PID {p.pid}")
|
|
736
913
|
|
|
737
914
|
_registry(request).add(proc)
|
|
@@ -767,6 +944,7 @@ async def api_start(req: StartRequest, request: Request) -> dict[str, Any]:
|
|
|
767
944
|
"pid": p.pid,
|
|
768
945
|
"command": cmd,
|
|
769
946
|
"provider_sync": provider_sync,
|
|
947
|
+
"speculative": spec_tokens,
|
|
770
948
|
}
|
|
771
949
|
|
|
772
950
|
|
|
@@ -971,6 +1149,20 @@ _LAUNCHER_HTML = r"""<!doctype html>
|
|
|
971
1149
|
<div id="profile-args" class="mt-1 text-xs font-mono text-slate-400 bg-slate-800/50 rounded p-2 hidden"></div>
|
|
972
1150
|
</div>
|
|
973
1151
|
|
|
1152
|
+
<div class="grid grid-cols-2 gap-2">
|
|
1153
|
+
<div>
|
|
1154
|
+
<label class="block text-xs text-slate-400 mb-1">MTP/draft gguf (空欄で自動検出)</label>
|
|
1155
|
+
<input id="f-draft" type="text" placeholder="companion .gguf (任意)" />
|
|
1156
|
+
</div>
|
|
1157
|
+
<div>
|
|
1158
|
+
<label class="block text-xs text-slate-400 mb-1">MTP</label>
|
|
1159
|
+
<select id="f-mtp">
|
|
1160
|
+
<option value="auto">auto</option>
|
|
1161
|
+
<option value="off">off</option>
|
|
1162
|
+
</select>
|
|
1163
|
+
</div>
|
|
1164
|
+
</div>
|
|
1165
|
+
|
|
974
1166
|
<div>
|
|
975
1167
|
<div class="flex items-center justify-between mb-1">
|
|
976
1168
|
<label class="block text-xs text-slate-400">追加オプション(自由入力)</label>
|
|
@@ -1292,6 +1484,8 @@ _LAUNCHER_HTML = r"""<!doctype html>
|
|
|
1292
1484
|
const backend = document.getElementById("f-backend").value;
|
|
1293
1485
|
const model = document.getElementById("f-model").value.trim();
|
|
1294
1486
|
const extra = document.getElementById("f-extra").value.trim();
|
|
1487
|
+
const draft = document.getElementById("f-draft").value.trim();
|
|
1488
|
+
const mtp = document.getElementById("f-mtp").value;
|
|
1295
1489
|
|
|
1296
1490
|
if (!name) { showLaunchErr("名前を入力してください"); return; }
|
|
1297
1491
|
if (!model) { showLaunchErr("モデルパスを入力してください"); return; }
|
|
@@ -1306,7 +1500,8 @@ _LAUNCHER_HTML = r"""<!doctype html>
|
|
|
1306
1500
|
method: "POST",
|
|
1307
1501
|
headers: authHeaders({"Content-Type": "application/json"}),
|
|
1308
1502
|
body: JSON.stringify({name, backend, model_path: model, port,
|
|
1309
|
-
options: selectedProfileArgs(), extra_args: extra
|
|
1503
|
+
options: selectedProfileArgs(), extra_args: extra,
|
|
1504
|
+
draft_model_path: draft || null, mtp_mode: mtp}),
|
|
1310
1505
|
});
|
|
1311
1506
|
const d = await res.json();
|
|
1312
1507
|
if (!res.ok) { showLaunchErr(d.detail || "起動失敗"); return; }
|