turbollm 0.6.2 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +38 -6
- package/dist/cli.js +819 -165
- package/dist/webdist/assets/index-B-GrbX4x.css +1 -0
- package/dist/webdist/assets/index-CCGiP_Qh.js +149 -0
- package/dist/webdist/index.html +2 -2
- package/package.json +1 -1
- package/dist/webdist/assets/index-BGb2zWKi.js +0 -127
- package/dist/webdist/assets/index-DeRFSZA-.css +0 -1
package/dist/cli.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
|
|
3
3
|
// src/cli.ts
|
|
4
|
-
import { spawn as
|
|
4
|
+
import { spawn as spawn4 } from "child_process";
|
|
5
5
|
import { openSync as openSync3, readFileSync as readFileSync8 } from "fs";
|
|
6
6
|
import { serve } from "@hono/node-server";
|
|
7
7
|
import { dirname as dirname7, join as join14 } from "path";
|
|
@@ -88,7 +88,9 @@ function defaultConfig() {
|
|
|
88
88
|
modelDefaults: { ctx: 8192, ngl: 99, imageMaxTokens: 0, maxTokens: 0 },
|
|
89
89
|
featuredOverrideUrl: "",
|
|
90
90
|
comfyui: { enabled: false, gatePath: "", url: "", reverseGate: false, cachePersist: false },
|
|
91
|
-
gateway: { autoSwap: true, keepN: 1 }
|
|
91
|
+
gateway: { autoSwap: true, keepN: 1 },
|
|
92
|
+
tools: {},
|
|
93
|
+
mcp: { servers: [] }
|
|
92
94
|
};
|
|
93
95
|
}
|
|
94
96
|
var ConfigStore = class _ConfigStore {
|
|
@@ -234,6 +236,15 @@ function normalize(c) {
|
|
|
234
236
|
autoSwap: gw.autoSwap !== false,
|
|
235
237
|
keepN: typeof gw.keepN === "number" && gw.keepN >= 1 ? Math.min(Math.floor(gw.keepN), 4) : 1
|
|
236
238
|
};
|
|
239
|
+
const tl = c.tools ?? {};
|
|
240
|
+
c.tools = {};
|
|
241
|
+
if (tl.tavily && typeof tl.tavily.apiKey === "string") {
|
|
242
|
+
c.tools.tavily = { apiKey: tl.tavily.apiKey };
|
|
243
|
+
}
|
|
244
|
+
const mc = c.mcp ?? {};
|
|
245
|
+
c.mcp = {
|
|
246
|
+
servers: Array.isArray(mc.servers) ? mc.servers.filter((s) => typeof s === "object" && s !== null && typeof s.id === "string" && typeof s.name === "string" && (s.transport === "stdio" || s.transport === "sse")) : []
|
|
247
|
+
};
|
|
237
248
|
c.telemetry.level = normalizeTelemetryLevel(c.telemetry.level);
|
|
238
249
|
for (const e of c.engines) {
|
|
239
250
|
e.capabilities ??= { kvTypes: [], flags: [] };
|
|
@@ -2597,10 +2608,10 @@ function safeJson(s) {
|
|
|
2597
2608
|
}
|
|
2598
2609
|
}
|
|
2599
2610
|
function rowToConv(r) {
|
|
2600
|
-
return { id: r.id, title: r.title, systemPrompt: r.system_prompt, modelKey: r.model_key, sampling: safeJson(r.sampling), expertMode: r.expert_mode === 1, createdAt: r.created_at, updatedAt: r.updated_at };
|
|
2611
|
+
return { id: r.id, title: r.title, systemPrompt: r.system_prompt, modelKey: r.model_key, sampling: safeJson(r.sampling), expertMode: r.expert_mode === 1, toolPolicy: r.tool_policy ?? void 0, createdAt: r.created_at, updatedAt: r.updated_at };
|
|
2601
2612
|
}
|
|
2602
2613
|
function rowToMsg(r) {
|
|
2603
|
-
return { id: r.id, convId: r.conv_id, seq: r.seq, role: r.role, content: r.content, reasoning: r.reasoning, attachments: safeJson(r.attachments), textAttachments: r.text_attachments ? safeJson(r.text_attachments) : [], stats: safeJson(r.stats), createdAt: r.created_at };
|
|
2614
|
+
return { id: r.id, convId: r.conv_id, seq: r.seq, role: r.role, content: r.content, reasoning: r.reasoning, attachments: safeJson(r.attachments), textAttachments: r.text_attachments ? safeJson(r.text_attachments) : [], toolCalls: r.tool_calls ? safeJson(r.tool_calls) : [], stats: safeJson(r.stats), createdAt: r.created_at };
|
|
2604
2615
|
}
|
|
2605
2616
|
var ConversationStore = class {
|
|
2606
2617
|
db;
|
|
@@ -2648,6 +2659,18 @@ var ConversationStore = class {
|
|
|
2648
2659
|
PRAGMA user_version = 4;
|
|
2649
2660
|
`);
|
|
2650
2661
|
}
|
|
2662
|
+
if (v < 5) {
|
|
2663
|
+
this.db.exec(`
|
|
2664
|
+
ALTER TABLE messages ADD COLUMN tool_calls TEXT;
|
|
2665
|
+
PRAGMA user_version = 5;
|
|
2666
|
+
`);
|
|
2667
|
+
}
|
|
2668
|
+
if (v < 6) {
|
|
2669
|
+
this.db.exec(`
|
|
2670
|
+
ALTER TABLE conversations ADD COLUMN tool_policy TEXT;
|
|
2671
|
+
PRAGMA user_version = 6;
|
|
2672
|
+
`);
|
|
2673
|
+
}
|
|
2651
2674
|
}
|
|
2652
2675
|
listConversations(q) {
|
|
2653
2676
|
if (q) {
|
|
@@ -2664,7 +2687,7 @@ var ConversationStore = class {
|
|
|
2664
2687
|
createConversation(partial) {
|
|
2665
2688
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
2666
2689
|
const id = randomUUID3();
|
|
2667
|
-
this.db.prepare(`INSERT INTO conversations (id,title,system_prompt,model_key,sampling,expert_mode,created_at,updated_at) VALUES ($id,$title,$sp,$mk,$samp,$expert,$now,$now)`).run({ $id: id, $title: partial?.title ?? "New chat", $sp: partial?.systemPrompt ?? "", $mk: partial?.modelKey ?? "", $samp: JSON.stringify(partial?.sampling ?? {}), $expert: partial?.expertMode ? 1 : 0, $now: now });
|
|
2690
|
+
this.db.prepare(`INSERT INTO conversations (id,title,system_prompt,model_key,sampling,expert_mode,tool_policy,created_at,updated_at) VALUES ($id,$title,$sp,$mk,$samp,$expert,$tp,$now,$now)`).run({ $id: id, $title: partial?.title ?? "New chat", $sp: partial?.systemPrompt ?? "", $mk: partial?.modelKey ?? "", $samp: JSON.stringify(partial?.sampling ?? {}), $expert: partial?.expertMode ? 1 : 0, $tp: partial?.toolPolicy ?? null, $now: now });
|
|
2668
2691
|
return this.getConversation(id);
|
|
2669
2692
|
}
|
|
2670
2693
|
getConversation(id, withMessages = false) {
|
|
@@ -2707,7 +2730,8 @@ var ConversationStore = class {
|
|
|
2707
2730
|
const row = this.db.prepare(`SELECT COALESCE(MAX(seq),0) AS ms FROM messages WHERE conv_id = $id`).get({ $id: convId });
|
|
2708
2731
|
const modelKey = role === "assistant" ? this.conversationModelKey(convId) : null;
|
|
2709
2732
|
const textAttachments = extra?.textAttachments?.length ? JSON.stringify(extra.textAttachments) : null;
|
|
2710
|
-
|
|
2733
|
+
const toolCalls = extra?.toolCalls?.length ? JSON.stringify(extra.toolCalls) : null;
|
|
2734
|
+
this.db.prepare(`INSERT INTO messages (id,conv_id,seq,role,content,reasoning,attachments,text_attachments,tool_calls,stats,model_key,created_at) VALUES ($id,$cid,$seq,$role,$content,$reasoning,$attachments,$ta,$tc,$stats,$mk,$now)`).run({ $id: id, $cid: convId, $seq: row.ms + 1, $role: role, $content: content, $reasoning: extra?.reasoning ?? "", $attachments: JSON.stringify(extra?.attachments ?? []), $ta: textAttachments, $tc: toolCalls, $stats: JSON.stringify(extra?.stats ?? {}), $mk: modelKey, $now: now });
|
|
2711
2735
|
this.touchConversation(convId);
|
|
2712
2736
|
return this.getMessage(id);
|
|
2713
2737
|
}
|
|
@@ -2731,6 +2755,10 @@ var ConversationStore = class {
|
|
|
2731
2755
|
sets.push("reasoning = $reasoning");
|
|
2732
2756
|
params.$reasoning = patch.reasoning;
|
|
2733
2757
|
}
|
|
2758
|
+
if (patch.toolCalls !== void 0) {
|
|
2759
|
+
sets.push("tool_calls = $tc");
|
|
2760
|
+
params.$tc = JSON.stringify(patch.toolCalls);
|
|
2761
|
+
}
|
|
2734
2762
|
if (patch.stats !== void 0) {
|
|
2735
2763
|
sets.push("stats = $stats");
|
|
2736
2764
|
params.$stats = JSON.stringify(patch.stats);
|
|
@@ -3859,8 +3887,497 @@ function sleep3(ms) {
|
|
|
3859
3887
|
return new Promise((r) => setTimeout(r, ms));
|
|
3860
3888
|
}
|
|
3861
3889
|
|
|
3862
|
-
// src/
|
|
3890
|
+
// src/tools/builtin.ts
|
|
3891
|
+
import { runInNewContext } from "vm";
|
|
3892
|
+
var WEB_SEARCH_TOOL = {
|
|
3893
|
+
type: "function",
|
|
3894
|
+
function: {
|
|
3895
|
+
name: "web_search",
|
|
3896
|
+
description: "Search the web for real-time information. Call this BEFORE answering any question that depends on current events, recent data, prices, specific facts, or anything your training data may not cover accurately. Formulate a precise, keyword-focused query \u2014 include names, dates, or version numbers when relevant. Run multiple focused searches rather than one broad one for complex questions.",
|
|
3897
|
+
parameters: {
|
|
3898
|
+
type: "object",
|
|
3899
|
+
properties: {
|
|
3900
|
+
query: {
|
|
3901
|
+
type: "string",
|
|
3902
|
+
description: 'A precise, specific search query. Use key terms and identifiers. Good: "Python 3.13 release date new features". Bad: "Python new stuff". Good: "NVIDIA RTX 5090 benchmark 2025". Bad: "new GPU benchmarks".'
|
|
3903
|
+
},
|
|
3904
|
+
max_results: { type: "number", description: "Number of results to return (default 5, max 10)." }
|
|
3905
|
+
},
|
|
3906
|
+
required: ["query"]
|
|
3907
|
+
}
|
|
3908
|
+
}
|
|
3909
|
+
};
|
|
3910
|
+
var FETCH_URL_TOOL = {
|
|
3911
|
+
type: "function",
|
|
3912
|
+
function: {
|
|
3913
|
+
name: "fetch_url",
|
|
3914
|
+
description: "Fetch the text content of a URL. Returns the main text of the page, stripped of HTML.",
|
|
3915
|
+
parameters: {
|
|
3916
|
+
type: "object",
|
|
3917
|
+
properties: {
|
|
3918
|
+
url: { type: "string", description: "The URL to fetch." }
|
|
3919
|
+
},
|
|
3920
|
+
required: ["url"]
|
|
3921
|
+
}
|
|
3922
|
+
}
|
|
3923
|
+
};
|
|
3924
|
+
var RUN_CODE_TOOL = {
|
|
3925
|
+
type: "function",
|
|
3926
|
+
function: {
|
|
3927
|
+
name: "run_code",
|
|
3928
|
+
description: "Execute a JavaScript snippet and return the result. Useful for calculations, data transformation, and logic. No network, file, or process access.",
|
|
3929
|
+
parameters: {
|
|
3930
|
+
type: "object",
|
|
3931
|
+
properties: {
|
|
3932
|
+
code: { type: "string", description: "JavaScript code to execute. The last expression is the return value." }
|
|
3933
|
+
},
|
|
3934
|
+
required: ["code"]
|
|
3935
|
+
}
|
|
3936
|
+
}
|
|
3937
|
+
};
|
|
3938
|
+
async function execWebSearch(args, tavilyApiKey) {
|
|
3939
|
+
const query = String(args.query ?? "");
|
|
3940
|
+
if (!query.trim()) return "Error: query is required.";
|
|
3941
|
+
const maxResults = Math.min(10, Math.max(1, Number(args.max_results ?? 5) || 5));
|
|
3942
|
+
let resp;
|
|
3943
|
+
try {
|
|
3944
|
+
resp = await fetch("https://api.tavily.com/search", {
|
|
3945
|
+
method: "POST",
|
|
3946
|
+
headers: { "Content-Type": "application/json" },
|
|
3947
|
+
body: JSON.stringify({
|
|
3948
|
+
api_key: tavilyApiKey,
|
|
3949
|
+
query,
|
|
3950
|
+
max_results: maxResults,
|
|
3951
|
+
search_depth: "advanced",
|
|
3952
|
+
include_answer: true
|
|
3953
|
+
}),
|
|
3954
|
+
signal: AbortSignal.timeout(2e4)
|
|
3955
|
+
});
|
|
3956
|
+
} catch (e) {
|
|
3957
|
+
return `Error: could not reach Tavily \u2014 ${e.message}`;
|
|
3958
|
+
}
|
|
3959
|
+
if (!resp.ok) {
|
|
3960
|
+
const text = await resp.text().catch(() => "");
|
|
3961
|
+
return `Error: Tavily returned ${resp.status}${text ? ` \u2014 ${text.slice(0, 200)}` : ""}`;
|
|
3962
|
+
}
|
|
3963
|
+
const data = await resp.json();
|
|
3964
|
+
const results = data.results ?? [];
|
|
3965
|
+
if (results.length === 0) return "No results found.";
|
|
3966
|
+
const lines = [];
|
|
3967
|
+
if (data.answer) lines.push(`ANSWER: ${data.answer}
|
|
3968
|
+
`);
|
|
3969
|
+
lines.push(`SOURCES (${results.length} results for "${query}"):`);
|
|
3970
|
+
for (const [i, r] of results.entries()) {
|
|
3971
|
+
lines.push(`
|
|
3972
|
+
[${i + 1}] ${r.title}`);
|
|
3973
|
+
lines.push(`Source: ${r.url}`);
|
|
3974
|
+
lines.push(r.content.slice(0, 700));
|
|
3975
|
+
}
|
|
3976
|
+
return lines.join("\n").trim();
|
|
3977
|
+
}
|
|
3978
|
+
async function execFetchUrl(args) {
|
|
3979
|
+
const url = String(args.url ?? "").trim();
|
|
3980
|
+
if (!url) return "Error: url is required.";
|
|
3981
|
+
if (!/^https?:\/\//i.test(url)) return "Error: URL must start with http:// or https://";
|
|
3982
|
+
let resp;
|
|
3983
|
+
try {
|
|
3984
|
+
resp = await fetch(url, {
|
|
3985
|
+
headers: { "User-Agent": "TurboLLM/0.7 (tool-fetch)" },
|
|
3986
|
+
signal: AbortSignal.timeout(15e3)
|
|
3987
|
+
});
|
|
3988
|
+
} catch (e) {
|
|
3989
|
+
return `Error: could not fetch URL \u2014 ${e.message}`;
|
|
3990
|
+
}
|
|
3991
|
+
const contentType2 = resp.headers.get("content-type") ?? "";
|
|
3992
|
+
const text = await resp.text().catch(() => "");
|
|
3993
|
+
let content;
|
|
3994
|
+
if (contentType2.includes("text/html")) {
|
|
3995
|
+
content = text.replace(/<script[\s\S]*?<\/script>/gi, "").replace(/<style[\s\S]*?<\/style>/gi, "").replace(/<[^>]+>/g, " ").replace(/ /g, " ").replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">").replace(/"/g, '"').replace(/\s{2,}/g, " ").trim();
|
|
3996
|
+
} else {
|
|
3997
|
+
content = text.trim();
|
|
3998
|
+
}
|
|
3999
|
+
if (content.length > 4e3) content = content.slice(0, 4e3) + "\n[truncated]";
|
|
4000
|
+
return content || "(empty response)";
|
|
4001
|
+
}
|
|
4002
|
+
function execRunCode(args) {
|
|
4003
|
+
const code = String(args.code ?? "").trim();
|
|
4004
|
+
if (!code) return "Error: code is required.";
|
|
4005
|
+
const output = [];
|
|
4006
|
+
const sandbox = {
|
|
4007
|
+
console: {
|
|
4008
|
+
log: (...a) => output.push(a.map(String).join(" ")),
|
|
4009
|
+
error: (...a) => output.push("ERROR: " + a.map(String).join(" ")),
|
|
4010
|
+
warn: (...a) => output.push("WARN: " + a.map(String).join(" "))
|
|
4011
|
+
},
|
|
4012
|
+
Math,
|
|
4013
|
+
JSON,
|
|
4014
|
+
Array,
|
|
4015
|
+
Object,
|
|
4016
|
+
String,
|
|
4017
|
+
Number,
|
|
4018
|
+
Boolean,
|
|
4019
|
+
Date,
|
|
4020
|
+
RegExp,
|
|
4021
|
+
parseInt,
|
|
4022
|
+
parseFloat,
|
|
4023
|
+
isNaN,
|
|
4024
|
+
isFinite,
|
|
4025
|
+
encodeURIComponent,
|
|
4026
|
+
decodeURIComponent
|
|
4027
|
+
};
|
|
4028
|
+
let result;
|
|
4029
|
+
try {
|
|
4030
|
+
result = runInNewContext(`(function(){${code}})()`, sandbox, { timeout: 5e3 });
|
|
4031
|
+
} catch (e) {
|
|
4032
|
+
return `Error: ${e.message}`;
|
|
4033
|
+
}
|
|
4034
|
+
const parts = [];
|
|
4035
|
+
if (output.length > 0) parts.push(output.join("\n"));
|
|
4036
|
+
if (result !== void 0) {
|
|
4037
|
+
try {
|
|
4038
|
+
parts.push(typeof result === "string" ? result : JSON.stringify(result, null, 2));
|
|
4039
|
+
} catch {
|
|
4040
|
+
parts.push(String(result));
|
|
4041
|
+
}
|
|
4042
|
+
}
|
|
4043
|
+
return parts.join("\n") || "(no output)";
|
|
4044
|
+
}
|
|
4045
|
+
|
|
4046
|
+
// src/tools/mcp-client.ts
|
|
3863
4047
|
import { spawn as spawn2 } from "child_process";
|
|
4048
|
+
import { EventEmitter } from "events";
|
|
4049
|
+
var StdioMcpClient = class {
|
|
4050
|
+
serverId;
|
|
4051
|
+
serverName;
|
|
4052
|
+
command;
|
|
4053
|
+
args;
|
|
4054
|
+
env;
|
|
4055
|
+
proc = null;
|
|
4056
|
+
pending = /* @__PURE__ */ new Map();
|
|
4057
|
+
buf = "";
|
|
4058
|
+
msgId = 0;
|
|
4059
|
+
initialized = false;
|
|
4060
|
+
constructor(opts) {
|
|
4061
|
+
this.serverId = opts.id;
|
|
4062
|
+
this.serverName = opts.name;
|
|
4063
|
+
this.command = opts.command;
|
|
4064
|
+
this.args = opts.args ?? [];
|
|
4065
|
+
this.env = opts.env ?? {};
|
|
4066
|
+
}
|
|
4067
|
+
async connect() {
|
|
4068
|
+
if (this.proc) return;
|
|
4069
|
+
this.proc = spawn2(this.command, this.args, {
|
|
4070
|
+
env: { ...process.env, ...this.env },
|
|
4071
|
+
stdio: ["pipe", "pipe", "pipe"]
|
|
4072
|
+
});
|
|
4073
|
+
this.proc.stdout?.on("data", (chunk) => {
|
|
4074
|
+
this.buf += chunk.toString("utf8");
|
|
4075
|
+
const lines = this.buf.split("\n");
|
|
4076
|
+
this.buf = lines.pop() ?? "";
|
|
4077
|
+
for (const line of lines) {
|
|
4078
|
+
const trimmed = line.trim();
|
|
4079
|
+
if (!trimmed) continue;
|
|
4080
|
+
try {
|
|
4081
|
+
const msg = JSON.parse(trimmed);
|
|
4082
|
+
if ("id" in msg && msg.id != null) {
|
|
4083
|
+
const pending = this.pending.get(msg.id);
|
|
4084
|
+
if (pending) {
|
|
4085
|
+
this.pending.delete(msg.id);
|
|
4086
|
+
pending.resolve(msg);
|
|
4087
|
+
}
|
|
4088
|
+
}
|
|
4089
|
+
} catch {
|
|
4090
|
+
}
|
|
4091
|
+
}
|
|
4092
|
+
});
|
|
4093
|
+
this.proc.on("error", () => this.cleanup());
|
|
4094
|
+
this.proc.on("exit", () => this.cleanup());
|
|
4095
|
+
await this.sendRequest("initialize", {
|
|
4096
|
+
protocolVersion: "2024-11-05",
|
|
4097
|
+
capabilities: { tools: {} },
|
|
4098
|
+
clientInfo: { name: "turbollm", version: "0.7.0" }
|
|
4099
|
+
});
|
|
4100
|
+
await this.sendNotification("notifications/initialized", {});
|
|
4101
|
+
this.initialized = true;
|
|
4102
|
+
}
|
|
4103
|
+
disconnect() {
|
|
4104
|
+
this.cleanup();
|
|
4105
|
+
}
|
|
4106
|
+
async listTools() {
|
|
4107
|
+
if (!this.initialized) await this.connect();
|
|
4108
|
+
const result = await this.sendRequest("tools/list", {});
|
|
4109
|
+
return result.tools ?? [];
|
|
4110
|
+
}
|
|
4111
|
+
async callTool(name, args) {
|
|
4112
|
+
if (!this.initialized) await this.connect();
|
|
4113
|
+
const result = await this.sendRequest("tools/call", { name, arguments: args });
|
|
4114
|
+
const text = (result.content ?? []).filter((c) => c.type === "text" && c.text).map((c) => c.text).join("\n");
|
|
4115
|
+
return result.isError ? `Error: ${text}` : text || "(no output)";
|
|
4116
|
+
}
|
|
4117
|
+
sendRequest(method, params) {
|
|
4118
|
+
const id = ++this.msgId;
|
|
4119
|
+
const req = { jsonrpc: "2.0", id, method, params };
|
|
4120
|
+
return new Promise((resolve2, reject) => {
|
|
4121
|
+
const timer = setTimeout(() => {
|
|
4122
|
+
this.pending.delete(id);
|
|
4123
|
+
reject(new Error(`MCP request timeout: ${method}`));
|
|
4124
|
+
}, 3e4);
|
|
4125
|
+
this.pending.set(id, {
|
|
4126
|
+
resolve: (r) => {
|
|
4127
|
+
clearTimeout(timer);
|
|
4128
|
+
if (r.error) reject(new Error(r.error.message));
|
|
4129
|
+
else resolve2(r.result);
|
|
4130
|
+
}
|
|
4131
|
+
});
|
|
4132
|
+
this.write(req);
|
|
4133
|
+
});
|
|
4134
|
+
}
|
|
4135
|
+
sendNotification(method, params) {
|
|
4136
|
+
const msg = { jsonrpc: "2.0", method, params };
|
|
4137
|
+
this.write(msg);
|
|
4138
|
+
}
|
|
4139
|
+
write(msg) {
|
|
4140
|
+
if (!this.proc?.stdin?.writable) return;
|
|
4141
|
+
try {
|
|
4142
|
+
this.proc.stdin.write(JSON.stringify(msg) + "\n");
|
|
4143
|
+
} catch {
|
|
4144
|
+
}
|
|
4145
|
+
}
|
|
4146
|
+
cleanup() {
|
|
4147
|
+
for (const p of this.pending.values()) {
|
|
4148
|
+
p.resolve({ jsonrpc: "2.0", id: -1, error: { code: -1, message: "MCP server disconnected" } });
|
|
4149
|
+
}
|
|
4150
|
+
this.pending.clear();
|
|
4151
|
+
try {
|
|
4152
|
+
this.proc?.kill();
|
|
4153
|
+
} catch {
|
|
4154
|
+
}
|
|
4155
|
+
this.proc = null;
|
|
4156
|
+
this.initialized = false;
|
|
4157
|
+
}
|
|
4158
|
+
};
|
|
4159
|
+
var SseMcpClient = class {
|
|
4160
|
+
serverId;
|
|
4161
|
+
serverName;
|
|
4162
|
+
baseUrl;
|
|
4163
|
+
msgId = 0;
|
|
4164
|
+
sseEndpoint = "";
|
|
4165
|
+
sseAbort = null;
|
|
4166
|
+
emitter = new EventEmitter();
|
|
4167
|
+
initialized = false;
|
|
4168
|
+
constructor(opts) {
|
|
4169
|
+
this.serverId = opts.id;
|
|
4170
|
+
this.serverName = opts.name;
|
|
4171
|
+
this.baseUrl = opts.url.replace(/\/$/, "");
|
|
4172
|
+
}
|
|
4173
|
+
async connect() {
|
|
4174
|
+
if (this.initialized) return;
|
|
4175
|
+
this.sseAbort = new AbortController();
|
|
4176
|
+
const endpoint = await new Promise((resolve2, reject) => {
|
|
4177
|
+
const timer = setTimeout(() => reject(new Error("SSE connect timeout")), 15e3);
|
|
4178
|
+
fetch(`${this.baseUrl}/sse`, { signal: this.sseAbort.signal }).then(async (resp) => {
|
|
4179
|
+
if (!resp.ok || !resp.body) {
|
|
4180
|
+
clearTimeout(timer);
|
|
4181
|
+
reject(new Error(`SSE connect failed: ${resp.status}`));
|
|
4182
|
+
return;
|
|
4183
|
+
}
|
|
4184
|
+
const reader = resp.body.getReader();
|
|
4185
|
+
const decoder = new TextDecoder();
|
|
4186
|
+
let buf = "";
|
|
4187
|
+
let resolved = false;
|
|
4188
|
+
while (true) {
|
|
4189
|
+
const { done, value } = await reader.read();
|
|
4190
|
+
if (done) break;
|
|
4191
|
+
buf += decoder.decode(value, { stream: true });
|
|
4192
|
+
const lines = buf.split("\n");
|
|
4193
|
+
buf = lines.pop() ?? "";
|
|
4194
|
+
for (const line of lines) {
|
|
4195
|
+
if (line.startsWith("data:")) {
|
|
4196
|
+
const data = line.slice(5).trim();
|
|
4197
|
+
if (!resolved) {
|
|
4198
|
+
resolved = true;
|
|
4199
|
+
clearTimeout(timer);
|
|
4200
|
+
resolve2(data);
|
|
4201
|
+
}
|
|
4202
|
+
try {
|
|
4203
|
+
const msg = JSON.parse(data);
|
|
4204
|
+
if (msg.id != null) this.emitter.emit(`rpc:${msg.id}`, msg);
|
|
4205
|
+
} catch {
|
|
4206
|
+
}
|
|
4207
|
+
}
|
|
4208
|
+
}
|
|
4209
|
+
}
|
|
4210
|
+
}).catch((e) => {
|
|
4211
|
+
clearTimeout(timer);
|
|
4212
|
+
if (!this.sseAbort?.signal.aborted) reject(e);
|
|
4213
|
+
});
|
|
4214
|
+
});
|
|
4215
|
+
this.sseEndpoint = endpoint.startsWith("http") ? endpoint : `${this.baseUrl}${endpoint}`;
|
|
4216
|
+
await this.sendRequest("initialize", {
|
|
4217
|
+
protocolVersion: "2024-11-05",
|
|
4218
|
+
capabilities: { tools: {} },
|
|
4219
|
+
clientInfo: { name: "turbollm", version: "0.7.0" }
|
|
4220
|
+
});
|
|
4221
|
+
await this.post({ jsonrpc: "2.0", method: "notifications/initialized", params: {} });
|
|
4222
|
+
this.initialized = true;
|
|
4223
|
+
}
|
|
4224
|
+
disconnect() {
|
|
4225
|
+
this.sseAbort?.abort();
|
|
4226
|
+
this.sseAbort = null;
|
|
4227
|
+
this.initialized = false;
|
|
4228
|
+
}
|
|
4229
|
+
async listTools() {
|
|
4230
|
+
if (!this.initialized) await this.connect();
|
|
4231
|
+
const result = await this.sendRequest("tools/list", {});
|
|
4232
|
+
return result.tools ?? [];
|
|
4233
|
+
}
|
|
4234
|
+
async callTool(name, args) {
|
|
4235
|
+
if (!this.initialized) await this.connect();
|
|
4236
|
+
const result = await this.sendRequest("tools/call", { name, arguments: args });
|
|
4237
|
+
const text = (result.content ?? []).filter((c) => c.type === "text" && c.text).map((c) => c.text).join("\n");
|
|
4238
|
+
return result.isError ? `Error: ${text}` : text || "(no output)";
|
|
4239
|
+
}
|
|
4240
|
+
async sendRequest(method, params) {
|
|
4241
|
+
const id = ++this.msgId;
|
|
4242
|
+
const req = { jsonrpc: "2.0", id, method, params };
|
|
4243
|
+
return new Promise((resolve2, reject) => {
|
|
4244
|
+
const timer = setTimeout(() => {
|
|
4245
|
+
this.emitter.removeAllListeners(`rpc:${id}`);
|
|
4246
|
+
reject(new Error(`MCP SSE request timeout: ${method}`));
|
|
4247
|
+
}, 3e4);
|
|
4248
|
+
this.emitter.once(`rpc:${id}`, (msg) => {
|
|
4249
|
+
clearTimeout(timer);
|
|
4250
|
+
if (msg.error) reject(new Error(msg.error.message));
|
|
4251
|
+
else resolve2(msg.result);
|
|
4252
|
+
});
|
|
4253
|
+
this.post(req).catch((e) => {
|
|
4254
|
+
clearTimeout(timer);
|
|
4255
|
+
reject(e);
|
|
4256
|
+
});
|
|
4257
|
+
});
|
|
4258
|
+
}
|
|
4259
|
+
async post(msg) {
|
|
4260
|
+
const url = this.sseEndpoint || `${this.baseUrl}/messages`;
|
|
4261
|
+
await fetch(url, {
|
|
4262
|
+
method: "POST",
|
|
4263
|
+
headers: { "Content-Type": "application/json" },
|
|
4264
|
+
body: JSON.stringify(msg),
|
|
4265
|
+
signal: AbortSignal.timeout(1e4)
|
|
4266
|
+
});
|
|
4267
|
+
}
|
|
4268
|
+
};
|
|
4269
|
+
function createMcpClient(server2) {
|
|
4270
|
+
if (server2.transport === "sse") {
|
|
4271
|
+
if (!server2.url) throw new Error(`MCP server "${server2.name}" has transport=sse but no url`);
|
|
4272
|
+
return new SseMcpClient({ id: server2.id, name: server2.name, url: server2.url });
|
|
4273
|
+
}
|
|
4274
|
+
if (!server2.command) throw new Error(`MCP server "${server2.name}" has transport=stdio but no command`);
|
|
4275
|
+
return new StdioMcpClient({
|
|
4276
|
+
id: server2.id,
|
|
4277
|
+
name: server2.name,
|
|
4278
|
+
command: server2.command,
|
|
4279
|
+
args: server2.args,
|
|
4280
|
+
env: server2.env
|
|
4281
|
+
});
|
|
4282
|
+
}
|
|
4283
|
+
|
|
4284
|
+
// src/tools/tool-registry.ts
|
|
4285
|
+
var ToolRegistry = class {
|
|
4286
|
+
toolsCfg;
|
|
4287
|
+
mcpClients = /* @__PURE__ */ new Map();
|
|
4288
|
+
constructor(toolsCfg) {
|
|
4289
|
+
this.toolsCfg = toolsCfg;
|
|
4290
|
+
}
|
|
4291
|
+
/** Update config (called on settings change without restart). */
|
|
4292
|
+
updateConfig(toolsCfg) {
|
|
4293
|
+
this.toolsCfg = toolsCfg;
|
|
4294
|
+
}
|
|
4295
|
+
/** Connect/disconnect MCP servers to match the current config list. */
|
|
4296
|
+
async syncMcpServers(servers) {
|
|
4297
|
+
const enabledIds = new Set(servers.filter((s) => s.enabled).map((s) => s.id));
|
|
4298
|
+
for (const [id, client] of this.mcpClients) {
|
|
4299
|
+
if (!enabledIds.has(id)) {
|
|
4300
|
+
client.disconnect();
|
|
4301
|
+
this.mcpClients.delete(id);
|
|
4302
|
+
}
|
|
4303
|
+
}
|
|
4304
|
+
for (const srv of servers) {
|
|
4305
|
+
if (!srv.enabled || this.mcpClients.has(srv.id)) continue;
|
|
4306
|
+
try {
|
|
4307
|
+
const client = createMcpClient(srv);
|
|
4308
|
+
await client.connect();
|
|
4309
|
+
this.mcpClients.set(srv.id, client);
|
|
4310
|
+
} catch {
|
|
4311
|
+
}
|
|
4312
|
+
}
|
|
4313
|
+
}
|
|
4314
|
+
disconnectAll() {
|
|
4315
|
+
for (const client of this.mcpClients.values()) client.disconnect();
|
|
4316
|
+
this.mcpClients.clear();
|
|
4317
|
+
}
|
|
4318
|
+
/** Build the full tools array to send to the engine. Returns [] when no tools are available. */
|
|
4319
|
+
async buildToolDefinitions() {
|
|
4320
|
+
const defs = [];
|
|
4321
|
+
if (this.toolsCfg.tavily?.apiKey) defs.push(WEB_SEARCH_TOOL);
|
|
4322
|
+
defs.push(FETCH_URL_TOOL);
|
|
4323
|
+
defs.push(RUN_CODE_TOOL);
|
|
4324
|
+
for (const client of this.mcpClients.values()) {
|
|
4325
|
+
try {
|
|
4326
|
+
const mcpTools = await client.listTools();
|
|
4327
|
+
for (const t of mcpTools) {
|
|
4328
|
+
defs.push({
|
|
4329
|
+
type: "function",
|
|
4330
|
+
function: {
|
|
4331
|
+
name: `mcp__${client.serverId.replace(/-/g, "_")}__${t.name}`,
|
|
4332
|
+
description: `[${client.serverName}] ${t.description ?? ""}`,
|
|
4333
|
+
parameters: t.inputSchema ?? { type: "object", properties: {} }
|
|
4334
|
+
}
|
|
4335
|
+
});
|
|
4336
|
+
}
|
|
4337
|
+
} catch {
|
|
4338
|
+
}
|
|
4339
|
+
}
|
|
4340
|
+
return defs;
|
|
4341
|
+
}
|
|
4342
|
+
/** Execute a single tool call. Returns the result string. */
|
|
4343
|
+
async executeTool(call) {
|
|
4344
|
+
const name = call.name;
|
|
4345
|
+
const args = call.args;
|
|
4346
|
+
if (name === "web_search") {
|
|
4347
|
+
const key = this.toolsCfg.tavily?.apiKey;
|
|
4348
|
+
if (!key) return "Error: Tavily API key not configured. Add it in Settings \u2192 Tools.";
|
|
4349
|
+
return execWebSearch(args, key);
|
|
4350
|
+
}
|
|
4351
|
+
if (name === "fetch_url") return execFetchUrl(args);
|
|
4352
|
+
if (name === "run_code") return execRunCode(args);
|
|
4353
|
+
const mcpMatch = name.match(/^mcp__([^_]+(?:_[^_]+)*)__(.+)$/);
|
|
4354
|
+
if (mcpMatch) {
|
|
4355
|
+
const rawServerId = mcpMatch[1].replace(/_/g, "-");
|
|
4356
|
+
const toolName = mcpMatch[2];
|
|
4357
|
+
let client = this.mcpClients.get(rawServerId);
|
|
4358
|
+
if (!client) {
|
|
4359
|
+
for (const [id, c] of this.mcpClients) {
|
|
4360
|
+
if (id.replace(/-/g, "_") === mcpMatch[1]) {
|
|
4361
|
+
client = c;
|
|
4362
|
+
break;
|
|
4363
|
+
}
|
|
4364
|
+
}
|
|
4365
|
+
}
|
|
4366
|
+
if (!client) return `Error: MCP server "${rawServerId}" not connected.`;
|
|
4367
|
+
return client.callTool(toolName, args);
|
|
4368
|
+
}
|
|
4369
|
+
return `Error: unknown tool "${name}"`;
|
|
4370
|
+
}
|
|
4371
|
+
/** Whether any tools are currently available (determines if tools should be sent to engine). */
|
|
4372
|
+
hasTools() {
|
|
4373
|
+
if (this.toolsCfg.tavily?.apiKey) return true;
|
|
4374
|
+
if (this.mcpClients.size > 0) return true;
|
|
4375
|
+
return true;
|
|
4376
|
+
}
|
|
4377
|
+
};
|
|
4378
|
+
|
|
4379
|
+
// src/cli-launch.ts
|
|
4380
|
+
import { spawn as spawn3 } from "child_process";
|
|
3864
4381
|
var SUPPORTED = {
|
|
3865
4382
|
claude: { bin: "claude", label: "Claude Code", install: "npm install -g @anthropic-ai/claude-code" }
|
|
3866
4383
|
};
|
|
@@ -3897,7 +4414,7 @@ Open ${base2} \u2192 Models \u2192 Load a model, then run this again.
|
|
|
3897
4414
|
const model = status.model.name;
|
|
3898
4415
|
process.stdout.write(`\u25B8 Launching ${spec.label} \u2192 TurboLLM (model: ${model}, ${base2})
|
|
3899
4416
|
`);
|
|
3900
|
-
const child =
|
|
4417
|
+
const child = spawn3(spec.bin, passthrough, {
|
|
3901
4418
|
stdio: "inherit",
|
|
3902
4419
|
// On Windows the CLI is usually a `.cmd`/`.ps1` shim; a shell resolves it via PATHEXT.
|
|
3903
4420
|
shell: process.platform === "win32",
|
|
@@ -4781,7 +5298,14 @@ function registerApi(app2, d) {
|
|
|
4781
5298
|
if (b.autoLoadOnStart !== void 0) cfg2.autoLoadOnStart = !!b.autoLoadOnStart;
|
|
4782
5299
|
if (telemetryLevel !== void 0) cfg2.telemetry.level = telemetryLevel;
|
|
4783
5300
|
if (b.hfToken !== void 0) cfg2.hf.token = String(b.hfToken).trim();
|
|
5301
|
+
if (b.tavilyApiKey !== void 0) {
|
|
5302
|
+
const key = String(b.tavilyApiKey).trim();
|
|
5303
|
+
cfg2.tools.tavily = key ? { apiKey: key } : void 0;
|
|
5304
|
+
}
|
|
4784
5305
|
});
|
|
5306
|
+
if (b.tavilyApiKey !== void 0) {
|
|
5307
|
+
d.tools?.updateConfig(d.store.snapshot().tools);
|
|
5308
|
+
}
|
|
4785
5309
|
const after = d.store.snapshot().daemon;
|
|
4786
5310
|
const lanChanged = after.lanBind !== before.lanBind;
|
|
4787
5311
|
const portChanged = after.port !== before.port;
|
|
@@ -4793,6 +5317,59 @@ function registerApi(app2, d) {
|
|
|
4793
5317
|
}
|
|
4794
5318
|
return c.json({ ...settingsPayload(d), rebind });
|
|
4795
5319
|
});
|
|
5320
|
+
app2.post("/api/v1/mcp/servers", async (c) => {
|
|
5321
|
+
const b = await body(c);
|
|
5322
|
+
const transport = b.transport === "sse" ? "sse" : "stdio";
|
|
5323
|
+
if (!b.name?.trim()) return err(c, 400, "invalid_config_value", "name is required.");
|
|
5324
|
+
if (transport === "stdio" && !b.command?.trim()) return err(c, 400, "invalid_config_value", "command is required for stdio transport.");
|
|
5325
|
+
if (transport === "sse" && !b.url?.trim()) return err(c, 400, "invalid_config_value", "url is required for sse transport.");
|
|
5326
|
+
if (transport === "sse" && !/^https?:\/\//i.test(b.url ?? "")) return err(c, 400, "invalid_config_value", "url must be an http(s):// address.");
|
|
5327
|
+
const { randomUUID: randomUUID8 } = await import("crypto");
|
|
5328
|
+
const server2 = {
|
|
5329
|
+
id: randomUUID8(),
|
|
5330
|
+
name: b.name.trim(),
|
|
5331
|
+
transport,
|
|
5332
|
+
enabled: b.enabled !== false,
|
|
5333
|
+
...transport === "stdio" ? { command: b.command.trim(), args: b.args ?? [], env: b.env ?? {} } : { url: b.url.trim() }
|
|
5334
|
+
};
|
|
5335
|
+
d.store.update((cfg2) => {
|
|
5336
|
+
cfg2.mcp.servers.push(server2);
|
|
5337
|
+
});
|
|
5338
|
+
if (server2.enabled) await d.tools?.syncMcpServers(d.store.snapshot().mcp.servers).catch(() => {
|
|
5339
|
+
});
|
|
5340
|
+
return c.json(server2, 201);
|
|
5341
|
+
});
|
|
5342
|
+
app2.put("/api/v1/mcp/servers/:id", async (c) => {
|
|
5343
|
+
const id = c.req.param("id");
|
|
5344
|
+
const b = await body(c);
|
|
5345
|
+
const cfg2 = d.store.snapshot();
|
|
5346
|
+
if (!cfg2.mcp.servers.some((s) => s.id === id)) return err(c, 404, "not_found", "MCP server not found.");
|
|
5347
|
+
if (b.transport && b.transport !== "stdio" && b.transport !== "sse") return err(c, 400, "invalid_config_value", "transport must be stdio or sse.");
|
|
5348
|
+
if (b.url && !/^https?:\/\//i.test(b.url)) return err(c, 400, "invalid_config_value", "url must be an http(s):// address.");
|
|
5349
|
+
d.store.update((c2) => {
|
|
5350
|
+
const s = c2.mcp.servers.find((x) => x.id === id);
|
|
5351
|
+
if (b.name !== void 0) s.name = b.name.trim();
|
|
5352
|
+
if (b.transport !== void 0) s.transport = b.transport;
|
|
5353
|
+
if (b.enabled !== void 0) s.enabled = !!b.enabled;
|
|
5354
|
+
if (b.command !== void 0) s.command = b.command;
|
|
5355
|
+
if (b.args !== void 0) s.args = b.args;
|
|
5356
|
+
if (b.env !== void 0) s.env = b.env;
|
|
5357
|
+
if (b.url !== void 0) s.url = b.url;
|
|
5358
|
+
});
|
|
5359
|
+
await d.tools?.syncMcpServers(d.store.snapshot().mcp.servers).catch(() => {
|
|
5360
|
+
});
|
|
5361
|
+
return c.json(d.store.snapshot().mcp.servers.find((s) => s.id === id));
|
|
5362
|
+
});
|
|
5363
|
+
app2.delete("/api/v1/mcp/servers/:id", (c) => {
|
|
5364
|
+
const id = c.req.param("id");
|
|
5365
|
+
const cfg2 = d.store.snapshot();
|
|
5366
|
+
if (!cfg2.mcp.servers.some((s) => s.id === id)) return err(c, 404, "not_found", "MCP server not found.");
|
|
5367
|
+
d.store.update((c2) => {
|
|
5368
|
+
c2.mcp.servers = c2.mcp.servers.filter((s) => s.id !== id);
|
|
5369
|
+
});
|
|
5370
|
+
void d.tools?.syncMcpServers(d.store.snapshot().mcp.servers);
|
|
5371
|
+
return c.json({ ok: true });
|
|
5372
|
+
});
|
|
4796
5373
|
app2.get("/api/v1/telemetry/preview", (c) => {
|
|
4797
5374
|
const raw = (c.req.query("level") ?? "").trim();
|
|
4798
5375
|
const level = ["off", "anon", "full"].includes(raw) ? raw : "off";
|
|
@@ -5045,7 +5622,10 @@ function settingsPayload(d) {
|
|
|
5045
5622
|
gateway: cfg2.gateway,
|
|
5046
5623
|
// The HF token is write-only over the wire (spec 10 §4): we never echo it back,
|
|
5047
5624
|
// only whether one is set, so the UI can show "configured" without leaking it.
|
|
5048
|
-
hfTokenSet: cfg2.hf.token.length > 0
|
|
5625
|
+
hfTokenSet: cfg2.hf.token.length > 0,
|
|
5626
|
+
// Tavily API key is write-only: expose only whether it is set.
|
|
5627
|
+
tavilyKeySet: !!cfg2.tools.tavily?.apiKey,
|
|
5628
|
+
mcp: cfg2.mcp
|
|
5049
5629
|
};
|
|
5050
5630
|
}
|
|
5051
5631
|
function telemetryPreview(level, version2) {
|
|
@@ -5326,7 +5906,7 @@ function registerChatRoutes(app2, d) {
|
|
|
5326
5906
|
});
|
|
5327
5907
|
app2.post("/api/v1/conversations", async (c) => {
|
|
5328
5908
|
const b = await body2(c);
|
|
5329
|
-
const conv = db2.createConversation({ title: b.title, systemPrompt: b.systemPrompt, modelKey: b.modelKey });
|
|
5909
|
+
const conv = db2.createConversation({ title: b.title, systemPrompt: b.systemPrompt, modelKey: b.modelKey, toolPolicy: b.toolPolicy });
|
|
5330
5910
|
return c.json(conv, 201);
|
|
5331
5911
|
});
|
|
5332
5912
|
app2.post("/api/v1/conversations/expert", async (c) => {
|
|
@@ -5466,7 +6046,7 @@ ${content}` : b.docContext : content;
|
|
|
5466
6046
|
}
|
|
5467
6047
|
async function runGeneration(d, stream, ctx) {
|
|
5468
6048
|
const { db: db2 } = d;
|
|
5469
|
-
const { convId, conv,
|
|
6049
|
+
const { convId, conv, assistantMsg, ms, target, ac, disableThinking } = ctx;
|
|
5470
6050
|
const convS = conv.sampling ?? {};
|
|
5471
6051
|
const SAMPLING_KEYS = {
|
|
5472
6052
|
temp: "temperature",
|
|
@@ -5484,179 +6064,247 @@ async function runGeneration(d, stream, ctx) {
|
|
|
5484
6064
|
for (const [k, v] of Object.entries(convS)) {
|
|
5485
6065
|
if (!(k in SAMPLING_KEYS) && k !== "stop") samplingOverride[k] = v;
|
|
5486
6066
|
}
|
|
5487
|
-
const reqBody = {
|
|
5488
|
-
// mlx-lm / vLLM serve under a fixed alias and 404 on TurboLLM's internal key;
|
|
5489
|
-
// llama.cpp ignores the field. engineModelAlias() returns the right value per kind.
|
|
5490
|
-
model: engineModelAlias(d.registry.active()?.kind ?? "") ?? ms.model.key,
|
|
5491
|
-
messages: engineMessages,
|
|
5492
|
-
stream: true,
|
|
5493
|
-
stream_options: { include_usage: true },
|
|
5494
|
-
return_progress: true,
|
|
5495
|
-
...samplingOverride
|
|
5496
|
-
};
|
|
5497
6067
|
const stopStrings = convS.stop;
|
|
5498
|
-
if (stopStrings?.length) reqBody.stop = stopStrings;
|
|
5499
6068
|
const maxLimit = d.store.snapshot().modelDefaults.maxTokens ?? 0;
|
|
5500
|
-
const
|
|
5501
|
-
|
|
5502
|
-
|
|
5503
|
-
|
|
5504
|
-
|
|
5505
|
-
|
|
5506
|
-
}
|
|
6069
|
+
const iterMessages = ctx.engineMessages.map((m) => ({ role: m.role, content: m.content }));
|
|
6070
|
+
const MAX_TOOL_ITER = 10;
|
|
6071
|
+
let toolIter = 0;
|
|
6072
|
+
let fullContent = "";
|
|
6073
|
+
let fullReasoning = "";
|
|
6074
|
+
const allToolCalls = [];
|
|
5507
6075
|
const requestStart = Date.now();
|
|
5508
6076
|
let ttftMs = 0;
|
|
5509
|
-
let firstDelta = false;
|
|
5510
6077
|
let thinkStart = 0;
|
|
5511
6078
|
let thinkEnd = 0;
|
|
5512
|
-
let fullContent = "";
|
|
5513
|
-
let fullReasoning = "";
|
|
5514
|
-
let inThink = false;
|
|
5515
|
-
let pendingThinkBuf = "";
|
|
5516
6079
|
let finalUsage = {};
|
|
5517
6080
|
let finalTimings = {};
|
|
5518
6081
|
let aborted = false;
|
|
5519
6082
|
let liveOut = 0;
|
|
5520
6083
|
d.manager.generationStart();
|
|
5521
6084
|
try {
|
|
5522
|
-
const
|
|
5523
|
-
|
|
5524
|
-
|
|
5525
|
-
|
|
5526
|
-
|
|
5527
|
-
|
|
5528
|
-
|
|
5529
|
-
|
|
5530
|
-
|
|
5531
|
-
|
|
5532
|
-
|
|
5533
|
-
|
|
5534
|
-
|
|
5535
|
-
|
|
5536
|
-
|
|
5537
|
-
|
|
5538
|
-
|
|
5539
|
-
|
|
5540
|
-
|
|
5541
|
-
|
|
5542
|
-
|
|
5543
|
-
|
|
5544
|
-
|
|
5545
|
-
|
|
5546
|
-
|
|
5547
|
-
|
|
5548
|
-
|
|
5549
|
-
|
|
5550
|
-
|
|
5551
|
-
|
|
5552
|
-
|
|
5553
|
-
|
|
5554
|
-
|
|
5555
|
-
|
|
5556
|
-
|
|
5557
|
-
|
|
5558
|
-
|
|
5559
|
-
|
|
5560
|
-
|
|
5561
|
-
|
|
5562
|
-
|
|
5563
|
-
|
|
5564
|
-
|
|
5565
|
-
|
|
5566
|
-
|
|
5567
|
-
|
|
5568
|
-
|
|
5569
|
-
|
|
5570
|
-
|
|
5571
|
-
|
|
5572
|
-
|
|
5573
|
-
|
|
5574
|
-
|
|
5575
|
-
|
|
5576
|
-
|
|
5577
|
-
|
|
5578
|
-
|
|
5579
|
-
|
|
5580
|
-
|
|
5581
|
-
|
|
5582
|
-
|
|
5583
|
-
|
|
5584
|
-
|
|
5585
|
-
|
|
5586
|
-
|
|
5587
|
-
|
|
5588
|
-
|
|
5589
|
-
|
|
5590
|
-
|
|
5591
|
-
|
|
5592
|
-
|
|
5593
|
-
|
|
5594
|
-
|
|
5595
|
-
|
|
5596
|
-
|
|
6085
|
+
const toolDefs = d.tools ? await d.tools.buildToolDefinitions() : [];
|
|
6086
|
+
outerLoop: while (toolIter <= MAX_TOOL_ITER) {
|
|
6087
|
+
toolIter++;
|
|
6088
|
+
const reqBody = {
|
|
6089
|
+
model: engineModelAlias(d.registry.active()?.kind ?? "") ?? ms.model.key,
|
|
6090
|
+
messages: iterMessages,
|
|
6091
|
+
stream: true,
|
|
6092
|
+
stream_options: { include_usage: true },
|
|
6093
|
+
return_progress: true,
|
|
6094
|
+
...samplingOverride
|
|
6095
|
+
};
|
|
6096
|
+
if (stopStrings?.length) reqBody.stop = stopStrings;
|
|
6097
|
+
const cappedMax = clampMaxTokens(reqBody.max_tokens, maxLimit);
|
|
6098
|
+
if (cappedMax != null) reqBody.max_tokens = cappedMax;
|
|
6099
|
+
else delete reqBody.max_tokens;
|
|
6100
|
+
if (disableThinking) {
|
|
6101
|
+
reqBody.reasoning_budget = 0;
|
|
6102
|
+
reqBody.chat_template_kwargs = { enable_thinking: false };
|
|
6103
|
+
}
|
|
6104
|
+
if (toolDefs.length > 0) reqBody.tools = toolDefs;
|
|
6105
|
+
if (conv.toolPolicy === "force_web_search" && toolIter <= 2 && toolDefs.some((t) => t.function.name === "web_search")) {
|
|
6106
|
+
reqBody.tool_choice = { type: "function", function: { name: "web_search" } };
|
|
6107
|
+
}
|
|
6108
|
+
const res = await fetch(`${target}/v1/chat/completions`, {
|
|
6109
|
+
method: "POST",
|
|
6110
|
+
headers: { "Content-Type": "application/json" },
|
|
6111
|
+
body: JSON.stringify(reqBody),
|
|
6112
|
+
signal: ac.signal,
|
|
6113
|
+
duplex: "half"
|
|
6114
|
+
});
|
|
6115
|
+
if (!res.ok || !res.body) {
|
|
6116
|
+
await stream.writeSSE({ event: "error", data: JSON.stringify({ code: "engine_error", message: `Engine returned ${res.status}` }) });
|
|
6117
|
+
return;
|
|
6118
|
+
}
|
|
6119
|
+
const reader = res.body.getReader();
|
|
6120
|
+
const decoder = new TextDecoder();
|
|
6121
|
+
let buf = "";
|
|
6122
|
+
const cancelReader = () => void reader.cancel();
|
|
6123
|
+
if (ac.signal.aborted) {
|
|
6124
|
+
cancelReader();
|
|
6125
|
+
} else {
|
|
6126
|
+
ac.signal.addEventListener("abort", cancelReader, { once: true });
|
|
6127
|
+
}
|
|
6128
|
+
let roundContent = "";
|
|
6129
|
+
let inThink = false;
|
|
6130
|
+
let pendingThinkBuf = "";
|
|
6131
|
+
let firstDelta = false;
|
|
6132
|
+
let finishReason = "";
|
|
6133
|
+
const pendingToolCalls = /* @__PURE__ */ new Map();
|
|
6134
|
+
roundLoop: while (true) {
|
|
6135
|
+
const { done, value } = await reader.read();
|
|
6136
|
+
if (done) break;
|
|
6137
|
+
buf += decoder.decode(value, { stream: true });
|
|
6138
|
+
const lines = buf.split("\n");
|
|
6139
|
+
buf = lines.pop() ?? "";
|
|
6140
|
+
for (const line of lines) {
|
|
6141
|
+
if (!line.startsWith("data: ")) continue;
|
|
6142
|
+
const raw = line.slice(6).trim();
|
|
6143
|
+
if (raw === "[DONE]") break roundLoop;
|
|
6144
|
+
let chunk;
|
|
6145
|
+
try {
|
|
6146
|
+
chunk = JSON.parse(raw);
|
|
6147
|
+
} catch {
|
|
6148
|
+
continue;
|
|
6149
|
+
}
|
|
6150
|
+
const pp = chunk.prompt_progress;
|
|
6151
|
+
if (pp && pp.total) {
|
|
6152
|
+
const pct = Math.round((pp.processed ?? 0) / pp.total * 100);
|
|
6153
|
+
d.manager.setLiveGen({ phase: "prompt", pct, outputTokens: 0 });
|
|
6154
|
+
await stream.writeSSE({ event: "progress", data: JSON.stringify({ phase: "prompt", processed: pp.processed, total: pp.total, pct, tps: pp.tps ?? 0 }) });
|
|
6155
|
+
continue;
|
|
6156
|
+
}
|
|
6157
|
+
if (chunk.usage) finalUsage = chunk.usage;
|
|
6158
|
+
if (chunk.timings) finalTimings = chunk.timings;
|
|
6159
|
+
const choices = chunk.choices;
|
|
6160
|
+
if (!choices?.length) continue;
|
|
6161
|
+
if (choices[0].finish_reason) finishReason = choices[0].finish_reason;
|
|
6162
|
+
const delta = choices[0].delta ?? {};
|
|
6163
|
+
if (delta.tool_calls?.length) {
|
|
6164
|
+
for (const tc of delta.tool_calls) {
|
|
6165
|
+
if (!pendingToolCalls.has(tc.index)) {
|
|
6166
|
+
pendingToolCalls.set(tc.index, { id: "", name: "", argsBuffer: "" });
|
|
5597
6167
|
}
|
|
5598
|
-
|
|
5599
|
-
|
|
5600
|
-
if (!
|
|
5601
|
-
|
|
5602
|
-
toProcess = "";
|
|
5603
|
-
} else if (pendingThinkBuf.length > 20) {
|
|
5604
|
-
const flush = pendingThinkBuf;
|
|
5605
|
-
pendingThinkBuf = "";
|
|
5606
|
-
fullContent += flush;
|
|
5607
|
-
firstDelta = true;
|
|
5608
|
-
if (!ttftMs) ttftMs = Date.now() - requestStart;
|
|
5609
|
-
await stream.writeSSE({ event: "delta", data: JSON.stringify({ delta: flush }) });
|
|
5610
|
-
toProcess = "";
|
|
5611
|
-
} else {
|
|
5612
|
-
toProcess = "";
|
|
6168
|
+
const entry = pendingToolCalls.get(tc.index);
|
|
6169
|
+
if (tc.id && !entry.id) entry.id = tc.id;
|
|
6170
|
+
if (tc.function?.name && !entry.name) entry.name = tc.function.name;
|
|
6171
|
+
if (tc.function?.arguments) entry.argsBuffer += tc.function.arguments;
|
|
5613
6172
|
}
|
|
5614
|
-
|
|
5615
|
-
|
|
5616
|
-
|
|
5617
|
-
|
|
5618
|
-
|
|
5619
|
-
|
|
5620
|
-
|
|
5621
|
-
|
|
5622
|
-
|
|
5623
|
-
|
|
5624
|
-
|
|
5625
|
-
|
|
5626
|
-
|
|
5627
|
-
|
|
5628
|
-
|
|
5629
|
-
|
|
6173
|
+
continue;
|
|
6174
|
+
}
|
|
6175
|
+
if (delta.reasoning_content) {
|
|
6176
|
+
const rc = delta.reasoning_content;
|
|
6177
|
+
if (!thinkStart) thinkStart = Date.now();
|
|
6178
|
+
thinkEnd = Date.now();
|
|
6179
|
+
fullReasoning += rc;
|
|
6180
|
+
await stream.writeSSE({ event: "reasoning", data: JSON.stringify({ delta: rc }) });
|
|
6181
|
+
continue;
|
|
6182
|
+
}
|
|
6183
|
+
const raw_content = delta.content ?? "";
|
|
6184
|
+
if (!raw_content) continue;
|
|
6185
|
+
let toProcess = raw_content;
|
|
6186
|
+
while (toProcess.length > 0) {
|
|
6187
|
+
if (!inThink && !firstDelta) {
|
|
6188
|
+
pendingThinkBuf += toProcess;
|
|
6189
|
+
const openIdx = pendingThinkBuf.indexOf("<think>");
|
|
6190
|
+
if (openIdx === 0) {
|
|
6191
|
+
inThink = true;
|
|
6192
|
+
if (!thinkStart) thinkStart = Date.now();
|
|
6193
|
+
pendingThinkBuf = pendingThinkBuf.slice(7);
|
|
6194
|
+
toProcess = "";
|
|
6195
|
+
} else if (openIdx > 0) {
|
|
6196
|
+
const before = pendingThinkBuf.slice(0, openIdx);
|
|
6197
|
+
fullContent += before;
|
|
6198
|
+
roundContent += before;
|
|
5630
6199
|
firstDelta = true;
|
|
5631
6200
|
if (!ttftMs) ttftMs = Date.now() - requestStart;
|
|
5632
|
-
await stream.writeSSE({ event: "delta", data: JSON.stringify({ delta:
|
|
6201
|
+
await stream.writeSSE({ event: "delta", data: JSON.stringify({ delta: before }) });
|
|
6202
|
+
inThink = true;
|
|
6203
|
+
if (!thinkStart) thinkStart = Date.now();
|
|
6204
|
+
pendingThinkBuf = pendingThinkBuf.slice(openIdx + 7);
|
|
6205
|
+
toProcess = "";
|
|
6206
|
+
} else if (pendingThinkBuf.length > 20) {
|
|
6207
|
+
const flush = pendingThinkBuf;
|
|
5633
6208
|
pendingThinkBuf = "";
|
|
6209
|
+
fullContent += flush;
|
|
6210
|
+
roundContent += flush;
|
|
6211
|
+
firstDelta = true;
|
|
6212
|
+
if (!ttftMs) ttftMs = Date.now() - requestStart;
|
|
6213
|
+
await stream.writeSSE({ event: "delta", data: JSON.stringify({ delta: flush }) });
|
|
6214
|
+
toProcess = "";
|
|
6215
|
+
} else {
|
|
6216
|
+
toProcess = "";
|
|
6217
|
+
}
|
|
6218
|
+
} else if (inThink) {
|
|
6219
|
+
pendingThinkBuf += toProcess;
|
|
6220
|
+
const closeIdx = pendingThinkBuf.indexOf("</think>");
|
|
6221
|
+
if (closeIdx >= 0) {
|
|
6222
|
+
const thinkChunk = pendingThinkBuf.slice(0, closeIdx);
|
|
6223
|
+
if (thinkChunk) {
|
|
6224
|
+
thinkEnd = Date.now();
|
|
6225
|
+
fullReasoning += thinkChunk;
|
|
6226
|
+
await stream.writeSSE({ event: "reasoning", data: JSON.stringify({ delta: thinkChunk }) });
|
|
6227
|
+
}
|
|
6228
|
+
inThink = false;
|
|
6229
|
+
thinkEnd = Date.now();
|
|
6230
|
+
pendingThinkBuf = pendingThinkBuf.slice(closeIdx + 8);
|
|
6231
|
+
toProcess = "";
|
|
6232
|
+
if (pendingThinkBuf) {
|
|
6233
|
+
fullContent += pendingThinkBuf;
|
|
6234
|
+
roundContent += pendingThinkBuf;
|
|
6235
|
+
firstDelta = true;
|
|
6236
|
+
if (!ttftMs) ttftMs = Date.now() - requestStart;
|
|
6237
|
+
await stream.writeSSE({ event: "delta", data: JSON.stringify({ delta: pendingThinkBuf }) });
|
|
6238
|
+
pendingThinkBuf = "";
|
|
6239
|
+
}
|
|
6240
|
+
} else {
|
|
6241
|
+
thinkEnd = Date.now();
|
|
6242
|
+
fullReasoning += pendingThinkBuf;
|
|
6243
|
+
await stream.writeSSE({ event: "reasoning", data: JSON.stringify({ delta: pendingThinkBuf }) });
|
|
6244
|
+
pendingThinkBuf = "";
|
|
6245
|
+
toProcess = "";
|
|
5634
6246
|
}
|
|
5635
6247
|
} else {
|
|
5636
|
-
|
|
5637
|
-
|
|
5638
|
-
|
|
5639
|
-
|
|
6248
|
+
if (!ttftMs) ttftMs = Date.now() - requestStart;
|
|
6249
|
+
firstDelta = true;
|
|
6250
|
+
fullContent += toProcess;
|
|
6251
|
+
roundContent += toProcess;
|
|
6252
|
+
d.manager.setLiveGen({ phase: "gen", pct: 0, outputTokens: ++liveOut });
|
|
6253
|
+
await stream.writeSSE({ event: "delta", data: JSON.stringify({ delta: toProcess }) });
|
|
5640
6254
|
toProcess = "";
|
|
5641
6255
|
}
|
|
5642
|
-
} else {
|
|
5643
|
-
if (!ttftMs) ttftMs = Date.now() - requestStart;
|
|
5644
|
-
firstDelta = true;
|
|
5645
|
-
fullContent += toProcess;
|
|
5646
|
-
d.manager.setLiveGen({ phase: "gen", pct: 0, outputTokens: ++liveOut });
|
|
5647
|
-
await stream.writeSSE({ event: "delta", data: JSON.stringify({ delta: toProcess }) });
|
|
5648
|
-
toProcess = "";
|
|
5649
6256
|
}
|
|
5650
6257
|
}
|
|
5651
6258
|
}
|
|
5652
|
-
|
|
5653
|
-
|
|
5654
|
-
|
|
5655
|
-
|
|
5656
|
-
|
|
5657
|
-
|
|
5658
|
-
|
|
5659
|
-
|
|
6259
|
+
ac.signal.removeEventListener("abort", cancelReader);
|
|
6260
|
+
if (pendingThinkBuf && inThink) {
|
|
6261
|
+
fullReasoning += pendingThinkBuf;
|
|
6262
|
+
await stream.writeSSE({ event: "reasoning", data: JSON.stringify({ delta: pendingThinkBuf }) });
|
|
6263
|
+
} else if (pendingThinkBuf) {
|
|
6264
|
+
fullContent += pendingThinkBuf;
|
|
6265
|
+
roundContent += pendingThinkBuf;
|
|
6266
|
+
await stream.writeSSE({ event: "delta", data: JSON.stringify({ delta: pendingThinkBuf }) });
|
|
6267
|
+
}
|
|
6268
|
+
if ((finishReason === "tool_calls" || pendingToolCalls.size > 0) && d.tools && toolIter <= MAX_TOOL_ITER) {
|
|
6269
|
+
const roundToolCalls = Array.from(pendingToolCalls.values());
|
|
6270
|
+
iterMessages.push({
|
|
6271
|
+
role: "assistant",
|
|
6272
|
+
content: roundContent || null,
|
|
6273
|
+
tool_calls: roundToolCalls.map((tc) => ({
|
|
6274
|
+
id: tc.id,
|
|
6275
|
+
type: "function",
|
|
6276
|
+
function: { name: tc.name, arguments: tc.argsBuffer }
|
|
6277
|
+
}))
|
|
6278
|
+
});
|
|
6279
|
+
for (const tc of roundToolCalls) {
|
|
6280
|
+
let parsedArgs;
|
|
6281
|
+
try {
|
|
6282
|
+
parsedArgs = JSON.parse(tc.argsBuffer || "{}");
|
|
6283
|
+
} catch {
|
|
6284
|
+
parsedArgs = {};
|
|
6285
|
+
}
|
|
6286
|
+
await stream.writeSSE({
|
|
6287
|
+
event: "tool_call",
|
|
6288
|
+
data: JSON.stringify({ id: tc.id, name: tc.name, args: parsedArgs, status: "pending" })
|
|
6289
|
+
});
|
|
6290
|
+
let result = "";
|
|
6291
|
+
let callError;
|
|
6292
|
+
try {
|
|
6293
|
+
result = await d.tools.executeTool({ id: tc.id, name: tc.name, args: parsedArgs });
|
|
6294
|
+
} catch (e) {
|
|
6295
|
+
callError = e.message;
|
|
6296
|
+
result = `Error: ${callError}`;
|
|
6297
|
+
}
|
|
6298
|
+
allToolCalls.push({ id: tc.id, name: tc.name, args: parsedArgs, result: callError ? void 0 : result, error: callError });
|
|
6299
|
+
await stream.writeSSE({
|
|
6300
|
+
event: "tool_call",
|
|
6301
|
+
data: JSON.stringify({ id: tc.id, name: tc.name, args: parsedArgs, status: callError ? "error" : "done", result })
|
|
6302
|
+
});
|
|
6303
|
+
iterMessages.push({ role: "tool", content: result, tool_call_id: tc.id });
|
|
6304
|
+
}
|
|
6305
|
+
continue outerLoop;
|
|
6306
|
+
}
|
|
6307
|
+
break outerLoop;
|
|
5660
6308
|
}
|
|
5661
6309
|
} catch (e) {
|
|
5662
6310
|
const isAbort = e?.name === "AbortError";
|
|
@@ -5698,7 +6346,7 @@ async function runGeneration(d, stream, ctx) {
|
|
|
5698
6346
|
stats.tps = stats.genMs > 0 ? Math.round(stats.genTokens / stats.genMs * 1e3 * 10) / 10 : 0;
|
|
5699
6347
|
stats.cachedTokens = cachedExplicit ?? 0;
|
|
5700
6348
|
}
|
|
5701
|
-
db2.updateMessage(assistantMsg.id, { content: fullContent, reasoning: fullReasoning, stats });
|
|
6349
|
+
db2.updateMessage(assistantMsg.id, { content: fullContent, reasoning: fullReasoning, toolCalls: allToolCalls, stats });
|
|
5702
6350
|
db2.touchConversation(convId);
|
|
5703
6351
|
try {
|
|
5704
6352
|
d.manager.recordCompletion({
|
|
@@ -5713,7 +6361,7 @@ async function runGeneration(d, stream, ctx) {
|
|
|
5713
6361
|
await stream.writeSSE({ event: "done", data: JSON.stringify({ message: finalMsg }) });
|
|
5714
6362
|
if (!aborted && conv.title === "New chat" && d.store.snapshot().daemon.autoGenerateTitles) {
|
|
5715
6363
|
setTimeout(() => {
|
|
5716
|
-
void autoTitle(d, convId, engineMessages, fullContent, target);
|
|
6364
|
+
void autoTitle(d, convId, ctx.engineMessages, fullContent, target);
|
|
5717
6365
|
}, 1e3);
|
|
5718
6366
|
}
|
|
5719
6367
|
}
|
|
@@ -6503,8 +7151,13 @@ var downloads = new DownloadManager(store, () => void scanner.rescan(), () => hf
|
|
|
6503
7151
|
var bench = new BenchRunner(manager, store, scanner, registry, version);
|
|
6504
7152
|
var comfy = new ComfyGuard(store, manager);
|
|
6505
7153
|
var modelRouter = new ModelRouter(store, registry, manager, scanner, comfy);
|
|
7154
|
+
var toolRegistry = new ToolRegistry(store.snapshot().tools);
|
|
7155
|
+
void (async () => {
|
|
7156
|
+
const cfg2 = store.snapshot();
|
|
7157
|
+
await toolRegistry.syncMcpServers(cfg2.mcp.servers);
|
|
7158
|
+
})();
|
|
6506
7159
|
var startedAt = Date.now();
|
|
6507
|
-
var deps = { store, registry, manager, scanner, hashes, db, provision, hf, downloads, bench, modelRouter, comfy, version, startedAt };
|
|
7160
|
+
var deps = { store, registry, manager, scanner, hashes, db, provision, hf, downloads, bench, modelRouter, comfy, tools: toolRegistry, version, startedAt };
|
|
6508
7161
|
var app = createApp(deps);
|
|
6509
7162
|
var cfg = store.snapshot();
|
|
6510
7163
|
var defaultHost = cfg.daemon.lanBind ? "0.0.0.0" : cfg.daemon.host || "127.0.0.1";
|
|
@@ -6531,7 +7184,7 @@ function openBrowser(url) {
|
|
|
6531
7184
|
cmd = "xdg-open";
|
|
6532
7185
|
args = [url];
|
|
6533
7186
|
}
|
|
6534
|
-
const child =
|
|
7187
|
+
const child = spawn4(cmd, args, { detached: true, stdio: "ignore" });
|
|
6535
7188
|
child.unref();
|
|
6536
7189
|
child.on("error", () => {
|
|
6537
7190
|
console.log(` Could not open browser automatically. Visit the URL above manually.`);
|
|
@@ -6622,7 +7275,7 @@ function spawnReplacement() {
|
|
|
6622
7275
|
} catch {
|
|
6623
7276
|
out = "ignore";
|
|
6624
7277
|
}
|
|
6625
|
-
const child =
|
|
7278
|
+
const child = spawn4(process.execPath, process.argv.slice(1), {
|
|
6626
7279
|
cwd: process.cwd(),
|
|
6627
7280
|
detached: true,
|
|
6628
7281
|
stdio: ["ignore", out, out]
|
|
@@ -6715,6 +7368,7 @@ for (const sig of ["SIGINT", "SIGTERM"]) {
|
|
|
6715
7368
|
shuttingDown = true;
|
|
6716
7369
|
console.log("shutting down");
|
|
6717
7370
|
comfy.stop();
|
|
7371
|
+
toolRegistry.disconnectAll();
|
|
6718
7372
|
void manager.shutdown().finally(() => {
|
|
6719
7373
|
db.close();
|
|
6720
7374
|
server.close(() => process.exit(0));
|