reasonix 0.50.1 → 0.51.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dashboard/dist/app.css +1 -1
- package/dashboard/dist/app.js +24 -22
- package/dashboard/dist/app.js.map +1 -1
- package/dist/cli/{acp-6B25WIFF.js → acp-XEUHGG7X.js} +34 -31
- package/dist/cli/acp-XEUHGG7X.js.map +1 -0
- package/dist/cli/chat-NJ2Q5KHG.js +50 -0
- package/dist/cli/{chunk-OPGWCKKU.js → chunk-2HVTBFCI.js} +3 -3
- package/dist/cli/{chunk-AJIZ5KFK.js → chunk-2WUEAI2I.js} +3 -3
- package/dist/cli/{chunk-I4Q3QT4W.js → chunk-36BM7INR.js} +2 -2
- package/dist/cli/{chunk-3RNFYDDM.js → chunk-3BTK5BHI.js} +11 -7
- package/dist/cli/chunk-3BTK5BHI.js.map +1 -0
- package/dist/cli/{chunk-GMSAB2TC.js → chunk-3YRTIWFX.js} +2 -2
- package/dist/cli/{chunk-NLRC3DWQ.js → chunk-544J4PXD.js} +5 -5
- package/dist/cli/{chunk-7WITYWKN.js → chunk-5AIDYVH2.js} +2 -2
- package/dist/cli/{chunk-ALCOQP6R.js → chunk-5BBC6YMV.js} +5 -5
- package/dist/cli/{chunk-S4XVGLRW.js → chunk-6UNHNVJR.js} +72 -5
- package/dist/cli/chunk-6UNHNVJR.js.map +1 -0
- package/dist/cli/{chunk-IK6WWRIX.js → chunk-6XWXIVQ3.js} +38 -22
- package/dist/cli/chunk-6XWXIVQ3.js.map +1 -0
- package/dist/cli/{chunk-AAHB2PFX.js → chunk-7YB26OQO.js} +4 -4
- package/dist/cli/chunk-7YB26OQO.js.map +1 -0
- package/dist/cli/{chunk-MXWPAPZW.js → chunk-A5PBEIJ7.js} +53 -10
- package/dist/cli/chunk-A5PBEIJ7.js.map +1 -0
- package/dist/cli/{chunk-FQSQFCBI.js → chunk-BA5R6BAE.js} +2 -2
- package/dist/cli/{chunk-XWPZHWC2.js → chunk-BM6BBFAV.js} +2 -2
- package/dist/cli/{chunk-CAGKEGNE.js → chunk-BOWSNGQC.js} +52 -140
- package/dist/cli/chunk-BOWSNGQC.js.map +1 -0
- package/dist/cli/{chunk-EZ57UEZQ.js → chunk-C2MRSJTV.js} +2 -2
- package/dist/cli/{chunk-PYIZZAVQ.js → chunk-DVD67FXQ.js} +1716 -4
- package/dist/cli/chunk-DVD67FXQ.js.map +1 -0
- package/dist/cli/{chunk-ZAXMJANP.js → chunk-EAMXOWUW.js} +3 -3
- package/dist/cli/{chunk-TX652NBA.js → chunk-EWVFGYT6.js} +2 -2
- package/dist/cli/{chunk-IBRTU5WO.js → chunk-FP7IOWBQ.js} +18 -1182
- package/dist/cli/chunk-FP7IOWBQ.js.map +1 -0
- package/dist/cli/{chunk-I6FBSTTR.js → chunk-HGK57NBN.js} +9 -353
- package/dist/cli/chunk-HGK57NBN.js.map +1 -0
- package/dist/cli/chunk-JHWQDJZA.js +80 -0
- package/dist/cli/chunk-JHWQDJZA.js.map +1 -0
- package/dist/cli/{chunk-X2BQZQEE.js → chunk-K3QJ3GKI.js} +3 -3
- package/dist/cli/{chunk-GPUH2BNM.js → chunk-K4YQFULP.js} +612 -254
- package/dist/cli/chunk-K4YQFULP.js.map +1 -0
- package/dist/cli/chunk-L3VPEESB.js +31 -0
- package/dist/cli/chunk-L3VPEESB.js.map +1 -0
- package/dist/cli/{chunk-ENFBF6HI.js → chunk-N4SEBLU4.js} +383 -5
- package/dist/cli/chunk-N4SEBLU4.js.map +1 -0
- package/dist/cli/chunk-NRROJXXT.js +879 -0
- package/dist/cli/chunk-NRROJXXT.js.map +1 -0
- package/dist/cli/{chunk-3KRRTLC5.js → chunk-R6KIHEF3.js} +1619 -1036
- package/dist/cli/chunk-R6KIHEF3.js.map +1 -0
- package/dist/cli/{chunk-VVMY4M7J.js → chunk-SBHF5NWD.js} +27 -4
- package/dist/cli/chunk-SBHF5NWD.js.map +1 -0
- package/dist/cli/{chunk-OWA42BKS.js → chunk-SXSAWOB7.js} +14 -14
- package/dist/cli/{chunk-6IUMTRFP.js → chunk-UMZ6KHTS.js} +2 -2
- package/dist/cli/{chunk-7X4JJOO7.js → chunk-UO6E7FN3.js} +69 -5
- package/dist/cli/{chunk-7X4JJOO7.js.map → chunk-UO6E7FN3.js.map} +1 -1
- package/dist/cli/{chunk-3ZZXQ3CZ.js → chunk-UPW544V3.js} +2 -2
- package/dist/cli/{chunk-XJZWMU5P.js → chunk-WPOKBW5E.js} +2 -2
- package/dist/cli/{chunk-WSBFVOCO.js → chunk-Z3MKG7MQ.js} +2 -2
- package/dist/cli/{code-TBK2TASK.js → code-BMXLBC7D.js} +37 -36
- package/dist/cli/{code-TBK2TASK.js.map → code-BMXLBC7D.js.map} +1 -1
- package/dist/cli/{commands-NXTKSQTN.js → commands-E4RZXMF6.js} +5 -5
- package/dist/cli/{commit-IR5SPP7A.js → commit-KSRQ64IL.js} +3 -3
- package/dist/cli/{config-XK5WQGTS.js → config-QNDONOTU.js} +4 -2
- package/dist/cli/{desktop-5NTQBADL.js → desktop-H3ZHIMDA.js} +83 -37
- package/dist/cli/desktop-H3ZHIMDA.js.map +1 -0
- package/dist/cli/{diff-JNYX5BSZ.js → diff-I4PYI43W.js} +9 -9
- package/dist/cli/{doctor-IKYLUFXX.js → doctor-Y2E4MY2F.js} +12 -12
- package/dist/cli/{events-HSC57ONU.js → events-47HOT7ZA.js} +5 -5
- package/dist/cli/find-in-code-YLEIK5FK.js +145 -0
- package/dist/cli/find-in-code-YLEIK5FK.js.map +1 -0
- package/dist/cli/index.js +95 -44
- package/dist/cli/index.js.map +1 -1
- package/dist/cli/{mcp-BDJJWOCD.js → mcp-76DK63ZB.js} +3 -3
- package/dist/cli/{mcp-browse-NJRZDI6V.js → mcp-browse-SDNUGO74.js} +3 -3
- package/dist/cli/{mcp-inspect-Y62NWZQL.js → mcp-inspect-BL5DEO5M.js} +3 -3
- package/dist/cli/{prompt-UTOIFUQC.js → prompt-JLATI3P7.js} +5 -5
- package/dist/cli/{prune-sessions-UCUD4XAP.js → prune-sessions-WHZDFUKD.js} +4 -4
- package/dist/cli/{replay-VVIN64MN.js → replay-MHXS7C7Z.js} +10 -10
- package/dist/cli/{run-76OBDZFB.js → run-SXNCPRJE.js} +22 -22
- package/dist/cli/{server-SZZDKTH2.js → server-GEHOE6CO.js} +61 -35
- package/dist/cli/server-GEHOE6CO.js.map +1 -0
- package/dist/cli/{sessions-FZTGRCM5.js → sessions-EPBFYISL.js} +18 -18
- package/dist/cli/{setup-4UNENGOE.js → setup-IW2XR5XI.js} +8 -7
- package/dist/cli/setup-IW2XR5XI.js.map +1 -0
- package/dist/cli/{stats-F4NDOD7D.js → stats-4WB4XHBP.js} +6 -6
- package/dist/cli/symbols-UQ274IOB.js +167 -0
- package/dist/cli/symbols-UQ274IOB.js.map +1 -0
- package/dist/cli/version-4SP3DLLH.js +33 -0
- package/dist/index.d.ts +25 -6
- package/dist/index.js +2700 -578
- package/dist/index.js.map +1 -1
- package/package.json +4 -2
- package/scripts/postinstall.mjs +3 -5
- package/dist/cli/acp-6B25WIFF.js.map +0 -1
- package/dist/cli/chat-7WASPB4O.js +0 -50
- package/dist/cli/chunk-3KRRTLC5.js.map +0 -1
- package/dist/cli/chunk-3RNFYDDM.js.map +0 -1
- package/dist/cli/chunk-AAHB2PFX.js.map +0 -1
- package/dist/cli/chunk-CAGKEGNE.js.map +0 -1
- package/dist/cli/chunk-ENFBF6HI.js.map +0 -1
- package/dist/cli/chunk-GPUH2BNM.js.map +0 -1
- package/dist/cli/chunk-I6FBSTTR.js.map +0 -1
- package/dist/cli/chunk-IBRTU5WO.js.map +0 -1
- package/dist/cli/chunk-IK6WWRIX.js.map +0 -1
- package/dist/cli/chunk-MXWPAPZW.js.map +0 -1
- package/dist/cli/chunk-PYIZZAVQ.js.map +0 -1
- package/dist/cli/chunk-S4XVGLRW.js.map +0 -1
- package/dist/cli/chunk-VVMY4M7J.js.map +0 -1
- package/dist/cli/desktop-5NTQBADL.js.map +0 -1
- package/dist/cli/server-SZZDKTH2.js.map +0 -1
- package/dist/cli/setup-4UNENGOE.js.map +0 -1
- package/dist/cli/version-LUVTWHLL.js +0 -33
- /package/dist/cli/{chat-7WASPB4O.js.map → chat-NJ2Q5KHG.js.map} +0 -0
- /package/dist/cli/{chunk-OPGWCKKU.js.map → chunk-2HVTBFCI.js.map} +0 -0
- /package/dist/cli/{chunk-AJIZ5KFK.js.map → chunk-2WUEAI2I.js.map} +0 -0
- /package/dist/cli/{chunk-I4Q3QT4W.js.map → chunk-36BM7INR.js.map} +0 -0
- /package/dist/cli/{chunk-GMSAB2TC.js.map → chunk-3YRTIWFX.js.map} +0 -0
- /package/dist/cli/{chunk-NLRC3DWQ.js.map → chunk-544J4PXD.js.map} +0 -0
- /package/dist/cli/{chunk-7WITYWKN.js.map → chunk-5AIDYVH2.js.map} +0 -0
- /package/dist/cli/{chunk-ALCOQP6R.js.map → chunk-5BBC6YMV.js.map} +0 -0
- /package/dist/cli/{chunk-FQSQFCBI.js.map → chunk-BA5R6BAE.js.map} +0 -0
- /package/dist/cli/{chunk-XWPZHWC2.js.map → chunk-BM6BBFAV.js.map} +0 -0
- /package/dist/cli/{chunk-EZ57UEZQ.js.map → chunk-C2MRSJTV.js.map} +0 -0
- /package/dist/cli/{chunk-ZAXMJANP.js.map → chunk-EAMXOWUW.js.map} +0 -0
- /package/dist/cli/{chunk-TX652NBA.js.map → chunk-EWVFGYT6.js.map} +0 -0
- /package/dist/cli/{chunk-X2BQZQEE.js.map → chunk-K3QJ3GKI.js.map} +0 -0
- /package/dist/cli/{chunk-OWA42BKS.js.map → chunk-SXSAWOB7.js.map} +0 -0
- /package/dist/cli/{chunk-6IUMTRFP.js.map → chunk-UMZ6KHTS.js.map} +0 -0
- /package/dist/cli/{chunk-3ZZXQ3CZ.js.map → chunk-UPW544V3.js.map} +0 -0
- /package/dist/cli/{chunk-XJZWMU5P.js.map → chunk-WPOKBW5E.js.map} +0 -0
- /package/dist/cli/{chunk-WSBFVOCO.js.map → chunk-Z3MKG7MQ.js.map} +0 -0
- /package/dist/cli/{commands-NXTKSQTN.js.map → commands-E4RZXMF6.js.map} +0 -0
- /package/dist/cli/{commit-IR5SPP7A.js.map → commit-KSRQ64IL.js.map} +0 -0
- /package/dist/cli/{config-XK5WQGTS.js.map → config-QNDONOTU.js.map} +0 -0
- /package/dist/cli/{diff-JNYX5BSZ.js.map → diff-I4PYI43W.js.map} +0 -0
- /package/dist/cli/{doctor-IKYLUFXX.js.map → doctor-Y2E4MY2F.js.map} +0 -0
- /package/dist/cli/{events-HSC57ONU.js.map → events-47HOT7ZA.js.map} +0 -0
- /package/dist/cli/{mcp-BDJJWOCD.js.map → mcp-76DK63ZB.js.map} +0 -0
- /package/dist/cli/{mcp-browse-NJRZDI6V.js.map → mcp-browse-SDNUGO74.js.map} +0 -0
- /package/dist/cli/{mcp-inspect-Y62NWZQL.js.map → mcp-inspect-BL5DEO5M.js.map} +0 -0
- /package/dist/cli/{prompt-UTOIFUQC.js.map → prompt-JLATI3P7.js.map} +0 -0
- /package/dist/cli/{prune-sessions-UCUD4XAP.js.map → prune-sessions-WHZDFUKD.js.map} +0 -0
- /package/dist/cli/{replay-VVIN64MN.js.map → replay-MHXS7C7Z.js.map} +0 -0
- /package/dist/cli/{run-76OBDZFB.js.map → run-SXNCPRJE.js.map} +0 -0
- /package/dist/cli/{sessions-FZTGRCM5.js.map → sessions-EPBFYISL.js.map} +0 -0
- /package/dist/cli/{stats-F4NDOD7D.js.map → stats-4WB4XHBP.js.map} +0 -0
- /package/dist/cli/{version-LUVTWHLL.js.map → version-4SP3DLLH.js.map} +0 -0
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
import { createRequire as __cr } from 'node:module'; if (typeof globalThis.require === 'undefined') { globalThis.require = __cr(import.meta.url); }
|
|
3
3
|
import {
|
|
4
4
|
loadPricingOverride
|
|
5
|
-
} from "./chunk-
|
|
5
|
+
} from "./chunk-A5PBEIJ7.js";
|
|
6
6
|
|
|
7
7
|
// src/telemetry/stats.ts
|
|
8
8
|
var DEEPSEEK_PRICING = {
|
|
@@ -199,4 +199,4 @@ export {
|
|
|
199
199
|
claudeEquivalentCost,
|
|
200
200
|
SessionStats
|
|
201
201
|
};
|
|
202
|
-
//# sourceMappingURL=chunk-
|
|
202
|
+
//# sourceMappingURL=chunk-BA5R6BAE.js.map
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
import { createRequire as __cr } from 'node:module'; if (typeof globalThis.require === 'undefined') { globalThis.require = __cr(import.meta.url); }
|
|
3
3
|
import {
|
|
4
4
|
SkillStore
|
|
5
|
-
} from "./chunk-
|
|
5
|
+
} from "./chunk-N4SEBLU4.js";
|
|
6
6
|
|
|
7
7
|
// src/core/event-redaction.ts
|
|
8
8
|
var SECRET_KEY_RE = /(secret|token|password|passphrase|api[-_]?key|authorization|cookie|credential|passwd|pwd)/i;
|
|
@@ -600,4 +600,4 @@ export {
|
|
|
600
600
|
shouldAutoResolveCheckpoint,
|
|
601
601
|
autoResolveVerdict
|
|
602
602
|
};
|
|
603
|
-
//# sourceMappingURL=chunk-
|
|
603
|
+
//# sourceMappingURL=chunk-BM6BBFAV.js.map
|
|
@@ -1,5 +1,8 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
import { createRequire as __cr } from 'node:module'; if (typeof globalThis.require === 'undefined') { globalThis.require = __cr(import.meta.url); }
|
|
3
|
+
import {
|
|
4
|
+
LruCache
|
|
5
|
+
} from "./chunk-6UNHNVJR.js";
|
|
3
6
|
|
|
4
7
|
// src/tokenizer.ts
|
|
5
8
|
import { existsSync, readFileSync } from "fs";
|
|
@@ -84,6 +87,9 @@ function loadTokenizer() {
|
|
|
84
87
|
};
|
|
85
88
|
return cached;
|
|
86
89
|
}
|
|
90
|
+
function warmupTokenizer() {
|
|
91
|
+
loadTokenizer();
|
|
92
|
+
}
|
|
87
93
|
function escapeRegex(s) {
|
|
88
94
|
return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
89
95
|
}
|
|
@@ -109,10 +115,13 @@ function byteLevelEncode(s, byteToChar) {
|
|
|
109
115
|
for (let i = 0; i < bytes.length; i++) out += byteToChar[bytes[i]];
|
|
110
116
|
return out;
|
|
111
117
|
}
|
|
118
|
+
var bpeCache = new LruCache(8192);
|
|
112
119
|
function bpeEncode(piece, mergeRank) {
|
|
113
120
|
if (piece.length <= 1) return piece ? [piece] : [];
|
|
114
|
-
|
|
115
|
-
|
|
121
|
+
const cached2 = bpeCache.get(piece);
|
|
122
|
+
if (cached2 !== void 0) return cached2;
|
|
123
|
+
const word = Array.from(piece);
|
|
124
|
+
while (word.length > 1) {
|
|
116
125
|
let bestIdx = -1;
|
|
117
126
|
let bestRank = Number.POSITIVE_INFINITY;
|
|
118
127
|
for (let i = 0; i < word.length - 1; i++) {
|
|
@@ -125,13 +134,9 @@ function bpeEncode(piece, mergeRank) {
|
|
|
125
134
|
}
|
|
126
135
|
}
|
|
127
136
|
if (bestIdx < 0) break;
|
|
128
|
-
word
|
|
129
|
-
...word.slice(0, bestIdx),
|
|
130
|
-
word[bestIdx] + word[bestIdx + 1],
|
|
131
|
-
...word.slice(bestIdx + 2)
|
|
132
|
-
];
|
|
133
|
-
if (word.length === 1) break;
|
|
137
|
+
word.splice(bestIdx, 2, word[bestIdx] + word[bestIdx + 1]);
|
|
134
138
|
}
|
|
139
|
+
bpeCache.set(piece, word);
|
|
135
140
|
return word;
|
|
136
141
|
}
|
|
137
142
|
function encode(text) {
|
|
@@ -186,10 +191,6 @@ function countTokensBounded(text, maxChars = DEFAULT_BOUNDED_TOKENIZE_CHARS) {
|
|
|
186
191
|
const ratio = sampleChars > 0 ? sampleTokens / sampleChars : 0.3;
|
|
187
192
|
return Math.max(1, Math.ceil(text.length * ratio));
|
|
188
193
|
}
|
|
189
|
-
var BOS = "<\uFF5Cbegin\u2581of\u2581sentence\uFF5C>";
|
|
190
|
-
var EOS = "<\uFF5Cend\u2581of\u2581sentence\uFF5C>";
|
|
191
|
-
var USER_SP = "<\uFF5CUser\uFF5C>";
|
|
192
|
-
var ASSISTANT_SP = "<\uFF5CAssistant\uFF5C>";
|
|
193
194
|
var THINK_START = "<think>";
|
|
194
195
|
var THINK_END = "</think>";
|
|
195
196
|
var DSML = "\uFF5CDSML\uFF5C";
|
|
@@ -198,7 +199,6 @@ var TC_END = `</${DSML}tool_calls>`;
|
|
|
198
199
|
var INVOKE_BEGIN = `<${DSML}invoke name="`;
|
|
199
200
|
var INVOKE_END = `</${DSML}invoke>`;
|
|
200
201
|
var PARAM_TEMPLATE = `<${DSML}parameter name="{key}" string="{is_str}">{value}</${DSML}parameter>`;
|
|
201
|
-
var TOOL_RESULT_TEMPLATE = "<tool_result>{content}</tool_result>";
|
|
202
202
|
var toolsTemplateCache = /* @__PURE__ */ new WeakMap();
|
|
203
203
|
function renderTools(tools) {
|
|
204
204
|
const cached2 = toolsTemplateCache.get(tools);
|
|
@@ -235,142 +235,53 @@ You MUST strictly follow the above defined tool name and parameter schemas to in
|
|
|
235
235
|
toolsTemplateCache.set(tools, rendered);
|
|
236
236
|
return rendered;
|
|
237
237
|
}
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
).join("\n");
|
|
248
|
-
}
|
|
249
|
-
function renderToolCallsDsml(toolCalls) {
|
|
250
|
-
const invokes = toolCalls.map((tc) => {
|
|
251
|
-
const name = tc.function?.name ?? "";
|
|
252
|
-
const argsJson = tc.function?.arguments ?? "{}";
|
|
253
|
-
return `${INVOKE_BEGIN + name}">
|
|
254
|
-
${encodeArgumentsToDsml(argsJson)}
|
|
255
|
-
${INVOKE_END}`;
|
|
256
|
-
}).join("\n");
|
|
257
|
-
return `
|
|
258
|
-
|
|
259
|
-
${TC_BEGIN}
|
|
260
|
-
${invokes}
|
|
261
|
-
${TC_END}`;
|
|
238
|
+
var PER_MESSAGE_TEMPLATE_TOKENS = 6;
|
|
239
|
+
var contentTokenCache = new LruCache(4096);
|
|
240
|
+
function cachedBoundedTokens(s) {
|
|
241
|
+
if (s.length === 0) return 0;
|
|
242
|
+
const cached2 = contentTokenCache.get(s);
|
|
243
|
+
if (cached2 !== void 0) return cached2;
|
|
244
|
+
const n = countTokensBounded(s);
|
|
245
|
+
contentTokenCache.set(s, n);
|
|
246
|
+
return n;
|
|
262
247
|
}
|
|
263
|
-
function
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
if (role === "tool") {
|
|
268
|
-
const toolBlock = TOOL_RESULT_TEMPLATE.replace("{content}", msg.content ?? "");
|
|
269
|
-
const last = merged[merged.length - 1];
|
|
270
|
-
if (last && last.role === "user" && Array.isArray(last._toolBlocks) && Array.isArray(last._textParts)) {
|
|
271
|
-
last._toolBlocks.push(toolBlock);
|
|
272
|
-
last.content = `${last._textParts.join("\n\n")}
|
|
273
|
-
|
|
274
|
-
${last._toolBlocks.join("\n")}`.replace(
|
|
275
|
-
/^\n\n/,
|
|
276
|
-
""
|
|
277
|
-
);
|
|
278
|
-
} else {
|
|
279
|
-
merged.push({
|
|
280
|
-
role: "user",
|
|
281
|
-
content: toolBlock,
|
|
282
|
-
_textParts: [],
|
|
283
|
-
_toolBlocks: [toolBlock]
|
|
284
|
-
});
|
|
285
|
-
}
|
|
286
|
-
} else if (role === "user") {
|
|
287
|
-
const text = msg.content ?? "";
|
|
288
|
-
const last = merged[merged.length - 1];
|
|
289
|
-
if (last && last.role === "user" && Array.isArray(last._toolBlocks) && Array.isArray(last._textParts)) {
|
|
290
|
-
last._textParts.push(text);
|
|
291
|
-
last.content = `${last._textParts.join("\n\n")}
|
|
292
|
-
|
|
293
|
-
${last._toolBlocks.join("\n\n")}`.replace(
|
|
294
|
-
/^\n\n/,
|
|
295
|
-
""
|
|
296
|
-
);
|
|
297
|
-
} else {
|
|
298
|
-
merged.push({
|
|
299
|
-
...msg,
|
|
300
|
-
role: "user",
|
|
301
|
-
content: text,
|
|
302
|
-
_textParts: [text],
|
|
303
|
-
_toolBlocks: []
|
|
304
|
-
});
|
|
305
|
-
}
|
|
306
|
-
} else {
|
|
307
|
-
merged.push({ ...msg });
|
|
308
|
-
}
|
|
309
|
-
}
|
|
310
|
-
for (const m of merged) {
|
|
311
|
-
m._textParts = void 0;
|
|
312
|
-
m._toolBlocks = void 0;
|
|
248
|
+
function tokensForMessage(m, dropThisReasoning) {
|
|
249
|
+
let n = 0;
|
|
250
|
+
if (typeof m.content === "string" && m.content.length > 0) {
|
|
251
|
+
n += cachedBoundedTokens(m.content);
|
|
313
252
|
}
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
let lastUserIdx = -1;
|
|
318
|
-
for (let i = messages.length - 1; i >= 0; i--) {
|
|
319
|
-
const role = messages[i].role;
|
|
320
|
-
if (role === "user" || role === "developer") {
|
|
321
|
-
lastUserIdx = i;
|
|
322
|
-
break;
|
|
253
|
+
if (m.role === "assistant") {
|
|
254
|
+
if (!dropThisReasoning && typeof m.reasoning_content === "string" && m.reasoning_content.length > 0) {
|
|
255
|
+
n += cachedBoundedTokens(m.reasoning_content);
|
|
323
256
|
}
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
for (let i = 0; i < messages.length; i++) {
|
|
328
|
-
const msg = messages[i];
|
|
329
|
-
if (i < lastUserIdx && msg.role === "developer") continue;
|
|
330
|
-
if (i < lastUserIdx && msg.role === "assistant") {
|
|
331
|
-
result.push({ ...msg, reasoning_content: null });
|
|
332
|
-
} else {
|
|
333
|
-
result.push(msg);
|
|
257
|
+
const tcs = m.tool_calls;
|
|
258
|
+
if (Array.isArray(tcs) && tcs.length > 0) {
|
|
259
|
+
n += cachedBoundedTokens(JSON.stringify(tcs));
|
|
334
260
|
}
|
|
335
261
|
}
|
|
336
|
-
return
|
|
262
|
+
return n;
|
|
337
263
|
}
|
|
338
|
-
function
|
|
339
|
-
if (messages.length === 0) return
|
|
340
|
-
let
|
|
264
|
+
function estimateConversationTokens(messages, drop_thinking = false) {
|
|
265
|
+
if (messages.length === 0) return 0;
|
|
266
|
+
let lastUserOrDev = -1;
|
|
341
267
|
if (drop_thinking) {
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
const msg = merged[i];
|
|
348
|
-
const role = msg.role ?? "user";
|
|
349
|
-
const nextRole = i + 1 < merged.length ? merged[i + 1].role ?? "user" : null;
|
|
350
|
-
if (role === "system") {
|
|
351
|
-
prompt += msg.content ?? "";
|
|
352
|
-
} else if (role === "user") {
|
|
353
|
-
prompt += USER_SP + (msg.content ?? "");
|
|
354
|
-
if (nextRole === "assistant" || nextRole === null) {
|
|
355
|
-
prompt += ASSISTANT_SP + THINK_END;
|
|
356
|
-
}
|
|
357
|
-
} else if (role === "assistant") {
|
|
358
|
-
if (msg.reasoning_content) {
|
|
359
|
-
prompt += THINK_START + msg.reasoning_content + THINK_END;
|
|
268
|
+
for (let i = messages.length - 1; i >= 0; i--) {
|
|
269
|
+
const r = messages[i].role;
|
|
270
|
+
if (r === "user" || r === "developer") {
|
|
271
|
+
lastUserOrDev = i;
|
|
272
|
+
break;
|
|
360
273
|
}
|
|
361
|
-
if (msg.content) prompt += msg.content;
|
|
362
|
-
const tcs = msg.tool_calls;
|
|
363
|
-
if (Array.isArray(tcs) && tcs.length > 0) {
|
|
364
|
-
prompt += renderToolCallsDsml(tcs);
|
|
365
|
-
}
|
|
366
|
-
prompt += EOS;
|
|
367
274
|
}
|
|
368
275
|
}
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
276
|
+
let total = 2;
|
|
277
|
+
for (let i = 0; i < messages.length; i++) {
|
|
278
|
+
const m = messages[i];
|
|
279
|
+
if (drop_thinking && i < lastUserOrDev && m.role === "developer") continue;
|
|
280
|
+
total += PER_MESSAGE_TEMPLATE_TOKENS;
|
|
281
|
+
const dropReasoning = drop_thinking && i < lastUserOrDev && m.role === "assistant";
|
|
282
|
+
total += tokensForMessage(m, dropReasoning);
|
|
283
|
+
}
|
|
284
|
+
return total;
|
|
374
285
|
}
|
|
375
286
|
function estimateRequestTokens(messages, toolSpecs, drop_thinking = false) {
|
|
376
287
|
let total = estimateConversationTokens(messages, drop_thinking);
|
|
@@ -382,8 +293,9 @@ function estimateRequestTokens(messages, toolSpecs, drop_thinking = false) {
|
|
|
382
293
|
|
|
383
294
|
export {
|
|
384
295
|
resolveDataPath,
|
|
296
|
+
warmupTokenizer,
|
|
385
297
|
countTokens,
|
|
386
298
|
countTokensBounded,
|
|
387
299
|
estimateRequestTokens
|
|
388
300
|
};
|
|
389
|
-
//# sourceMappingURL=chunk-
|
|
301
|
+
//# sourceMappingURL=chunk-BOWSNGQC.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../../src/tokenizer.ts"],"sourcesContent":["/** Encode-only DeepSeek V4 tokenizer port. Applies V4 chat template so token count tracks API `prompt_tokens`. */\n\nimport { existsSync, readFileSync } from \"node:fs\";\nimport { createRequire } from \"node:module\";\nimport { dirname, join } from \"node:path\";\nimport { fileURLToPath } from \"node:url\";\nimport { gunzipSync } from \"node:zlib\";\nimport { LruCache } from \"./core/lru.js\";\n\ninterface AddedToken {\n id: number;\n content: string;\n special: boolean;\n normalized: boolean;\n}\n\ninterface SplitPretokenizer {\n type: \"Split\";\n pattern: { Regex: string };\n behavior: \"Isolated\" | \"Removed\" | string;\n invert: boolean;\n}\n\ninterface ByteLevelPretokenizer {\n type: \"ByteLevel\";\n add_prefix_space: boolean;\n trim_offsets: boolean;\n use_regex: boolean;\n}\n\ntype Pretokenizer = SplitPretokenizer | ByteLevelPretokenizer;\n\ninterface TokenizerData {\n added_tokens: AddedToken[];\n pre_tokenizer: {\n type: \"Sequence\";\n pretokenizers: Pretokenizer[];\n };\n model: {\n type: \"BPE\";\n vocab: Record<string, number>;\n merges: string[];\n };\n}\n\ninterface LoadedTokenizer {\n vocab: Record<string, number>;\n mergeRank: Map<string, number>;\n splitRegexes: RegExp[];\n byteToChar: string[];\n /** Non-special added tokens only — special tokens in user text tokenize byte-by-byte (HF default). */\n addedPattern: RegExp | null;\n addedMap: Map<string, number>;\n}\n\n/** GPT-2 byte→unicode map; lets byte-level BPE vocab serialize as readable JSON strings. */\nfunction buildByteToChar(): string[] {\n const result: string[] = new Array(256);\n const bs: number[] = [];\n for (let b = 33; b <= 126; b++) bs.push(b);\n for (let b = 161; b <= 172; b++) bs.push(b);\n for (let b = 174; b <= 255; b++) bs.push(b);\n const cs = bs.slice();\n let n = 0;\n for (let b = 0; b < 256; b++) {\n if (!bs.includes(b)) {\n bs.push(b);\n cs.push(256 + n);\n n++;\n }\n }\n for (let i = 0; i < bs.length; i++) {\n result[bs[i]!] = String.fromCodePoint(cs[i]!);\n }\n return result;\n}\n\nlet cached: LoadedTokenizer | null = null;\n\n/** Two ../data candidates needed: dist/index.js AND dist/cli/index.js resolve to different roots. */\nexport function resolveDataPath(): string {\n if (process.env.REASONIX_TOKENIZER_PATH) return process.env.REASONIX_TOKENIZER_PATH;\n const candidates: string[] = [];\n try {\n const here = dirname(fileURLToPath(import.meta.url));\n candidates.push(join(here, \"..\", \"data\", \"deepseek-tokenizer.json.gz\"));\n candidates.push(join(here, \"..\", \"..\", \"data\", \"deepseek-tokenizer.json.gz\"));\n } catch {\n /* import.meta.url unavailable — skip to the package resolution step. */\n }\n try {\n const req = createRequire(import.meta.url);\n candidates.push(\n join(dirname(req.resolve(\"reasonix/package.json\")), \"data\", \"deepseek-tokenizer.json.gz\"),\n );\n } catch {\n /* Not installed as `reasonix/` — the earlier candidates still may hit. */\n }\n for (const p of candidates) {\n if (existsSync(p)) return p;\n }\n // Nothing exists — return the first candidate anyway so readFileSync\n // surfaces a concrete path in the ENOENT message (better than silent miss).\n return candidates[0] ?? join(process.cwd(), \"data\", \"deepseek-tokenizer.json.gz\");\n}\n\nfunction loadTokenizer(): LoadedTokenizer {\n if (cached) return cached;\n const buf = readFileSync(resolveDataPath());\n const json = gunzipSync(buf).toString(\"utf8\");\n const data = JSON.parse(json) as TokenizerData;\n\n const mergeRank = new Map<string, number>();\n for (let i = 0; i < data.model.merges.length; i++) {\n mergeRank.set(data.model.merges[i]!, i);\n }\n\n const splitRegexes: RegExp[] = [];\n for (const p of data.pre_tokenizer.pretokenizers) {\n if (p.type === \"Split\") {\n // All three Split rules use Isolated — matches become their own\n // pre-tokens and so do the in-between stretches. The ByteLevel\n // stage in the Sequence does no extra splitting here\n // (use_regex:false), so our 3 Split regexes are the whole story.\n splitRegexes.push(new RegExp(p.pattern.Regex, \"gu\"));\n }\n }\n\n const addedMap = new Map<string, number>();\n const addedContents: string[] = [];\n for (const t of data.added_tokens) {\n if (!t.special) {\n addedMap.set(t.content, t.id);\n addedContents.push(t.content);\n }\n }\n // Longest-first ensures greedy matching doesn't lose a longer token\n // to a shorter prefix (e.g. `<think>` before `<`).\n addedContents.sort((a, b) => b.length - a.length);\n const addedPattern = addedContents.length\n ? new RegExp(addedContents.map(escapeRegex).join(\"|\"), \"g\")\n : null;\n\n cached = {\n vocab: data.model.vocab,\n mergeRank,\n splitRegexes,\n byteToChar: buildByteToChar(),\n addedPattern,\n addedMap,\n };\n return cached;\n}\n\n/** Force the BPE vocab to load now (gunzip + JSON.parse + Map build ≈ 100ms, 35MB heap).\n * Idempotent. Call once at idle after first paint so the first user turn doesn't pay it. */\nexport function warmupTokenizer(): void {\n loadTokenizer();\n}\n\nfunction escapeRegex(s: string): string {\n return s.replace(/[.*+?^${}()|[\\]\\\\]/g, \"\\\\$&\");\n}\n\nfunction applySplit(chunks: string[], re: RegExp): string[] {\n const out: string[] = [];\n for (const chunk of chunks) {\n if (!chunk) continue;\n // Reset lastIndex — reusing a /g regex across matchAll iterations\n // is safe (matchAll internally advances), but across different\n // input strings we want a clean start.\n re.lastIndex = 0;\n let last = 0;\n for (const m of chunk.matchAll(re)) {\n const idx = m.index ?? 0;\n if (idx > last) out.push(chunk.slice(last, idx));\n if (m[0].length > 0) out.push(m[0]);\n last = idx + m[0].length;\n }\n if (last < chunk.length) out.push(chunk.slice(last));\n }\n return out;\n}\n\n/** UTF-8 bytes of `s`, each mapped to its byte-level visible char. */\nfunction byteLevelEncode(s: string, byteToChar: string[]): string {\n const bytes = new TextEncoder().encode(s);\n let out = \"\";\n for (let i = 0; i < bytes.length; i++) out += byteToChar[bytes[i]!];\n return out;\n}\n\n/** Repetitive tool output / identifier chunks re-encode thousands of times per session; LRU bounds at ~400KB. */\nconst bpeCache = new LruCache<string, string[]>(8192);\n\nfunction bpeEncode(piece: string, mergeRank: Map<string, number>): string[] {\n if (piece.length <= 1) return piece ? [piece] : [];\n const cached = bpeCache.get(piece);\n if (cached !== undefined) return cached;\n const word: string[] = Array.from(piece);\n while (word.length > 1) {\n let bestIdx = -1;\n let bestRank = Number.POSITIVE_INFINITY;\n for (let i = 0; i < word.length - 1; i++) {\n const pair = `${word[i]} ${word[i + 1]}`;\n const rank = mergeRank.get(pair);\n if (rank !== undefined && rank < bestRank) {\n bestRank = rank;\n bestIdx = i;\n if (rank === 0) break;\n }\n }\n if (bestIdx < 0) break;\n word.splice(bestIdx, 2, word[bestIdx]! + word[bestIdx + 1]!);\n }\n bpeCache.set(piece, word);\n return word;\n}\n\nexport function encode(text: string): number[] {\n if (!text) return [];\n const t = loadTokenizer();\n const ids: number[] = [];\n\n const process = (segment: string) => {\n if (!segment) return;\n let chunks: string[] = [segment];\n for (const re of t.splitRegexes) chunks = applySplit(chunks, re);\n for (const chunk of chunks) {\n if (!chunk) continue;\n const byteLevel = byteLevelEncode(chunk, t.byteToChar);\n const pieces = bpeEncode(byteLevel, t.mergeRank);\n for (const p of pieces) {\n const id = t.vocab[p];\n // If not in vocab we silently skip: shouldn't happen for\n // byte-level BPE (every single byte has its own vocab entry),\n // but if a future tokenizer update breaks that invariant we'd\n // rather under-count than throw from a UI gauge.\n if (id !== undefined) ids.push(id);\n }\n }\n };\n\n if (t.addedPattern) {\n t.addedPattern.lastIndex = 0;\n let last = 0;\n for (const m of text.matchAll(t.addedPattern)) {\n const idx = m.index ?? 0;\n if (idx > last) process(text.slice(last, idx));\n const id = t.addedMap.get(m[0]);\n if (id !== undefined) ids.push(id);\n last = idx + m[0].length;\n }\n if (last < text.length) process(text.slice(last));\n } else {\n process(text);\n }\n return ids;\n}\n\nexport function countTokens(text: string): number {\n return encode(text).length;\n}\n\nexport const DEFAULT_BOUNDED_TOKENIZE_CHARS = 2 * 1024;\n\nexport function countTokensBounded(\n text: string,\n maxChars = DEFAULT_BOUNDED_TOKENIZE_CHARS,\n): number {\n if (text.length === 0) return 0;\n const cap = Math.floor(maxChars);\n if (cap > 0 && text.length <= cap) return countTokens(text);\n if (cap <= 0) return Math.max(1, Math.ceil(text.length * 0.3));\n\n const headChars = Math.ceil(cap / 2);\n const tailChars = Math.floor(cap / 2);\n const head = text.slice(0, headChars);\n const tail = tailChars > 0 ? text.slice(-tailChars) : \"\";\n const sampleChars = head.length + tail.length;\n const sampleTokens = countTokens(head) + countTokens(tail);\n const ratio = sampleChars > 0 ? sampleTokens / sampleChars : 0.3;\n return Math.max(1, Math.ceil(text.length * ratio));\n}\n\nconst BOS = \"<|begin▁of▁sentence|>\";\nconst EOS = \"<|end▁of▁sentence|>\";\nconst USER_SP = \"<|User|>\";\nconst ASSISTANT_SP = \"<|Assistant|>\";\nconst THINK_START = \"<think>\";\nconst THINK_END = \"</think>\";\n\nconst DSML = \"|DSML|\";\nconst TC_BEGIN = `<${DSML}tool_calls>`;\nconst TC_END = `</${DSML}tool_calls>`;\nconst INVOKE_BEGIN = `<${DSML}invoke name=\"`;\nconst INVOKE_END = `</${DSML}invoke>`;\nconst PARAM_TEMPLATE = `<${DSML}parameter name=\"{key}\" string=\"{is_str}\">{value}</${DSML}parameter>`;\nconst TOOL_RESULT_TEMPLATE = \"<tool_result>{content}</tool_result>\";\n\n/** Keyed by `ImmutablePrefix._toolSpecs` identity — stable for the prefix's lifetime. */\nconst toolsTemplateCache = new WeakMap<ReadonlyArray<unknown>, string>();\n\nfunction renderTools(tools: ReadonlyArray<unknown>): string {\n const cached = toolsTemplateCache.get(tools);\n if (cached !== undefined) return cached;\n\n const schemas = tools\n .map((t) => {\n const fn = (t as { function?: unknown }).function ?? t;\n return JSON.stringify(fn);\n })\n .join(\"\\n\");\n const rendered = `## Tools\\n\\nYou have access to a set of tools to help answer the user's question. You can invoke tools by writing a \\\"<${DSML}tool_calls>\" block like the following:\\n\\n<${DSML}tool_calls>\\n<${DSML}invoke name=\"$TOOL_NAME\">\\n<${DSML}parameter name=\"$PARAMETER_NAME\" string=\"true|false\">$PARAMETER_VALUE</${DSML}parameter>\\n...\\n</${DSML}invoke>\\n<${DSML}invoke name=\"$TOOL_NAME2\">\\n...\\n</${DSML}invoke>\\n</${DSML}tool_calls>\\n\\nString parameters should be specified as is and set \\`string=\"true\"\\`. For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set \\`string=\"false\"\\`.\\n\\nIf thinking_mode is enabled (triggered by ${THINK_START}), you MUST output your complete reasoning inside ${THINK_START}...${THINK_END} BEFORE any tool calls or final response.\\n\\nOtherwise, output directly after ${THINK_END} with tool calls or final response.\\n\\n### Available Tool Schemas\\n\\n${schemas}\\n\\nYou MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls.`;\n\n toolsTemplateCache.set(tools, rendered);\n return rendered;\n}\n\ninterface ToolCall {\n function?: { name?: string; arguments?: string };\n [k: string]: unknown;\n}\n\ninterface V4Message {\n role?: string;\n content?: string | null;\n tool_calls?: ToolCall[];\n tool_call_id?: string;\n reasoning_content?: string | null;\n _toolBlocks?: string[];\n _textParts?: string[];\n}\n\nfunction encodeArgumentsToDsml(argsJson: string): string {\n let args: Record<string, unknown>;\n try {\n args = JSON.parse(argsJson) as Record<string, unknown>;\n } catch {\n args = { arguments: argsJson };\n }\n return Object.entries(args)\n .map(([k, v]) =>\n PARAM_TEMPLATE.replace(\"{key}\", k)\n .replace(\"{is_str}\", typeof v === \"string\" ? \"true\" : \"false\")\n .replace(\"{value}\", typeof v === \"string\" ? v : JSON.stringify(v)),\n )\n .join(\"\\n\");\n}\n\nfunction renderToolCallsDsml(toolCalls: ToolCall[]): string {\n const invokes = toolCalls\n .map((tc) => {\n const name = tc.function?.name ?? \"\";\n const argsJson = tc.function?.arguments ?? \"{}\";\n return `${INVOKE_BEGIN + name}\">\\n${encodeArgumentsToDsml(argsJson)}\\n${INVOKE_END}`;\n })\n .join(\"\\n\");\n return `\\n\\n${TC_BEGIN}\\n${invokes}\\n${TC_END}`;\n}\n\nfunction mergeToolMessages(messages: V4Message[]): V4Message[] {\n const merged: V4Message[] = [];\n for (const msg of messages) {\n const role = msg.role ?? \"user\";\n if (role === \"tool\") {\n const toolBlock = TOOL_RESULT_TEMPLATE.replace(\"{content}\", msg.content ?? \"\");\n const last = merged[merged.length - 1];\n if (\n last &&\n last.role === \"user\" &&\n Array.isArray(last._toolBlocks) &&\n Array.isArray(last._textParts)\n ) {\n last._toolBlocks.push(toolBlock);\n last.content = `${last._textParts.join(\"\\n\\n\")}\\n\\n${last._toolBlocks.join(\"\\n\")}`.replace(\n /^\\n\\n/,\n \"\",\n );\n } else {\n merged.push({\n role: \"user\",\n content: toolBlock,\n _textParts: [],\n _toolBlocks: [toolBlock],\n });\n }\n } else if (role === \"user\") {\n const text = msg.content ?? \"\";\n const last = merged[merged.length - 1];\n if (\n last &&\n last.role === \"user\" &&\n Array.isArray(last._toolBlocks) &&\n Array.isArray(last._textParts)\n ) {\n last._textParts.push(text);\n last.content =\n `${last._textParts.join(\"\\n\\n\")}\\n\\n${last._toolBlocks.join(\"\\n\\n\")}`.replace(\n /^\\n\\n/,\n \"\",\n );\n } else {\n merged.push({\n ...msg,\n role: \"user\",\n content: text,\n _textParts: [text],\n _toolBlocks: [],\n });\n }\n } else {\n merged.push({ ...msg });\n }\n }\n for (const m of merged) {\n m._textParts = undefined;\n m._toolBlocks = undefined;\n }\n return merged;\n}\n\n/** Drop `reasoning_content` from assistant messages before the last user/developer message. Matches Python `_drop_thinking_messages`. */\nfunction dropThinkingMessages(messages: V4Message[]): V4Message[] {\n let lastUserIdx = -1;\n for (let i = messages.length - 1; i >= 0; i--) {\n const role = messages[i]!.role;\n if (role === \"user\" || role === \"developer\") {\n lastUserIdx = i;\n break;\n }\n }\n if (lastUserIdx < 0) return messages;\n\n // Match Python `_drop_thinking_messages`:\n // - developer messages before lastUserIdx are dropped entirely\n // - assistant messages before lastUserIdx keep content & tool_calls\n // but have reasoning_content stripped\n const result: V4Message[] = [];\n for (let i = 0; i < messages.length; i++) {\n const msg = messages[i]!;\n if (i < lastUserIdx && msg.role === \"developer\") continue;\n if (i < lastUserIdx && msg.role === \"assistant\") {\n result.push({ ...msg, reasoning_content: null });\n } else {\n result.push(msg);\n }\n }\n return result;\n}\n\n/** Apply DeepSeek V4 chat template. Matches `encoding_dsv4.py`: tool results merged into user messages, assistant tool_calls in DSML, generation suffix appended. */\nexport function formatDeepSeekPrompt(\n messages: Array<{\n role?: string;\n content?: string | null;\n tool_calls?: unknown;\n tool_call_id?: string;\n reasoning_content?: string | null;\n }>,\n drop_thinking = false,\n): string {\n if (messages.length === 0) return ASSISTANT_SP + THINK_END;\n\n let msgs = messages as V4Message[];\n if (drop_thinking) {\n msgs = dropThinkingMessages(msgs);\n }\n const merged = mergeToolMessages(msgs);\n\n let prompt = BOS;\n\n for (let i = 0; i < merged.length; i++) {\n const msg = merged[i]!;\n const role = msg.role ?? \"user\";\n const nextRole = i + 1 < merged.length ? (merged[i + 1]!.role ?? \"user\") : null;\n\n if (role === \"system\") {\n prompt += msg.content ?? \"\";\n } else if (role === \"user\") {\n prompt += USER_SP + (msg.content ?? \"\");\n if (nextRole === \"assistant\" || nextRole === null) {\n prompt += ASSISTANT_SP + THINK_END;\n }\n } else if (role === \"assistant\") {\n if (msg.reasoning_content) {\n prompt += THINK_START + msg.reasoning_content + THINK_END;\n }\n if (msg.content) prompt += msg.content;\n const tcs = msg.tool_calls;\n if (Array.isArray(tcs) && tcs.length > 0) {\n prompt += renderToolCallsDsml(tcs);\n }\n prompt += EOS;\n }\n }\n\n return prompt;\n}\n\nconst PER_MESSAGE_TEMPLATE_TOKENS = 6;\n\n/** Keyed by content string — WeakMap-on-message can't be used because callers spread `{...e}` defensive copies, breaking identity every turn. */\nconst contentTokenCache = new LruCache<string, number>(4096);\n\nfunction cachedBoundedTokens(s: string): number {\n if (s.length === 0) return 0;\n const cached = contentTokenCache.get(s);\n if (cached !== undefined) return cached;\n const n = countTokensBounded(s);\n contentTokenCache.set(s, n);\n return n;\n}\n\nfunction tokensForMessage(\n m: {\n role?: string;\n content?: string | null;\n tool_calls?: unknown;\n reasoning_content?: string | null;\n },\n dropThisReasoning: boolean,\n): number {\n let n = 0;\n if (typeof m.content === \"string\" && m.content.length > 0) {\n n += cachedBoundedTokens(m.content);\n }\n if (m.role === \"assistant\") {\n if (\n !dropThisReasoning &&\n typeof m.reasoning_content === \"string\" &&\n m.reasoning_content.length > 0\n ) {\n n += cachedBoundedTokens(m.reasoning_content);\n }\n const tcs = m.tool_calls;\n if (Array.isArray(tcs) && tcs.length > 0) {\n n += cachedBoundedTokens(JSON.stringify(tcs));\n }\n }\n return n;\n}\n\n/** Per-message bounded sum, not a full-prompt rebuild — used for fold-threshold checks where ±5% slop is fine. */\nexport function estimateConversationTokens(\n messages: Array<{\n role?: string;\n content?: string | null;\n tool_calls?: unknown;\n tool_call_id?: string;\n reasoning_content?: string | null;\n }>,\n drop_thinking = false,\n): number {\n if (messages.length === 0) return 0;\n let lastUserOrDev = -1;\n if (drop_thinking) {\n for (let i = messages.length - 1; i >= 0; i--) {\n const r = messages[i]!.role;\n if (r === \"user\" || r === \"developer\") {\n lastUserOrDev = i;\n break;\n }\n }\n }\n let total = 2;\n for (let i = 0; i < messages.length; i++) {\n const m = messages[i]!;\n if (drop_thinking && i < lastUserOrDev && m.role === \"developer\") continue;\n total += PER_MESSAGE_TEMPLATE_TOKENS;\n const dropReasoning = drop_thinking && i < lastUserOrDev && m.role === \"assistant\";\n total += tokensForMessage(m, dropReasoning);\n }\n return total;\n}\n\n/** Total request tokens (messages + tool specs) as the API counts them. Tool specs rendered via V4 TOOLS_TEMPLATE and added to message token count. */\nexport function estimateRequestTokens(\n messages: Array<{\n role?: string;\n content?: string | null;\n tool_calls?: unknown;\n tool_call_id?: string;\n reasoning_content?: string | null;\n }>,\n toolSpecs?: ReadonlyArray<unknown> | null,\n drop_thinking = false,\n): number {\n let total = estimateConversationTokens(messages, drop_thinking);\n if (toolSpecs && toolSpecs.length > 0) {\n total += countTokensBounded(renderTools(toolSpecs));\n }\n return total;\n}\n\n/** Exposed for tests — resets the lazy-load singleton. */\nexport function _resetForTests(): void {\n cached = null;\n}\n"],"mappings":";;;;;;;AAEA,SAAS,YAAY,oBAAoB;AACzC,SAAS,qBAAqB;AAC9B,SAAS,SAAS,YAAY;AAC9B,SAAS,qBAAqB;AAC9B,SAAS,kBAAkB;AAkD3B,SAAS,kBAA4B;AACnC,QAAM,SAAmB,IAAI,MAAM,GAAG;AACtC,QAAM,KAAe,CAAC;AACtB,WAAS,IAAI,IAAI,KAAK,KAAK,IAAK,IAAG,KAAK,CAAC;AACzC,WAAS,IAAI,KAAK,KAAK,KAAK,IAAK,IAAG,KAAK,CAAC;AAC1C,WAAS,IAAI,KAAK,KAAK,KAAK,IAAK,IAAG,KAAK,CAAC;AAC1C,QAAM,KAAK,GAAG,MAAM;AACpB,MAAI,IAAI;AACR,WAAS,IAAI,GAAG,IAAI,KAAK,KAAK;AAC5B,QAAI,CAAC,GAAG,SAAS,CAAC,GAAG;AACnB,SAAG,KAAK,CAAC;AACT,SAAG,KAAK,MAAM,CAAC;AACf;AAAA,IACF;AAAA,EACF;AACA,WAAS,IAAI,GAAG,IAAI,GAAG,QAAQ,KAAK;AAClC,WAAO,GAAG,CAAC,CAAE,IAAI,OAAO,cAAc,GAAG,CAAC,CAAE;AAAA,EAC9C;AACA,SAAO;AACT;AAEA,IAAI,SAAiC;AAG9B,SAAS,kBAA0B;AACxC,MAAI,QAAQ,IAAI,wBAAyB,QAAO,QAAQ,IAAI;AAC5D,QAAM,aAAuB,CAAC;AAC9B,MAAI;AACF,UAAM,OAAO,QAAQ,cAAc,YAAY,GAAG,CAAC;AACnD,eAAW,KAAK,KAAK,MAAM,MAAM,QAAQ,4BAA4B,CAAC;AACtE,eAAW,KAAK,KAAK,MAAM,MAAM,MAAM,QAAQ,4BAA4B,CAAC;AAAA,EAC9E,QAAQ;AAAA,EAER;AACA,MAAI;AACF,UAAM,MAAM,cAAc,YAAY,GAAG;AACzC,eAAW;AAAA,MACT,KAAK,QAAQ,IAAI,QAAQ,uBAAuB,CAAC,GAAG,QAAQ,4BAA4B;AAAA,IAC1F;AAAA,EACF,QAAQ;AAAA,EAER;AACA,aAAW,KAAK,YAAY;AAC1B,QAAI,WAAW,CAAC,EAAG,QAAO;AAAA,EAC5B;AAGA,SAAO,WAAW,CAAC,KAAK,KAAK,QAAQ,IAAI,GAAG,QAAQ,4BAA4B;AAClF;AAEA,SAAS,gBAAiC;AACxC,MAAI,OAAQ,QAAO;AACnB,QAAM,MAAM,aAAa,gBAAgB,CAAC;AAC1C,QAAM,OAAO,WAAW,GAAG,EAAE,SAAS,MAAM;AAC5C,QAAM,OAAO,KAAK,MAAM,IAAI;AAE5B,QAAM,YAAY,oBAAI,IAAoB;AAC1C,WAAS,IAAI,GAAG,IAAI,KAAK,MAAM,OAAO,QAAQ,KAAK;AACjD,cAAU,IAAI,KAAK,MAAM,OAAO,CAAC,GAAI,CAAC;AAAA,EACxC;AAEA,QAAM,eAAyB,CAAC;AAChC,aAAW,KAAK,KAAK,cAAc,eAAe;AAChD,QAAI,EAAE,SAAS,SAAS;AAKtB,mBAAa,KAAK,IAAI,OAAO,EAAE,QAAQ,OAAO,IAAI,CAAC;AAAA,IACrD;AAAA,EACF;AAEA,QAAM,WAAW,oBAAI,IAAoB;AACzC,QAAM,gBAA0B,CAAC;AACjC,aAAW,KAAK,KAAK,cAAc;AACjC,QAAI,CAAC,EAAE,SAAS;AACd,eAAS,IAAI,EAAE,SAAS,EAAE,EAAE;AAC5B,oBAAc,KAAK,EAAE,OAAO;AAAA,IAC9B;AAAA,EACF;AAGA,gBAAc,KAAK,CAAC,GAAG,MAAM,EAAE,SAAS,EAAE,MAAM;AAChD,QAAM,eAAe,cAAc,SAC/B,IAAI,OAAO,cAAc,IAAI,WAAW,EAAE,KAAK,GAAG,GAAG,GAAG,IACxD;AAEJ,WAAS;AAAA,IACP,OAAO,KAAK,MAAM;AAAA,IAClB;AAAA,IACA;AAAA,IACA,YAAY,gBAAgB;AAAA,IAC5B;AAAA,IACA;AAAA,EACF;AACA,SAAO;AACT;AAIO,SAAS,kBAAwB;AACtC,gBAAc;AAChB;AAEA,SAAS,YAAY,GAAmB;AACtC,SAAO,EAAE,QAAQ,uBAAuB,MAAM;AAChD;AAEA,SAAS,WAAW,QAAkB,IAAsB;AAC1D,QAAM,MAAgB,CAAC;AACvB,aAAW,SAAS,QAAQ;AAC1B,QAAI,CAAC,MAAO;AAIZ,OAAG,YAAY;AACf,QAAI,OAAO;AACX,eAAW,KAAK,MAAM,SAAS,EAAE,GAAG;AAClC,YAAM,MAAM,EAAE,SAAS;AACvB,UAAI,MAAM,KAAM,KAAI,KAAK,MAAM,MAAM,MAAM,GAAG,CAAC;AAC/C,UAAI,EAAE,CAAC,EAAE,SAAS,EAAG,KAAI,KAAK,EAAE,CAAC,CAAC;AAClC,aAAO,MAAM,EAAE,CAAC,EAAE;AAAA,IACpB;AACA,QAAI,OAAO,MAAM,OAAQ,KAAI,KAAK,MAAM,MAAM,IAAI,CAAC;AAAA,EACrD;AACA,SAAO;AACT;AAGA,SAAS,gBAAgB,GAAW,YAA8B;AAChE,QAAM,QAAQ,IAAI,YAAY,EAAE,OAAO,CAAC;AACxC,MAAI,MAAM;AACV,WAAS,IAAI,GAAG,IAAI,MAAM,QAAQ,IAAK,QAAO,WAAW,MAAM,CAAC,CAAE;AAClE,SAAO;AACT;AAGA,IAAM,WAAW,IAAI,SAA2B,IAAI;AAEpD,SAAS,UAAU,OAAe,WAA0C;AAC1E,MAAI,MAAM,UAAU,EAAG,QAAO,QAAQ,CAAC,KAAK,IAAI,CAAC;AACjD,QAAMA,UAAS,SAAS,IAAI,KAAK;AACjC,MAAIA,YAAW,OAAW,QAAOA;AACjC,QAAM,OAAiB,MAAM,KAAK,KAAK;AACvC,SAAO,KAAK,SAAS,GAAG;AACtB,QAAI,UAAU;AACd,QAAI,WAAW,OAAO;AACtB,aAAS,IAAI,GAAG,IAAI,KAAK,SAAS,GAAG,KAAK;AACxC,YAAM,OAAO,GAAG,KAAK,CAAC,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC;AACtC,YAAM,OAAO,UAAU,IAAI,IAAI;AAC/B,UAAI,SAAS,UAAa,OAAO,UAAU;AACzC,mBAAW;AACX,kBAAU;AACV,YAAI,SAAS,EAAG;AAAA,MAClB;AAAA,IACF;AACA,QAAI,UAAU,EAAG;AACjB,SAAK,OAAO,SAAS,GAAG,KAAK,OAAO,IAAK,KAAK,UAAU,CAAC,CAAE;AAAA,EAC7D;AACA,WAAS,IAAI,OAAO,IAAI;AACxB,SAAO;AACT;AAEO,SAAS,OAAO,MAAwB;AAC7C,MAAI,CAAC,KAAM,QAAO,CAAC;AACnB,QAAM,IAAI,cAAc;AACxB,QAAM,MAAgB,CAAC;AAEvB,QAAMC,WAAU,CAAC,YAAoB;AACnC,QAAI,CAAC,QAAS;AACd,QAAI,SAAmB,CAAC,OAAO;AAC/B,eAAW,MAAM,EAAE,aAAc,UAAS,WAAW,QAAQ,EAAE;AAC/D,eAAW,SAAS,QAAQ;AAC1B,UAAI,CAAC,MAAO;AACZ,YAAM,YAAY,gBAAgB,OAAO,EAAE,UAAU;AACrD,YAAM,SAAS,UAAU,WAAW,EAAE,SAAS;AAC/C,iBAAW,KAAK,QAAQ;AACtB,cAAM,KAAK,EAAE,MAAM,CAAC;AAKpB,YAAI,OAAO,OAAW,KAAI,KAAK,EAAE;AAAA,MACnC;AAAA,IACF;AAAA,EACF;AAEA,MAAI,EAAE,cAAc;AAClB,MAAE,aAAa,YAAY;AAC3B,QAAI,OAAO;AACX,eAAW,KAAK,KAAK,SAAS,EAAE,YAAY,GAAG;AAC7C,YAAM,MAAM,EAAE,SAAS;AACvB,UAAI,MAAM,KAAM,CAAAA,SAAQ,KAAK,MAAM,MAAM,GAAG,CAAC;AAC7C,YAAM,KAAK,EAAE,SAAS,IAAI,EAAE,CAAC,CAAC;AAC9B,UAAI,OAAO,OAAW,KAAI,KAAK,EAAE;AACjC,aAAO,MAAM,EAAE,CAAC,EAAE;AAAA,IACpB;AACA,QAAI,OAAO,KAAK,OAAQ,CAAAA,SAAQ,KAAK,MAAM,IAAI,CAAC;AAAA,EAClD,OAAO;AACL,IAAAA,SAAQ,IAAI;AAAA,EACd;AACA,SAAO;AACT;AAEO,SAAS,YAAY,MAAsB;AAChD,SAAO,OAAO,IAAI,EAAE;AACtB;AAEO,IAAM,iCAAiC,IAAI;AAE3C,SAAS,mBACd,MACA,WAAW,gCACH;AACR,MAAI,KAAK,WAAW,EAAG,QAAO;AAC9B,QAAM,MAAM,KAAK,MAAM,QAAQ;AAC/B,MAAI,MAAM,KAAK,KAAK,UAAU,IAAK,QAAO,YAAY,IAAI;AAC1D,MAAI,OAAO,EAAG,QAAO,KAAK,IAAI,GAAG,KAAK,KAAK,KAAK,SAAS,GAAG,CAAC;AAE7D,QAAM,YAAY,KAAK,KAAK,MAAM,CAAC;AACnC,QAAM,YAAY,KAAK,MAAM,MAAM,CAAC;AACpC,QAAM,OAAO,KAAK,MAAM,GAAG,SAAS;AACpC,QAAM,OAAO,YAAY,IAAI,KAAK,MAAM,CAAC,SAAS,IAAI;AACtD,QAAM,cAAc,KAAK,SAAS,KAAK;AACvC,QAAM,eAAe,YAAY,IAAI,IAAI,YAAY,IAAI;AACzD,QAAM,QAAQ,cAAc,IAAI,eAAe,cAAc;AAC7D,SAAO,KAAK,IAAI,GAAG,KAAK,KAAK,KAAK,SAAS,KAAK,CAAC;AACnD;AAMA,IAAM,cAAc;AACpB,IAAM,YAAY;AAElB,IAAM,OAAO;AACb,IAAM,WAAW,IAAI,IAAI;AACzB,IAAM,SAAS,KAAK,IAAI;AACxB,IAAM,eAAe,IAAI,IAAI;AAC7B,IAAM,aAAa,KAAK,IAAI;AAC5B,IAAM,iBAAiB,IAAI,IAAI,qDAAqD,IAAI;AAIxF,IAAM,qBAAqB,oBAAI,QAAwC;AAEvE,SAAS,YAAY,OAAuC;AAC1D,QAAMC,UAAS,mBAAmB,IAAI,KAAK;AAC3C,MAAIA,YAAW,OAAW,QAAOA;AAEjC,QAAM,UAAU,MACb,IAAI,CAAC,MAAM;AACV,UAAM,KAAM,EAA6B,YAAY;AACrD,WAAO,KAAK,UAAU,EAAE;AAAA,EAC1B,CAAC,EACA,KAAK,IAAI;AACZ,QAAM,WAAW;AAAA;AAAA,4GAA0H,IAAI;AAAA;AAAA,GAA8C,IAAI;AAAA,GAAiB,IAAI;AAAA,GAA+B,IAAI,0EAA0E,IAAI;AAAA;AAAA,IAAsB,IAAI;AAAA,GAAa,IAAI;AAAA;AAAA,IAAsC,IAAI;AAAA,IAAc,IAAI;AAAA;AAAA;AAAA;AAAA,4CAA0P,WAAW,qDAAqD,WAAW,MAAM,SAAS;AAAA;AAAA,mCAAiF,SAAS;AAAA;AAAA;AAAA;AAAA,EAAwE,OAAO;AAAA;AAAA;AAE36B,qBAAmB,IAAI,OAAO,QAAQ;AACtC,SAAO;AACT;AAuLA,IAAM,8BAA8B;AAGpC,IAAM,oBAAoB,IAAI,SAAyB,IAAI;AAE3D,SAAS,oBAAoB,GAAmB;AAC9C,MAAI,EAAE,WAAW,EAAG,QAAO;AAC3B,QAAMC,UAAS,kBAAkB,IAAI,CAAC;AACtC,MAAIA,YAAW,OAAW,QAAOA;AACjC,QAAM,IAAI,mBAAmB,CAAC;AAC9B,oBAAkB,IAAI,GAAG,CAAC;AAC1B,SAAO;AACT;AAEA,SAAS,iBACP,GAMA,mBACQ;AACR,MAAI,IAAI;AACR,MAAI,OAAO,EAAE,YAAY,YAAY,EAAE,QAAQ,SAAS,GAAG;AACzD,SAAK,oBAAoB,EAAE,OAAO;AAAA,EACpC;AACA,MAAI,EAAE,SAAS,aAAa;AAC1B,QACE,CAAC,qBACD,OAAO,EAAE,sBAAsB,YAC/B,EAAE,kBAAkB,SAAS,GAC7B;AACA,WAAK,oBAAoB,EAAE,iBAAiB;AAAA,IAC9C;AACA,UAAM,MAAM,EAAE;AACd,QAAI,MAAM,QAAQ,GAAG,KAAK,IAAI,SAAS,GAAG;AACxC,WAAK,oBAAoB,KAAK,UAAU,GAAG,CAAC;AAAA,IAC9C;AAAA,EACF;AACA,SAAO;AACT;AAGO,SAAS,2BACd,UAOA,gBAAgB,OACR;AACR,MAAI,SAAS,WAAW,EAAG,QAAO;AAClC,MAAI,gBAAgB;AACpB,MAAI,eAAe;AACjB,aAAS,IAAI,SAAS,SAAS,GAAG,KAAK,GAAG,KAAK;AAC7C,YAAM,IAAI,SAAS,CAAC,EAAG;AACvB,UAAI,MAAM,UAAU,MAAM,aAAa;AACrC,wBAAgB;AAChB;AAAA,MACF;AAAA,IACF;AAAA,EACF;AACA,MAAI,QAAQ;AACZ,WAAS,IAAI,GAAG,IAAI,SAAS,QAAQ,KAAK;AACxC,UAAM,IAAI,SAAS,CAAC;AACpB,QAAI,iBAAiB,IAAI,iBAAiB,EAAE,SAAS,YAAa;AAClE,aAAS;AACT,UAAM,gBAAgB,iBAAiB,IAAI,iBAAiB,EAAE,SAAS;AACvE,aAAS,iBAAiB,GAAG,aAAa;AAAA,EAC5C;AACA,SAAO;AACT;AAGO,SAAS,sBACd,UAOA,WACA,gBAAgB,OACR;AACR,MAAI,QAAQ,2BAA2B,UAAU,aAAa;AAC9D,MAAI,aAAa,UAAU,SAAS,GAAG;AACrC,aAAS,mBAAmB,YAAY,SAAS,CAAC;AAAA,EACpD;AACA,SAAO;AACT;","names":["cached","process","cached","cached"]}
|
|
@@ -5,7 +5,7 @@ import {
|
|
|
5
5
|
} from "./chunk-25T6CVUP.js";
|
|
6
6
|
import {
|
|
7
7
|
loadRateLimit
|
|
8
|
-
} from "./chunk-
|
|
8
|
+
} from "./chunk-A5PBEIJ7.js";
|
|
9
9
|
|
|
10
10
|
// src/retry.ts
|
|
11
11
|
var DEFAULT_RETRYABLE_STATUSES = [408, 429, 500, 502, 503, 504];
|
|
@@ -357,4 +357,4 @@ export {
|
|
|
357
357
|
pickPrimaryBalance,
|
|
358
358
|
DeepSeekClient
|
|
359
359
|
};
|
|
360
|
-
//# sourceMappingURL=chunk-
|
|
360
|
+
//# sourceMappingURL=chunk-C2MRSJTV.js.map
|