reasonix 0.33.1 → 0.33.2

package/dist/cli/{chat-Q5ZCVIOO.js → chat-ZMSAXE77.js}

@@ -1,12 +1,12 @@
 #!/usr/bin/env node
 import {
   chatCommand
-} from "./chunk-MDHVWCJ4.js";
+} from "./chunk-OW7IHE6M.js";
 import "./chunk-BQNUJJN7.js";
 import "./chunk-RFX7TYVV.js";
 import "./chunk-63KAV5DX.js";
 import "./chunk-CPOV2O73.js";
-import "./chunk-Q6YFXW7H.js";
+import "./chunk-G7M3QWEN.js";
 import "./chunk-I6YIAK6C.js";
 import "./chunk-XJLZ4HKU.js";
 import "./chunk-XHQIK7B6.js";
@@ -14,7 +14,8 @@ import "./chunk-6TMHAK5D.js";
 import "./chunk-SDE5U32Z.js";
 import "./chunk-ZPTSJGX5.js";
 import "./chunk-MHDNZXJJ.js";
-import "./chunk-D5DKXIP5.js";
+import "./chunk-WVJL7ZO2.js";
+import "./chunk-DAEAAVDF.js";
 import "./chunk-KMWKGPFZ.js";
 import "./chunk-3Q3C4W66.js";
 import "./chunk-4DCHFFEY.js";
@@ -36,4 +37,4 @@ import "./chunk-ORM6PK57.js";
 export {
   chatCommand
 };
-//# sourceMappingURL=chat-Q5ZCVIOO.js.map
+//# sourceMappingURL=chat-ZMSAXE77.js.map

package/dist/cli/chunk-DAEAAVDF.js

@@ -0,0 +1,199 @@
+#!/usr/bin/env node
+
+// src/tokenizer.ts
+import { existsSync, readFileSync } from "fs";
+import { createRequire } from "module";
+import { dirname, join } from "path";
+import { fileURLToPath } from "url";
+import { gunzipSync } from "zlib";
+function buildByteToChar() {
+  const result = new Array(256);
+  const bs = [];
+  for (let b = 33; b <= 126; b++) bs.push(b);
+  for (let b = 161; b <= 172; b++) bs.push(b);
+  for (let b = 174; b <= 255; b++) bs.push(b);
+  const cs = bs.slice();
+  let n = 0;
+  for (let b = 0; b < 256; b++) {
+    if (!bs.includes(b)) {
+      bs.push(b);
+      cs.push(256 + n);
+      n++;
+    }
+  }
+  for (let i = 0; i < bs.length; i++) {
+    result[bs[i]] = String.fromCodePoint(cs[i]);
+  }
+  return result;
+}
+var cached = null;
+function resolveDataPath() {
+  if (process.env.REASONIX_TOKENIZER_PATH) return process.env.REASONIX_TOKENIZER_PATH;
+  const candidates = [];
+  try {
+    const here = dirname(fileURLToPath(import.meta.url));
+    candidates.push(join(here, "..", "data", "deepseek-tokenizer.json.gz"));
+    candidates.push(join(here, "..", "..", "data", "deepseek-tokenizer.json.gz"));
+  } catch {
+  }
+  try {
+    const req = createRequire(import.meta.url);
+    candidates.push(
+      join(dirname(req.resolve("reasonix/package.json")), "data", "deepseek-tokenizer.json.gz")
+    );
+  } catch {
+  }
+  for (const p of candidates) {
+    if (existsSync(p)) return p;
+  }
+  return candidates[0] ?? join(process.cwd(), "data", "deepseek-tokenizer.json.gz");
+}
+function loadTokenizer() {
+  if (cached) return cached;
+  const buf = readFileSync(resolveDataPath());
+  const json = gunzipSync(buf).toString("utf8");
+  const data = JSON.parse(json);
+  const mergeRank = /* @__PURE__ */ new Map();
+  for (let i = 0; i < data.model.merges.length; i++) {
+    mergeRank.set(data.model.merges[i], i);
+  }
+  const splitRegexes = [];
+  for (const p of data.pre_tokenizer.pretokenizers) {
+    if (p.type === "Split") {
+      splitRegexes.push(new RegExp(p.pattern.Regex, "gu"));
+    }
+  }
+  const addedMap = /* @__PURE__ */ new Map();
+  const addedContents = [];
+  for (const t of data.added_tokens) {
+    if (!t.special) {
+      addedMap.set(t.content, t.id);
+      addedContents.push(t.content);
+    }
+  }
+  addedContents.sort((a, b) => b.length - a.length);
+  const addedPattern = addedContents.length ? new RegExp(addedContents.map(escapeRegex).join("|"), "g") : null;
+  cached = {
+    vocab: data.model.vocab,
+    mergeRank,
+    splitRegexes,
+    byteToChar: buildByteToChar(),
+    addedPattern,
+    addedMap
+  };
+  return cached;
+}
+function escapeRegex(s) {
+  return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+function applySplit(chunks, re) {
+  const out = [];
+  for (const chunk of chunks) {
+    if (!chunk) continue;
+    re.lastIndex = 0;
+    let last = 0;
+    for (const m of chunk.matchAll(re)) {
+      const idx = m.index ?? 0;
+      if (idx > last) out.push(chunk.slice(last, idx));
+      if (m[0].length > 0) out.push(m[0]);
+      last = idx + m[0].length;
+    }
+    if (last < chunk.length) out.push(chunk.slice(last));
+  }
+  return out;
+}
+function byteLevelEncode(s, byteToChar) {
+  const bytes = new TextEncoder().encode(s);
+  let out = "";
+  for (let i = 0; i < bytes.length; i++) out += byteToChar[bytes[i]];
+  return out;
+}
+function bpeEncode(piece, mergeRank) {
+  if (piece.length <= 1) return piece ? [piece] : [];
+  let word = Array.from(piece);
+  while (true) {
+    let bestIdx = -1;
+    let bestRank = Number.POSITIVE_INFINITY;
+    for (let i = 0; i < word.length - 1; i++) {
+      const pair = `${word[i]} ${word[i + 1]}`;
+      const rank = mergeRank.get(pair);
+      if (rank !== void 0 && rank < bestRank) {
+        bestRank = rank;
+        bestIdx = i;
+        if (rank === 0) break;
+      }
+    }
+    if (bestIdx < 0) break;
+    word = [
+      ...word.slice(0, bestIdx),
+      word[bestIdx] + word[bestIdx + 1],
+      ...word.slice(bestIdx + 2)
+    ];
+    if (word.length === 1) break;
+  }
+  return word;
+}
+function encode(text) {
+  if (!text) return [];
+  const t = loadTokenizer();
+  const ids = [];
+  const process2 = (segment) => {
+    if (!segment) return;
+    let chunks = [segment];
+    for (const re of t.splitRegexes) chunks = applySplit(chunks, re);
+    for (const chunk of chunks) {
+      if (!chunk) continue;
+      const byteLevel = byteLevelEncode(chunk, t.byteToChar);
+      const pieces = bpeEncode(byteLevel, t.mergeRank);
+      for (const p of pieces) {
+        const id = t.vocab[p];
+        if (id !== void 0) ids.push(id);
+      }
+    }
+  };
+  if (t.addedPattern) {
+    t.addedPattern.lastIndex = 0;
+    let last = 0;
+    for (const m of text.matchAll(t.addedPattern)) {
+      const idx = m.index ?? 0;
+      if (idx > last) process2(text.slice(last, idx));
+      const id = t.addedMap.get(m[0]);
+      if (id !== void 0) ids.push(id);
+      last = idx + m[0].length;
+    }
+    if (last < text.length) process2(text.slice(last));
+  } else {
+    process2(text);
+  }
+  return ids;
+}
+function countTokens(text) {
+  return encode(text).length;
+}
+function estimateConversationTokens(messages) {
+  let total = 0;
+  for (const m of messages) {
+    if (typeof m.content === "string" && m.content) {
+      total += countTokens(m.content);
+    }
+    if (m.tool_calls && Array.isArray(m.tool_calls) && m.tool_calls.length > 0) {
+      total += countTokens(JSON.stringify(m.tool_calls));
+    }
+  }
+  return total;
+}
+function estimateRequestTokens(messages, toolSpecs) {
+  let total = estimateConversationTokens(messages);
+  if (toolSpecs && toolSpecs.length > 0) {
+    total += countTokens(JSON.stringify(toolSpecs));
+  }
+  return total;
+}
+
+export {
+  resolveDataPath,
+  countTokens,
+  estimateConversationTokens,
+  estimateRequestTokens
+};
+//# sourceMappingURL=chunk-DAEAAVDF.js.map

package/dist/cli/chunk-DAEAAVDF.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/tokenizer.ts"],"sourcesContent":["/** Encode-only DeepSeek V3 tokenizer port; ~3% drift vs API (chat-template framing not replayed). */\n\nimport { existsSync, readFileSync } from \"node:fs\";\nimport { createRequire } from \"node:module\";\nimport { dirname, join } from \"node:path\";\nimport { fileURLToPath } from \"node:url\";\nimport { gunzipSync } from \"node:zlib\";\n\ninterface AddedToken {\n  id: number;\n  content: string;\n  special: boolean;\n  normalized: boolean;\n}\n\ninterface SplitPretokenizer {\n  type: \"Split\";\n  pattern: { Regex: string };\n  behavior: \"Isolated\" | \"Removed\" | string;\n  invert: boolean;\n}\n\ninterface ByteLevelPretokenizer {\n  type: \"ByteLevel\";\n  add_prefix_space: boolean;\n  trim_offsets: boolean;\n  use_regex: boolean;\n}\n\ntype Pretokenizer = SplitPretokenizer | ByteLevelPretokenizer;\n\ninterface TokenizerData {\n  added_tokens: AddedToken[];\n  pre_tokenizer: {\n    type: \"Sequence\";\n    pretokenizers: Pretokenizer[];\n  };\n  model: {\n    type: \"BPE\";\n    vocab: Record<string, number>;\n    merges: string[];\n  };\n}\n\ninterface LoadedTokenizer {\n  vocab: Record<string, number>;\n  mergeRank: Map<string, number>;\n  splitRegexes: RegExp[];\n  byteToChar: string[];\n  /** Non-special added tokens only — special tokens in user text tokenize byte-by-byte (HF default). */\n  addedPattern: RegExp | null;\n  addedMap: Map<string, number>;\n}\n\n/** GPT-2 byte→unicode map; lets byte-level BPE vocab serialize as readable JSON strings. */\nfunction buildByteToChar(): string[] {\n  const result: string[] = new Array(256);\n  const bs: number[] = [];\n  for (let b = 33; b <= 126; b++) bs.push(b);\n  for (let b = 161; b <= 172; b++) bs.push(b);\n  for (let b = 174; b <= 255; b++) bs.push(b);\n  const cs = bs.slice();\n  let n = 0;\n  for (let b = 0; b < 256; b++) {\n    if (!bs.includes(b)) {\n      bs.push(b);\n      cs.push(256 + n);\n      n++;\n    }\n  }\n  for (let i = 0; i < bs.length; i++) {\n    result[bs[i]!] = String.fromCodePoint(cs[i]!);\n  }\n  return result;\n}\n\nlet cached: LoadedTokenizer | null = null;\n\n/** Two ../data candidates needed: dist/index.js AND dist/cli/index.js resolve to different roots. */\nexport function resolveDataPath(): string {\n  if (process.env.REASONIX_TOKENIZER_PATH) return process.env.REASONIX_TOKENIZER_PATH;\n  const candidates: string[] = [];\n  try {\n    const here = dirname(fileURLToPath(import.meta.url));\n    candidates.push(join(here, \"..\", \"data\", \"deepseek-tokenizer.json.gz\"));\n    candidates.push(join(here, \"..\", \"..\", \"data\", \"deepseek-tokenizer.json.gz\"));\n  } catch {\n    /* import.meta.url unavailable — skip to the package resolution step. */\n  }\n  try {\n    const req = createRequire(import.meta.url);\n    candidates.push(\n      join(dirname(req.resolve(\"reasonix/package.json\")), \"data\", \"deepseek-tokenizer.json.gz\"),\n    );\n  } catch {\n    /* Not installed as `reasonix/` — the earlier candidates still may hit. */\n  }\n  for (const p of candidates) {\n    if (existsSync(p)) return p;\n  }\n  // Nothing exists — return the first candidate anyway so readFileSync\n  // surfaces a concrete path in the ENOENT message (better than silent miss).\n  return candidates[0] ?? join(process.cwd(), \"data\", \"deepseek-tokenizer.json.gz\");\n}\n\nfunction loadTokenizer(): LoadedTokenizer {\n  if (cached) return cached;\n  const buf = readFileSync(resolveDataPath());\n  const json = gunzipSync(buf).toString(\"utf8\");\n  const data = JSON.parse(json) as TokenizerData;\n\n  const mergeRank = new Map<string, number>();\n  for (let i = 0; i < data.model.merges.length; i++) {\n    mergeRank.set(data.model.merges[i]!, i);\n  }\n\n  const splitRegexes: RegExp[] = [];\n  for (const p of data.pre_tokenizer.pretokenizers) {\n    if (p.type === \"Split\") {\n      // All three Split rules use Isolated — matches become their own\n      // pre-tokens and so do the in-between stretches. The ByteLevel\n      // stage in the Sequence does no extra splitting here\n      // (use_regex:false), so our 3 Split regexes are the whole story.\n      splitRegexes.push(new RegExp(p.pattern.Regex, \"gu\"));\n    }\n  }\n\n  const addedMap = new Map<string, number>();\n  const addedContents: string[] = [];\n  for (const t of data.added_tokens) {\n    if (!t.special) {\n      addedMap.set(t.content, t.id);\n      addedContents.push(t.content);\n    }\n  }\n  // Longest-first ensures greedy matching doesn't lose a longer token\n  // to a shorter prefix (e.g. `<think>` before `<`).\n  addedContents.sort((a, b) => b.length - a.length);\n  const addedPattern = addedContents.length\n    ? new RegExp(addedContents.map(escapeRegex).join(\"|\"), \"g\")\n    : null;\n\n  cached = {\n    vocab: data.model.vocab,\n    mergeRank,\n    splitRegexes,\n    byteToChar: buildByteToChar(),\n    addedPattern,\n    addedMap,\n  };\n  return cached;\n}\n\nfunction escapeRegex(s: string): string {\n  return s.replace(/[.*+?^${}()|[\\]\\\\]/g, \"\\\\$&\");\n}\n\nfunction applySplit(chunks: string[], re: RegExp): string[] {\n  const out: string[] = [];\n  for (const chunk of chunks) {\n    if (!chunk) continue;\n    // Reset lastIndex — reusing a /g regex across matchAll iterations\n    // is safe (matchAll internally advances), but across different\n    // input strings we want a clean start.\n    re.lastIndex = 0;\n    let last = 0;\n    for (const m of chunk.matchAll(re)) {\n      const idx = m.index ?? 0;\n      if (idx > last) out.push(chunk.slice(last, idx));\n      if (m[0].length > 0) out.push(m[0]);\n      last = idx + m[0].length;\n    }\n    if (last < chunk.length) out.push(chunk.slice(last));\n  }\n  return out;\n}\n\n/** UTF-8 bytes of `s`, each mapped to its byte-level visible char. */\nfunction byteLevelEncode(s: string, byteToChar: string[]): string {\n  const bytes = new TextEncoder().encode(s);\n  let out = \"\";\n  for (let i = 0; i < bytes.length; i++) out += byteToChar[bytes[i]!];\n  return out;\n}\n\nfunction bpeEncode(piece: string, mergeRank: Map<string, number>): string[] {\n  if (piece.length <= 1) return piece ? [piece] : [];\n  let word: string[] = Array.from(piece);\n  while (true) {\n    let bestIdx = -1;\n    let bestRank = Number.POSITIVE_INFINITY;\n    for (let i = 0; i < word.length - 1; i++) {\n      const pair = `${word[i]} ${word[i + 1]}`;\n      const rank = mergeRank.get(pair);\n      if (rank !== undefined && rank < bestRank) {\n        bestRank = rank;\n        bestIdx = i;\n        if (rank === 0) break; // 0 is already the best possible\n      }\n    }\n    if (bestIdx < 0) break;\n    word = [\n      ...word.slice(0, bestIdx),\n      word[bestIdx]! + word[bestIdx + 1]!,\n      ...word.slice(bestIdx + 2),\n    ];\n    if (word.length === 1) break;\n  }\n  return word;\n}\n\nexport function encode(text: string): number[] {\n  if (!text) return [];\n  const t = loadTokenizer();\n  const ids: number[] = [];\n\n  const process = (segment: string) => {\n    if (!segment) return;\n    let chunks: string[] = [segment];\n    for (const re of t.splitRegexes) chunks = applySplit(chunks, re);\n    for (const chunk of chunks) {\n      if (!chunk) continue;\n      const byteLevel = byteLevelEncode(chunk, t.byteToChar);\n      const pieces = bpeEncode(byteLevel, t.mergeRank);\n      for (const p of pieces) {\n        const id = t.vocab[p];\n        // If not in vocab we silently skip: shouldn't happen for\n        // byte-level BPE (every single byte has its own vocab entry),\n        // but if a future tokenizer update breaks that invariant we'd\n        // rather under-count than throw from a UI gauge.\n        if (id !== undefined) ids.push(id);\n      }\n    }\n  };\n\n  if (t.addedPattern) {\n    t.addedPattern.lastIndex = 0;\n    let last = 0;\n    for (const m of text.matchAll(t.addedPattern)) {\n      const idx = m.index ?? 0;\n      if (idx > last) process(text.slice(last, idx));\n      const id = t.addedMap.get(m[0]);\n      if (id !== undefined) ids.push(id);\n      last = idx + m[0].length;\n    }\n    if (last < text.length) process(text.slice(last));\n  } else {\n    process(text);\n  }\n  return ids;\n}\n\nexport function countTokens(text: string): number {\n  return encode(text).length;\n}\n\n/** Doesn't add chat-template framing overhead; under-counts ~3-6% vs real `prompt_tokens`. */\nexport function estimateConversationTokens(\n  messages: Array<{ content?: string | null; tool_calls?: unknown }>,\n): number {\n  let total = 0;\n  for (const m of messages) {\n    if (typeof m.content === \"string\" && m.content) {\n      total += countTokens(m.content);\n    }\n    // Tool-call arguments are serialized as JSON in the prompt by the\n    // chat template; their bytes WILL count upstream, so we count\n    // them too. Stringify-once is cheap relative to the tokenize.\n    if (m.tool_calls && Array.isArray(m.tool_calls) && m.tool_calls.length > 0) {\n      total += countTokens(JSON.stringify(m.tool_calls));\n    }\n  }\n  return total;\n}\n\n/** Tool specs ride in a separate request blob; must be counted separately for an accurate preflight. */\nexport function estimateRequestTokens(\n  messages: Array<{ content?: string | null; tool_calls?: unknown }>,\n  toolSpecs?: ReadonlyArray<unknown> | null,\n): number {\n  let total = estimateConversationTokens(messages);\n  if (toolSpecs && toolSpecs.length > 0) {\n    total += countTokens(JSON.stringify(toolSpecs));\n  }\n  return total;\n}\n\n/** Exposed for tests — resets the lazy-load singleton. */\nexport function _resetForTests(): void {\n  cached = null;\n}\n"],"mappings":";;;AAEA,SAAS,YAAY,oBAAoB;AACzC,SAAS,qBAAqB;AAC9B,SAAS,SAAS,YAAY;AAC9B,SAAS,qBAAqB;AAC9B,SAAS,kBAAkB;AAiD3B,SAAS,kBAA4B;AACnC,QAAM,SAAmB,IAAI,MAAM,GAAG;AACtC,QAAM,KAAe,CAAC;AACtB,WAAS,IAAI,IAAI,KAAK,KAAK,IAAK,IAAG,KAAK,CAAC;AACzC,WAAS,IAAI,KAAK,KAAK,KAAK,IAAK,IAAG,KAAK,CAAC;AAC1C,WAAS,IAAI,KAAK,KAAK,KAAK,IAAK,IAAG,KAAK,CAAC;AAC1C,QAAM,KAAK,GAAG,MAAM;AACpB,MAAI,IAAI;AACR,WAAS,IAAI,GAAG,IAAI,KAAK,KAAK;AAC5B,QAAI,CAAC,GAAG,SAAS,CAAC,GAAG;AACnB,SAAG,KAAK,CAAC;AACT,SAAG,KAAK,MAAM,CAAC;AACf;AAAA,IACF;AAAA,EACF;AACA,WAAS,IAAI,GAAG,IAAI,GAAG,QAAQ,KAAK;AAClC,WAAO,GAAG,CAAC,CAAE,IAAI,OAAO,cAAc,GAAG,CAAC,CAAE;AAAA,EAC9C;AACA,SAAO;AACT;AAEA,IAAI,SAAiC;AAG9B,SAAS,kBAA0B;AACxC,MAAI,QAAQ,IAAI,wBAAyB,QAAO,QAAQ,IAAI;AAC5D,QAAM,aAAuB,CAAC;AAC9B,MAAI;AACF,UAAM,OAAO,QAAQ,cAAc,YAAY,GAAG,CAAC;AACnD,eAAW,KAAK,KAAK,MAAM,MAAM,QAAQ,4BAA4B,CAAC;AACtE,eAAW,KAAK,KAAK,MAAM,MAAM,MAAM,QAAQ,4BAA4B,CAAC;AAAA,EAC9E,QAAQ;AAAA,EAER;AACA,MAAI;AACF,UAAM,MAAM,cAAc,YAAY,GAAG;AACzC,eAAW;AAAA,MACT,KAAK,QAAQ,IAAI,QAAQ,uBAAuB,CAAC,GAAG,QAAQ,4BAA4B;AAAA,IAC1F;AAAA,EACF,QAAQ;AAAA,EAER;AACA,aAAW,KAAK,YAAY;AAC1B,QAAI,WAAW,CAAC,EAAG,QAAO;AAAA,EAC5B;AAGA,SAAO,WAAW,CAAC,KAAK,KAAK,QAAQ,IAAI,GAAG,QAAQ,4BAA4B;AAClF;AAEA,SAAS,gBAAiC;AACxC,MAAI,OAAQ,QAAO;AACnB,QAAM,MAAM,aAAa,gBAAgB,CAAC;AAC1C,QAAM,OAAO,WAAW,GAAG,EAAE,SAAS,MAAM;AAC5C,QAAM,OAAO,KAAK,MAAM,IAAI;AAE5B,QAAM,YAAY,oBAAI,IAAoB;AAC1C,WAAS,IAAI,GAAG,IAAI,KAAK,MAAM,OAAO,QAAQ,KAAK;AACjD,cAAU,IAAI,KAAK,MAAM,OAAO,CAAC,GAAI,CAAC;AAAA,EACxC;AAEA,QAAM,eAAyB,CAAC;AAChC,aAAW,KAAK,KAAK,cAAc,eAAe;AAChD,QAAI,EAAE,SAAS,SAAS;AAKtB,mBAAa,KAAK,IAAI,OAAO,EAAE,QAAQ,OAAO,IAAI,CAAC;AAAA,IACrD;AAAA,EACF;AAEA,QAAM,WAAW,oBAAI,IAAoB;AACzC,QAAM,gBAA0B,CAAC;AACjC,aAAW,KAAK,KAAK,cAAc;AACjC,QAAI,CAAC,EAAE,SAAS;AACd,eAAS,IAAI,EAAE,SAAS,EAAE,EAAE;AAC5B,oBAAc,KAAK,EAAE,OAAO;AAAA,IAC9B;AAAA,EACF;AAGA,gBAAc,KAAK,CAAC,GAAG,MAAM,EAAE,SAAS,EAAE,MAAM;AAChD,QAAM,eAAe,cAAc,SAC/B,IAAI,OAAO,cAAc,IAAI,WAAW,EAAE,KAAK,GAAG,GAAG,GAAG,IACxD;AAEJ,WAAS;AAAA,IACP,OAAO,KAAK,MAAM;AAAA,IAClB;AAAA,IACA;AAAA,IACA,YAAY,gBAAgB;AAAA,IAC5B;AAAA,IACA;AAAA,EACF;AACA,SAAO;AACT;AAEA,SAAS,YAAY,GAAmB;AACtC,SAAO,EAAE,QAAQ,uBAAuB,MAAM;AAChD;AAEA,SAAS,WAAW,QAAkB,IAAsB;AAC1D,QAAM,MAAgB,CAAC;AACvB,aAAW,SAAS,QAAQ;AAC1B,QAAI,CAAC,MAAO;AAIZ,OAAG,YAAY;AACf,QAAI,OAAO;AACX,eAAW,KAAK,MAAM,SAAS,EAAE,GAAG;AAClC,YAAM,MAAM,EAAE,SAAS;AACvB,UAAI,MAAM,KAAM,KAAI,KAAK,MAAM,MAAM,MAAM,GAAG,CAAC;AAC/C,UAAI,EAAE,CAAC,EAAE,SAAS,EAAG,KAAI,KAAK,EAAE,CAAC,CAAC;AAClC,aAAO,MAAM,EAAE,CAAC,EAAE;AAAA,IACpB;AACA,QAAI,OAAO,MAAM,OAAQ,KAAI,KAAK,MAAM,MAAM,IAAI,CAAC;AAAA,EACrD;AACA,SAAO;AACT;AAGA,SAAS,gBAAgB,GAAW,YAA8B;AAChE,QAAM,QAAQ,IAAI,YAAY,EAAE,OAAO,CAAC;AACxC,MAAI,MAAM;AACV,WAAS,IAAI,GAAG,IAAI,MAAM,QAAQ,IAAK,QAAO,WAAW,MAAM,CAAC,CAAE;AAClE,SAAO;AACT;AAEA,SAAS,UAAU,OAAe,WAA0C;AAC1E,MAAI,MAAM,UAAU,EAAG,QAAO,QAAQ,CAAC,KAAK,IAAI,CAAC;AACjD,MAAI,OAAiB,MAAM,KAAK,KAAK;AACrC,SAAO,MAAM;AACX,QAAI,UAAU;AACd,QAAI,WAAW,OAAO;AACtB,aAAS,IAAI,GAAG,IAAI,KAAK,SAAS,GAAG,KAAK;AACxC,YAAM,OAAO,GAAG,KAAK,CAAC,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC;AACtC,YAAM,OAAO,UAAU,IAAI,IAAI;AAC/B,UAAI,SAAS,UAAa,OAAO,UAAU;AACzC,mBAAW;AACX,kBAAU;AACV,YAAI,SAAS,EAAG;AAAA,MAClB;AAAA,IACF;AACA,QAAI,UAAU,EAAG;AACjB,WAAO;AAAA,MACL,GAAG,KAAK,MAAM,GAAG,OAAO;AAAA,MACxB,KAAK,OAAO,IAAK,KAAK,UAAU,CAAC;AAAA,MACjC,GAAG,KAAK,MAAM,UAAU,CAAC;AAAA,IAC3B;AACA,QAAI,KAAK,WAAW,EAAG;AAAA,EACzB;AACA,SAAO;AACT;AAEO,SAAS,OAAO,MAAwB;AAC7C,MAAI,CAAC,KAAM,QAAO,CAAC;AACnB,QAAM,IAAI,cAAc;AACxB,QAAM,MAAgB,CAAC;AAEvB,QAAMA,WAAU,CAAC,YAAoB;AACnC,QAAI,CAAC,QAAS;AACd,QAAI,SAAmB,CAAC,OAAO;AAC/B,eAAW,MAAM,EAAE,aAAc,UAAS,WAAW,QAAQ,EAAE;AAC/D,eAAW,SAAS,QAAQ;AAC1B,UAAI,CAAC,MAAO;AACZ,YAAM,YAAY,gBAAgB,OAAO,EAAE,UAAU;AACrD,YAAM,SAAS,UAAU,WAAW,EAAE,SAAS;AAC/C,iBAAW,KAAK,QAAQ;AACtB,cAAM,KAAK,EAAE,MAAM,CAAC;AAKpB,YAAI,OAAO,OAAW,KAAI,KAAK,EAAE;AAAA,MACnC;AAAA,IACF;AAAA,EACF;AAEA,MAAI,EAAE,cAAc;AAClB,MAAE,aAAa,YAAY;AAC3B,QAAI,OAAO;AACX,eAAW,KAAK,KAAK,SAAS,EAAE,YAAY,GAAG;AAC7C,YAAM,MAAM,EAAE,SAAS;AACvB,UAAI,MAAM,KAAM,CAAAA,SAAQ,KAAK,MAAM,MAAM,GAAG,CAAC;AAC7C,YAAM,KAAK,EAAE,SAAS,IAAI,EAAE,CAAC,CAAC;AAC9B,UAAI,OAAO,OAAW,KAAI,KAAK,EAAE;AACjC,aAAO,MAAM,EAAE,CAAC,EAAE;AAAA,IACpB;AACA,QAAI,OAAO,KAAK,OAAQ,CAAAA,SAAQ,KAAK,MAAM,IAAI,CAAC;AAAA,EAClD,OAAO;AACL,IAAAA,SAAQ,IAAI;AAAA,EACd;AACA,SAAO;AACT;AAEO,SAAS,YAAY,MAAsB;AAChD,SAAO,OAAO,IAAI,EAAE;AACtB;AAGO,SAAS,2BACd,UACQ;AACR,MAAI,QAAQ;AACZ,aAAW,KAAK,UAAU;AACxB,QAAI,OAAO,EAAE,YAAY,YAAY,EAAE,SAAS;AAC9C,eAAS,YAAY,EAAE,OAAO;AAAA,IAChC;AAIA,QAAI,EAAE,cAAc,MAAM,QAAQ,EAAE,UAAU,KAAK,EAAE,WAAW,SAAS,GAAG;AAC1E,eAAS,YAAY,KAAK,UAAU,EAAE,UAAU,CAAC;AAAA,IACnD;AAAA,EACF;AACA,SAAO;AACT;AAGO,SAAS,sBACd,UACA,WACQ;AACR,MAAI,QAAQ,2BAA2B,QAAQ;AAC/C,MAAI,aAAa,UAAU,SAAS,GAAG;AACrC,aAAS,YAAY,KAAK,UAAU,SAAS,CAAC;AAAA,EAChD;AACA,SAAO;AACT;","names":["process"]}

package/dist/cli/{chunk-Q6YFXW7H.js → chunk-G7M3QWEN.js}

@@ -3,6 +3,11 @@ import {
   MemoryStore,
   sanitizeMemoryName
 } from "./chunk-6TMHAK5D.js";
+import {
+  countTokens,
+  estimateConversationTokens,
+  estimateRequestTokens
+} from "./chunk-DAEAAVDF.js";
 import {
   Usage
 } from "./chunk-KMWKGPFZ.js";
@@ -43,196 +48,6 @@ import {
   SessionStats
 } from "./chunk-ORM6PK57.js";
 
-// src/tokenizer.ts
-import { existsSync, readFileSync } from "fs";
-import { createRequire } from "module";
-import { dirname, join } from "path";
-import { fileURLToPath } from "url";
-import { gunzipSync } from "zlib";
-function buildByteToChar() {
-  const result = new Array(256);
-  const bs = [];
-  for (let b = 33; b <= 126; b++) bs.push(b);
-  for (let b = 161; b <= 172; b++) bs.push(b);
-  for (let b = 174; b <= 255; b++) bs.push(b);
-  const cs = bs.slice();
-  let n = 0;
-  for (let b = 0; b < 256; b++) {
-    if (!bs.includes(b)) {
-      bs.push(b);
-      cs.push(256 + n);
-      n++;
-    }
-  }
-  for (let i = 0; i < bs.length; i++) {
-    result[bs[i]] = String.fromCodePoint(cs[i]);
-  }
-  return result;
-}
-var cached = null;
-function resolveDataPath() {
-  if (process.env.REASONIX_TOKENIZER_PATH) return process.env.REASONIX_TOKENIZER_PATH;
-  const candidates = [];
-  try {
-    const here = dirname(fileURLToPath(import.meta.url));
-    candidates.push(join(here, "..", "data", "deepseek-tokenizer.json.gz"));
-    candidates.push(join(here, "..", "..", "data", "deepseek-tokenizer.json.gz"));
-  } catch {
-  }
-  try {
-    const req = createRequire(import.meta.url);
-    candidates.push(
-      join(dirname(req.resolve("reasonix/package.json")), "data", "deepseek-tokenizer.json.gz")
-    );
-  } catch {
-  }
-  for (const p of candidates) {
-    if (existsSync(p)) return p;
-  }
-  return candidates[0] ?? join(process.cwd(), "data", "deepseek-tokenizer.json.gz");
-}
-function loadTokenizer() {
-  if (cached) return cached;
-  const buf = readFileSync(resolveDataPath());
-  const json = gunzipSync(buf).toString("utf8");
-  const data = JSON.parse(json);
-  const mergeRank = /* @__PURE__ */ new Map();
-  for (let i = 0; i < data.model.merges.length; i++) {
-    mergeRank.set(data.model.merges[i], i);
-  }
-  const splitRegexes = [];
-  for (const p of data.pre_tokenizer.pretokenizers) {
-    if (p.type === "Split") {
-      splitRegexes.push(new RegExp(p.pattern.Regex, "gu"));
-    }
-  }
-  const addedMap = /* @__PURE__ */ new Map();
-  const addedContents = [];
-  for (const t2 of data.added_tokens) {
-    if (!t2.special) {
-      addedMap.set(t2.content, t2.id);
-      addedContents.push(t2.content);
-    }
-  }
-  addedContents.sort((a, b) => b.length - a.length);
-  const addedPattern = addedContents.length ? new RegExp(addedContents.map(escapeRegex).join("|"), "g") : null;
-  cached = {
-    vocab: data.model.vocab,
-    mergeRank,
-    splitRegexes,
-    byteToChar: buildByteToChar(),
-    addedPattern,
-    addedMap
-  };
-  return cached;
-}
-function escapeRegex(s) {
-  return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
-}
-function applySplit(chunks, re) {
-  const out = [];
-  for (const chunk of chunks) {
-    if (!chunk) continue;
-    re.lastIndex = 0;
-    let last = 0;
-    for (const m of chunk.matchAll(re)) {
-      const idx = m.index ?? 0;
-      if (idx > last) out.push(chunk.slice(last, idx));
-      if (m[0].length > 0) out.push(m[0]);
-      last = idx + m[0].length;
-    }
-    if (last < chunk.length) out.push(chunk.slice(last));
-  }
-  return out;
-}
-function byteLevelEncode(s, byteToChar) {
-  const bytes = new TextEncoder().encode(s);
-  let out = "";
-  for (let i = 0; i < bytes.length; i++) out += byteToChar[bytes[i]];
-  return out;
-}
-function bpeEncode(piece, mergeRank) {
-  if (piece.length <= 1) return piece ? [piece] : [];
-  let word = Array.from(piece);
-  while (true) {
-    let bestIdx = -1;
-    let bestRank = Number.POSITIVE_INFINITY;
-    for (let i = 0; i < word.length - 1; i++) {
-      const pair = `${word[i]} ${word[i + 1]}`;
-      const rank = mergeRank.get(pair);
-      if (rank !== void 0 && rank < bestRank) {
-        bestRank = rank;
-        bestIdx = i;
-        if (rank === 0) break;
-      }
-    }
-    if (bestIdx < 0) break;
-    word = [
-      ...word.slice(0, bestIdx),
-      word[bestIdx] + word[bestIdx + 1],
-      ...word.slice(bestIdx + 2)
-    ];
-    if (word.length === 1) break;
-  }
-  return word;
-}
-function encode(text) {
-  if (!text) return [];
-  const t2 = loadTokenizer();
-  const ids = [];
-  const process2 = (segment) => {
-    if (!segment) return;
-    let chunks = [segment];
-    for (const re of t2.splitRegexes) chunks = applySplit(chunks, re);
-    for (const chunk of chunks) {
-      if (!chunk) continue;
-      const byteLevel = byteLevelEncode(chunk, t2.byteToChar);
-      const pieces = bpeEncode(byteLevel, t2.mergeRank);
-      for (const p of pieces) {
-        const id = t2.vocab[p];
-        if (id !== void 0) ids.push(id);
-      }
-    }
-  };
-  if (t2.addedPattern) {
-    t2.addedPattern.lastIndex = 0;
-    let last = 0;
-    for (const m of text.matchAll(t2.addedPattern)) {
-      const idx = m.index ?? 0;
-      if (idx > last) process2(text.slice(last, idx));
-      const id = t2.addedMap.get(m[0]);
-      if (id !== void 0) ids.push(id);
-      last = idx + m[0].length;
-    }
-    if (last < text.length) process2(text.slice(last));
-  } else {
-    process2(text);
-  }
-  return ids;
-}
-function countTokens(text) {
-  return encode(text).length;
-}
-function estimateConversationTokens(messages) {
-  let total = 0;
-  for (const m of messages) {
-    if (typeof m.content === "string" && m.content) {
-      total += countTokens(m.content);
-    }
-    if (m.tool_calls && Array.isArray(m.tool_calls) && m.tool_calls.length > 0) {
-      total += countTokens(JSON.stringify(m.tool_calls));
-    }
-  }
-  return total;
-}
-function estimateRequestTokens(messages, toolSpecs) {
-  let total = estimateConversationTokens(messages);
-  if (toolSpecs && toolSpecs.length > 0) {
-    total += countTokens(JSON.stringify(toolSpecs));
-  }
-  return total;
-}
-
 // src/mcp/latency.ts
 var SAMPLE_SIZE = 5;
 var DEFAULT_THRESHOLD_MS = 4e3;
@@ -4021,9 +3836,9 @@ ${i + 1}. ${r.title}`);
 }
 
 // src/at-mentions.ts
-import { existsSync as existsSync2, readFileSync as readFileSync2, readdirSync, statSync } from "fs";
+import { existsSync, readFileSync, readdirSync, statSync } from "fs";
 import { readdir, stat } from "fs/promises";
-import { isAbsolute as isAbsolute2, join as join5, relative as relative5, resolve as resolve2 } from "path";
+import { isAbsolute as isAbsolute2, join as join4, relative as relative5, resolve as resolve2 } from "path";
 var DEFAULT_AT_MENTION_MAX_BYTES = 64 * 1024;
 var DEFAULT_AT_DIR_MAX_ENTRIES = 200;
 var DEFAULT_PICKER_IGNORE_DIRS = [
@@ -4068,7 +3883,7 @@ function listFilesWithStatsSync(root, opts = {}) {
     for (const ent of entries) {
       if (out.length >= maxResults) return;
       const relPath = dirRel ? `${dirRel}/${ent.name}` : ent.name;
-      const absPath = join5(dirAbs, ent.name);
+      const absPath = join4(dirAbs, ent.name);
       if (ent.isDirectory()) {
         if (ent.name.startsWith(".") || ignoreDirs.has(ent.name)) continue;
         if (ignoredByLayers(effectiveLayers, absPath, true)) continue;
@@ -4121,7 +3936,7 @@ async function listFilesWithStatsAsync(root, opts = {}) {
     for (const ent of entries) {
       if (out.length >= maxResults) break;
       const relPath = dirRel ? `${dirRel}/${ent.name}` : ent.name;
-      const absPath = join5(dirAbs, ent.name);
+      const absPath = join4(dirAbs, ent.name);
       if (ent.isDirectory()) {
         if (ent.name.startsWith(".") || ignoreDirs.has(ent.name)) continue;
         if (ignoredByLayers(effectiveLayers, absPath, true)) continue;
@@ -4146,12 +3961,12 @@ async function statBatch(ents, dirAbs, dirRel, out, maxResults, layers) {
   const accepted = [];
   for (const e of ents) {
     if (out.length + accepted.length >= maxResults) break;
-    if (ignoredByLayers(layers, join5(dirAbs, e.name), false)) continue;
+    if (ignoredByLayers(layers, join4(dirAbs, e.name), false)) continue;
     accepted.push(e);
   }
   const stats = await Promise.all(
     accepted.map(
-      (e) => stat(join5(dirAbs, e.name)).then((s) => ({ mtimeMs: s.mtimeMs, isFile: s.isFile() })).catch(() => null)
+      (e) => stat(join4(dirAbs, e.name)).then((s) => ({ mtimeMs: s.mtimeMs, isFile: s.isFile() })).catch(() => null)
     )
   );
   for (let i = 0; i < accepted.length; i++) {
@@ -4342,7 +4157,7 @@ function readSafe(root, rawPath, fs5) {
   }
 }
 var defaultFs = {
-  exists: (p) => existsSync2(p),
+  exists: (p) => existsSync(p),
   isFile: (p) => {
     try {
       return statSync(p).isFile();
@@ -4375,7 +4190,7 @@ var defaultFs = {
       return 0;
     }
   },
-  read: (p) => readFileSync2(p, "utf8")
+  read: (p) => readFileSync(p, "utf8")
 };
 var AT_URL_PATTERN = /(?<=^|\s)@(https?:\/\/\S+)/g;
 var DEFAULT_AT_URL_MAX_CHARS = 32e3;
@@ -4393,10 +4208,10 @@ async function expandAtUrls(text, opts = {}) {
     const url = stripUrlTail(rawUrl);
     if (!url) continue;
     if (seen.has(url)) continue;
-    const cached2 = opts.cache?.get(url);
-    if (cached2) {
-      seen.set(url, cached2);
-      if (cached2.body) bodies.set(url, cached2.body);
+    const cached = opts.cache?.get(url);
+    if (cached) {
+      seen.set(url, cached);
+      if (cached.body) bodies.set(url, cached.body);
       order.push(url);
       continue;
     }
@@ -4770,18 +4585,18 @@ function forkRegistryWithAllowList(parent, allow, alsoExclude) {
 // src/code/edit-blocks.ts
 import {
   closeSync,
-  existsSync as existsSync3,
+  existsSync as existsSync2,
   fstatSync,
   ftruncateSync,
   mkdirSync,
   openSync,
-  readFileSync as readFileSync3,
+  readFileSync as readFileSync2,
   readSync,
   unlinkSync,
   writeFileSync,
   writeSync
 } from "fs";
-import { dirname as dirname3, resolve as resolve3 } from "path";
+import { dirname as dirname2, resolve as resolve3 } from "path";
 var BLOCK_RE = /^(\S[^\n]*)\n<{7} SEARCH\n([\s\S]*?)\n?={7}\n([\s\S]*?)\n?>{7} REPLACE/gm;
 function parseEditBlocks(text) {
   const out = [];
@@ -4811,7 +4626,7 @@ function applyEditBlock(block, rootDir) {
   const searchEmpty = block.search.length === 0;
   if (searchEmpty) {
     try {
-      mkdirSync(dirname3(absTarget), { recursive: true });
+      mkdirSync(dirname2(absTarget), { recursive: true });
       const fd = openSync(absTarget, "wx");
       try {
         writeSync(fd, block.replace);
@@ -4889,9 +4704,9 @@ function applyEditBlocks(blocks, rootDir) {
 function toWholeFileEditBlock(path, content, rootDir) {
   const abs = resolve3(rootDir, path);
   let search = "";
-  if (existsSync3(abs)) {
+  if (existsSync2(abs)) {
     try {
-      search = readFileSync3(abs, "utf8");
+      search = readFileSync2(abs, "utf8");
     } catch {
       search = "";
     }
@@ -4906,12 +4721,12 @@ function snapshotBeforeEdits(blocks, rootDir) {
     if (seen.has(b.path)) continue;
     seen.add(b.path);
     const abs = resolve3(absRoot, b.path);
-    if (!existsSync3(abs)) {
+    if (!existsSync2(abs)) {
       snapshots.push({ path: b.path, prevContent: null });
       continue;
     }
     try {
-      snapshots.push({ path: b.path, prevContent: readFileSync3(abs, "utf8") });
+      snapshots.push({ path: b.path, prevContent: readFileSync2(abs, "utf8") });
     } catch {
       snapshots.push({ path: b.path, prevContent: null });
     }
@@ -4931,7 +4746,7 @@ function restoreSnapshots(snapshots, rootDir) {
     }
     try {
       if (snap.prevContent === null) {
-        if (existsSync3(abs)) unlinkSync(abs);
+        if (existsSync2(abs)) unlinkSync(abs);
         return {
           path: snap.path,
           status: "applied",
@@ -4957,7 +4772,6 @@ function lineEndingOf(text) {
 }
 
 export {
-  countTokens,
   ToolRegistry,
   registerSingleMcpTool,
   bridgeMcpTools,
@@ -4983,4 +4797,4 @@ export {
   snapshotBeforeEdits,
   restoreSnapshots
 };
-//# sourceMappingURL=chunk-Q6YFXW7H.js.map
+//# sourceMappingURL=chunk-G7M3QWEN.js.map