reasonix 0.5.0 → 0.5.2

package/dist/cli/index.js

@@ -696,6 +696,170 @@ async function runHooks(opts) {
   return { event, outcomes, blocked };
 }
 
+// src/tokenizer.ts
+import { readFileSync as readFileSync3 } from "fs";
+import { createRequire } from "module";
+import { dirname as dirname2, join as join3 } from "path";
+import { fileURLToPath } from "url";
+import { gunzipSync } from "zlib";
+function buildByteToChar() {
+  const result = new Array(256);
+  const bs = [];
+  for (let b = 33; b <= 126; b++) bs.push(b);
+  for (let b = 161; b <= 172; b++) bs.push(b);
+  for (let b = 174; b <= 255; b++) bs.push(b);
+  const cs = bs.slice();
+  let n = 0;
+  for (let b = 0; b < 256; b++) {
+    if (!bs.includes(b)) {
+      bs.push(b);
+      cs.push(256 + n);
+      n++;
+    }
+  }
+  for (let i = 0; i < bs.length; i++) {
+    result[bs[i]] = String.fromCodePoint(cs[i]);
+  }
+  return result;
+}
+var cached = null;
+function resolveDataPath() {
+  if (process.env.REASONIX_TOKENIZER_PATH) return process.env.REASONIX_TOKENIZER_PATH;
+  try {
+    const here = dirname2(fileURLToPath(import.meta.url));
+    return join3(here, "..", "data", "deepseek-tokenizer.json.gz");
+  } catch {
+    const req = createRequire(import.meta.url);
+    return join3(
+      dirname2(req.resolve("reasonix/package.json")),
+      "data",
+      "deepseek-tokenizer.json.gz"
+    );
+  }
+}
+function loadTokenizer() {
+  if (cached) return cached;
+  const buf = readFileSync3(resolveDataPath());
+  const json = gunzipSync(buf).toString("utf8");
+  const data = JSON.parse(json);
+  const mergeRank = /* @__PURE__ */ new Map();
+  for (let i = 0; i < data.model.merges.length; i++) {
+    mergeRank.set(data.model.merges[i], i);
+  }
+  const splitRegexes = [];
+  for (const p of data.pre_tokenizer.pretokenizers) {
+    if (p.type === "Split") {
+      splitRegexes.push(new RegExp(p.pattern.Regex, "gu"));
+    }
+  }
+  const addedMap = /* @__PURE__ */ new Map();
+  const addedContents = [];
+  for (const t of data.added_tokens) {
+    if (!t.special) {
+      addedMap.set(t.content, t.id);
+      addedContents.push(t.content);
+    }
+  }
+  addedContents.sort((a, b) => b.length - a.length);
+  const addedPattern = addedContents.length ? new RegExp(addedContents.map(escapeRegex).join("|"), "g") : null;
+  cached = {
+    vocab: data.model.vocab,
+    mergeRank,
+    splitRegexes,
+    byteToChar: buildByteToChar(),
+    addedPattern,
+    addedMap
+  };
+  return cached;
+}
+function escapeRegex(s) {
+  return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+function applySplit(chunks, re) {
+  const out = [];
+  for (const chunk of chunks) {
+    if (!chunk) continue;
+    re.lastIndex = 0;
+    let last = 0;
+    for (const m of chunk.matchAll(re)) {
+      const idx = m.index ?? 0;
+      if (idx > last) out.push(chunk.slice(last, idx));
+      if (m[0].length > 0) out.push(m[0]);
+      last = idx + m[0].length;
+    }
+    if (last < chunk.length) out.push(chunk.slice(last));
+  }
+  return out;
+}
+function byteLevelEncode(s, byteToChar) {
+  const bytes = new TextEncoder().encode(s);
+  let out = "";
+  for (let i = 0; i < bytes.length; i++) out += byteToChar[bytes[i]];
+  return out;
+}
+function bpeEncode(piece, mergeRank) {
+  if (piece.length <= 1) return piece ? [piece] : [];
+  let word = Array.from(piece);
+  while (true) {
+    let bestIdx = -1;
+    let bestRank = Number.POSITIVE_INFINITY;
+    for (let i = 0; i < word.length - 1; i++) {
+      const pair = `${word[i]} ${word[i + 1]}`;
+      const rank = mergeRank.get(pair);
+      if (rank !== void 0 && rank < bestRank) {
+        bestRank = rank;
+        bestIdx = i;
+        if (rank === 0) break;
+      }
+    }
+    if (bestIdx < 0) break;
+    word = [
+      ...word.slice(0, bestIdx),
+      word[bestIdx] + word[bestIdx + 1],
+      ...word.slice(bestIdx + 2)
+    ];
+    if (word.length === 1) break;
+  }
+  return word;
+}
+function encode(text) {
+  if (!text) return [];
+  const t = loadTokenizer();
+  const ids = [];
+  const process2 = (segment) => {
+    if (!segment) return;
+    let chunks = [segment];
+    for (const re of t.splitRegexes) chunks = applySplit(chunks, re);
+    for (const chunk of chunks) {
+      if (!chunk) continue;
+      const byteLevel = byteLevelEncode(chunk, t.byteToChar);
+      const pieces = bpeEncode(byteLevel, t.mergeRank);
+      for (const p of pieces) {
+        const id = t.vocab[p];
+        if (id !== void 0) ids.push(id);
+      }
+    }
+  };
+  if (t.addedPattern) {
+    t.addedPattern.lastIndex = 0;
+    let last = 0;
+    for (const m of text.matchAll(t.addedPattern)) {
+      const idx = m.index ?? 0;
+      if (idx > last) process2(text.slice(last, idx));
+      const id = t.addedMap.get(m[0]);
+      if (id !== void 0) ids.push(id);
+      last = idx + m[0].length;
+    }
+    if (last < text.length) process2(text.slice(last));
+  } else {
+    process2(text);
+  }
+  return ids;
+}
+function countTokens(text) {
+  return encode(text).length;
+}
+
 // src/repair/flatten.ts
 function analyzeSchema(schema) {
   if (!schema) return { shouldFlatten: false, leafCount: 0, maxDepth: 0 };
@@ -846,7 +1010,14 @@ var ToolRegistry = class {
     try {
       const result = await tool.fn(args, { signal: opts.signal });
       const str = typeof result === "string" ? result : JSON.stringify(result);
-      return opts.maxResultChars ? truncateForModel(str, opts.maxResultChars) : str;
+      let clipped = str;
+      if (opts.maxResultTokens !== void 0) {
+        clipped = truncateForModelByTokens(clipped, opts.maxResultTokens);
+      }
+      if (opts.maxResultChars !== void 0) {
+        clipped = truncateForModel(clipped, opts.maxResultChars);
+      }
+      return clipped;
     } catch (err) {
       const e = err;
       if (typeof e.toToolResult === "function") {
@@ -880,6 +1051,7 @@ function hasDotKey(obj) {
 
 // src/mcp/registry.ts
 var DEFAULT_MAX_RESULT_CHARS = 32e3;
+var DEFAULT_MAX_RESULT_TOKENS = 8e3;
 async function bridgeMcpTools(client, opts = {}) {
   const registry = opts.registry ?? new ToolRegistry({ autoFlatten: opts.autoFlatten });
   const prefix = opts.namePrefix ?? "";
@@ -936,6 +1108,61 @@ function truncateForModel(s, maxChars) {
 
 ${tail}`;
 }
+function truncateForModelByTokens(s, maxTokens) {
+  if (maxTokens <= 0) return "";
+  if (s.length <= maxTokens) return s;
+  if (s.length <= maxTokens * 4) {
+    const tokens = countTokens(s);
+    if (tokens <= maxTokens) return s;
+  }
+  const markerOverhead = 48;
+  const contentBudget = Math.max(0, maxTokens - markerOverhead);
+  const tailBudget = Math.min(256, Math.floor(contentBudget * 0.1));
+  const headBudget = Math.max(0, contentBudget - tailBudget);
+  const head = sizePrefixToTokens(s, headBudget);
+  const tail = sizeSuffixToTokens(s, tailBudget);
+  const droppedChars = s.length - head.length - tail.length;
+  const headTokens = head ? countTokens(head) : 0;
+  const tailTokens = tail ? countTokens(tail) : 0;
+  const sampleChars = head.length + tail.length;
+  const sampleTokens = headTokens + tailTokens;
+  const ratio = sampleChars > 0 ? sampleTokens / sampleChars : 0.3;
+  const estTotalTokens = Math.ceil(s.length * ratio);
+  const droppedTokens = Math.max(0, estTotalTokens - sampleTokens);
+  return `${head}
+
+[\u2026truncated ~${droppedTokens} tokens (${droppedChars} chars) \u2014 raise BridgeOptions.maxResultTokens, or call the tool with a narrower scope (filter, head, pagination)\u2026]
+
+${tail}`;
+}
+function sizePrefixToTokens(s, budget) {
+  if (budget <= 0 || s.length === 0) return "";
+  let size = Math.min(s.length, budget * 4);
+  for (let iter = 0; iter < 6; iter++) {
+    if (size <= 0) return "";
+    const slice = s.slice(0, size);
+    const count = countTokens(slice);
+    if (count <= budget) return slice;
+    const next = Math.floor(size * (budget / count) * 0.95);
+    if (next >= size) return s.slice(0, Math.max(0, size - 1));
+    size = next;
+  }
+  return s.slice(0, Math.max(0, size));
+}
+function sizeSuffixToTokens(s, budget) {
+  if (budget <= 0 || s.length === 0) return "";
+  let size = Math.min(s.length, budget * 4);
+  for (let iter = 0; iter < 6; iter++) {
+    if (size <= 0) return "";
+    const slice = s.slice(-size);
+    const count = countTokens(slice);
+    if (count <= budget) return slice;
+    const next = Math.floor(size * (budget / count) * 0.95);
+    if (next >= size) return s.slice(-Math.max(0, size - 1));
+    size = next;
+  }
+  return s.slice(-Math.max(0, size));
+}
 function blockToString(block) {
   if (block.type === "text") return block.text;
   if (block.type === "image") return `[image ${block.mimeType}, ${block.data.length} chars base64]`;
@@ -1321,19 +1548,19 @@ import {
   chmodSync as chmodSync2,
   existsSync as existsSync2,
   mkdirSync as mkdirSync2,
-  readFileSync as readFileSync3,
+  readFileSync as readFileSync4,
   readdirSync,
   statSync,
   unlinkSync,
   writeFileSync as writeFileSync2
 } from "fs";
 import { homedir as homedir3 } from "os";
-import { dirname as dirname2, join as join3 } from "path";
+import { dirname as dirname3, join as join4 } from "path";
 function sessionsDir() {
-  return join3(homedir3(), ".reasonix", "sessions");
+  return join4(homedir3(), ".reasonix", "sessions");
 }
 function sessionPath(name) {
-  return join3(sessionsDir(), `${sanitizeName(name)}.jsonl`);
+  return join4(sessionsDir(), `${sanitizeName(name)}.jsonl`);
 }
 function sanitizeName(name) {
   const cleaned = name.replace(/[^\w\-\u4e00-\u9fa5]/g, "_").slice(0, 64);
@@ -1343,7 +1570,7 @@ function loadSessionMessages(name) {
   const path = sessionPath(name);
   if (!existsSync2(path)) return [];
   try {
-    const raw = readFileSync3(path, "utf8");
+    const raw = readFileSync4(path, "utf8");
     const out = [];
     for (const line of raw.split(/\r?\n/)) {
       const trimmed = line.trim();
@@ -1361,7 +1588,7 @@ function loadSessionMessages(name) {
 }
 function appendSessionMessage(name, message) {
   const path = sessionPath(name);
-  mkdirSync2(dirname2(path), { recursive: true });
+  mkdirSync2(dirname3(path), { recursive: true });
   appendFileSync(path, `${JSON.stringify(message)}
 `, "utf8");
   try {
@@ -1375,7 +1602,7 @@ function listSessions() {
   try {
     const files = readdirSync(dir).filter((f) => f.endsWith(".jsonl"));
     return files.map((file) => {
-      const path = join3(dir, file);
+      const path = join4(dir, file);
       const stat = statSync(path);
       const name = file.replace(/\.jsonl$/, "");
       const messageCount = countLines(path);
@@ -1396,7 +1623,7 @@ function deleteSession(name) {
 }
 function rewriteSession(name, messages) {
   const path = sessionPath(name);
-  mkdirSync2(dirname2(path), { recursive: true });
+  mkdirSync2(dirname3(path), { recursive: true });
   const body = messages.map((m) => JSON.stringify(m)).join("\n");
   writeFileSync2(path, body ? `${body}
 ` : "", "utf8");
@@ -1407,7 +1634,7 @@ function rewriteSession(name, messages) {
 }
 function countLines(path) {
   try {
-    const raw = readFileSync3(path, "utf8");
+    const raw = readFileSync4(path, "utf8");
     return raw.split(/\r?\n/).filter((l) => l.trim()).length;
   } catch {
     return 0;
@@ -2054,7 +2281,7 @@ ${reason}`;
         } else {
           result = await this.tools.dispatch(name, args, {
             signal,
-            maxResultChars: DEFAULT_MAX_RESULT_CHARS
+            maxResultTokens: DEFAULT_MAX_RESULT_TOKENS
           });
           const postReport = await runHooks({
             hooks: this.hooks,
@@ -3162,6 +3389,50 @@ function tokenizeCommand(cmd) {
   if (cur.length > 0) out.push(cur);
   return out;
 }
+function detectShellOperator(cmd) {
+  const opPrefix = /^(?:2>&1|&>|\|{1,2}|&{1,2}|2>{1,2}|>{1,2}|<{1,2})/;
+  let cur = "";
+  let curQuoted = false;
+  let quote = null;
+  const check = () => {
+    if (cur.length === 0 && !curQuoted) return null;
+    if (!curQuoted) {
+      const m = opPrefix.exec(cur);
+      if (m) return m[0] ?? null;
+    }
+    return null;
+  };
+  for (let i = 0; i < cmd.length; i++) {
+    const ch = cmd[i];
+    if (quote) {
+      if (ch === quote) {
+        quote = null;
+      } else if (ch === "\\" && quote === '"' && i + 1 < cmd.length) {
+        cur += cmd[++i];
+        curQuoted = true;
+      } else {
+        cur += ch;
+        curQuoted = true;
+      }
+      continue;
+    }
+    if (ch === '"' || ch === "'") {
+      quote = ch;
+      curQuoted = true;
+      continue;
+    }
+    if (ch === " " || ch === "	") {
+      const op = check();
+      if (op) return op;
+      cur = "";
+      curQuoted = false;
+      continue;
+    }
+    cur += ch;
+  }
+  if (quote) return null;
+  return check();
+}
 function isAllowed(cmd, extra = []) {
   const normalized = cmd.trim().replace(/\s+/g, " ");
   const allowlist = [...BUILTIN_ALLOWLIST, ...extra];
@@ -3174,6 +3445,12 @@ function isAllowed(cmd, extra = []) {
 async function runCommand(cmd, opts) {
   const argv = tokenizeCommand(cmd);
   if (argv.length === 0) throw new Error("run_command: empty command");
+  const operator = detectShellOperator(cmd);
+  if (operator !== null) {
+    throw new Error(
+      `run_command: shell operator "${operator}" is not supported \u2014 this tool spawns one process, no shell expansion. Split into separate run_command calls and combine the output in your reasoning (e.g. instead of \`grep foo *.ts | wc -l\`, call \`grep -c foo *.ts\` or two separate commands). To pass "${operator}" as a literal argument, wrap it in quotes.`
+    );
+  }
   const timeoutMs = (opts.timeoutSec ?? DEFAULT_TIMEOUT_SEC) * 1e3;
   const maxChars = opts.maxOutputChars ?? DEFAULT_MAX_OUTPUT_CHARS;
   const spawnOpts = {
@@ -3351,7 +3628,7 @@ function registerShellTools(registry, opts) {
       properties: {
         command: {
           type: "string",
-          description: "Full command line. Tokenized with POSIX-ish quoting; no shell expansion, no pipes, no redirects."
+          description: 'Full command line. Tokenized with POSIX-ish quoting; no shell expansion. Pipes (`|`), redirects (`>`, `<`, `2>`), and `&&`/`||` chaining are rejected with an error \u2014 split into separate calls instead. To pass an operator character as a literal argument (e.g. a regex), wrap it in quotes: `grep "a|b" file.txt`.'
         },
         timeoutSec: {
           type: "integer",
@@ -3570,12 +3847,12 @@ ${i + 1}. ${r.title}`);
 }
 
 // src/env.ts
-import { readFileSync as readFileSync4 } from "fs";
+import { readFileSync as readFileSync5 } from "fs";
 import { resolve as resolve3 } from "path";
 function loadDotenv(path = ".env") {
   let raw;
   try {
-    raw = readFileSync4(resolve3(process.cwd(), path), "utf8");
+    raw = readFileSync5(resolve3(process.cwd(), path), "utf8");
   } catch {
     return;
   }
@@ -3594,7 +3871,7 @@ function loadDotenv(path = ".env") {
 }
 
 // src/transcript.ts
-import { createWriteStream, readFileSync as readFileSync5 } from "fs";
+import { createWriteStream, readFileSync as readFileSync6 } from "fs";
 function recordFromLoopEvent(ev, extra) {
   const rec = {
     ts: (/* @__PURE__ */ new Date()).toISOString(),
@@ -3645,7 +3922,7 @@ function openTranscriptFile(path, meta) {
   return stream;
 }
 function readTranscript(path) {
-  const raw = readFileSync5(path, "utf8");
+  const raw = readFileSync6(path, "utf8");
   return parseTranscript(raw);
 }
 function isPlanStateEmptyShape(s) {
@@ -4731,8 +5008,8 @@ async function trySection(load) {
 }
 
 // src/code/edit-blocks.ts
-import { existsSync as existsSync4, mkdirSync as mkdirSync3, readFileSync as readFileSync6, unlinkSync as unlinkSync2, writeFileSync as writeFileSync3 } from "fs";
-import { dirname as dirname4, resolve as resolve4 } from "path";
+import { existsSync as existsSync4, mkdirSync as mkdirSync3, readFileSync as readFileSync7, unlinkSync as unlinkSync2, writeFileSync as writeFileSync3 } from "fs";
+import { dirname as dirname5, resolve as resolve4 } from "path";
 var BLOCK_RE = /^(\S[^\n]*)\n<{7} SEARCH\n([\s\S]*?)\n?={7}\n([\s\S]*?)\n?>{7} REPLACE/gm;
 function parseEditBlocks(text) {
   const out = [];
@@ -4770,11 +5047,11 @@ function applyEditBlock(block, rootDir) {
           message: "file does not exist; to create it, use an empty SEARCH block"
         };
       }
-      mkdirSync3(dirname4(absTarget), { recursive: true });
+      mkdirSync3(dirname5(absTarget), { recursive: true });
       writeFileSync3(absTarget, block.replace, "utf8");
       return { path: block.path, status: "created" };
     }
-    const content = readFileSync6(absTarget, "utf8");
+    const content = readFileSync7(absTarget, "utf8");
     if (searchEmpty) {
       return {
         path: block.path,
@@ -4813,7 +5090,7 @@ function snapshotBeforeEdits(blocks, rootDir) {
       continue;
     }
     try {
-      snapshots.push({ path: b.path, prevContent: readFileSync6(abs, "utf8") });
+      snapshots.push({ path: b.path, prevContent: readFileSync7(abs, "utf8") });
     } catch {
       snapshots.push({ path: b.path, prevContent: null });
     }
@@ -4856,25 +5133,25 @@ function sep() {
 }
 
 // src/version.ts
-import { existsSync as existsSync5, mkdirSync as mkdirSync4, readFileSync as readFileSync7, writeFileSync as writeFileSync4 } from "fs";
+import { existsSync as existsSync5, mkdirSync as mkdirSync4, readFileSync as readFileSync8, writeFileSync as writeFileSync4 } from "fs";
 import { homedir as homedir4 } from "os";
-import { dirname as dirname5, join as join5 } from "path";
-import { fileURLToPath } from "url";
+import { dirname as dirname6, join as join6 } from "path";
+import { fileURLToPath as fileURLToPath2 } from "url";
 var REGISTRY_URL = "https://registry.npmjs.org/reasonix/latest";
 var LATEST_CACHE_TTL_MS = 24 * 60 * 60 * 1e3;
 var LATEST_FETCH_TIMEOUT_MS = 2e3;
 function readPackageVersion() {
   try {
-    let dir = dirname5(fileURLToPath(import.meta.url));
+    let dir = dirname6(fileURLToPath2(import.meta.url));
     for (let i = 0; i < 6; i++) {
-      const p = join5(dir, "package.json");
+      const p = join6(dir, "package.json");
       if (existsSync5(p)) {
-        const pkg = JSON.parse(readFileSync7(p, "utf8"));
+        const pkg = JSON.parse(readFileSync8(p, "utf8"));
         if (pkg?.name === "reasonix" && typeof pkg.version === "string") {
           return pkg.version;
         }
       }
-      const parent = dirname5(dir);
+      const parent = dirname6(dir);
       if (parent === dir) break;
       dir = parent;
     }
@@ -4884,11 +5161,11 @@ function readPackageVersion() {
 }
 var VERSION = readPackageVersion();
 function cachePath(homeDirOverride) {
-  return join5(homeDirOverride ?? homedir4(), ".reasonix", "version-cache.json");
+  return join6(homeDirOverride ?? homedir4(), ".reasonix", "version-cache.json");
 }
 function readCache(homeDirOverride) {
   try {
-    const raw = readFileSync7(cachePath(homeDirOverride), "utf8");
+    const raw = readFileSync8(cachePath(homeDirOverride), "utf8");
     const parsed = JSON.parse(raw);
     if (parsed && typeof parsed.version === "string" && typeof parsed.checkedAt === "number") {
       return parsed;
@@ -4900,7 +5177,7 @@ function readCache(homeDirOverride) {
 function writeCache(entry, homeDirOverride) {
   try {
     const p = cachePath(homeDirOverride);
-    mkdirSync4(dirname5(p), { recursive: true });
+    mkdirSync4(dirname6(p), { recursive: true });
     writeFileSync4(p, JSON.stringify(entry), "utf8");
   } catch {
   }
@@ -4957,11 +5234,11 @@ function isNpxInstall() {
 }
 
 // src/usage.ts
-import { appendFileSync as appendFileSync2, existsSync as existsSync6, mkdirSync as mkdirSync5, readFileSync as readFileSync8, statSync as statSync3 } from "fs";
+import { appendFileSync as appendFileSync2, existsSync as existsSync6, mkdirSync as mkdirSync5, readFileSync as readFileSync9, statSync as statSync3 } from "fs";
 import { homedir as homedir5 } from "os";
-import { dirname as dirname6, join as join6 } from "path";
+import { dirname as dirname7, join as join7 } from "path";
 function defaultUsageLogPath(homeDirOverride) {
-  return join6(homeDirOverride ?? homedir5(), ".reasonix", "usage.jsonl");
+  return join7(homeDirOverride ?? homedir5(), ".reasonix", "usage.jsonl");
 }
 function appendUsage(input) {
   const record = {
@@ -4977,7 +5254,7 @@ function appendUsage(input) {
   };
   const path = input.path ?? defaultUsageLogPath();
   try {
-    mkdirSync5(dirname6(path), { recursive: true });
+    mkdirSync5(dirname7(path), { recursive: true });
     appendFileSync2(path, `${JSON.stringify(record)}
 `, "utf8");
   } catch {
@@ -4988,7 +5265,7 @@ function readUsageLog(path = defaultUsageLogPath()) {
   if (!existsSync6(path)) return [];
   let raw;
   try {
-    raw = readFileSync8(path, "utf8");
+    raw = readFileSync9(path, "utf8");
   } catch {
     return [];
   }
@@ -5181,8 +5458,8 @@ function PlanStateBlock({ planState }) {
 }
 
 // src/cli/ui/markdown.tsx
-import { readFileSync as readFileSync9, statSync as statSync4 } from "fs";
-import { isAbsolute as isAbsolute3, join as join7 } from "path";
+import { readFileSync as readFileSync10, statSync as statSync4 } from "fs";
+import { isAbsolute as isAbsolute3, join as join8 } from "path";
 import { Box as Box2, Text as Text2 } from "ink";
 import React2 from "react";
 var SUPERSCRIPT = {
@@ -5260,7 +5537,7 @@ function parseCitationUrl(url) {
 function validateCitation(url, projectRoot) {
   const parts = parseCitationUrl(url);
   if (!parts || !parts.path) return { ok: false, reason: "empty path" };
-  const fullPath = isAbsolute3(parts.path) ? parts.path : join7(projectRoot, parts.path);
+  const fullPath = isAbsolute3(parts.path) ? parts.path : join8(projectRoot, parts.path);
   let stat;
   try {
     stat = statSync4(fullPath);
@@ -5271,7 +5548,7 @@ function validateCitation(url, projectRoot) {
   if (parts.startLine === void 0) return { ok: true };
   let lineCount;
   try {
-    lineCount = readFileSync9(fullPath, "utf8").split("\n").length;
+    lineCount = readFileSync10(fullPath, "utf8").split("\n").length;
   } catch {
     return { ok: false, reason: "unreadable" };
   }
@@ -6450,170 +6727,6 @@ function formatTokens(n) {
 // src/cli/ui/slash.ts
 import { spawnSync } from "child_process";
 
-// src/tokenizer.ts
-import { readFileSync as readFileSync10 } from "fs";
-import { createRequire } from "module";
-import { dirname as dirname7, join as join8 } from "path";
-import { fileURLToPath as fileURLToPath2 } from "url";
-import { gunzipSync } from "zlib";
-function buildByteToChar() {
-  const result = new Array(256);
-  const bs = [];
-  for (let b = 33; b <= 126; b++) bs.push(b);
-  for (let b = 161; b <= 172; b++) bs.push(b);
-  for (let b = 174; b <= 255; b++) bs.push(b);
-  const cs = bs.slice();
-  let n = 0;
-  for (let b = 0; b < 256; b++) {
-    if (!bs.includes(b)) {
-      bs.push(b);
-      cs.push(256 + n);
-      n++;
-    }
-  }
-  for (let i = 0; i < bs.length; i++) {
-    result[bs[i]] = String.fromCodePoint(cs[i]);
-  }
-  return result;
-}
-var cached = null;
-function resolveDataPath() {
-  if (process.env.REASONIX_TOKENIZER_PATH) return process.env.REASONIX_TOKENIZER_PATH;
-  try {
-    const here = dirname7(fileURLToPath2(import.meta.url));
-    return join8(here, "..", "data", "deepseek-tokenizer.json.gz");
-  } catch {
-    const req = createRequire(import.meta.url);
-    return join8(
-      dirname7(req.resolve("reasonix/package.json")),
-      "data",
-      "deepseek-tokenizer.json.gz"
-    );
-  }
-}
-function loadTokenizer() {
-  if (cached) return cached;
-  const buf = readFileSync10(resolveDataPath());
-  const json = gunzipSync(buf).toString("utf8");
-  const data = JSON.parse(json);
-  const mergeRank = /* @__PURE__ */ new Map();
-  for (let i = 0; i < data.model.merges.length; i++) {
-    mergeRank.set(data.model.merges[i], i);
-  }
-  const splitRegexes = [];
-  for (const p of data.pre_tokenizer.pretokenizers) {
-    if (p.type === "Split") {
-      splitRegexes.push(new RegExp(p.pattern.Regex, "gu"));
-    }
-  }
-  const addedMap = /* @__PURE__ */ new Map();
-  const addedContents = [];
-  for (const t of data.added_tokens) {
-    if (!t.special) {
-      addedMap.set(t.content, t.id);
-      addedContents.push(t.content);
-    }
-  }
-  addedContents.sort((a, b) => b.length - a.length);
-  const addedPattern = addedContents.length ? new RegExp(addedContents.map(escapeRegex).join("|"), "g") : null;
-  cached = {
-    vocab: data.model.vocab,
-    mergeRank,
-    splitRegexes,
-    byteToChar: buildByteToChar(),
-    addedPattern,
-    addedMap
-  };
-  return cached;
-}
-function escapeRegex(s) {
-  return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
-}
-function applySplit(chunks, re) {
-  const out = [];
-  for (const chunk of chunks) {
-    if (!chunk) continue;
-    re.lastIndex = 0;
-    let last = 0;
-    for (const m of chunk.matchAll(re)) {
-      const idx = m.index ?? 0;
-      if (idx > last) out.push(chunk.slice(last, idx));
-      if (m[0].length > 0) out.push(m[0]);
-      last = idx + m[0].length;
-    }
-    if (last < chunk.length) out.push(chunk.slice(last));
-  }
-  return out;
-}
-function byteLevelEncode(s, byteToChar) {
-  const bytes = new TextEncoder().encode(s);
-  let out = "";
-  for (let i = 0; i < bytes.length; i++) out += byteToChar[bytes[i]];
-  return out;
-}
-function bpeEncode(piece, mergeRank) {
-  if (piece.length <= 1) return piece ? [piece] : [];
-  let word = Array.from(piece);
-  while (true) {
-    let bestIdx = -1;
-    let bestRank = Number.POSITIVE_INFINITY;
-    for (let i = 0; i < word.length - 1; i++) {
-      const pair = `${word[i]} ${word[i + 1]}`;
-      const rank = mergeRank.get(pair);
-      if (rank !== void 0 && rank < bestRank) {
-        bestRank = rank;
-        bestIdx = i;
-        if (rank === 0) break;
-      }
-    }
-    if (bestIdx < 0) break;
-    word = [
-      ...word.slice(0, bestIdx),
-      word[bestIdx] + word[bestIdx + 1],
-      ...word.slice(bestIdx + 2)
-    ];
-    if (word.length === 1) break;
-  }
-  return word;
-}
-function encode(text) {
-  if (!text) return [];
-  const t = loadTokenizer();
-  const ids = [];
-  const process2 = (segment) => {
-    if (!segment) return;
-    let chunks = [segment];
-    for (const re of t.splitRegexes) chunks = applySplit(chunks, re);
-    for (const chunk of chunks) {
-      if (!chunk) continue;
-      const byteLevel = byteLevelEncode(chunk, t.byteToChar);
-      const pieces = bpeEncode(byteLevel, t.mergeRank);
-      for (const p of pieces) {
-        const id = t.vocab[p];
-        if (id !== void 0) ids.push(id);
-      }
-    }
-  };
-  if (t.addedPattern) {
-    t.addedPattern.lastIndex = 0;
-    let last = 0;
-    for (const m of text.matchAll(t.addedPattern)) {
-      const idx = m.index ?? 0;
-      if (idx > last) process2(text.slice(last, idx));
-      const id = t.addedMap.get(m[0]);
-      if (id !== void 0) ids.push(id);
-      last = idx + m[0].length;
-    }
-    if (last < text.length) process2(text.slice(last));
-  } else {
-    process2(text);
-  }
-  return ids;
-}
-function countTokens(text) {
-  return encode(text).length;
-}
-
 // src/cli/commands/stats.ts
 import { existsSync as existsSync7, readFileSync as readFileSync11 } from "fs";
 function statsCommand(opts) {