slash-tokens 1.5.1 → 1.5.3

package/dist/cli.js

@@ -28,10 +28,21 @@ function getInstance() {
   _instance = new WebAssembly.Instance(module);
   return _instance;
 }
+var MAX_WASM_INPUT_BYTES = 32 * 1024 * 1024;
+var WASM_PAGE_BYTES = 65536;
+function ensureCapacity(memory, neededBytes) {
+  const cappedNeeded = Math.min(neededBytes, MAX_WASM_INPUT_BYTES);
+  if (memory.buffer.byteLength >= cappedNeeded)
+    return;
+  const additionalPages = Math.ceil((cappedNeeded - memory.buffer.byteLength) / WASM_PAGE_BYTES);
+  if (additionalPages > 0)
+    memory.grow(additionalPages);
+}
 function writeToMemory(content) {
   const instance = getInstance();
   const bytes = encoder.encode(content);
   const memory = instance.exports.memory;
+  ensureCapacity(memory, WASM_INPUT_OFFSET + bytes.length + 1);
   const buffer = new Uint8Array(memory.buffer);
   const maxLen = Math.min(bytes.length, buffer.length - WASM_INPUT_OFFSET - 1);
   buffer.set(bytes.subarray(0, maxLen), WASM_INPUT_OFFSET);
@@ -45,8 +56,8 @@ var CALIBRATION = {
   "claude-opus-4.7": 2.05,
   "claude-sonnet": 2.05,
   "claude-haiku": 1.45,
-  "gemini-3.1-pro": 1.4,
-  "gemini-2.5-flash": 1.4,
+  "gemini-3.1-pro": 1.45,
+  "gemini-2.5-flash": 1.45,
   "grok-4.20": 1.15,
   "grok-4-1-fast": 1.15,
   "gpt-5.4": 1.15,

package/dist/slash.js

@@ -1,4 +1,4 @@
-import { getInstance, writeToMemory } from './wasm.js';
+import { getInstance, writeToMemory, ensureCapacity } from './wasm.js';
 const WASM_INPUT_OFFSET = 4096;
 /**
  * Per-model calibration factors.
@@ -36,20 +36,24 @@ const WASM_INPUT_OFFSET = 4096;
  * claude-haiku (routed to claude-haiku-4.5): 1.45 (was 1.40) — drifts
  *             less than the two above but still worsened under the
  *             bigger corpus (min ratio 0.744, was 0.762)
- * gemini-3.1-pro / gemini-2.5-flash: 1.40 — measured 2026-08-23 against
- *             Google's free countTokens endpoint via the gemini-pro-latest
- *             / gemini-flash-latest aliases (min ratio 0.768, identical
- *             for both — pro/flash share one tokenizer this generation).
+ * gemini-3.1-pro / gemini-2.5-flash: 1.45 (was 1.40) — re-verified
+ *             2026-08-23 against the 29-sample corpus via Google's free
+ *             countTokens endpoint (gemini-pro-latest / gemini-flash-latest
+ *             aliases; identical ratios for both — pro/flash share one
+ *             tokenizer this generation). Worst case shifted from the
+ *             original 9-sample corpus: now graphql-response JSON (ratio
+ *             0.749, needs >=1.335), not the prose sample that mattered
+ *             before. 1.40 was technically still safe here (unlike
+ *             Claude's factor, which actually failed) but at the thinnest
+ *             margin of any provider (4.9%) — bumped to match the ~6-8%
+ *             band used everywhere else rather than leave the one factor
+ *             most likely to fail next time this corpus grows again.
  *             Note: 'gemini-3.1-pro' as a literal model ID does not exist
  *             on the live API (404) — same stale-hardcoded-ID class of bug
  *             as the retired Claude snapshots, caught the same day. See
  *             intercept.ts MODEL_API_NAMES: the wire name is now the
  *             '-latest' alias, which sidesteps this whole bug class going
  *             forward since Google repoints it, not us.
- *             ⚠ NOT YET RE-VERIFIED against the 29-sample corpus — measured
- *             only against the original 9. Given Claude's factor needed a
- *             ~11% bump once Spanish/SQL/bash samples were added, treat
- *             this number as provisional until re-run.
  * gpt-5.4 / gpt-5.4-mini / gpt-5.4-nano: 1.15 — measured 2026-08-23
  *             against js-tiktoken's o200k_base (free, local, exact — same
  *             encoding for the whole GPT-5.4 family per OpenAI's own
@@ -61,31 +65,28 @@ const WASM_INPUT_OFFSET = 4096;
  *             DEFAULT_UNKNOWN_MODEL_FACTOR (1.85), a ~60% larger correction
  *             than it needed. Still safe either way (1.85 > required
  *             minimum), just needlessly inflated for real GPT users.
- * grok-4.20 / grok-4-1-fast: 1.15 — measured 2026-08-23 (min ratio 0.928,
- *             the least drift of any provider tested — Grok's tokenizer
- *             runs fewer tokens per content than the others, so raw WASM
- *             already over-reports on most samples). xAI has no free
- *             count-tokens endpoint; this used real chat completions
- *             (max_tokens: 1) with a per-model baseline subtracted to
- *             remove xAI's fixed system-preamble overhead (~185-193
- *             tokens, mostly cached) from the content-only count.
+ * grok-4.20 / grok-4-1-fast: 1.15 — re-verified 2026-08-23 against the
+ *             29-sample corpus (20 new samples: more languages, Spanish/
+ *             Japanese prose, more JSON shapes) and UNCHANGED — same
+ *             worst case (technical-docs prose, ratio 0.928) as the
+ *             original 9-sample measurement. The only one of the three
+ *             non-GPT providers whose factor held without a bump; Claude
+ *             and Gemini both needed one, on different content types each
+ *             (Spanish prose, then JSON). xAI has no free count-tokens
+ *             endpoint; this used real chat completions (max_tokens: 1)
+ *             with a per-model baseline subtracted to remove xAI's fixed
+ *             system-preamble overhead (~185-193 tokens, mostly cached)
+ *             from the content-only count.
  *             Both literal API IDs ('grok-4.20', 'grok-4-1-fast') are
  *             fully retired — not just old snapshots, gone entirely —
  *             remapped to grok-4.20-0309-non-reasoning / grok-4.3 in
  *             intercept.ts MODEL_API_NAMES the same day this was found.
- *             ⚠ NOT YET RE-VERIFIED against the 29-sample corpus — same
- *             caveat as Gemini above. Costs real (small) money per run,
- *             unlike Gemini/Anthropic, so re-verify deliberately, not
- *             reflexively, but don't skip it — Grok has not been tested
- *             against non-English content or SQL/bash at all yet, and
- *             those are exactly what broke Claude's factor.
  *
- * Known limitation: Claude's factors are now verified against a 29-sample
- * corpus (expanded 2026-08-23 from the original 9, which was mostly
- * slash-tokens' own code/docs — not representative third-party content).
- * Gemini and Grok above are still only verified against the old 9-sample
- * set — re-run bench/calibrate-gemini.ts and bench/calibrate-grok.ts
- * before trusting those two numbers as final.
+ * All four providers (Claude, Gemini, Grok, GPT) are now verified against
+ * the same 29-sample corpus (expanded 2026-08-23 from the original 9,
+ * which was mostly slash-tokens' own code/docs — not representative
+ * third-party content). Re-verify again if the corpus grows further —
+ * Claude and Gemini both needed real bumps the first time it grew.
  *
  * Slash must NEVER under-report. Over-reporting is safe (go/no-go only).
  */
@@ -94,8 +95,8 @@ const CALIBRATION = {
     'claude-opus-4.7': 2.05,
     'claude-sonnet': 2.05,
     'claude-haiku': 1.45,
-    'gemini-3.1-pro': 1.40,
-    'gemini-2.5-flash': 1.40,
+    'gemini-3.1-pro': 1.45,
+    'gemini-2.5-flash': 1.45,
     'grok-4.20': 1.15,
     'grok-4-1-fast': 1.15,
     'gpt-5.4': 1.15,
@@ -139,6 +140,7 @@ export function slashBytes(bytes) {
         return 0;
     const instance = getInstance();
     const memory = instance.exports.memory;
+    ensureCapacity(memory, WASM_INPUT_OFFSET + bytes.length + 1);
     const buffer = new Uint8Array(memory.buffer);
     const maxLen = Math.min(bytes.length, buffer.length - WASM_INPUT_OFFSET - 1);
     buffer.set(bytes.subarray(0, maxLen), WASM_INPUT_OFFSET);

package/dist/wasm.d.ts

@@ -1,2 +1,18 @@
 export declare function getInstance(): WebAssembly.Instance;
+/**
+ * Grow WASM linear memory if needed so `neededBytes` of input (plus the
+ * offset reserved for Zig's stack/globals) fits without truncation.
+ *
+ * Fixed 2026-08-23: writeToMemory() used to silently truncate any input
+ * larger than the WASM module's initial memory (~1.06MB usable) BEFORE
+ * estimate_tokens ever ran — no error, no flag, just a token count
+ * computed over a truncated prefix. This defeated the "never
+ * under-report" invariant exactly where it matters most: calls near a
+ * model's real context ceiling. WebAssembly.Memory is growable at
+ * runtime (confirmed: `memory.grow()` works on this module, 17 pages ->
+ * 117 pages tested directly) — growing does not move or invalidate
+ * existing data, it only appends new zeroed pages, so this is safe to
+ * do lazily on every call with no effect on correctness for small inputs.
+ */
+export declare function ensureCapacity(memory: WebAssembly.Memory, neededBytes: number): void;
 export declare function writeToMemory(content: string): number;

package/dist/wasm.js

@@ -23,10 +23,46 @@ export function getInstance() {
     _instance = new WebAssembly.Instance(module);
     return _instance;
 }
+// Safety cap on how far we'll grow WASM linear memory for one input.
+// 32 MB is a large margin over any real model's context window today
+// (the biggest is 1M tokens, which even for dense multi-byte scripts
+// like CJK doesn't approach 32MB of UTF-8 bytes) — so growth alone
+// eliminates silent truncation for every realistic real-world call.
+// Above this cap we still truncate (see below), but a truncated
+// estimate at 32MB+ of input is still far larger than any model's
+// context ceiling, so the overflow gate still fires correctly either
+// way — the cap exists only to bound memory growth against a
+// pathological/abusive input, not to protect estimate correctness.
+const MAX_WASM_INPUT_BYTES = 32 * 1024 * 1024;
+const WASM_PAGE_BYTES = 65536;
+/**
+ * Grow WASM linear memory if needed so `neededBytes` of input (plus the
+ * offset reserved for Zig's stack/globals) fits without truncation.
+ *
+ * Fixed 2026-08-23: writeToMemory() used to silently truncate any input
+ * larger than the WASM module's initial memory (~1.06MB usable) BEFORE
+ * estimate_tokens ever ran — no error, no flag, just a token count
+ * computed over a truncated prefix. This defeated the "never
+ * under-report" invariant exactly where it matters most: calls near a
+ * model's real context ceiling. WebAssembly.Memory is growable at
+ * runtime (confirmed: `memory.grow()` works on this module, 17 pages ->
+ * 117 pages tested directly) — growing does not move or invalidate
+ * existing data, it only appends new zeroed pages, so this is safe to
+ * do lazily on every call with no effect on correctness for small inputs.
+ */
+export function ensureCapacity(memory, neededBytes) {
+    const cappedNeeded = Math.min(neededBytes, MAX_WASM_INPUT_BYTES);
+    if (memory.buffer.byteLength >= cappedNeeded)
+        return;
+    const additionalPages = Math.ceil((cappedNeeded - memory.buffer.byteLength) / WASM_PAGE_BYTES);
+    if (additionalPages > 0)
+        memory.grow(additionalPages);
+}
 export function writeToMemory(content) {
     const instance = getInstance();
     const bytes = encoder.encode(content);
     const memory = instance.exports.memory;
+    ensureCapacity(memory, WASM_INPUT_OFFSET + bytes.length + 1);
     const buffer = new Uint8Array(memory.buffer);
     const maxLen = Math.min(bytes.length, buffer.length - WASM_INPUT_OFFSET - 1);
     buffer.set(bytes.subarray(0, maxLen), WASM_INPUT_OFFSET);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "slash-tokens",
-  "version": "1.5.1",
+  "version": "1.5.3",
   "description": "Token Optimization for Context Engineers. 4.8 KB WASM. Sub-millisecond. Zero dependencies.",
   "main": "dist/index.js",
   "module": "dist/index.js",