slash-tokens 1.4.1 → 1.5.0

package/README.md

@@ -1,6 +1,7 @@
 # /slash-tokens
 
 [![CI/CD](https://github.com/Wolfe-Jam/slash-tokens/actions/workflows/test.yml/badge.svg?branch=main)](https://github.com/Wolfe-Jam/slash-tokens/actions/workflows/test.yml)
+[![FAF Trophy 100%](https://img.shields.io/badge/FAF-%F0%9F%8F%86%20100%25-000000?labelColor=FF6B35)](https://faf.one)
 [![npm version](https://img.shields.io/npm/v/slash-tokens?style=flat&color=cb3837)](https://www.npmjs.com/package/slash-tokens)
 [![npm downloads](https://img.shields.io/npm/dm/slash-tokens?style=flat&color=brightgreen)](https://www.npmjs.com/package/slash-tokens)
 [![WASM size](https://img.shields.io/badge/WASM-4.8_KB-blue?style=flat)](https://bundlephobia.com/package/slash-tokens)
@@ -60,7 +61,7 @@ Full docs, examples, and model pricing at **[GitHub](https://github.com/Wolfe-Ja
 
 ## License
 
-**Code: MIT.** Fork it, ship it, change it, sell it.
+**Code: MIT.** Fork it, ship it, change it, show it, share it, sell it.
 
 **Brand: reserved.** The slash-tokens name, ⚡ mark, and red/gold colors stay with the project. If you're building on top of the SDK, ship under your own name and colors — don't represent your app as Slash.
 

package/dist/cli.js

@@ -41,11 +41,16 @@ function writeToMemory(content) {
 // src/slash.ts
 var WASM_INPUT_OFFSET2 = 4096;
 var CALIBRATION = {
-  "claude-opus": 1.15,
-  "claude-opus-4.7": 1.5,
-  "claude-sonnet": 1.15,
-  "claude-haiku": 1.15
+  "claude-opus": 1.85,
+  "claude-opus-4.7": 1.85,
+  "claude-sonnet": 1.85,
+  "claude-haiku": 1.4,
+  "gemini-3.1-pro": 1.4,
+  "gemini-2.5-flash": 1.4,
+  "grok-4.20": 1.15,
+  "grok-4-1-fast": 1.15
 };
+var DEFAULT_UNKNOWN_MODEL_FACTOR = 1.85;
 function slash(content, model) {
   if (!content)
     return 0;
@@ -54,7 +59,7 @@ function slash(content, model) {
   const raw = instance.exports.estimate_tokens(WASM_INPUT_OFFSET2, len);
   if (!model)
     return raw;
-  const factor = CALIBRATION[model] ?? 1;
+  const factor = CALIBRATION[model] ?? DEFAULT_UNKNOWN_MODEL_FACTOR;
   return factor === 1 ? raw : Math.ceil(raw * factor);
 }
 

package/dist/intercept.js

@@ -5,16 +5,16 @@ import { PROVIDER_MODELS } from './providers.js';
 // Reverse lookup: model name → provider model names in the API
 // (what to put back in the request body)
 const MODEL_API_NAMES = {
-    'claude-opus': 'claude-opus-4-20250514',
-    'claude-sonnet': 'claude-sonnet-4-20250514',
+    'claude-opus': 'claude-opus-5',
+    'claude-sonnet': 'claude-sonnet-5',
     'claude-haiku': 'claude-haiku-4-5-20251001',
     'gpt-5.4': 'gpt-5.4',
     'gpt-5.4-mini': 'gpt-5.4-mini',
     'gpt-5.4-nano': 'gpt-5.4-nano',
-    'grok-4.20': 'grok-4.20',
-    'grok-4-1-fast': 'grok-4-1-fast',
-    'gemini-3.1-pro': 'gemini-3.1-pro',
-    'gemini-2.5-flash': 'gemini-2.5-flash',
+    'grok-4.20': 'grok-4.20-0309-non-reasoning',
+    'grok-4-1-fast': 'grok-4.3',
+    'gemini-3.1-pro': 'gemini-pro-latest',
+    'gemini-2.5-flash': 'gemini-flash-latest',
 };
 // AI API endpoint detection
 const AI_ENDPOINTS = [
@@ -140,7 +140,7 @@ export function patchFetch() {
                     const content = extractContent(body);
                     const rawModel = match.modelExtractor(body, url);
                     const originalModel = normalizeModel(rawModel);
-                    const tokens = slash(content);
+                    const tokens = slash(content, originalModel);
                     const originalInfo = getModel(originalModel);
                     const originalCost = originalInfo ? Math.round(((tokens / 1000000) * originalInfo.input) * 1000000) / 1000000 : 0;
                     const fits = originalInfo ? tokens <= originalInfo.context : true;

package/dist/models.js

@@ -1,4 +1,13 @@
 // Pricing as of April 2026 — USD per million tokens
+// xAI pricing re-derived 2026-08-23: grok-4.20 and grok-4-1-fast (the literal
+// API IDs, not just the generic keys below) were both fully retired — not
+// just old snapshots, they 404 on the live API. Current lineup has no cheap
+// tier at all; grok-4.20 (generic) now targets grok-4.20-0309-non-reasoning
+// and grok-4-1-fast (generic) targets grok-4.3, both $1.25/$2.50/1M — see
+// intercept.ts MODEL_API_NAMES. There is currently no xAI model cheaper than
+// $1.25/M input, so routing between these two generic keys yields zero
+// savings (findCheapestRoute requires strictly cheaper — this is honest,
+// not a bug: the old $0.20/M "fast" tier no longer exists).
 export const MODELS = {
     // Anthropic
     'claude-opus': { input: 5.00, output: 25.00, context: 1000000 },
@@ -6,8 +15,8 @@ export const MODELS = {
     'claude-sonnet': { input: 3.00, output: 15.00, context: 1000000 },
     'claude-haiku': { input: 1.00, output: 5.00, context: 200000 },
     // xAI
-    'grok-4.20': { input: 2.00, output: 6.00, context: 2000000 },
-    'grok-4-1-fast': { input: 0.20, output: 0.50, context: 2000000 },
+    'grok-4.20': { input: 1.25, output: 2.50, context: 1000000 },
+    'grok-4-1-fast': { input: 1.25, output: 2.50, context: 1000000 },
     // Google
     'gemini-3.1-pro': { input: 2.00, output: 12.00, context: 1000000 },
     'gemini-2.5-flash': { input: 0.30, output: 2.50, context: 1000000 },

package/dist/preflight.js

@@ -34,7 +34,7 @@ function buildAlternative(model, originalCost, tokens, info) {
  *     entries (e.g. given model='claude-opus', options[0]?.model CAN be 'grok-...').
  */
 export function preflight(content, model) {
-    const tokens = slash(content);
+    const tokens = slash(content, model);
     const info = getModel(model);
     if (!info) {
         throw new Error(`Unknown model: "${model}". Available: ${Object.keys(MODELS).join(', ')}`);
@@ -74,7 +74,7 @@ export function preflight(content, model) {
  *     both functions return the same model name.
  */
 export function preflightRoute(content, model) {
-    const tokens = slash(content);
+    const tokens = slash(content, model);
     const info = getModel(model);
     if (!info)
         return null;

package/dist/slash.js

@@ -2,28 +2,83 @@ import { getInstance, writeToMemory } from './wasm.js';
 const WASM_INPUT_OFFSET = 4096;
 /**
  * Per-model calibration factors.
- * WASM estimator was tuned for Opus 4.6. Newer tokenizers may drift.
  * Factor > 1.0 means "model uses more tokens than WASM predicts."
  * Applied as: estimate = wasm_estimate * factor (rounded up).
  *
  * Default 1.0 = no adjustment. Update after running bench/calibrate.ts.
- */
-/**
- * Calibration factors derived from bench/calibrate.ts (2026-04-16).
- * WASM under-reports 4.6 by ~12% median. 4.7 tokenizer uses 1.29x more
- * tokens than 4.6 median. Combined correction with safety margin:
  *
- * 4.6: 1.15 (corrects -12% under-report, adds ~3% safety)
- * 4.7: 1.50 (corrects -28% under-report, adds ~5% safety)
+ * Re-derived 2026-08-23 from a fresh benchmark against real Anthropic
+ * count_tokens ground truth (the 2026-04-16 factors were measured
+ * against claude-opus-4-20250514, since retired — that baseline no
+ * longer exists to verify against). Factor chosen as
+ * 1 / min(observed_ratio) with a small safety margin, so it does not
+ * under-report even against the worst sample in this benchmark:
+ *
+ * opus-4.7:   1.85 (was 1.50 — that value was insufficient: min observed
+ *             ratio 0.571 needs >=1.75 just to avoid under-report, before
+ *             any margin. Previously the factor also went UNUSED in
+ *             production — see intercept.ts/preflight.ts history, fixed
+ *             2026-08-23 same day)
+ * claude-opus (generic, now routed to claude-opus-5): 1.85 — carried
+ *             over from opus-4.7's measurement as the closest tested
+ *             sibling; opus-5 itself has not been directly benchmarked
+ * claude-sonnet (routed to claude-sonnet-5): 1.85 — measured directly,
+ *             nearly identical drift to opus-4.7 (min ratio 0.580)
+ * claude-haiku (routed to claude-haiku-4.5): 1.40 — measured directly,
+ *             drifts less than the two above (min ratio 0.762)
+ * gemini-3.1-pro / gemini-2.5-flash: 1.40 — measured 2026-08-23 against
+ *             Google's free countTokens endpoint via the gemini-pro-latest
+ *             / gemini-flash-latest aliases (min ratio 0.768, identical
+ *             for both — pro/flash share one tokenizer this generation).
+ *             Note: 'gemini-3.1-pro' as a literal model ID does not exist
+ *             on the live API (404) — same stale-hardcoded-ID class of bug
+ *             as the retired Claude snapshots, caught the same day. See
+ *             intercept.ts MODEL_API_NAMES: the wire name is now the
+ *             '-latest' alias, which sidesteps this whole bug class going
+ *             forward since Google repoints it, not us.
+ * grok-4.20 / grok-4-1-fast: 1.15 — measured 2026-08-23 (min ratio 0.928,
+ *             the least drift of any provider tested — Grok's tokenizer
+ *             runs fewer tokens per content than the others, so raw WASM
+ *             already over-reports on most samples). xAI has no free
+ *             count-tokens endpoint; this used real chat completions
+ *             (max_tokens: 1) with a per-model baseline subtracted to
+ *             remove xAI's fixed system-preamble overhead (~185-193
+ *             tokens, mostly cached) from the content-only count.
+ *             Both literal API IDs ('grok-4.20', 'grok-4-1-fast') are
+ *             fully retired — not just old snapshots, gone entirely —
+ *             remapped to grok-4.20-0309-non-reasoning / grok-4.3 in
+ *             intercept.ts MODEL_API_NAMES the same day this was found.
+ *
+ * Known limitation: derived from a 9-sample benchmark corpus, several
+ * of which are slash-tokens' own code/docs (not representative
+ * third-party content). Re-run bench/calibrate.ts with a larger,
+ * more diverse corpus before treating these as final.
  *
  * Slash must NEVER under-report. Over-reporting is safe (go/no-go only).
  */
 const CALIBRATION = {
-    'claude-opus': 1.15,
-    'claude-opus-4.7': 1.50,
-    'claude-sonnet': 1.15,
-    'claude-haiku': 1.15,
+    'claude-opus': 1.85,
+    'claude-opus-4.7': 1.85,
+    'claude-sonnet': 1.85,
+    'claude-haiku': 1.40,
+    'gemini-3.1-pro': 1.40,
+    'gemini-2.5-flash': 1.40,
+    'grok-4.20': 1.15,
+    'grok-4-1-fast': 1.15,
 };
+/**
+ * Fallback factor for any model not in CALIBRATION — new model IDs,
+ * providers we haven't benchmarked yet, anything unrecognized.
+ *
+ * Deliberately the highest known-safe factor, not 1.0. "Unknown" must
+ * never mean "no correction" — that's the most dangerous case, worse
+ * than any calibrated model, because it silently assumes a brand-new
+ * tokenizer behaves exactly like the raw WASM heuristic with zero
+ * evidence either way. Slash must NEVER under-report; for a model we
+ * haven't measured, the only safe assumption is the worst one we've
+ * actually observed (see CALIBRATION comment above).
+ */
+const DEFAULT_UNKNOWN_MODEL_FACTOR = 1.85;
 /**
  * Estimate token count for a string.
  * Sub-millisecond. Zero allocations in WASM.
@@ -37,7 +92,7 @@ export function slash(content, model) {
     const raw = instance.exports.estimate_tokens(WASM_INPUT_OFFSET, len);
     if (!model)
         return raw;
-    const factor = CALIBRATION[model] ?? 1.0;
+    const factor = CALIBRATION[model] ?? DEFAULT_UNKNOWN_MODEL_FACTOR;
     return factor === 1.0 ? raw : Math.ceil(raw * factor);
 }
 /**

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "slash-tokens",
-  "version": "1.4.1",
+  "version": "1.5.0",
   "description": "Token Optimization for Context Engineers. 4.8 KB WASM. Sub-millisecond. Zero dependencies.",
   "main": "dist/index.js",
   "module": "dist/index.js",
@@ -50,6 +50,7 @@
   "homepage": "https://slashtokens.com",
   "devDependencies": {
     "@types/node": "^25.5.0",
+    "js-tiktoken": "^1.0.21",
     "typescript": "^6.0.2"
   }
 }