slash-tokens 1.2.0 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -1
- package/dist/cli.js +12 -2
- package/dist/models.js +1 -0
- package/dist/slash.d.ts +2 -1
- package/dist/slash.js +31 -2
- package/dist/transact.d.ts +1 -1
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -1,8 +1,17 @@
|
|
|
1
1
|
# /slash-tokens
|
|
2
2
|
|
|
3
|
+
[](https://github.com/Wolfe-Jam/slash-tokens/actions/workflows/test.yml)
|
|
4
|
+
[](https://www.npmjs.com/package/slash-tokens)
|
|
5
|
+
[](https://www.npmjs.com/package/slash-tokens)
|
|
6
|
+
[](https://bundlephobia.com/package/slash-tokens)
|
|
7
|
+
[](./LICENSE)
|
|
8
|
+
[](https://github.com/Wolfe-Jam/slash-tokens)
|
|
9
|
+
|
|
3
10
|
Token Optimization for Context Engineers.
|
|
4
11
|
For anyone building with LLMs. 4.8 KB WASM. Sub-millisecond. Zero dependencies.
|
|
5
12
|
|
|
13
|
+
> 🆕 **v1.3 — The Opus 4.7 Edition.** Same-day support for Claude Opus 4.7 with measured token calibration (1.16–1.51x). Plus Gemini proxy fix and benchmark harness for any upstream.
|
|
14
|
+
|
|
6
15
|
```bash
|
|
7
16
|
npm install slash-tokens
|
|
8
17
|
```
|
|
@@ -10,7 +19,7 @@ npm install slash-tokens
|
|
|
10
19
|
```js
|
|
11
20
|
import { preflight } from 'slash-tokens'
|
|
12
21
|
|
|
13
|
-
const check = preflight(prompt, 'claude-opus')
|
|
22
|
+
const check = preflight(prompt, 'claude-opus-4.7')
|
|
14
23
|
// tokens: 47,000 | cost: $0.71 | 11 cheaper options | save 99%
|
|
15
24
|
```
|
|
16
25
|
|
package/dist/cli.js
CHANGED
|
@@ -40,12 +40,22 @@ function writeToMemory(content) {
|
|
|
40
40
|
|
|
41
41
|
// src/slash.ts
|
|
42
42
|
var WASM_INPUT_OFFSET2 = 4096;
|
|
43
|
-
|
|
43
|
+
var CALIBRATION = {
|
|
44
|
+
"claude-opus": 1.15,
|
|
45
|
+
"claude-opus-4.7": 1.5,
|
|
46
|
+
"claude-sonnet": 1.15,
|
|
47
|
+
"claude-haiku": 1.15
|
|
48
|
+
};
|
|
49
|
+
function slash(content, model) {
|
|
44
50
|
if (!content)
|
|
45
51
|
return 0;
|
|
46
52
|
const len = writeToMemory(content);
|
|
47
53
|
const instance = getInstance();
|
|
48
|
-
|
|
54
|
+
const raw = instance.exports.estimate_tokens(WASM_INPUT_OFFSET2, len);
|
|
55
|
+
if (!model)
|
|
56
|
+
return raw;
|
|
57
|
+
const factor = CALIBRATION[model] ?? 1;
|
|
58
|
+
return factor === 1 ? raw : Math.ceil(raw * factor);
|
|
49
59
|
}
|
|
50
60
|
|
|
51
61
|
// src/patterns.ts
|
package/dist/models.js
CHANGED
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
export const MODELS = {
|
|
3
3
|
// Anthropic
|
|
4
4
|
'claude-opus': { input: 5.00, output: 25.00, context: 1000000 },
|
|
5
|
+
'claude-opus-4.7': { input: 5.00, output: 25.00, context: 1000000 },
|
|
5
6
|
'claude-sonnet': { input: 3.00, output: 15.00, context: 1000000 },
|
|
6
7
|
'claude-haiku': { input: 1.00, output: 5.00, context: 200000 },
|
|
7
8
|
// xAI
|
package/dist/slash.d.ts
CHANGED
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Estimate token count for a string.
|
|
3
3
|
* Sub-millisecond. Zero allocations in WASM.
|
|
4
|
+
* Optional model parameter applies per-model calibration.
|
|
4
5
|
*/
|
|
5
|
-
export declare function slash(content: string): number;
|
|
6
|
+
export declare function slash(content: string, model?: string): number;
|
|
6
7
|
/**
|
|
7
8
|
* Estimate token count from raw bytes.
|
|
8
9
|
*/
|
package/dist/slash.js
CHANGED
|
@@ -1,15 +1,44 @@
|
|
|
1
1
|
import { getInstance, writeToMemory } from './wasm.js';
|
|
2
2
|
const WASM_INPUT_OFFSET = 4096;
|
|
3
|
+
/**
|
|
4
|
+
* Per-model calibration factors.
|
|
5
|
+
* WASM estimator was tuned for Opus 4.6. Newer tokenizers may drift.
|
|
6
|
+
* Factor > 1.0 means "model uses more tokens than WASM predicts."
|
|
7
|
+
* Applied as: estimate = wasm_estimate * factor (rounded up).
|
|
8
|
+
*
|
|
9
|
+
* Default 1.0 = no adjustment. Update after running bench/calibrate.ts.
|
|
10
|
+
*/
|
|
11
|
+
/**
|
|
12
|
+
* Calibration factors derived from bench/calibrate.ts (2026-04-16).
|
|
13
|
+
* WASM under-reports 4.6 by ~12% median. 4.7 tokenizer uses 1.29x more
|
|
14
|
+
* tokens than 4.6 median. Combined correction with safety margin:
|
|
15
|
+
*
|
|
16
|
+
* 4.6: 1.15 (corrects -12% under-report, adds ~3% safety)
|
|
17
|
+
* 4.7: 1.50 (corrects -28% under-report, adds ~5% safety)
|
|
18
|
+
*
|
|
19
|
+
* Slash must NEVER under-report. Over-reporting is safe (go/no-go only).
|
|
20
|
+
*/
|
|
21
|
+
const CALIBRATION = {
|
|
22
|
+
'claude-opus': 1.15,
|
|
23
|
+
'claude-opus-4.7': 1.50,
|
|
24
|
+
'claude-sonnet': 1.15,
|
|
25
|
+
'claude-haiku': 1.15,
|
|
26
|
+
};
|
|
3
27
|
/**
|
|
4
28
|
* Estimate token count for a string.
|
|
5
29
|
* Sub-millisecond. Zero allocations in WASM.
|
|
30
|
+
* Optional model parameter applies per-model calibration.
|
|
6
31
|
*/
|
|
7
|
-
export function slash(content) {
|
|
32
|
+
export function slash(content, model) {
|
|
8
33
|
if (!content)
|
|
9
34
|
return 0;
|
|
10
35
|
const len = writeToMemory(content);
|
|
11
36
|
const instance = getInstance();
|
|
12
|
-
|
|
37
|
+
const raw = instance.exports.estimate_tokens(WASM_INPUT_OFFSET, len);
|
|
38
|
+
if (!model)
|
|
39
|
+
return raw;
|
|
40
|
+
const factor = CALIBRATION[model] ?? 1.0;
|
|
41
|
+
return factor === 1.0 ? raw : Math.ceil(raw * factor);
|
|
13
42
|
}
|
|
14
43
|
/**
|
|
15
44
|
* Estimate token count from raw bytes.
|
package/dist/transact.d.ts
CHANGED