slash-tokens 0.0.1 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +118 -2
- package/dist/cli.d.ts +2 -0
- package/dist/cli.js +236 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.js +57 -0
- package/dist/patterns.d.ts +7 -0
- package/dist/report.d.ts +2 -0
- package/dist/scanner.d.ts +11 -0
- package/dist/slash.d.ts +9 -0
- package/dist/wasm.d.ts +2 -0
- package/package.json +31 -3
package/README.md
CHANGED
|
@@ -1,5 +1,121 @@
|
|
|
1
|
-
# slash-tokens
|
|
1
|
+
# /slash-tokens
|
|
2
2
|
|
|
3
3
|
Token budget estimation in 4.8 KB WASM. Sub-millisecond. Zero dependencies.
|
|
4
4
|
|
|
5
|
-
|
|
5
|
+
I don't count tokens. I slash them.
|
|
6
|
+
|
|
7
|
+
## Install
|
|
8
|
+
|
|
9
|
+
```bash
|
|
10
|
+
npm install slash-tokens
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
## Use
|
|
14
|
+
|
|
15
|
+
```js
|
|
16
|
+
import { slash } from 'slash-tokens'
|
|
17
|
+
|
|
18
|
+
const tokens = slash('Your content here')
|
|
19
|
+
|
|
20
|
+
if (tokens > 128_000) trim(content) // Context window gate
|
|
21
|
+
const cost = tokens * PRICE_PER_TOKEN // Cost estimation
|
|
22
|
+
```
|
|
23
|
+
|
|
24
|
+
One function. Returns a number. That's the entire API.
|
|
25
|
+
|
|
26
|
+
## Scan your project
|
|
27
|
+
|
|
28
|
+
```bash
|
|
29
|
+
bunx slash-tokens
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
Finds every AI API call in your codebase and estimates your token burn:
|
|
33
|
+
|
|
34
|
+
```
|
|
35
|
+
⚡ /slash
|
|
36
|
+
Token burn analysis
|
|
37
|
+
|
|
38
|
+
Scanned 47 files in 3ms
|
|
39
|
+
|
|
40
|
+
CALL SITES
|
|
41
|
+
────────────────────────────────────────────────────────
|
|
42
|
+
src/api/chat.ts:24 Anthropic ~2,400 tok/call
|
|
43
|
+
src/api/summary.ts:11 OpenAI ~8,200 tok/call
|
|
44
|
+
src/lib/agent.ts:88 Vercel AI ~1,100 tok/call
|
|
45
|
+
|
|
46
|
+
DAILY BURN (100 calls/site/day)
|
|
47
|
+
────────────────────────────────────────────────────────
|
|
48
|
+
Input tokens: 1,520,000
|
|
49
|
+
Output tokens: 3,040,000 (estimated)
|
|
50
|
+
Total: 4,560,000 tokens/day
|
|
51
|
+
|
|
52
|
+
MONTHLY COST ($3/$15 per MTok)
|
|
53
|
+
────────────────────────────────────────────────────────
|
|
54
|
+
Total: $1,504.80/mo
|
|
55
|
+
|
|
56
|
+
⚡ SLASH SAVINGS (10% gate efficiency)
|
|
57
|
+
────────────────────────────────────────────────────────
|
|
58
|
+
Annual savings: $1,805.76/yr
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
Detects: OpenAI, Anthropic, Vercel AI, LangChain, Gemini, Bedrock, Grok, Cohere, Mistral.
|
|
62
|
+
|
|
63
|
+
## How it works
|
|
64
|
+
|
|
65
|
+
Single-pass byte classification over your content. No vocabulary tables. No BPE decode. Classifies content type (prose, code, JSON, YAML, mixed), applies a calibrated chars-per-token ratio, adds a safety margin.
|
|
66
|
+
|
|
67
|
+
Budget gate — better to say "doesn't fit" than overflow. The API returns exact counts after the call. Pre-call, you only need go/no-go.
|
|
68
|
+
|
|
69
|
+
## Numbers
|
|
70
|
+
|
|
71
|
+
| | js-tiktoken | gpt-tokenizer | slash-tokens |
|
|
72
|
+
|---|---|---|---|
|
|
73
|
+
| Size | ~1 MB | 300 KB+ | **4.8 KB** |
|
|
74
|
+
| Cold start | Heavy | Moderate | **Sub-ms** |
|
|
75
|
+
| Edge-ready | Painful | Difficult | **Native** |
|
|
76
|
+
| Models | GPT only | GPT only | **Any model** |
|
|
77
|
+
| Dependencies | npm tree | npm tree | **Zero** |
|
|
78
|
+
| Allocations | Many | Many | **Zero** |
|
|
79
|
+
|
|
80
|
+
## Runtime support
|
|
81
|
+
|
|
82
|
+
Works everywhere WASM runs:
|
|
83
|
+
|
|
84
|
+
- Node.js
|
|
85
|
+
- Bun
|
|
86
|
+
- Deno
|
|
87
|
+
- Cloudflare Workers
|
|
88
|
+
- Vercel Edge
|
|
89
|
+
- Browser
|
|
90
|
+
|
|
91
|
+
## API
|
|
92
|
+
|
|
93
|
+
### `slash(content: string): number`
|
|
94
|
+
|
|
95
|
+
Estimate token count for a string. Returns `u32`.
|
|
96
|
+
|
|
97
|
+
```js
|
|
98
|
+
slash('') // 0
|
|
99
|
+
slash('hello') // 1
|
|
100
|
+
slash(longDocument) // 4,283
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
### `slashBytes(bytes: Uint8Array): number`
|
|
104
|
+
|
|
105
|
+
Estimate token count from raw bytes. Skips `TextEncoder` overhead.
|
|
106
|
+
|
|
107
|
+
## What this is not
|
|
108
|
+
|
|
109
|
+
This is not a tokenizer. It doesn't decode BPE. It doesn't reproduce exact token counts. No tokenizer can — Claude, Grok, and Gemini vocabularies are not public.
|
|
110
|
+
|
|
111
|
+
This is a **budget gate**. It answers "does this fit?" with 96-98% accuracy in sub-millisecond time. The 2-4% margin is intentional — overestimate, never underestimate.
|
|
112
|
+
|
|
113
|
+
## Engine
|
|
114
|
+
|
|
115
|
+
4,865 bytes of Zig-compiled WASM (`wasm32-freestanding`). No allocator. No floats. No heap. Base64-embedded — no separate `.wasm` file to load.
|
|
116
|
+
|
|
117
|
+
Built from [xai-faf-zig](https://github.com/Wolfe-Jam/xai-faf-zig). 172 tests including 65 adversarial (CJK, emoji, binary, base64, threshold boundaries).
|
|
118
|
+
|
|
119
|
+
## License
|
|
120
|
+
|
|
121
|
+
MIT
|
package/dist/cli.d.ts
ADDED
package/dist/cli.js
ADDED
|
@@ -0,0 +1,236 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
|
|
3
|
+
// src/scanner.ts
|
|
4
|
+
import { readdirSync, readFileSync, statSync } from "fs";
|
|
5
|
+
import { join, extname } from "path";
|
|
6
|
+
|
|
7
|
+
// src/wasm.ts
|
|
8
|
+
var WASM_BASE64 = "AGFzbQEAAAABLAZgAn9/AX9gBH9/f38AYAZ/f39/f38AYAV/f39/fwBgBH9/f38Bf2ABfwF/AxEQAAABAQIDBAQBBAMAAAUABQUDAQARBgkBfwFBgIDAAAsHWQYGbWVtb3J5AgAJc2NvcmVfZmFmAAAUc2NvcmVfZmFmX2VudGVycHJpc2UACwx2YWxpZGF0ZV9mYWYADAhnZXRfdGllcgAND2VzdGltYXRlX3Rva2VucwAOCoEaEBAAIAAgARCBgICAAEH/AXELSgEBfyOAgICAAEEQayICJICAgIAAAkACQCABDQBBACEBDAELIAJBC2ogACABQQAQgoCAgAAgAi0ACyEBCyACQRBqJICAgIAAIAEL/wcBE38jgICAgABB0ABrIgQkgICAgABBqIHAgABBgIDAgAAgA0EBcSIFGyEGQSFBFSAFGyEHQQAhCEEAIQlBACEKA0ACQAJAAkAgCCAHRg0AIARBOGogBiAIQQN0aiIFKAIAIgsgBSgCBCIFQeeIwIAAEIOAgIAAAkAgBC0APA0AQQAhDAwCC0EBIQ0gBSAEKAI4Ig5BAWoiD2shECALIA9qIRFBACESQQAhEwNAQQAhDAJAAkAgDUH/AXENAEEAIQ1BACEFQQAhDwwBCyAEQcgAaiABIAIgEkH3iMCAAEEBEISAgIAAIAQoAkgiFCACIAQtAEwiFRsgEmshDyABIBJqIQUgFUEARyENIBRBAWpBACAVGyESCyAFRQ0CIARBMGogBSAPQfWIwIAAQQEQhYCAgAAgBCgCMCEWIAQoAjQiDEUNAEEAIQVBACEPA0ACQAJAIAwgBUYNAAJAIBYgBWotAAAiFEEJRg0AQQEhFSAUQSBHDQEMAgtBAiEVDAELAkAgDw0AIARBGGogFiAMQfmIwIAAQQIQhYCAgABBACETIAQoAhgiDyAEKAIcIgUgCyAOEIaAgIAAQQFxRQ0DIAUgDk0NAyAPIA5qLQAAQTpGIRMMAwsgEyAPQQFHcUUNAiAEQShqIBYgDEH5iMCAAEECEIWAgIAAQQEhEyAEKAIoIhUgBCgCLCIPIBEgEBCGgICAAEEBcUUNAiAPIBBNDQIgFSAQai0AAEE6Rw0CIARBwABqIBUgD0HliMCAABCDgICAACAELQBERQ0CQQAhDEEAIQUgDyAEKAJAQQFqIhRNDQUgBEEgaiAVIBRqIA8gFGtB8IjAgABBBBCFgICAACAEKAIgQQAgBCgCJCIFGyEMIAVBACAFGyEFDAULIAVBAWohBSAVIA9qIQ8MAAsLC0EhQRUgA0EBcRsiBSAJayEPQQAhFQJAIAUgCUH/AXFGDQAgCkH/AXFB5ABsIA9B/wFxIhVBAXZqIBVuIRULIAAgBToABCAAIA86AAMgACAJOgACIAAgCjoAASAAIBVB/wFxIgVB5AAgBUHkAEkbOgAAIARB0ABqJICAgIAADwtBACEFCwJAIAxFDQAgBEEQaiAMIAVB6YjAgABBBhCFgICAAAJAIAQoAhAgBCgCFEHKiMCAAEELEIeAgIAAQQFxDQAgBEEIaiAMIAVB6YjAgABBBhCFgICAACAEKAIMIgxFDQEgBCgCCCEWQbh/IQUCQANAIAVBCGoiD0UNASAFQciJwIAAaiEVIAVBxInAgABqIRQgDyEFIBYgDCAUKAIAIBUoAgAQh4CAgABBAXFFDQAMAwsLIApBAWohCgwBCyAJQQFqIQkLIAhBAWohCAwACws/AQF/I4CAgIAAQRBrIgQkgICAgAAgBEEIaiABIAJBACADQQEQhICAgAAgACAEKQMINwIAIARBEGokgICAgAALsQMDAn8BfgF/I4CAgIAAQZAIayIGJICAgIAAAkACQCACIAVPDQAgAEIANwIADAELAkACQAJAAkACQCAFQQFLDQAgBQ0BIABBAToABAwECyACQTRJDQEgBUEFSQ0BQQAhBwwCCyAGQQhqIAEgAiADIAQtAAAQioCAgAAgACAGKQMINwIADAMLIAIgBWshBwNAAkACQAJAIAMgB00NAEEAIQNCACEIDAELIAEgA2ogBSAEIAUQiYCAgABBAXFFDQFCgICAgBAhCAsgACAIIAOthDcCAAwECyADQQFqIQMMAAsLAkADQCAHQYAIRg0BIAZBEGogB2ogBTYCACAHQQRqIQcMAAsLIAVBf2ohByAEIQkDQAJAIAcNACACIAVrIQkDQAJAIAMgCU0NACAAQgA3AgAMBQsCQCABIANqIgcgBSAEIAUQiYCAgABBAXFFDQAgAEEBOgAEDAQLIAZBEGogByAFakF/ai0AAEECdGooAgAgA2ohAwwACwsgBkEQaiAJLQAAQQJ0aiAHNgIAIAlBAWohCSAHQX9qIQcMAAsLIAAgAzYCAAsgBkGQCGokgICAgAALrgEBBH8jgICAgABBEGsiBSSAgICAAEEAIQYCQANAAkAgAiAGRw0AIAIhBgwCCyAFIAMgBCABIAZqLQAAEIiAgIAAIAUtAARFDQEgBkEBaiEGDAALCyABQX9qIQcCQANAIAIiCCAGTQ0BIAVBCGogAyAEIAcgCGotAAAQiICAgAAgCEF/aiECIAUtAAwNAAsLIAAgCCAGazYCBCAAIAEgBmo2AgAgBUEQaiSAgICAAAskAQF/QQAhBAJAIAMgAUsNACAAIAMgAiADEImAgIAAIQQLIAQLeQECf0EAIQQCQCABIANHDQADQCABRSEEIAFFDQEgAi0AACEDIAAtAAAhBSACQQFqIQIgAEEBaiEAIAFBf2ohASAFQSBqIAUgBUG/f2pB/wFxQRpJG0H/AXEgA0EgaiADIANBv39qQf8BcUEaSRtB/wFxRg0ACwsgBAs9AQF/I4CAgIAAQRBrIgQkgICAgAAgBEEIaiABIAJBACADEIqAgIAAIAAgBCkDCDcCACAEQRBqJICAgIAAC/0CAQV/I4CAgIAAQRBrIgQkgICAgAACQAJAIAEgA0cNAEEBIQUgAUUNASAAQarVqtV6IAEbIgYgAkGq1arVeiADGyIHRg0BAkAgAUEQSw0AAkAgAUEETw0AIAAtAAAgBy0AAEYgACABQX9qIgNqLQAAIAcgA2otAABGcSAAIAFBAXYiA2otAAAgByADai0AAEZxIQUMAwtBACEDIARBADYCACAEIAFBfGoiBTYCBCAEIAFBAXZBDHEiADYCCCAEIAUgAGs2AgxBACEFAkADQCADQRBGDQEgByAEIANqKAIAIgBqKAAAIAYgAGooAABzIAVyIQUgA0EEaiEDDAALCyAFRSEFDAILIAFBf2pBAnZBAWohACAGIQMgByEFAkADQCAAQX9qIgBFDQEgBSgAACECIAMoAAAhCCADQQRqIQMgBUEEaiEFIAggAkcNAgwACwsgBiABQXxqIgNqKAAAIAcgA2ooAABGIQUMAQtBACEFCyAEQRBqJICAgIAAIAULWAACQCADIAJJDQAgAEIANwIADwsgBEH/AXEhBAJAA0AgAiADRg0BAkAgASADai0AACAERw0AIABBAToABCAAIAM2AgAPCyADQQFqIQMMAAsLIABCADcCAAtOAQF/I4CAgIAAQRBrIgIkgICAgAACQAJAIAENAEEAIQEMAQsgAkELaiAAIAFBARCCgICAACACLQALIQELIAJBEGokgICAgAAgAUH/AXELFAAgACABEIGAgIAAQf8BcUHjAEsLCAAgAEHjAEsL5gQBCH8jgICAgABBEGsiAiSAgICAAAJAAkAgAQ0AQQAhAwwBCyACIAAtAAAiBBCPgICAAEEHcSIDOgAPIANBBHNBB3FBAnQiA0GcisCAAGohBSADQfyJwIAAaiEGIANB3InAgABqIQcgA0G8icCAAGohA0EAIQgCQAJAAkAgBEGFf2oOAwECAQALIARBWGpBAk8NAQtBASEICyAFKAIAIQUgBigCACEJIAcoAgAhBiADKAIAIQcgAUF/aiEDIABBAWohAANAAkACQAJAAkACQAJAAkACQCADRQ0AIAAtAAAiBBCPgICAAEEHcQ4GBAQHAQIDBAsgCEHkAGwgAW4hCCAGQeQAbCABbiEDIAdB5ABsIAFuIQAgBUHkAGwgAW4hBCAJIAZqQeQAbCABbkEjSQ0EIANBBU8NBEE4IQMMBQsgCUEBaiEJDAULIAZBAWohBgwECyAHQQFqIQcMAwsgBUEBaiEFDAILAkAgBEHBAEkNACADQQJLDQAgAEEITw0AQTYhAwwBCwJAIAhBAkkNACADQQRJDQAgAEENTQ0AQRohAwwBCwJAIABBFU0NAEEcIQMMAQsCQCADQQRJDQAgAEENTQ0AQSAhAwwBCwJAIABBDkkNACAEQcgASw0AIAhBAk8NAEEeIQMMAQtBIUEjIABBBksbQSMgA0EESxshAwsgA0EBdiABQQpsaiADbiIDQSFuIANqIgNBASADQQFLGyEDDAILAkACQCAEQVhqQQJJDQAgBEGFf2oOAwABAAELIAhBAWohCAsgA0F/aiEDIABBAWohAAwACwsgAkEQaiSAgICAACADC3UBAX8CQCAAQZ9/akH/AXFBGk8NAEEADwsCQCAAQb9/akH/AXFBGk8NAEEBDwsCQCAAQVBqQf8BcUEKTw0AQQIPC0EDIQECQAJAAkACQCAAQf8BcSIAQXdqDgUDAgEBAgALIABBIEYNAgtBBQ8LQQQhAQsgAQsLxgoBAEGAgMAAC7wKvwMQAAwAAAAjAxAADAAAAP0DEAAVAAAAsQIQABEAAAAxAhAAEgAAAMEBEAARAAAAiQMQABMAAADoAhAAEgAAAPABEAARAAAAIQQQAA4AAABLAxAAEwAAALABEAAQAAAAGgIQABYAAAAwBBAADQAAAJ0DEAAOAAAAsQMQAA0AAAB6AxAADgAAAMsCEAAQAAAAbAMQAA0AAAA+BBAACwAAAFYEEAAKAAAAvwMQAAwAAAAjAxAADAAAAP0DEAAVAAAAsQIQABEAAAAxAhAAEgAAAMEBEAARAAAAiQMQABMAAADoAhAAEgAAAPABEAARAAAAIQQQAA4AAABLAxAAEwAAALABEAAQAAAAGgIQABYAAAAwBBAADQAAAJ0DEAAOAAAAsQMQAA0AAAB6AxAADgAAAMsCEAAQAAAAbAMQAA0AAAA+BBAACwAAAFYEEAAKAAAACgMQABMAAACbAhAAFQAAAG4CEAAQAAAAAgIQABcAAAB/AhAAGwAAANwCEAALAAAA8QMQAAsAAABfAxAADAAAABMEEAANAAAA0wEQABwAAABWAhAAFwAAANsDEAAVAAAAc3RhY2sudWlfbGlicmFyeQBodW1hbl9jb250ZXh0LndoeQBtb25vcmVwby52ZXJzaW9uaW5nX3N0cmF0ZWd5AGh1bWFuX2NvbnRleHQuaG93AG1vbm9yZXBvLnBhY2thZ2VzX2NvdW50AHN0YWNrLnN0YXRlX21hbmFnZW1lbnQAaHVtYW5fY29udGV4dC53aGF0AGRldmVsb3BtZW50IHRlYW1zAG1vbm9yZXBvLnNoYXJlZF9jb25maWdzAHN0YWNrLndvcmtzcGFjZXMAbW9ub3JlcG8uYnVpbGRfb3JjaGVzdHJhdG9yAHN0YWNrLnBhY2thZ2VfbWFuYWdlcgBodW1hbl9jb250ZXh0LndobwB1bmtub3duAHN0YWNrLmNvbm5lY3Rpb24Ac3RhY2suYWRtaW4AaHVtYW5fY29udGV4dC53aGVuAGNsb3VkIHBsYXRmb3JtAHN0YWNrLm1vbm9yZXBvX3Rvb2wAbnVsbABwcm9qZWN0LmdvYWwAZGVzY3JpYmUgeW91ciBwcm9qZWN0IGdvYWwAc3RhY2suY3NzX2ZyYW1ld29yawBzdGFjay5zZWFyY2gAc3RhY2suaG9zdGluZwBzdGFjay5kYXRhYmFzZQBodW1hbl9jb250ZXh0LndoZXJlAHN0YWNrLmFwaV90eXBlAG5vbmUAc3RhY2sucnVudGltZQBwcm9qZWN0Lm5hbWUAbm90IGFwcGxpY2FibGUAbW9ub3JlcG8ucmVtb3RlX2NhY2hlAHN0YWNrLmNhY2hlAHByb2plY3QubWFpbl9sYW5ndWFnZQBzdGFjay5zdG9yYWdlAHN0YWNrLmZyb250ZW5kAHN0YWNrLmJhY2tlbmQAc3RhY2suYnVpbGQAc2xvdGlnbm9yZWQAc3RhY2suY2ljZABuL2EAOgAuACAJDQoiJwAgCSInAA0ACgAgCQAwAxAAGgAAAEQCEAARAAAA+wIQAA4AAAAeAxAABAAAAKwDEAAEAAAAwwIQAAcAAABhBBAAAwAAAMwDEAAOAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAABAAAAAAAAAAAAAAA=";
|
|
9
|
+
var WASM_INPUT_OFFSET = 4096;
|
|
10
|
+
var encoder = new TextEncoder;
|
|
11
|
+
var _instance = null;
|
|
12
|
+
function decodeBase64(b64) {
|
|
13
|
+
if (typeof Buffer !== "undefined") {
|
|
14
|
+
return new Uint8Array(Buffer.from(b64, "base64"));
|
|
15
|
+
}
|
|
16
|
+
const binary = atob(b64);
|
|
17
|
+
const bytes = new Uint8Array(binary.length);
|
|
18
|
+
for (let i = 0;i < binary.length; i++) {
|
|
19
|
+
bytes[i] = binary.charCodeAt(i);
|
|
20
|
+
}
|
|
21
|
+
return bytes;
|
|
22
|
+
}
|
|
23
|
+
function getInstance() {
|
|
24
|
+
if (_instance)
|
|
25
|
+
return _instance;
|
|
26
|
+
const bytes = decodeBase64(WASM_BASE64);
|
|
27
|
+
const module = new WebAssembly.Module(bytes);
|
|
28
|
+
_instance = new WebAssembly.Instance(module);
|
|
29
|
+
return _instance;
|
|
30
|
+
}
|
|
31
|
+
function writeToMemory(content) {
|
|
32
|
+
const instance = getInstance();
|
|
33
|
+
const bytes = encoder.encode(content);
|
|
34
|
+
const memory = instance.exports.memory;
|
|
35
|
+
const buffer = new Uint8Array(memory.buffer);
|
|
36
|
+
const maxLen = Math.min(bytes.length, buffer.length - WASM_INPUT_OFFSET - 1);
|
|
37
|
+
buffer.set(bytes.subarray(0, maxLen), WASM_INPUT_OFFSET);
|
|
38
|
+
return maxLen;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
// src/slash.ts
|
|
42
|
+
var WASM_INPUT_OFFSET2 = 4096;
|
|
43
|
+
function slash(content) {
|
|
44
|
+
if (!content)
|
|
45
|
+
return 0;
|
|
46
|
+
const len = writeToMemory(content);
|
|
47
|
+
const instance = getInstance();
|
|
48
|
+
return instance.exports.estimate_tokens(WASM_INPUT_OFFSET2, len);
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
// src/patterns.ts
|
|
52
|
+
var AI_PATTERNS = [
|
|
53
|
+
{ name: "OpenAI", regex: /openai\.chat\.completions\.create|new OpenAI\(|from\s+['"]openai['"]/g },
|
|
54
|
+
{ name: "Anthropic", regex: /anthropic\.messages\.create|new Anthropic\(|from\s+['"]@anthropic-ai/g },
|
|
55
|
+
{ name: "Vercel AI", regex: /from\s+['"]ai['"]|generateText|streamText|generateObject/g },
|
|
56
|
+
{ name: "LangChain", regex: /from\s+['"]langchain|ChatOpenAI|ChatAnthropic|\.invoke\(/g },
|
|
57
|
+
{ name: "Gemini", regex: /GoogleGenerativeAI|generateContent|from\s+['"]@google\/generative/g },
|
|
58
|
+
{ name: "Bedrock", regex: /BedrockRuntimeClient|InvokeModelCommand/g },
|
|
59
|
+
{ name: "Grok", regex: /x\.ai\/api|xai\.chat|from\s+['"]grok/g },
|
|
60
|
+
{ name: "fetch (AI)", regex: /fetch\(.*api\.openai\.com|fetch\(.*api\.anthropic\.com|fetch\(.*generativelanguage\.googleapis/g },
|
|
61
|
+
{ name: "Cohere", regex: /from\s+['"]cohere|CohereClient|cohere\.chat/g },
|
|
62
|
+
{ name: "Mistral", regex: /from\s+['"]@mistralai|MistralClient/g }
|
|
63
|
+
];
|
|
64
|
+
var SKIP_DIRS = new Set([
|
|
65
|
+
"node_modules",
|
|
66
|
+
".git",
|
|
67
|
+
"dist",
|
|
68
|
+
"build",
|
|
69
|
+
".next",
|
|
70
|
+
".nuxt",
|
|
71
|
+
".svelte-kit",
|
|
72
|
+
"coverage",
|
|
73
|
+
".turbo",
|
|
74
|
+
".cache",
|
|
75
|
+
"__pycache__",
|
|
76
|
+
".venv",
|
|
77
|
+
"venv",
|
|
78
|
+
"target",
|
|
79
|
+
"zig-out",
|
|
80
|
+
"zig-cache"
|
|
81
|
+
]);
|
|
82
|
+
var SCAN_EXTENSIONS = new Set([
|
|
83
|
+
".ts",
|
|
84
|
+
".tsx",
|
|
85
|
+
".js",
|
|
86
|
+
".jsx",
|
|
87
|
+
".mjs",
|
|
88
|
+
".mts",
|
|
89
|
+
".py",
|
|
90
|
+
".rs"
|
|
91
|
+
]);
|
|
92
|
+
|
|
93
|
+
// src/scanner.ts
|
|
94
|
+
function walkDir(dir, files) {
|
|
95
|
+
let entries;
|
|
96
|
+
try {
|
|
97
|
+
entries = readdirSync(dir);
|
|
98
|
+
} catch {
|
|
99
|
+
return;
|
|
100
|
+
}
|
|
101
|
+
for (const entry of entries) {
|
|
102
|
+
if (SKIP_DIRS.has(entry))
|
|
103
|
+
continue;
|
|
104
|
+
const full = join(dir, entry);
|
|
105
|
+
let stat;
|
|
106
|
+
try {
|
|
107
|
+
stat = statSync(full);
|
|
108
|
+
} catch {
|
|
109
|
+
continue;
|
|
110
|
+
}
|
|
111
|
+
if (stat.isDirectory()) {
|
|
112
|
+
walkDir(full, files);
|
|
113
|
+
} else if (SCAN_EXTENSIONS.has(extname(entry))) {
|
|
114
|
+
files.push(full);
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
function findCallSites(filePath, content) {
|
|
119
|
+
const sites = [];
|
|
120
|
+
const lines = content.split(`
|
|
121
|
+
`);
|
|
122
|
+
for (const pattern of AI_PATTERNS) {
|
|
123
|
+
pattern.regex.lastIndex = 0;
|
|
124
|
+
let match;
|
|
125
|
+
while ((match = pattern.regex.exec(content)) !== null) {
|
|
126
|
+
const beforeMatch = content.substring(0, match.index);
|
|
127
|
+
const lineNum = beforeMatch.split(`
|
|
128
|
+
`).length;
|
|
129
|
+
const startLine = Math.max(0, lineNum - 5);
|
|
130
|
+
const endLine = Math.min(lines.length, lineNum + 20);
|
|
131
|
+
const context = lines.slice(startLine, endLine).join(`
|
|
132
|
+
`);
|
|
133
|
+
const tokensPerCall = slash(context);
|
|
134
|
+
if (!sites.some((s) => s.line === lineNum && s.sdk === pattern.name)) {
|
|
135
|
+
sites.push({
|
|
136
|
+
file: filePath,
|
|
137
|
+
line: lineNum,
|
|
138
|
+
sdk: pattern.name,
|
|
139
|
+
tokensPerCall
|
|
140
|
+
});
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
return sites;
|
|
145
|
+
}
|
|
146
|
+
function scan(dir) {
|
|
147
|
+
const start = performance.now();
|
|
148
|
+
const files = [];
|
|
149
|
+
walkDir(dir, files);
|
|
150
|
+
const sites = [];
|
|
151
|
+
for (const file of files) {
|
|
152
|
+
let content;
|
|
153
|
+
try {
|
|
154
|
+
content = readFileSync(file, "utf-8");
|
|
155
|
+
} catch {
|
|
156
|
+
continue;
|
|
157
|
+
}
|
|
158
|
+
sites.push(...findCallSites(file, content));
|
|
159
|
+
}
|
|
160
|
+
return {
|
|
161
|
+
sites,
|
|
162
|
+
filesScanned: files.length,
|
|
163
|
+
timeMs: Math.round(performance.now() - start)
|
|
164
|
+
};
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
// src/report.ts
|
|
168
|
+
var R = "\x1B[0m";
|
|
169
|
+
var B = "\x1B[1m";
|
|
170
|
+
var DIM = "\x1B[2m";
|
|
171
|
+
var ORANGE = "\x1B[38;2;255;68;0m";
|
|
172
|
+
var GOLD = "\x1B[38;2;230;161;65m";
|
|
173
|
+
var WHITE = "\x1B[97m";
|
|
174
|
+
var GRAY = "\x1B[90m";
|
|
175
|
+
function pad(s, n) {
|
|
176
|
+
return s.length >= n ? s : s + " ".repeat(n - s.length);
|
|
177
|
+
}
|
|
178
|
+
function formatNum(n) {
|
|
179
|
+
return n.toLocaleString("en-US");
|
|
180
|
+
}
|
|
181
|
+
function printReport(sites, filesScanned, timeMs, cwd) {
|
|
182
|
+
console.log("");
|
|
183
|
+
console.log(`${ORANGE}${B} ⚡ /slash${R}`);
|
|
184
|
+
console.log(`${GRAY} Token burn analysis${R}`);
|
|
185
|
+
console.log("");
|
|
186
|
+
console.log(`${DIM} Scanned ${filesScanned} files in ${timeMs}ms${R}`);
|
|
187
|
+
console.log("");
|
|
188
|
+
if (sites.length === 0) {
|
|
189
|
+
console.log(`${WHITE} No AI API call sites detected.${R}`);
|
|
190
|
+
console.log(`${GRAY} Supported: OpenAI, Anthropic, Vercel AI, LangChain, Gemini, Bedrock, Grok${R}`);
|
|
191
|
+
console.log("");
|
|
192
|
+
return;
|
|
193
|
+
}
|
|
194
|
+
console.log(`${ORANGE} CALL SITES${R}`);
|
|
195
|
+
console.log(`${DIM} ${"─".repeat(60)}${R}`);
|
|
196
|
+
for (const site of sites) {
|
|
197
|
+
const loc = site.file.replace(cwd + "/", "") + ":" + site.line;
|
|
198
|
+
console.log(`${WHITE} ${pad(loc, 40)} ${GOLD}${pad(site.sdk, 12)} ${ORANGE}~${formatNum(site.tokensPerCall)} tok/call${R}`);
|
|
199
|
+
}
|
|
200
|
+
console.log("");
|
|
201
|
+
const callsPerDay = 100;
|
|
202
|
+
const dailyInput = sites.reduce((sum, s) => sum + s.tokensPerCall * callsPerDay, 0);
|
|
203
|
+
const dailyOutput = dailyInput * 2;
|
|
204
|
+
const dailyTotal = dailyInput + dailyOutput;
|
|
205
|
+
const monthlyTotal = dailyTotal * 30;
|
|
206
|
+
const monthlyCostInput = dailyInput * 30 / 1e6 * 3;
|
|
207
|
+
const monthlyCostOutput = dailyOutput * 30 / 1e6 * 15;
|
|
208
|
+
const monthlyCost = monthlyCostInput + monthlyCostOutput;
|
|
209
|
+
console.log(`${ORANGE} DAILY BURN ${GRAY}(${callsPerDay} calls/site/day)${R}`);
|
|
210
|
+
console.log(`${DIM} ${"─".repeat(60)}${R}`);
|
|
211
|
+
console.log(`${WHITE} Input tokens: ${B}${formatNum(dailyInput)}${R}`);
|
|
212
|
+
console.log(`${WHITE} Output tokens: ${B}${formatNum(dailyOutput)}${R} ${GRAY}(estimated)${R}`);
|
|
213
|
+
console.log(`${WHITE} Total: ${ORANGE}${B}${formatNum(dailyTotal)} tokens/day${R}`);
|
|
214
|
+
console.log("");
|
|
215
|
+
console.log(`${ORANGE} MONTHLY COST ${GRAY}($3/$15 per MTok)${R}`);
|
|
216
|
+
console.log(`${DIM} ${"─".repeat(60)}${R}`);
|
|
217
|
+
console.log(`${WHITE} Input: ${B}$${monthlyCostInput.toFixed(2)}/mo${R}`);
|
|
218
|
+
console.log(`${WHITE} Output: ${B}$${monthlyCostOutput.toFixed(2)}/mo${R}`);
|
|
219
|
+
console.log(`${WHITE} Total: ${ORANGE}${B}$${monthlyCost.toFixed(2)}/mo${R}`);
|
|
220
|
+
console.log("");
|
|
221
|
+
const savingsPct = 10;
|
|
222
|
+
const monthlySaved = monthlyCost * savingsPct / 100;
|
|
223
|
+
console.log(`${GOLD} ⚡ SLASH SAVINGS ${GRAY}(${savingsPct}% gate efficiency)${R}`);
|
|
224
|
+
console.log(`${DIM} ${"─".repeat(60)}${R}`);
|
|
225
|
+
console.log(`${WHITE} Monthly savings: ${GOLD}${B}$${monthlySaved.toFixed(2)}/mo${R}`);
|
|
226
|
+
console.log(`${WHITE} Annual savings: ${GOLD}${B}$${(monthlySaved * 12).toFixed(2)}/yr${R}`);
|
|
227
|
+
console.log("");
|
|
228
|
+
console.log(`${GRAY} 4.8 KB WASM · sub-ms · zero deps${R}`);
|
|
229
|
+
console.log(`${GRAY} → ${ORANGE}slashtokens.com${R}`);
|
|
230
|
+
console.log("");
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
// src/cli.ts
|
|
234
|
+
var cwd = process.cwd();
|
|
235
|
+
var { sites, filesScanned, timeMs } = scan(cwd);
|
|
236
|
+
printReport(sites, filesScanned, timeMs, cwd);
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export { slash, slashBytes } from './slash';
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
// src/wasm.ts
|
|
2
|
+
var WASM_BASE64 = "AGFzbQEAAAABLAZgAn9/AX9gBH9/f38AYAZ/f39/f38AYAV/f39/fwBgBH9/f38Bf2ABfwF/AxEQAAABAQIDBAQBBAMAAAUABQUDAQARBgkBfwFBgIDAAAsHWQYGbWVtb3J5AgAJc2NvcmVfZmFmAAAUc2NvcmVfZmFmX2VudGVycHJpc2UACwx2YWxpZGF0ZV9mYWYADAhnZXRfdGllcgAND2VzdGltYXRlX3Rva2VucwAOCoEaEBAAIAAgARCBgICAAEH/AXELSgEBfyOAgICAAEEQayICJICAgIAAAkACQCABDQBBACEBDAELIAJBC2ogACABQQAQgoCAgAAgAi0ACyEBCyACQRBqJICAgIAAIAEL/wcBE38jgICAgABB0ABrIgQkgICAgABBqIHAgABBgIDAgAAgA0EBcSIFGyEGQSFBFSAFGyEHQQAhCEEAIQlBACEKA0ACQAJAAkAgCCAHRg0AIARBOGogBiAIQQN0aiIFKAIAIgsgBSgCBCIFQeeIwIAAEIOAgIAAAkAgBC0APA0AQQAhDAwCC0EBIQ0gBSAEKAI4Ig5BAWoiD2shECALIA9qIRFBACESQQAhEwNAQQAhDAJAAkAgDUH/AXENAEEAIQ1BACEFQQAhDwwBCyAEQcgAaiABIAIgEkH3iMCAAEEBEISAgIAAIAQoAkgiFCACIAQtAEwiFRsgEmshDyABIBJqIQUgFUEARyENIBRBAWpBACAVGyESCyAFRQ0CIARBMGogBSAPQfWIwIAAQQEQhYCAgAAgBCgCMCEWIAQoAjQiDEUNAEEAIQVBACEPA0ACQAJAIAwgBUYNAAJAIBYgBWotAAAiFEEJRg0AQQEhFSAUQSBHDQEMAgtBAiEVDAELAkAgDw0AIARBGGogFiAMQfmIwIAAQQIQhYCAgABBACETIAQoAhgiDyAEKAIcIgUgCyAOEIaAgIAAQQFxRQ0DIAUgDk0NAyAPIA5qLQAAQTpGIRMMAwsgEyAPQQFHcUUNAiAEQShqIBYgDEH5iMCAAEECEIWAgIAAQQEhEyAEKAIoIhUgBCgCLCIPIBEgEBCGgICAAEEBcUUNAiAPIBBNDQIgFSAQai0AAEE6Rw0CIARBwABqIBUgD0HliMCAABCDgICAACAELQBERQ0CQQAhDEEAIQUgDyAEKAJAQQFqIhRNDQUgBEEgaiAVIBRqIA8gFGtB8IjAgABBBBCFgICAACAEKAIgQQAgBCgCJCIFGyEMIAVBACAFGyEFDAULIAVBAWohBSAVIA9qIQ8MAAsLC0EhQRUgA0EBcRsiBSAJayEPQQAhFQJAIAUgCUH/AXFGDQAgCkH/AXFB5ABsIA9B/wFxIhVBAXZqIBVuIRULIAAgBToABCAAIA86AAMgACAJOgACIAAgCjoAASAAIBVB/wFxIgVB5AAgBUHkAEkbOgAAIARB0ABqJICAgIAADwtBACEFCwJAIAxFDQAgBEEQaiAMIAVB6YjAgABBBhCFgICAAAJAIAQoAhAgBCgCFEHKiMCAAEELEIeAgIAAQQFxDQAgBEEIaiAMIAVB6YjAgABBBhCFgICAACAEKAIMIgxFDQEgBCgCCCEWQbh/IQUCQANAIAVBCGoiD0UNASAFQciJwIAAaiEVIAVBxInAgABqIRQgDyEFIBYgDCAUKAIAIBUoAgAQh4CAgABBAXFFDQAMAwsLIApBAWohCgwBCyAJQQFqIQkLIAhBAWohCAwACws/AQF/I4CAgIAAQRBrIgQkgICAgAAgBEEIaiABIAJBACADQQEQhICAgAAgACAEKQMINwIAIARBEGokgICAgAALsQMDAn8BfgF/I4CAgIAAQZAIayIGJICAgIAAAkACQCACIAVPDQAgAEIANwIADAELAkACQAJAAkACQCAFQQFLDQAgBQ0BIABBAToABAwECyACQTRJDQEgBUEFSQ0BQQAhBwwCCyAGQQhqIAEgAiADIAQtAAAQioCAgAAgACAGKQMINwIADAMLIAIgBWshBwNAAkACQAJAIAMgB00NAEEAIQNCACEIDAELIAEgA2ogBSAEIAUQiYCAgABBAXFFDQFCgICAgBAhCAsgACAIIAOthDcCAAwECyADQQFqIQMMAAsLAkADQCAHQYAIRg0BIAZBEGogB2ogBTYCACAHQQRqIQcMAAsLIAVBf2ohByAEIQkDQAJAIAcNACACIAVrIQkDQAJAIAMgCU0NACAAQgA3AgAMBQsCQCABIANqIgcgBSAEIAUQiYCAgABBAXFFDQAgAEEBOgAEDAQLIAZBEGogByAFakF/ai0AAEECdGooAgAgA2ohAwwACwsgBkEQaiAJLQAAQQJ0aiAHNgIAIAlBAWohCSAHQX9qIQcMAAsLIAAgAzYCAAsgBkGQCGokgICAgAALrgEBBH8jgICAgABBEGsiBSSAgICAAEEAIQYCQANAAkAgAiAGRw0AIAIhBgwCCyAFIAMgBCABIAZqLQAAEIiAgIAAIAUtAARFDQEgBkEBaiEGDAALCyABQX9qIQcCQANAIAIiCCAGTQ0BIAVBCGogAyAEIAcgCGotAAAQiICAgAAgCEF/aiECIAUtAAwNAAsLIAAgCCAGazYCBCAAIAEgBmo2AgAgBUEQaiSAgICAAAskAQF/QQAhBAJAIAMgAUsNACAAIAMgAiADEImAgIAAIQQLIAQLeQECf0EAIQQCQCABIANHDQADQCABRSEEIAFFDQEgAi0AACEDIAAtAAAhBSACQQFqIQIgAEEBaiEAIAFBf2ohASAFQSBqIAUgBUG/f2pB/wFxQRpJG0H/AXEgA0EgaiADIANBv39qQf8BcUEaSRtB/wFxRg0ACwsgBAs9AQF/I4CAgIAAQRBrIgQkgICAgAAgBEEIaiABIAJBACADEIqAgIAAIAAgBCkDCDcCACAEQRBqJICAgIAAC/0CAQV/I4CAgIAAQRBrIgQkgICAgAACQAJAIAEgA0cNAEEBIQUgAUUNASAAQarVqtV6IAEbIgYgAkGq1arVeiADGyIHRg0BAkAgAUEQSw0AAkAgAUEETw0AIAAtAAAgBy0AAEYgACABQX9qIgNqLQAAIAcgA2otAABGcSAAIAFBAXYiA2otAAAgByADai0AAEZxIQUMAwtBACEDIARBADYCACAEIAFBfGoiBTYCBCAEIAFBAXZBDHEiADYCCCAEIAUgAGs2AgxBACEFAkADQCADQRBGDQEgByAEIANqKAIAIgBqKAAAIAYgAGooAABzIAVyIQUgA0EEaiEDDAALCyAFRSEFDAILIAFBf2pBAnZBAWohACAGIQMgByEFAkADQCAAQX9qIgBFDQEgBSgAACECIAMoAAAhCCADQQRqIQMgBUEEaiEFIAggAkcNAgwACwsgBiABQXxqIgNqKAAAIAcgA2ooAABGIQUMAQtBACEFCyAEQRBqJICAgIAAIAULWAACQCADIAJJDQAgAEIANwIADwsgBEH/AXEhBAJAA0AgAiADRg0BAkAgASADai0AACAERw0AIABBAToABCAAIAM2AgAPCyADQQFqIQMMAAsLIABCADcCAAtOAQF/I4CAgIAAQRBrIgIkgICAgAACQAJAIAENAEEAIQEMAQsgAkELaiAAIAFBARCCgICAACACLQALIQELIAJBEGokgICAgAAgAUH/AXELFAAgACABEIGAgIAAQf8BcUHjAEsLCAAgAEHjAEsL5gQBCH8jgICAgABBEGsiAiSAgICAAAJAAkAgAQ0AQQAhAwwBCyACIAAtAAAiBBCPgICAAEEHcSIDOgAPIANBBHNBB3FBAnQiA0GcisCAAGohBSADQfyJwIAAaiEGIANB3InAgABqIQcgA0G8icCAAGohA0EAIQgCQAJAAkAgBEGFf2oOAwECAQALIARBWGpBAk8NAQtBASEICyAFKAIAIQUgBigCACEJIAcoAgAhBiADKAIAIQcgAUF/aiEDIABBAWohAANAAkACQAJAAkACQAJAAkACQCADRQ0AIAAtAAAiBBCPgICAAEEHcQ4GBAQHAQIDBAsgCEHkAGwgAW4hCCAGQeQAbCABbiEDIAdB5ABsIAFuIQAgBUHkAGwgAW4hBCAJIAZqQeQAbCABbkEjSQ0EIANBBU8NBEE4IQMMBQsgCUEBaiEJDAULIAZBAWohBgwECyAHQQFqIQcMAwsgBUEBaiEFDAILAkAgBEHBAEkNACADQQJLDQAgAEEITw0AQTYhAwwBCwJAIAhBAkkNACADQQRJDQAgAEENTQ0AQRohAwwBCwJAIABBFU0NAEEcIQMMAQsCQCADQQRJDQAgAEENTQ0AQSAhAwwBCwJAIABBDkkNACAEQcgASw0AIAhBAk8NAEEeIQMMAQtBIUEjIABBBksbQSMgA0EESxshAwsgA0EBdiABQQpsaiADbiIDQSFuIANqIgNBASADQQFLGyEDDAILAkACQCAEQVhqQQJJDQAgBEGFf2oOAwABAAELIAhBAWohCAsgA0F/aiEDIABBAWohAAwACwsgAkEQaiSAgICAACADC3UBAX8CQCAAQZ9/akH/AXFBGk8NAEEADwsCQCAAQb9/akH/AXFBGk8NAEEBDwsCQCAAQVBqQf8BcUEKTw0AQQIPC0EDIQECQAJAAkACQCAAQf8BcSIAQXdqDgUDAgEBAgALIABBIEYNAgtBBQ8LQQQhAQsgAQsLxgoBAEGAgMAAC7wKvwMQAAwAAAAjAxAADAAAAP0DEAAVAAAAsQIQABEAAAAxAhAAEgAAAMEBEAARAAAAiQMQABMAAADoAhAAEgAAAPABEAARAAAAIQQQAA4AAABLAxAAEwAAALABEAAQAAAAGgIQABYAAAAwBBAADQAAAJ0DEAAOAAAAsQMQAA0AAAB6AxAADgAAAMsCEAAQAAAAbAMQAA0AAAA+BBAACwAAAFYEEAAKAAAAvwMQAAwAAAAjAxAADAAAAP0DEAAVAAAAsQIQABEAAAAxAhAAEgAAAMEBEAARAAAAiQMQABMAAADoAhAAEgAAAPABEAARAAAAIQQQAA4AAABLAxAAEwAAALABEAAQAAAAGgIQABYAAAAwBBAADQAAAJ0DEAAOAAAAsQMQAA0AAAB6AxAADgAAAMsCEAAQAAAAbAMQAA0AAAA+BBAACwAAAFYEEAAKAAAACgMQABMAAACbAhAAFQAAAG4CEAAQAAAAAgIQABcAAAB/AhAAGwAAANwCEAALAAAA8QMQAAsAAABfAxAADAAAABMEEAANAAAA0wEQABwAAABWAhAAFwAAANsDEAAVAAAAc3RhY2sudWlfbGlicmFyeQBodW1hbl9jb250ZXh0LndoeQBtb25vcmVwby52ZXJzaW9uaW5nX3N0cmF0ZWd5AGh1bWFuX2NvbnRleHQuaG93AG1vbm9yZXBvLnBhY2thZ2VzX2NvdW50AHN0YWNrLnN0YXRlX21hbmFnZW1lbnQAaHVtYW5fY29udGV4dC53aGF0AGRldmVsb3BtZW50IHRlYW1zAG1vbm9yZXBvLnNoYXJlZF9jb25maWdzAHN0YWNrLndvcmtzcGFjZXMAbW9ub3JlcG8uYnVpbGRfb3JjaGVzdHJhdG9yAHN0YWNrLnBhY2thZ2VfbWFuYWdlcgBodW1hbl9jb250ZXh0LndobwB1bmtub3duAHN0YWNrLmNvbm5lY3Rpb24Ac3RhY2suYWRtaW4AaHVtYW5fY29udGV4dC53aGVuAGNsb3VkIHBsYXRmb3JtAHN0YWNrLm1vbm9yZXBvX3Rvb2wAbnVsbABwcm9qZWN0LmdvYWwAZGVzY3JpYmUgeW91ciBwcm9qZWN0IGdvYWwAc3RhY2suY3NzX2ZyYW1ld29yawBzdGFjay5zZWFyY2gAc3RhY2suaG9zdGluZwBzdGFjay5kYXRhYmFzZQBodW1hbl9jb250ZXh0LndoZXJlAHN0YWNrLmFwaV90eXBlAG5vbmUAc3RhY2sucnVudGltZQBwcm9qZWN0Lm5hbWUAbm90IGFwcGxpY2FibGUAbW9ub3JlcG8ucmVtb3RlX2NhY2hlAHN0YWNrLmNhY2hlAHByb2plY3QubWFpbl9sYW5ndWFnZQBzdGFjay5zdG9yYWdlAHN0YWNrLmZyb250ZW5kAHN0YWNrLmJhY2tlbmQAc3RhY2suYnVpbGQAc2xvdGlnbm9yZWQAc3RhY2suY2ljZABuL2EAOgAuACAJDQoiJwAgCSInAA0ACgAgCQAwAxAAGgAAAEQCEAARAAAA+wIQAA4AAAAeAxAABAAAAKwDEAAEAAAAwwIQAAcAAABhBBAAAwAAAMwDEAAOAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAABAAAAAAAAAAAAAAA=";
|
|
3
|
+
var WASM_INPUT_OFFSET = 4096;
|
|
4
|
+
var encoder = new TextEncoder;
|
|
5
|
+
var _instance = null;
|
|
6
|
+
function decodeBase64(b64) {
|
|
7
|
+
if (typeof Buffer !== "undefined") {
|
|
8
|
+
return new Uint8Array(Buffer.from(b64, "base64"));
|
|
9
|
+
}
|
|
10
|
+
const binary = atob(b64);
|
|
11
|
+
const bytes = new Uint8Array(binary.length);
|
|
12
|
+
for (let i = 0;i < binary.length; i++) {
|
|
13
|
+
bytes[i] = binary.charCodeAt(i);
|
|
14
|
+
}
|
|
15
|
+
return bytes;
|
|
16
|
+
}
|
|
17
|
+
function getInstance() {
|
|
18
|
+
if (_instance)
|
|
19
|
+
return _instance;
|
|
20
|
+
const bytes = decodeBase64(WASM_BASE64);
|
|
21
|
+
const module = new WebAssembly.Module(bytes);
|
|
22
|
+
_instance = new WebAssembly.Instance(module);
|
|
23
|
+
return _instance;
|
|
24
|
+
}
|
|
25
|
+
function writeToMemory(content) {
|
|
26
|
+
const instance = getInstance();
|
|
27
|
+
const bytes = encoder.encode(content);
|
|
28
|
+
const memory = instance.exports.memory;
|
|
29
|
+
const buffer = new Uint8Array(memory.buffer);
|
|
30
|
+
const maxLen = Math.min(bytes.length, buffer.length - WASM_INPUT_OFFSET - 1);
|
|
31
|
+
buffer.set(bytes.subarray(0, maxLen), WASM_INPUT_OFFSET);
|
|
32
|
+
return maxLen;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
// src/slash.ts
|
|
36
|
+
var WASM_INPUT_OFFSET2 = 4096;
|
|
37
|
+
function slash(content) {
|
|
38
|
+
if (!content)
|
|
39
|
+
return 0;
|
|
40
|
+
const len = writeToMemory(content);
|
|
41
|
+
const instance = getInstance();
|
|
42
|
+
return instance.exports.estimate_tokens(WASM_INPUT_OFFSET2, len);
|
|
43
|
+
}
|
|
44
|
+
function slashBytes(bytes) {
|
|
45
|
+
if (!bytes.length)
|
|
46
|
+
return 0;
|
|
47
|
+
const instance = getInstance();
|
|
48
|
+
const memory = instance.exports.memory;
|
|
49
|
+
const buffer = new Uint8Array(memory.buffer);
|
|
50
|
+
const maxLen = Math.min(bytes.length, buffer.length - WASM_INPUT_OFFSET2 - 1);
|
|
51
|
+
buffer.set(bytes.subarray(0, maxLen), WASM_INPUT_OFFSET2);
|
|
52
|
+
return instance.exports.estimate_tokens(WASM_INPUT_OFFSET2, maxLen);
|
|
53
|
+
}
|
|
54
|
+
export {
|
|
55
|
+
slashBytes,
|
|
56
|
+
slash
|
|
57
|
+
};
|
package/dist/report.d.ts
ADDED
package/dist/slash.d.ts
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Estimate token count for a string.
|
|
3
|
+
* Sub-millisecond. Zero allocations in WASM.
|
|
4
|
+
*/
|
|
5
|
+
export declare function slash(content: string): number;
|
|
6
|
+
/**
|
|
7
|
+
* Estimate token count from raw bytes.
|
|
8
|
+
*/
|
|
9
|
+
export declare function slashBytes(bytes: Uint8Array): number;
|
package/dist/wasm.d.ts
ADDED
package/package.json
CHANGED
|
@@ -1,8 +1,36 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "slash-tokens",
|
|
3
|
-
"version": "0.0
|
|
4
|
-
"description": "Token budget estimation in 4.8 KB WASM. Sub-millisecond. Zero dependencies.",
|
|
3
|
+
"version": "0.1.0",
|
|
4
|
+
"description": "Token budget estimation in 4.8 KB WASM. Sub-millisecond. Zero dependencies. I don't count tokens. I slash them.",
|
|
5
|
+
"main": "dist/index.js",
|
|
6
|
+
"module": "dist/index.js",
|
|
7
|
+
"types": "dist/index.d.ts",
|
|
8
|
+
"type": "module",
|
|
9
|
+
"bin": {
|
|
10
|
+
"slash-tokens": "dist/cli.js"
|
|
11
|
+
},
|
|
12
|
+
"exports": {
|
|
13
|
+
".": {
|
|
14
|
+
"import": "./dist/index.js",
|
|
15
|
+
"types": "./dist/index.d.ts"
|
|
16
|
+
}
|
|
17
|
+
},
|
|
18
|
+
"files": ["dist"],
|
|
19
|
+
"sideEffects": false,
|
|
20
|
+
"scripts": {
|
|
21
|
+
"build": "bun build src/cli.ts --outfile=dist/cli.js --target=node && tsc --emitDeclarationOnly && bun build src/index.ts --outfile=dist/index.js --target=node",
|
|
22
|
+
"test": "bun test"
|
|
23
|
+
},
|
|
5
24
|
"author": "wolfejam",
|
|
6
25
|
"license": "MIT",
|
|
7
|
-
"keywords": ["tokens", "wasm", "edge", "budget", "estimation", "llm", "ai"]
|
|
26
|
+
"keywords": ["tokens", "wasm", "edge", "budget", "estimation", "llm", "ai", "openai", "anthropic", "claude", "grok", "gemini"],
|
|
27
|
+
"repository": {
|
|
28
|
+
"type": "git",
|
|
29
|
+
"url": "https://github.com/Wolfe-Jam/slash-tokens"
|
|
30
|
+
},
|
|
31
|
+
"homepage": "https://slashtokens.com",
|
|
32
|
+
"devDependencies": {
|
|
33
|
+
"@types/node": "^25.5.0",
|
|
34
|
+
"typescript": "^6.0.2"
|
|
35
|
+
}
|
|
8
36
|
}
|