tokenbuild 0.0.0-stage → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -1,3 +1,24 @@
1
- # Temporary Holding Version
1
+ # TokenBuild
2
2
 
3
- This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.
3
+ **Build More. Use Less.** Cut Claude Code token use with local tools, and prove the saving.
4
+
5
+ ```bash
6
+ npx tokenbuild audit # free: price the avoidable waste in your Claude Code logs (local, nothing uploaded)
7
+ npx tokenbuild login <key> # once: key from app.tokenbuild.ai/billing (needs an active trial or plan)
8
+ npx tokenbuild init # turn on the tools in this project, then restart Claude Code
9
+ ```
10
+
11
+ | Tool | What it does |
12
+ | --- | --- |
13
+ | Output Shrinker | `.tokenbuild/run <command>`: test, build and install logs condensed to errors and summary; full log saved |
14
+ | Waste Guard | Blocks lockfiles, build output, huge whole-file reads and unchanged re-reads |
15
+ | Repo Index | `tb_map`, `tb_find`, `tb_pack`: exact files and lines under a token budget |
16
+ | Model routing | Helper agents on a lighter model (`init --route balanced\|lean\|off`) |
17
+ | Auto-format | Your configured formatter runs on edited files once per turn |
18
+ | Local Boost | Optional local model (Ollama, this machine only) reads big files for Claude |
19
+
20
+ `tokenbuild stats` shows tokens kept out of context, tool by tool.
21
+
22
+ Your code never leaves your machine. TokenBuild never asks for your Claude login. Node 20+, zero dependencies.
23
+
24
+ [tokenbuild.ai](https://www.tokenbuild.ai) · [FAQ](https://www.tokenbuild.ai/faq.html) · [Privacy](https://www.tokenbuild.ai/privacy.html) · support@tokenbuild.ai
@@ -0,0 +1,171 @@
1
+ #!/usr/bin/env node
2
+ // tokenbuild — Build More. Use Less.
3
+ import fs from 'node:fs';
4
+ import path from 'node:path';
5
+
6
+ const [cmd = 'help', ...rest] = process.argv.slice(2);
7
+ if (cmd === 'run') { // raw argv: the command's own flags must not be parsed as ours
8
+ const { run } = await import('../src/shrink.mjs');
9
+ const { rootOf } = await import('../src/guard.mjs');
10
+ const { licensed, explain } = await import('../src/license.mjs');
11
+ const argv = rest[0] === '--' ? rest.slice(1) : rest;
12
+ const l = licensed();
13
+ process.exit(await run(argv, rootOf(process.cwd()), l.ok ? {} : { passthrough: explain(l) }));
14
+ }
15
+ const flags = {}; const args = [];
16
+ for (let i = 0; i < rest.length; i++) {
17
+ const a = rest[i];
18
+ if (a.startsWith('--')) { const [k, v] = a.slice(2).split('='); flags[k] = v ?? (rest[i + 1] && !rest[i + 1].startsWith('--') ? rest[++i] : true); }
19
+ else args.push(a);
20
+ }
21
+ const root = path.resolve(flags.root || process.cwd());
22
+ const out = (s) => process.stdout.write(s + '\n');
23
+
24
+ const HELP = `tokenbuild — Build More. Use Less.
25
+
26
+ audit [--days 30] [--project name] [--json] Free: price the token waste in your Claude Code logs
27
+ login [key] Connect this computer to your plan (key from app.tokenbuild.ai/billing)
28
+ logout Remove the saved key from this computer
29
+ init [--route balanced|lean|off] [--no-format] Add hooks, MCP tools, model routing and rules to this repo
30
+ run <command> Run a noisy command; print errors + summary, save the full log
31
+ index Build/refresh the local repo index
32
+ map [--budget 2500] Print the compact repo map
33
+ find <name> Locate a symbol or file (path:line)
34
+ pack "<task>" [--budget 4000] Context pack for a task
35
+ stats Tokens kept out of context in this repo
36
+ boost [on|off] [--model name] Local Boost: a local model (Ollama) reads big files for Claude
37
+ ask "<question>" <file> Ask the local model about a file (Local Boost)
38
+ sync Send daily counts (no code) to your dashboard
39
+ guard [--reset] (hook) Claude Code PreToolUse guard
40
+ format [--track] (hook) Format edited files with your configured formatter
41
+ mcp (server) MCP stdio server`;
42
+
43
+ // Paid tools need an active trial or plan (cached check; offline-safe). The audit is free.
44
+ // init and login check online; hooks, run and mcp degrade to doing nothing instead of failing.
45
+ const GATED = new Set(['index', 'map', 'find', 'pack', 'boost', 'ask']);
46
+ if (GATED.has(cmd)) {
47
+ const { licensed, explain } = await import('../src/license.mjs');
48
+ const l = licensed();
49
+ if (!l.ok) { process.stdout.write(explain(l) + '\n'); process.exit(1); }
50
+ }
51
+
52
+ const readLedger = () => { try { return fs.readFileSync(path.join(root, '.tokenbuild', 'ledger.jsonl'), 'utf8').trim().split('\n'); } catch { return []; } };
53
+
54
+ switch (cmd) {
55
+ case 'audit': {
56
+ const { audit, report } = await import('../src/audit.mjs');
57
+ const days = Number(flags.days || 30);
58
+ const t = audit({ days, project: flags.project || '' });
59
+ if (flags.json) { t.files = Object.fromEntries([...t.files].sort((a, b) => b[1] - a[1]).slice(0, 20)); out(JSON.stringify(t)); }
60
+ else out(report(t, days));
61
+ break;
62
+ }
63
+ case 'login': {
64
+ const L = await import('../src/license.mjs');
65
+ if (args[0]) L.saveKey(args[0]);
66
+ if (!L.readKey()) { out(`Create a CLI key at ${L.BILLING}, then run: tokenbuild login <key>`); process.exitCode = 1; break; }
67
+ const r = await L.check();
68
+ if (r.ok && r.active) out(`✓ Connected: ${r.plan === 'team' ? 'Team' : r.plan === 'comp' ? 'Complimentary' : 'Solo'} plan (${r.status}). Now run in a project: tokenbuild init`);
69
+ else if (r.error === 'offline') { out('Saved. Couldn\'t reach app.tokenbuild.ai to confirm your plan; run tokenbuild login again when online.'); process.exitCode = 1; }
70
+ else { out(L.explain(L.licensed())); process.exitCode = 1; }
71
+ break;
72
+ }
73
+ case 'logout': {
74
+ (await import('../src/license.mjs')).forget();
75
+ out('Removed the saved key and plan from this computer.');
76
+ break;
77
+ }
78
+ case 'init': {
79
+ const L = await import('../src/license.mjs');
80
+ if (!L.readKey()) { out(L.explain({ reason: 'no-license' })); process.exitCode = 1; break; }
81
+ const c = await L.check();
82
+ const l = L.licensed();
83
+ if (!l.ok) { out(c.error === 'offline' ? 'Couldn\'t reach app.tokenbuild.ai to confirm your plan. Check your connection and try again.' : L.explain(l)); process.exitCode = 1; break; }
84
+ const { init } = await import('../src/init.mjs');
85
+ const { buildIndex } = await import('../src/repo.mjs');
86
+ const route = flags.route || 'balanced';
87
+ if (!(route in (await import('../src/init.mjs')).ROUTES)) { out('--route must be balanced, lean or off'); process.exitCode = 1; break; }
88
+ for (const d of init(root, undefined, { route, format: !flags['no-format'] })) out('✓ ' + d);
89
+ const { total } = buildIndex(root);
90
+ out(`✓ Indexed ${total} source files\nRestart Claude Code in this folder to load the tools.`);
91
+ break;
92
+ }
93
+ case 'index': {
94
+ const { buildIndex } = await import('../src/repo.mjs');
95
+ const t0 = Date.now(); const { total, parsed } = buildIndex(root);
96
+ out(`Indexed ${total} files (${parsed} changed) in ${Date.now() - t0} ms`);
97
+ break;
98
+ }
99
+ case 'map': case 'find': case 'pack': {
100
+ const r = await import('../src/repo.mjs');
101
+ const idx = r.buildIndex(root).idx;
102
+ out(cmd === 'map' ? r.repoMap(idx, Number(flags.budget) || 2500)
103
+ : cmd === 'find' ? r.find(idx, args.join(' '))
104
+ : r.pack(idx, args.join(' '), Number(flags.budget) || 4000));
105
+ break;
106
+ }
107
+ case 'stats': {
108
+ const { num } = await import('../src/util.mjs');
109
+ const { tally } = await import('../src/ledger.mjs');
110
+ const t = tally(readLedger());
111
+ const total = t.tok_blocked + t.tok_shrunk + t.tok_local;
112
+ out([`~${num(total)} tokens kept out of Claude's context in this repo`,
113
+ ` Waste Guard ${num(t.blocks)} reads blocked, ~${num(t.tok_blocked)} tokens`,
114
+ ` Output Shrinker ${num(t.shrinks)} logs condensed, ~${num(t.tok_shrunk)} tokens`,
115
+ ` Local Boost ${num(t.local_calls)} files read locally, ~${num(t.tok_local)} tokens`,
116
+ ` Auto-format ${num(t.formats)} files formatted`,
117
+ ` Repo tools ${num(t.tool_calls)} calls (tb_map, tb_find, tb_pack)`].join('\n'));
118
+ break;
119
+ }
120
+ case 'sync': { // sends daily counts only: never code, file contents or paths
121
+ const key = flags.key || (await import('../src/license.mjs')).readKey();
122
+ if (!key) { out('Run tokenbuild login <key> first (key from app.tokenbuild.ai/billing).'); process.exitCode = 1; break; }
123
+ const { byDay } = await import('../src/ledger.mjs');
124
+ const body = { days: byDay(readLedger()) };
125
+ const r = await fetch((process.env.TOKENBUILD_API || 'https://app.tokenbuild.ai') + '/v1/usage', { method: 'POST', headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${key}` }, body: JSON.stringify(body) });
126
+ if (!r.ok) { out(`Sync failed: ${(await r.json().catch(() => ({}))).error || r.status}`); process.exitCode = 1; break; }
127
+ fs.renameSync(path.join(root, '.tokenbuild', 'ledger.jsonl'), path.join(root, '.tokenbuild', `ledger.${Date.now()}.synced`));
128
+ out(`Synced ${body.days.length} day(s).`);
129
+ break;
130
+ }
131
+ case 'format': {
132
+ const { track, flush } = await import('../src/format.mjs');
133
+ const { rootOf } = await import('../src/guard.mjs');
134
+ let raw = ''; for await (const c of process.stdin) raw += c;
135
+ let input = {}; try { input = JSON.parse(raw); } catch {}
136
+ const r = rootOf(input.cwd || process.cwd());
137
+ if (!(await import('../src/license.mjs')).licensed().ok) break;
138
+ try { if (flags.track) track(input, r); else flush(input, r); } catch {} // never break Claude's turn
139
+ break;
140
+ }
141
+ case 'boost': {
142
+ const L = await import('../src/local.mjs');
143
+ if (args[0] === 'on' || args[0] === 'off') L.setConfig(root, { boost: args[0] === 'on', ...(flags.model ? { model: flags.model } : {}) });
144
+ const c = L.config(root); const d = await L.detect(root, 1500);
145
+ out(`Local Boost: ${c.boost ? 'on' : 'off'}`);
146
+ if (!d.ok) out(` ${d.error}. Install Ollama (https://ollama.com), then: ollama pull ${L.SUGGEST}`);
147
+ else if (!d.model) out(` Ollama is running but has no model${c.model ? ` named ${c.model}` : ''}. Try: ollama pull ${L.SUGGEST}`);
148
+ else out(` Model: ${d.model}${d.models.length > 1 ? ` (installed: ${d.models.join(', ')})` : ''}`);
149
+ if (c.boost && d.model) out(' Claude gets the tb_digest tool after you restart Claude Code.');
150
+ else if (!c.boost) out(' Turn on: tokenbuild boost on');
151
+ break;
152
+ }
153
+ case 'ask': {
154
+ const { digest } = await import('../src/local.mjs');
155
+ if (args.length < 2) { out('Usage: tokenbuild ask "<question>" <file>'); process.exitCode = 1; break; }
156
+ out(await digest(root, args.at(-1), args.slice(0, -1).join(' ')));
157
+ break;
158
+ }
159
+ case 'guard': {
160
+ const { runGuard } = await import('../src/guard.mjs');
161
+ process.exitCode = await runGuard(flags.reset ? 'reset' : 'pre');
162
+ break;
163
+ }
164
+ case 'mcp': {
165
+ const { runMcp } = await import('../src/mcp.mjs');
166
+ runMcp(root);
167
+ break;
168
+ }
169
+ case '--version': case 'version': out('0.1.0'); break;
170
+ default: out(HELP);
171
+ }
package/package.json CHANGED
@@ -1,6 +1,40 @@
1
1
  {
2
2
  "name": "tokenbuild",
3
- "version": "0.0.0-stage",
4
- "stub": true,
5
- "description": "Temporary package placeholder for staged publishing"
6
- }
3
+ "version": "0.1.0",
4
+ "description": "Build More. Use Less. Cut Claude Code token use and prove the saving.",
5
+ "type": "module",
6
+ "bin": {
7
+ "tokenbuild": "bin/tokenbuild.mjs"
8
+ },
9
+ "files": [
10
+ "bin",
11
+ "src",
12
+ "README.md"
13
+ ],
14
+ "engines": {
15
+ "node": ">=20"
16
+ },
17
+ "scripts": {
18
+ "test": "node --test test/*.test.mjs"
19
+ },
20
+ "license": "UNLICENSED",
21
+ "homepage": "https://www.tokenbuild.ai",
22
+ "repository": {
23
+ "type": "git",
24
+ "url": "git+https://github.com/lidarstore/tokenbuild.git",
25
+ "directory": "packages/cli"
26
+ },
27
+ "bugs": {
28
+ "email": "support@tokenbuild.ai"
29
+ },
30
+ "keywords": [
31
+ "claude",
32
+ "claude-code",
33
+ "tokens",
34
+ "mcp",
35
+ "hooks",
36
+ "ai",
37
+ "developer-tools",
38
+ "cost"
39
+ ]
40
+ }
package/src/audit.mjs ADDED
@@ -0,0 +1,142 @@
1
+ // Waste Audit: reads local Claude Code session logs, prices avoidable waste. Zero tokens, zero network.
2
+ import fs from 'node:fs';
3
+ import path from 'node:path';
4
+ import os from 'node:os';
5
+ import { tokens, isJunk, usd, num } from './util.mjs';
6
+ import { rates, cost } from './prices.mjs';
7
+ import { shrink, isNoisy } from './shrink.mjs';
8
+
9
+ const OVERSIZED = 6000; // a Read returning more than this many tokens is "oversized"
10
+ const KEEP = 2000; // ...of which this much is assumed useful
11
+ const WRITES = new Set(['Edit', 'Write', 'MultiEdit', 'NotebookEdit']);
12
+ const SEARCH = new Set(['Grep', 'Glob', 'LS']);
13
+
14
+ export function logDirs() {
15
+ const base = process.env.CLAUDE_CONFIG_DIR || path.join(os.homedir(), '.claude');
16
+ const root = path.join(base, 'projects');
17
+ try { return fs.readdirSync(root).map((d) => path.join(root, d)); } catch { return []; }
18
+ }
19
+
20
+ const resultText = (c) => typeof c === 'string' ? c
21
+ : Array.isArray(c) ? c.map((b) => (b && b.type === 'text' ? b.text : b && b.type === 'image' ? 'x'.repeat(6000) : '')).join('') : '';
22
+
23
+ // "claude-opus-5-5-20261001" -> "opus-5-5"
24
+ export const shortModel = (m) => String(m || '').replace(/^claude-/, '').replace(/-\d{8}$/, '').replace(/^<synthetic>$/, '');
25
+
26
+ export function auditSession(lines) {
27
+ const usageById = new Map(); // dedupe streamed duplicates by message id
28
+ const turns = []; // assistant message ids in order
29
+ const uses = new Map(); // tool_use id -> {name, input, turn}
30
+ const events = []; // {kind, tok, turn, file}
31
+ const lastRead = new Map(); // file -> key of last read
32
+ let compactions = 0, model = '';
33
+ const segmentEnds = []; // turn indexes where context resets
34
+
35
+ for (const line of lines) {
36
+ let d; try { d = JSON.parse(line); } catch { continue; }
37
+ if (d.type === 'system' && /compact/.test(d.subtype || '') || d.isCompactSummary) {
38
+ compactions++; segmentEnds.push(turns.length); lastRead.clear(); continue;
39
+ }
40
+ const m = d.message;
41
+ if (!m || typeof m !== 'object') continue;
42
+ if (d.type === 'assistant') {
43
+ const id = m.id || d.uuid;
44
+ if (!usageById.has(id)) turns.push(id);
45
+ if (m.usage) usageById.set(id, { usage: m.usage, model: m.model });
46
+ if (m.model) model = m.model;
47
+ for (const b of Array.isArray(m.content) ? m.content : []) {
48
+ if (b.type !== 'tool_use') continue;
49
+ uses.set(b.id, { name: b.name, input: b.input || {}, turn: turns.length - 1 });
50
+ if (WRITES.has(b.name) && b.input?.file_path) lastRead.delete(b.input.file_path);
51
+ }
52
+ } else if (d.type === 'user') {
53
+ for (const b of Array.isArray(m.content) ? m.content : []) {
54
+ if (b.type !== 'tool_result') continue;
55
+ const u = uses.get(b.tool_use_id); if (!u) continue;
56
+ const tok = tokens(resultText(b.content));
57
+ const f = u.input.file_path || u.input.path || '';
58
+ if (u.name === 'Read') {
59
+ const key = `${u.input.offset || 0}:${u.input.limit || 0}`;
60
+ if (f && isJunk(f)) events.push({ kind: 'junk', tok, turn: u.turn, file: f });
61
+ else if (f && lastRead.get(f) === key) events.push({ kind: 'reread', tok, turn: u.turn, file: f });
62
+ else if (tok > OVERSIZED && !u.input.limit) events.push({ kind: 'oversized', tok: tok - KEEP, turn: u.turn, file: f });
63
+ if (f) lastRead.set(f, key);
64
+ } else if (u.name === 'Bash' && tok > 600 && isNoisy(u.input.command)) {
65
+ // Measured, not estimated: run the real shrinker on the logged output.
66
+ const saved = tok - tokens(shrink(resultText(b.content)).text) - 40;
67
+ if (saved > 0) events.push({ kind: 'logs', tok: saved, turn: u.turn, file: '$ ' + String(u.input.command).slice(0, 48) });
68
+ } else if (SEARCH.has(u.name)) {
69
+ events.push({ kind: 'search', tok, turn: u.turn });
70
+ if (tok > OVERSIZED) events.push({ kind: 'oversized', tok: tok - KEEP, turn: u.turn, file: u.input.pattern || '' });
71
+ }
72
+ }
73
+ }
74
+ }
75
+
76
+ // Tokens added to context are re-sent (cached) on every later turn until a compaction.
77
+ const r = rates(model);
78
+ const carry = (turn) => { const end = segmentEnds.find((e) => e > turn) ?? turns.length; return Math.max(0, end - turn - 1); };
79
+ const waste = { junk: 0, reread: 0, oversized: 0, logs: 0 }, wasteUsd = { junk: 0, reread: 0, oversized: 0, logs: 0 };
80
+ let searchTok = 0; const files = new Map();
81
+ for (const e of events) {
82
+ if (e.kind === 'search') { searchTok += e.tok; continue; }
83
+ const c = e.tok * (r.cacheWrite + carry(e.turn) * r.cacheRead) / 1e6;
84
+ waste[e.kind] += e.tok; wasteUsd[e.kind] += c;
85
+ if (e.file) files.set(e.file, (files.get(e.file) || 0) + c);
86
+ }
87
+ let inTok = 0, outTok = 0, cacheRead = 0, cacheWrite = 0, spend = 0; const byModel = new Map();
88
+ for (const { usage, model: mm } of usageById.values()) {
89
+ inTok += usage.input_tokens || 0; outTok += usage.output_tokens || 0;
90
+ cacheRead += usage.cache_read_input_tokens || 0; cacheWrite += usage.cache_creation_input_tokens || 0;
91
+ const c = cost(usage, mm || model); spend += c;
92
+ const name = shortModel(mm || model); if (name) byModel.set(name, (byModel.get(name) || 0) + c);
93
+ }
94
+ return { turns: turns.length, compactions, model, inTok, outTok, cacheRead, cacheWrite, spend, waste, wasteUsd, searchTok, files, byModel };
95
+ }
96
+
97
+ export function audit({ days = 30, project = '' } = {}) {
98
+ const since = Date.now() - days * 864e5;
99
+ const total = { sessions: 0, turns: 0, compactions: 0, inTok: 0, outTok: 0, cacheRead: 0, cacheWrite: 0, spend: 0, searchTok: 0,
100
+ waste: { junk: 0, reread: 0, oversized: 0, logs: 0 }, wasteUsd: { junk: 0, reread: 0, oversized: 0, logs: 0 }, files: new Map(), byModel: new Map() };
101
+ for (const dir of logDirs()) {
102
+ if (project && !dir.toLowerCase().includes(project.toLowerCase())) continue;
103
+ let names; try { names = fs.readdirSync(dir).filter((n) => n.endsWith('.jsonl')); } catch { continue; }
104
+ for (const n of names) {
105
+ const p = path.join(dir, n);
106
+ try { if (fs.statSync(p).mtimeMs < since) continue; } catch { continue; }
107
+ const s = auditSession(fs.readFileSync(p, 'utf8').split('\n'));
108
+ if (!s.turns) continue;
109
+ total.sessions++;
110
+ for (const k of ['turns', 'compactions', 'inTok', 'outTok', 'cacheRead', 'cacheWrite', 'spend', 'searchTok']) total[k] += s[k];
111
+ for (const k in s.waste) { total.waste[k] += s.waste[k]; total.wasteUsd[k] += s.wasteUsd[k]; }
112
+ for (const [f, c] of s.files) total.files.set(f, (total.files.get(f) || 0) + c);
113
+ for (const [m, c] of s.byModel) total.byModel.set(m, (total.byModel.get(m) || 0) + c);
114
+ }
115
+ }
116
+ return total;
117
+ }
118
+
119
+ export function report(t, days) {
120
+ if (!t.sessions) return `No Claude Code sessions found in the last ${days} days.\nLooked in ${path.join(process.env.CLAUDE_CONFIG_DIR || '~/.claude', 'projects')}.`;
121
+ const w = Object.values(t.wasteUsd).reduce((a, b) => a + b, 0);
122
+ const input = t.inTok + t.cacheRead + t.cacheWrite;
123
+ const hit = input ? (t.cacheRead / input) * 100 : 0;
124
+ const top = [...t.files].sort((a, b) => b[1] - a[1]).slice(0, 5);
125
+ const L = [
126
+ `TokenBuild Waste Audit — last ${days} days`,
127
+ `${t.sessions} sessions, ${num(t.turns)} turns, ${num(input + t.outTok)} tokens, ~${usd(t.spend)} at API list price`,
128
+ `Cache hit rate ${hit.toFixed(0)}% · compactions ${t.compactions} · search results ${num(t.searchTok)} tokens`,
129
+ '',
130
+ `Avoidable waste: ~${usd(w)} (${t.spend ? ((w / t.spend) * 100).toFixed(1) : 0}% of spend)`,
131
+ ` junk files read ${num(t.waste.junk).padStart(7)} tok ${usd(t.wasteUsd.junk)}`,
132
+ ` unchanged re-reads ${num(t.waste.reread).padStart(7)} tok ${usd(t.wasteUsd.reread)}`,
133
+ ` oversized results ${num(t.waste.oversized).padStart(7)} tok ${usd(t.wasteUsd.oversized)}`,
134
+ ` noisy command logs ${num(t.waste.logs).padStart(7)} tok ${usd(t.wasteUsd.logs)}`,
135
+ ];
136
+ const models = [...t.byModel].filter(([, c]) => c > 0).sort((a, b) => b[1] - a[1]);
137
+ if (models.length) L.push('', `Spend by model: ${models.map(([m, c]) => `${m} ${((c / t.spend) * 100).toFixed(0)}%`).join(' · ')}`);
138
+ if (top.length) L.push('', 'Costliest files:', ...top.map(([f, c]) => ` ${usd(c).padStart(7)} ${f}`));
139
+ L.push('', 'Cost counts each wasted token on every later turn it stayed in context.',
140
+ 'Fix most of it automatically: npx tokenbuild init');
141
+ return L.join('\n');
142
+ }
package/src/format.mjs ADDED
@@ -0,0 +1,84 @@
1
+ // Auto-format: Claude never spends turns on formatting. Uses only the formatter the project already
2
+ // has configured (never downloads one). Two hooks:
3
+ // PostToolUse (Edit|Write|MultiEdit) → `format --track`: remember the file (no formatting, ~1 ms of work)
4
+ // Stop → `format`: format every tracked file once, at the end of the turn
5
+ // Formatting at the end of the turn, not after each edit, avoids "file modified since read" re-reads
6
+ // in the middle of a run of edits. Silent: prints nothing, so it costs zero tokens.
7
+ import fs from 'node:fs';
8
+ import path from 'node:path';
9
+ import { spawnSync } from 'node:child_process';
10
+ import { tbDir, readJSON, writeJSON, appendLedger } from './util.mjs';
11
+
12
+ const exists = (p) => { try { fs.accessSync(p); return true; } catch { return false; } };
13
+ const read = (p) => { try { return fs.readFileSync(p, 'utf8'); } catch { return ''; } };
14
+ const onPath = (bin, root) => {
15
+ for (const d of [path.join(root, 'node_modules', '.bin'), path.join(root, '.venv', 'bin'), path.join(root, 'venv', 'bin'), ...(process.env.PATH || '').split(path.delimiter)]) {
16
+ const p = path.join(d, bin); if (d && exists(p)) return p;
17
+ }
18
+ return null;
19
+ };
20
+ const local = (root, bin) => { const p = path.join(root, 'node_modules', '.bin', bin); return exists(p) ? p : null; };
21
+ const any = (root, names) => names.some((n) => exists(path.join(root, n)));
22
+
23
+ const WEB = /\.(?:[cm]?[jt]sx?|json|jsonc|css|scss|less|html|vue|svelte|md|mdx|ya?ml|graphql)$/i;
24
+
25
+ // Which configured formatter handles this file? Returns [bin, ...args] or null.
26
+ export function formatterFor(file, root) {
27
+ const ext = path.extname(file).toLowerCase();
28
+ if (WEB.test(file)) {
29
+ const biome = local(root, 'biome');
30
+ if (biome && any(root, ['biome.json', 'biome.jsonc'])) return [biome, 'format', '--write', file];
31
+ const prettier = local(root, 'prettier');
32
+ const pkg = read(path.join(root, 'package.json'));
33
+ if (prettier && (any(root, ['.prettierrc', '.prettierrc.json', '.prettierrc.yaml', '.prettierrc.yml', '.prettierrc.js', '.prettierrc.cjs', '.prettierrc.mjs', 'prettier.config.js', 'prettier.config.cjs', 'prettier.config.mjs', '.prettierrc.toml']) || /"prettier"\s*:\s*[{"]/.test(pkg)))
34
+ return [prettier, '--write', '--log-level', 'silent', file];
35
+ return null;
36
+ }
37
+ if (ext === '.py') {
38
+ const py = read(path.join(root, 'pyproject.toml'));
39
+ const ruff = onPath('ruff', root);
40
+ if (ruff && (/\[tool\.ruff/.test(py) || any(root, ['ruff.toml', '.ruff.toml']))) return [ruff, 'format', '--quiet', file];
41
+ const black = onPath('black', root);
42
+ if (black && /\[tool\.black\]/.test(py)) return [black, '--quiet', file];
43
+ return null;
44
+ }
45
+ if (ext === '.go') { const g = onPath('gofmt', root); return g && exists(path.join(root, 'go.mod')) ? [g, '-w', file] : null; }
46
+ if (ext === '.rs') {
47
+ const r = onPath('rustfmt', root); const cargo = read(path.join(root, 'Cargo.toml'));
48
+ if (!r || !cargo) return null;
49
+ const ed = any(root, ['rustfmt.toml', '.rustfmt.toml']) ? [] : ['--edition', (cargo.match(/edition\s*=\s*"(\d{4})"/) || [, '2021'])[1]];
50
+ return [r, ...ed, file];
51
+ }
52
+ if (ext === '.swift') { const s = onPath('swift-format', root); return s && exists(path.join(root, '.swift-format')) ? [s, '--in-place', file] : null; }
53
+ return null;
54
+ }
55
+
56
+ const statePath = (root, sid) => path.join(tbDir(root), 'state', `fmt-${String(sid || 'none').replace(/[^\w-]/g, '')}.json`);
57
+
58
+ export function track(input, root) {
59
+ const f = input.tool_input?.file_path;
60
+ if (!f) return;
61
+ const abs = path.isAbsolute(f) ? f : path.join(root, f);
62
+ if (!abs.startsWith(root + path.sep)) return; // only this project's files
63
+ const p = statePath(root, input.session_id);
64
+ const s = readJSON(p, { files: [] });
65
+ if (!s.files.includes(abs)) { s.files.push(abs); writeJSON(p, s); }
66
+ }
67
+
68
+ // Format tracked files. Returns how many files the formatter actually changed.
69
+ export function flush(input, root) {
70
+ const p = statePath(root, input.session_id);
71
+ const { files = [] } = readJSON(p, {});
72
+ try { fs.rmSync(p); } catch {}
73
+ if (readJSON(path.join(tbDir(root), 'config.json'), {}).format === false) return 0;
74
+ let changed = 0;
75
+ for (const f of files) {
76
+ const cmd = exists(f) && formatterFor(f, root);
77
+ if (!cmd) continue;
78
+ const before = read(f);
79
+ const r = spawnSync(cmd[0], cmd.slice(1), { cwd: root, timeout: 15_000, stdio: 'ignore' });
80
+ if (r.status === 0 && read(f) !== before) changed++;
81
+ }
82
+ if (changed) appendLedger(root, { ev: 'format', n: changed });
83
+ return changed;
84
+ }
package/src/guard.mjs ADDED
@@ -0,0 +1,78 @@
1
+ // Waste Guard: Claude Code PreToolUse hook. Blocks known waste before it enters context.
2
+ // Contract: JSON on stdin; exit 2 + stderr message = block (Claude sees the message). Must stay <50 ms.
3
+ import fs from 'node:fs';
4
+ import path from 'node:path';
5
+ import { isJunk, readJSON, writeJSON, tbDir, appendLedger, num } from './util.mjs';
6
+ import { isNoisy } from './shrink.mjs';
7
+ import { licensed } from './license.mjs';
8
+
9
+ export const MAX_READ_BYTES = 60_000; // ~15K tokens
10
+
11
+ export function rootOf(dir) {
12
+ for (let d = dir; ; d = path.dirname(d)) {
13
+ if (fs.existsSync(path.join(d, '.tokenbuild')) || fs.existsSync(path.join(d, '.git'))) return d;
14
+ if (path.dirname(d) === d) return dir;
15
+ }
16
+ }
17
+
18
+ // Returns null to allow, or { reason, tok } to block.
19
+ export function decide(input, root) {
20
+ const { tool_name: tool, tool_input: ti = {}, session_id: sid = 'none' } = input;
21
+ const statePath = path.join(tbDir(root), 'state', `${String(sid).replace(/[^\w-]/g, '')}.json`);
22
+ const state = readJSON(statePath, { reads: {}, blocked: {} });
23
+ const save = () => writeJSON(statePath, state);
24
+ const abs = (p) => (p && !path.isAbsolute(p) ? path.join(root, p) : p);
25
+ const rel = (p) => path.relative(root, p || '') || p;
26
+
27
+ if (tool === 'Read' && ti.file_path) {
28
+ const f = abs(ti.file_path);
29
+ const key = `${f}|${ti.offset || 0}|${ti.limit || 0}`;
30
+ let st; try { st = fs.statSync(f); } catch { return null; }
31
+ const tok = Math.ceil(st.size / 4);
32
+ const once = (k, reason) => { // second identical attempt is allowed: Claude may truly need it
33
+ if (state.blocked[k]) { delete state.blocked[k]; save(); return null; }
34
+ state.blocked[k] = 1; save(); return { reason, tok };
35
+ };
36
+ if (isJunk(rel(f))) return once(key, `TokenBuild: ${rel(f)} is generated/vendored (~${num(tok)} tokens). Read the source file or package.json instead. Retry only if you truly need it.`);
37
+ if (st.size > MAX_READ_BYTES && !ti.limit) return once(key, `TokenBuild: ${rel(f)} is ~${num(tok)} tokens. Use tb_find to get line numbers, then Read with offset/limit. Retry to read it all anyway.`);
38
+ const sig = `${st.mtimeMs}:${st.size}`;
39
+ if (state.reads[key] === sig) return once(key + '#re', `TokenBuild: ${rel(f)} is unchanged since you last read it this session; it is already in your context. Retry if it was compacted away.`);
40
+ state.reads[key] = sig; save();
41
+ return null;
42
+ }
43
+ if (tool === 'Bash' && isNoisy(ti.command)) {
44
+ // Once per command per session: point Claude at the shrinker. Claude re-runs it wrapped, through
45
+ // the normal permission flow. Retrying the raw command unchanged is allowed.
46
+ const wrapper = path.join(tbDir(root), 'run');
47
+ if (!fs.existsSync(wrapper)) return null;
48
+ const k = 'bash|' + ti.command;
49
+ if (state.blocked[k]) return null;
50
+ state.blocked[k] = 1; save();
51
+ const cwd = input.cwd ? path.resolve(input.cwd) : root;
52
+ const w = cwd === root ? '.tokenbuild/run' : JSON.stringify(wrapper);
53
+ return { ev: 'hint', tok: 0, reason: `TokenBuild: this command prints a long log. Run it as:\n${w} ${ti.command}\nSame command and exit code; prints errors and the summary, saves the full log. Retry it unchanged only if you need the raw output.` };
54
+ }
55
+ if ((tool === 'Grep' || tool === 'Glob') && [ti.path, ti.pattern, ti.glob].some((p) => p && /node_modules|(^|\/)(dist|build|\.git)(\/|$)/.test(p))) {
56
+ return { reason: 'TokenBuild: searching vendored/generated folders wastes tokens. Narrow the path to source folders, or use tb_find.', tok: 0 };
57
+ }
58
+ return null;
59
+ }
60
+
61
+ export function reset(input, root) {
62
+ const sid = String(input.session_id || 'none').replace(/[^\w-]/g, '');
63
+ try { fs.rmSync(path.join(tbDir(root), 'state', `${sid}.json`)); } catch {}
64
+ }
65
+
66
+ export async function runGuard(mode = 'pre') {
67
+ let raw = '';
68
+ for await (const c of process.stdin) raw += c;
69
+ let input; try { input = JSON.parse(raw); } catch { return 0; } // never break Claude on bad input
70
+ const root = rootOf(input.cwd || process.cwd());
71
+ if (mode === 'reset') { reset(input, root); return 0; }
72
+ if (!licensed().ok) return 0; // no active plan: stay out of the way
73
+ const d = decide(input, root);
74
+ if (!d) return 0;
75
+ appendLedger(root, { ev: d.ev || 'block', tool: input.tool_name, tok: d.tok, sid: input.session_id });
76
+ process.stderr.write(d.reason + '\n');
77
+ return 2;
78
+ }
package/src/init.mjs ADDED
@@ -0,0 +1,100 @@
1
+ // Wires TokenBuild into a repo's Claude Code setup. Idempotent; touches only its own entries.
2
+ import fs from 'node:fs';
3
+ import path from 'node:path';
4
+ import { readJSON, tbDir } from './util.mjs';
5
+
6
+ // Kept tiny on purpose: CLAUDE.md is sent with every request (and to every subagent).
7
+ const BLOCK = (scout) => `<!-- tokenbuild -->
8
+ ## Token rules (TokenBuild)
9
+ - Start a task with tb_pack (or tb_map); use tb_find before Grep/Glob.
10
+ - Read line ranges (offset/limit) of large files. Don't re-read unchanged files.
11
+ - Run tests, builds, installs and linters as \`.tokenbuild/run <command>\`: errors + summary, full log saved.${scout ? '\n- Delegate searches and log reading to the tb-scout agent; it runs on a cheaper model.' : ''}
12
+ <!-- /tokenbuild -->`;
13
+
14
+ // Model routing presets: which model Claude Code's subagents use when they don't name one.
15
+ // The main session keeps whatever model the developer picked.
16
+ export const ROUTES = {
17
+ balanced: { subagents: 'sonnet', scout: 'haiku', about: 'subagents on Sonnet, tb-scout on Haiku' },
18
+ lean: { subagents: 'haiku', scout: 'haiku', about: 'all subagents on Haiku (cheapest; weaker on hard subtasks)' },
19
+ off: null,
20
+ };
21
+
22
+ const SCOUT = (model) => `---
23
+ name: tb-scout
24
+ description: Fast, low-cost explorer. Use for finding where things are, reading logs or large files, and answering "where/what/which" questions about the codebase. Returns paths with line numbers and short facts, not code dumps.
25
+ tools: Read, Grep, Glob, mcp__tokenbuild__tb_find, mcp__tokenbuild__tb_map, mcp__tokenbuild__tb_pack, mcp__tokenbuild__tb_digest
26
+ model: ${model}
27
+ ---
28
+ You are a read-only scout for the main agent. Find the answer with the fewest reads: tb_find or tb_pack first, then Read only the line ranges you need.
29
+ Reply in at most 15 lines: file:line references and the facts asked for. Quote code only when it is the answer, and then only the exact lines.
30
+ `;
31
+
32
+ export function cliCommand(self = process.argv[1]) {
33
+ if (!self) return 'tokenbuild';
34
+ if (/[\\/]_npx[\\/]/.test(self)) return 'npx -y tokenbuild'; // npx installs aren't on PATH
35
+ if (/[\\/]lib[\\/]node_modules[\\/]tokenbuild[\\/]/.test(self)) return 'tokenbuild'; // global install
36
+ return `node ${JSON.stringify(path.resolve(self))}`;
37
+ }
38
+
39
+ const ours = (h) => /tokenbuild/.test(h.command || '') && /\b(guard|format)\b/.test(h.command || '');
40
+
41
+ export function init(root, cmd = cliCommand(), { route = 'balanced', format = true } = {}) {
42
+ const done = [];
43
+ const plan = ROUTES[route] === undefined ? ROUTES.balanced : ROUTES[route];
44
+
45
+ // 1. Hooks (and model routing) in .claude/settings.json
46
+ const sp = path.join(root, '.claude', 'settings.json');
47
+ const s = readJSON(sp, {});
48
+ s.hooks ||= {};
49
+ const strip = (event) => { s.hooks[event] = (s.hooks[event] || []).map((g) => ({ ...g, hooks: (g.hooks || []).filter((h) => !ours(h)) })).filter((g) => g.hooks.length); };
50
+ const add = (event, matcher, command) => { s.hooks[event].push({ ...(matcher ? { matcher } : {}), hooks: [{ type: 'command', command }] }); };
51
+ for (const e of ['PreToolUse', 'PostToolUse', 'PreCompact', 'SessionStart', 'Stop']) strip(e);
52
+ add('PreToolUse', 'Read|Grep|Glob|Bash', `${cmd} guard`);
53
+ add('PreCompact', '', `${cmd} guard --reset`);
54
+ add('SessionStart', '', `${cmd} guard --reset`);
55
+ if (format) { add('PostToolUse', 'Edit|Write|MultiEdit', `${cmd} format --track`); add('Stop', '', `${cmd} format`); }
56
+ for (const e of Object.keys(s.hooks)) if (!s.hooks[e].length) delete s.hooks[e];
57
+ if (plan) {
58
+ s.env ||= {};
59
+ if (!s.env.CLAUDE_CODE_SUBAGENT_MODEL || s.env.TOKENBUILD_ROUTE) { s.env.CLAUDE_CODE_SUBAGENT_MODEL = plan.subagents; s.env.TOKENBUILD_ROUTE = route; }
60
+ } else if (s.env?.TOKENBUILD_ROUTE) { delete s.env.CLAUDE_CODE_SUBAGENT_MODEL; delete s.env.TOKENBUILD_ROUTE; if (!Object.keys(s.env).length) delete s.env; }
61
+ fs.mkdirSync(path.dirname(sp), { recursive: true });
62
+ fs.writeFileSync(sp, JSON.stringify(s, null, 2) + '\n');
63
+ done.push(`.claude/settings.json: Waste Guard${format ? ', auto-format' : ''} hooks`);
64
+ if (plan) done.push(s.env.TOKENBUILD_ROUTE ? `Model routing (${route}): ${plan.about}` : `Model routing: kept your CLAUDE_CODE_SUBAGENT_MODEL=${s.env.CLAUDE_CODE_SUBAGENT_MODEL}`);
65
+
66
+ // 2. tb-scout agent on a cheap model
67
+ const ap = path.join(root, '.claude', 'agents', 'tb-scout.md');
68
+ if (plan) { fs.mkdirSync(path.dirname(ap), { recursive: true }); fs.writeFileSync(ap, SCOUT(plan.scout)); done.push(`.claude/agents/tb-scout.md: explorer on ${plan.scout}`); }
69
+ else try { fs.rmSync(ap); } catch {}
70
+
71
+ // 3. MCP server in .mcp.json
72
+ const mp = path.join(root, '.mcp.json');
73
+ const m = readJSON(mp, {});
74
+ m.mcpServers ||= {};
75
+ const [command, ...pre] = cmd.startsWith('node ') ? ['node', JSON.parse(cmd.slice(5))] : cmd.split(' ');
76
+ m.mcpServers.tokenbuild = { command, args: [...pre, 'mcp'] };
77
+ fs.writeFileSync(mp, JSON.stringify(m, null, 2) + '\n');
78
+ done.push('.mcp.json: tb_map, tb_find, tb_pack tools');
79
+
80
+ // 4. .tokenbuild/run: the Output Shrinker, callable as a short path
81
+ const wp = path.join(tbDir(root), 'run');
82
+ fs.mkdirSync(path.dirname(wp), { recursive: true });
83
+ fs.writeFileSync(wp, `#!/bin/sh\n# TokenBuild Output Shrinker: same command and exit code, condensed output, full log saved.\nexec ${cmd} run -- "$@"\n`, { mode: 0o755 });
84
+ fs.chmodSync(wp, 0o755);
85
+ done.push('.tokenbuild/run: Output Shrinker');
86
+
87
+ // 5. CLAUDE.md rules block
88
+ const cp = path.join(root, 'CLAUDE.md');
89
+ let c = ''; try { c = fs.readFileSync(cp, 'utf8'); } catch {}
90
+ const block = BLOCK(!!plan);
91
+ c = c.includes('<!-- tokenbuild -->') ? c.replace(/<!-- tokenbuild -->[\s\S]*?<!-- \/tokenbuild -->/, block) : (c ? c.trimEnd() + '\n\n' : '') + block + '\n';
92
+ fs.writeFileSync(cp, c);
93
+ done.push(`CLAUDE.md: ${block.split('\n').length - 3}-line token rules block`);
94
+
95
+ // 6. Keep local state out of git
96
+ const gp = path.join(root, '.gitignore');
97
+ let g = ''; try { g = fs.readFileSync(gp, 'utf8'); } catch {}
98
+ if (!/^\.tokenbuild\/?$/m.test(g)) { fs.writeFileSync(gp, (g ? g.trimEnd() + '\n' : '') + '.tokenbuild/\n'); done.push('.gitignore: .tokenbuild/'); }
99
+ return done;
100
+ }
package/src/ledger.mjs ADDED
@@ -0,0 +1,20 @@
1
+ // Turns ledger events (.tokenbuild/ledger.jsonl) into counters. Counts only: what `sync` sends.
2
+ const EMPTY = () => ({ blocks: 0, tok_blocked: 0, tool_calls: 0, shrinks: 0, tok_shrunk: 0, formats: 0, local_calls: 0, tok_local: 0 });
3
+
4
+ function add(c, e) {
5
+ const tok = Math.max(0, Math.floor(e.tok || 0));
6
+ if (e.ev === 'block') { c.blocks++; c.tok_blocked += tok; }
7
+ else if (e.ev === 'tool') c.tool_calls++;
8
+ else if (e.ev === 'shrink') { c.shrinks++; c.tok_shrunk += tok; }
9
+ else if (e.ev === 'format') c.formats += e.n || 1;
10
+ else if (e.ev === 'local') { c.local_calls++; c.tok_local += tok; }
11
+ }
12
+ const parse = (lines) => lines.map((l) => { try { return JSON.parse(l); } catch { return null; } }).filter(Boolean);
13
+
14
+ export function tally(lines) { const c = EMPTY(); for (const e of parse(lines)) add(c, e); return c; }
15
+
16
+ export function byDay(lines) {
17
+ const days = {};
18
+ for (const e of parse(lines)) add((days[new Date(e.t || Date.now()).toISOString().slice(0, 10)] ||= EMPTY()), e);
19
+ return Object.entries(days).map(([day, v]) => ({ day, ...v }));
20
+ }
@@ -0,0 +1,71 @@
1
+ // Subscription check. The free Waste Audit needs nothing. Every other tool needs an active trial
2
+ // or plan, confirmed online with the user's CLI key and cached locally, so hooks stay offline and fast:
3
+ // online: `login`, `init`, `sync`, and the MCP server at the start of each Claude Code session
4
+ // offline: hooks and `run` only read the cache (a few microseconds)
5
+ // The cache is trusted for GRACE after the last successful check, so a flight or an outage
6
+ // doesn't switch the tools off; a cancelled plan stops them at the next online check.
7
+ import fs from 'node:fs';
8
+ import os from 'node:os';
9
+ import path from 'node:path';
10
+
11
+ export const GRACE = 14 * 864e5;
12
+ const REFRESH = 12 * 36e5;
13
+ const API = () => process.env.TOKENBUILD_API || 'https://app.tokenbuild.ai';
14
+ export const BILLING = 'https://app.tokenbuild.ai/billing';
15
+
16
+ export const home = () => process.env.TOKENBUILD_HOME || path.join(os.homedir(), '.tokenbuild');
17
+ const keyFile = () => path.join(home(), 'key');
18
+ const cacheFile = () => path.join(home(), 'license.json');
19
+
20
+ export function readKey() {
21
+ if (process.env.TOKENBUILD_KEY) return process.env.TOKENBUILD_KEY.trim();
22
+ try { return fs.readFileSync(keyFile(), 'utf8').trim(); } catch { return ''; }
23
+ }
24
+ export function saveKey(key) {
25
+ fs.mkdirSync(home(), { recursive: true, mode: 0o700 });
26
+ fs.writeFileSync(keyFile(), key.trim() + '\n', { mode: 0o600 });
27
+ }
28
+ export function forget() { for (const f of [keyFile(), cacheFile()]) try { fs.rmSync(f); } catch {} }
29
+
30
+ const readCache = () => { try { return JSON.parse(fs.readFileSync(cacheFile(), 'utf8')); } catch { return null; } };
31
+
32
+ // Offline: is the cached licence active and recent enough? { ok, reason }
33
+ export function licensed(now = Date.now()) {
34
+ const c = readCache();
35
+ if (!c) return { ok: false, reason: 'no-license' };
36
+ if (!c.active) return { ok: false, reason: 'inactive', status: c.status };
37
+ if (now - c.checkedAt > GRACE) return { ok: false, reason: 'stale' };
38
+ return { ok: true, plan: c.plan };
39
+ }
40
+
41
+ // Online: ask the API and update the cache. { ok, active, status, plan, error }
42
+ export async function check(key = readKey(), { timeout = 8000 } = {}) {
43
+ if (!key) return { ok: false, error: 'no-key' };
44
+ let r;
45
+ try { r = await fetch(API() + '/v1/license', { headers: { Authorization: `Bearer ${key}` }, signal: AbortSignal.timeout(timeout) }); }
46
+ catch { return { ok: false, error: 'offline' }; }
47
+ const j = await r.json().catch(() => ({}));
48
+ if (r.status === 401) { writeCache({ active: false, status: 'invalid-key' }); return { ok: false, error: 'bad-key', message: j.error }; }
49
+ if (!r.ok) return { ok: false, error: 'server', message: j.error };
50
+ writeCache({ active: !!j.active, status: j.status, plan: j.plan });
51
+ return { ok: true, active: !!j.active, status: j.status, plan: j.plan };
52
+ }
53
+ function writeCache(c) {
54
+ fs.mkdirSync(home(), { recursive: true, mode: 0o700 });
55
+ fs.writeFileSync(cacheFile(), JSON.stringify({ ...c, checkedAt: Date.now() }) + '\n', { mode: 0o600 });
56
+ }
57
+
58
+ // Refresh in the background when the cache is older than REFRESH (MCP server start).
59
+ export function refreshIfDue(now = Date.now()) {
60
+ const c = readCache();
61
+ if (!readKey() || (c && now - c.checkedAt < REFRESH)) return Promise.resolve(null);
62
+ return check(undefined, { timeout: 5000 }).catch(() => null);
63
+ }
64
+
65
+ // One short line explaining what to do. Kept short: it may land in Claude's context.
66
+ export function explain(l = licensed()) {
67
+ if (l.reason === 'no-license') return readKey() ? `TokenBuild: couldn't confirm your plan yet. Run: tokenbuild login` : `TokenBuild tools need an active trial or plan. Start one at ${BILLING}, then run: tokenbuild login <key>`;
68
+ if (l.reason === 'stale') return 'TokenBuild: your plan hasn\'t been confirmed for 14 days. Connect to the internet and run: tokenbuild login';
69
+ if (l.status === 'invalid-key') return `TokenBuild: this CLI key is no longer valid. Create a new one at ${BILLING}, then run: tokenbuild login <key>`;
70
+ return `TokenBuild: no active trial or plan (${l.status || 'none'}). Renew at ${BILLING}`;
71
+ }
package/src/local.mjs ADDED
@@ -0,0 +1,69 @@
1
+ // Local Boost: optional helper model running on the developer's own machine through Ollama.
2
+ // Claude asks it to read big files or logs and gets a short answer back, so the big text never
3
+ // enters Claude's context. Off unless turned on; local hosts only, so code never leaves the machine.
4
+ import fs from 'node:fs';
5
+ import path from 'node:path';
6
+ import { tbDir, readJSON, writeJSON, tokens, appendLedger, num } from './util.mjs';
7
+
8
+ const PREFER = [/qwen[\d.]*-?coder/i, /gpt-oss/i, /qwen3/i, /deepseek-coder/i, /codestral|devstral/i, /gemma/i, /llama3/i, /mistral/i, /phi/i];
9
+ export const SUGGEST = 'qwen2.5-coder:7b'; // ~5 GB, runs on most 16 GB laptops
10
+ const MAX_IN = 24_000; // tokens we hand the local model in one call
11
+
12
+ export function host() {
13
+ const h = process.env.OLLAMA_HOST || 'http://127.0.0.1:11434';
14
+ const url = new URL(/^https?:\/\//.test(h) ? h : `http://${h}`);
15
+ if (!/^(?:localhost|127\.0\.0\.1|\[::1\]|::1)$/.test(url.hostname)) throw new Error(`OLLAMA_HOST is ${url.hostname}; Local Boost only talks to this machine so your code never leaves it.`);
16
+ return url.origin;
17
+ }
18
+
19
+ const cfgPath = (root) => path.join(tbDir(root), 'config.json');
20
+ export const config = (root) => readJSON(cfgPath(root), {});
21
+ export function setConfig(root, patch) { const c = { ...config(root), ...patch }; writeJSON(cfgPath(root), c); return c; }
22
+
23
+ // { ok, models, model, error }
24
+ export async function detect(root, timeout = 400) {
25
+ try {
26
+ const r = await fetch(host() + '/api/tags', { signal: AbortSignal.timeout(timeout) });
27
+ const models = ((await r.json()).models || []).map((m) => m.name);
28
+ const want = process.env.TOKENBUILD_LOCAL_MODEL || config(root).model;
29
+ const model = want ? models.find((m) => m === want || m.startsWith(want + ':')) || null
30
+ : PREFER.map((re) => models.find((m) => re.test(m))).find(Boolean) || models[0] || null;
31
+ return { ok: true, models, model };
32
+ } catch (e) {
33
+ return { ok: false, models: [], model: null, error: /OLLAMA_HOST/.test(e.message) ? e.message : 'Ollama is not running' };
34
+ }
35
+ }
36
+
37
+ export async function ask(model, system, prompt, { maxOut = 400, timeout = 120_000 } = {}) {
38
+ const r = await fetch(host() + '/api/chat', {
39
+ method: 'POST', headers: { 'Content-Type': 'application/json' }, signal: AbortSignal.timeout(timeout),
40
+ body: JSON.stringify({ model, stream: false, messages: [{ role: 'system', content: system }, { role: 'user', content: prompt }],
41
+ options: { temperature: 0.1, num_predict: maxOut, num_ctx: Math.min(32_768, 2048 + Math.ceil(tokens(system + prompt) * 1.2)) } }),
42
+ });
43
+ if (!r.ok) throw new Error(`Local model error ${r.status}`);
44
+ return String((await r.json()).message?.content || '').replace(/<think>[\s\S]*?<\/think>/g, '').trim();
45
+ }
46
+
47
+ const SYSTEM = 'You answer questions about one file or log for a coding agent. Be exact and brief: at most 12 lines. Quote identifiers, error messages and line numbers exactly as they appear. If the text does not contain the answer, say "Not in this file." Never invent content.';
48
+
49
+ // Read a file locally and answer a question about it. Returns the text for Claude.
50
+ export async function digest(root, file, question) {
51
+ const abs = path.resolve(root, file);
52
+ if (!abs.startsWith(root + path.sep) && abs !== root) return 'tb_digest only reads files inside this project.';
53
+ let text; try { text = fs.readFileSync(abs, 'utf8'); } catch { return `Can't read ${file}.`; }
54
+ const d = await detect(root);
55
+ if (!d.ok || !d.model) return `Local Boost unavailable (${d.error || 'no model installed'}). Read the file with offset/limit instead.`;
56
+ const lines = text.split('\n');
57
+ let body = lines.map((l, i) => `${i + 1}: ${l}`).join('\n'), note = '';
58
+ if (tokens(body) > MAX_IN) { // keep the head and the tail, where logs and files usually matter most
59
+ const keep = Math.floor((MAX_IN * 4) / 2);
60
+ body = body.slice(0, keep) + '\n[… middle omitted …]\n' + body.slice(-keep);
61
+ note = ' (middle of file omitted)';
62
+ }
63
+ const t0 = Date.now();
64
+ let a; try { a = await ask(d.model, SYSTEM, `File: ${path.relative(root, abs)}\nQuestion: ${question}\n\n${body}`); }
65
+ catch (e) { return `Local model failed (${e.message}). Read the file with offset/limit instead.`; }
66
+ const read = tokens(text);
67
+ appendLedger(root, { ev: 'local', tok: Math.max(0, read - tokens(a)) });
68
+ return `${a}\n[Local Boost · ${d.model} read ${num(read)} tokens on this machine in ${((Date.now() - t0) / 1000).toFixed(1)}s${note}. Verify exact lines with Read offset/limit before editing.]`;
69
+ }
package/src/mcp.mjs ADDED
@@ -0,0 +1,55 @@
1
+ // Minimal MCP stdio server (JSON-RPC 2.0, newline-delimited). Tools answer from the local index: zero model tokens to compute.
2
+ import readline from 'node:readline';
3
+ import { loadIndex, buildIndex, repoMap, find, pack } from './repo.mjs';
4
+ import { appendLedger } from './util.mjs';
5
+ import { config, detect, digest } from './local.mjs';
6
+ import { licensed, explain, refreshIfDue } from './license.mjs';
7
+
8
+ const TOOLS = [
9
+ { name: 'tb_map', description: 'Compact map of this repo: most-imported files and their top symbols. Call once at the start instead of exploring with Glob/LS.',
10
+ inputSchema: { type: 'object', properties: { budget: { type: 'number', description: 'Max tokens (default 2500)' } } } },
11
+ { name: 'tb_find', description: 'Find a symbol or file by name. Returns path:line so you can Read just that range.',
12
+ inputSchema: { type: 'object', properties: { query: { type: 'string' } }, required: ['query'] } },
13
+ { name: 'tb_pack', description: 'Files and symbols a task most likely needs, with line numbers, under a token budget. Use before starting a task.',
14
+ inputSchema: { type: 'object', properties: { task: { type: 'string' }, budget: { type: 'number' } }, required: ['task'] } },
15
+ ];
16
+ // Listed only when Local Boost is on and a local model is available, so it costs nothing otherwise.
17
+ const DIGEST = { name: 'tb_digest', description: 'Ask a local model (on this machine, zero Claude tokens) a question about a large file or a .tokenbuild/logs log. Returns a short answer with line numbers. Use before reading a big file or log in full.',
18
+ inputSchema: { type: 'object', properties: { path: { type: 'string' }, question: { type: 'string' } }, required: ['path', 'question'] } };
19
+
20
+ export async function handle(msg, root, state) {
21
+ const { id, method, params = {} } = msg;
22
+ const ok = (result) => ({ jsonrpc: '2.0', id, result });
23
+ if (method === 'initialize') return ok({ protocolVersion: params.protocolVersion || '2025-06-18', capabilities: { tools: {} }, serverInfo: { name: 'tokenbuild', version: '0.1.0' } });
24
+ if (method === 'tools/list') return ok({ tools: state.boost ? [...TOOLS, DIGEST] : TOOLS });
25
+ if (method === 'tools/call') {
26
+ if (!state.idx || Date.now() - state.at > 30_000) { state.idx = state.idx ? buildIndex(root).idx : loadIndex(root); state.at = Date.now(); }
27
+ const a = params.arguments || {};
28
+ const l = licensed();
29
+ if (!l.ok) return ok({ content: [{ type: 'text', text: explain(l) }], isError: true });
30
+ if (params.name === 'tb_digest') return ok({ content: [{ type: 'text', text: await digest(root, String(a.path || ''), String(a.question || '')) }] });
31
+ const text = params.name === 'tb_map' ? repoMap(state.idx, a.budget)
32
+ : params.name === 'tb_find' ? find(state.idx, String(a.query || ''))
33
+ : params.name === 'tb_pack' ? pack(state.idx, String(a.task || ''), a.budget)
34
+ : null;
35
+ if (text === null) return { jsonrpc: '2.0', id, error: { code: -32602, message: 'Unknown tool' } };
36
+ appendLedger(root, { ev: 'tool', tool: params.name, out: Math.ceil(text.length / 4) });
37
+ return ok({ content: [{ type: 'text', text }] });
38
+ }
39
+ if (method === 'ping') return ok({});
40
+ if (id === undefined) return null; // notification
41
+ return { jsonrpc: '2.0', id, error: { code: -32601, message: 'Method not found' } };
42
+ }
43
+
44
+ export function runMcp(root) {
45
+ const state = { idx: null, at: 0, boost: false };
46
+ refreshIfDue(); // confirm the plan once per session start, in the background
47
+ const ready = config(root).boost ? detect(root).then((d) => { state.boost = d.ok && !!d.model; }) : Promise.resolve();
48
+ const rl = readline.createInterface({ input: process.stdin });
49
+ rl.on('line', async (line) => {
50
+ let msg; try { msg = JSON.parse(line); } catch { return; }
51
+ await ready;
52
+ const res = await handle(msg, root, state);
53
+ if (res) process.stdout.write(JSON.stringify(res) + '\n');
54
+ });
55
+ }
package/src/prices.mjs ADDED
@@ -0,0 +1,28 @@
1
+ // USD per million tokens. List prices, approximate; edit here only.
2
+ // cacheWrite = 1.25x input (5-min cache), cacheRead = 0.1x input.
3
+ const TABLE = [
4
+ [/opus-4/, 5, 25],
5
+ [/opus/, 4, 20],
6
+ [/sonnet-4/, 3, 15],
7
+ [/sonnet/, 2, 10],
8
+ [/haiku/, 1, 5],
9
+ [/fable|mythos/, 10, 50],
10
+ ];
11
+ const DEFAULT = [3, 15];
12
+
13
+ export function rates(model = '') {
14
+ const m = String(model).toLowerCase();
15
+ const hit = TABLE.find(([re]) => re.test(m));
16
+ const [input, output] = hit ? [hit[1], hit[2]] : DEFAULT;
17
+ return { input, output, cacheWrite: input * 1.25, cacheRead: input * 0.1 };
18
+ }
19
+
20
+ export function cost(usage, model) {
21
+ const r = rates(model);
22
+ return (
23
+ (usage.input_tokens || 0) * r.input +
24
+ (usage.output_tokens || 0) * r.output +
25
+ (usage.cache_creation_input_tokens || 0) * r.cacheWrite +
26
+ (usage.cache_read_input_tokens || 0) * r.cacheRead
27
+ ) / 1e6;
28
+ }
package/src/repo.mjs ADDED
@@ -0,0 +1,149 @@
1
+ // Repo Index, map, find and context packs. Regex symbol extraction: fast, zero deps, zero tokens.
2
+ import fs from 'node:fs';
3
+ import path from 'node:path';
4
+ import { walk, tokens, hash, readJSON, writeJSON, tbDir } from './util.mjs';
5
+
6
+ const LANG = { js: 'js', mjs: 'js', cjs: 'js', jsx: 'js', ts: 'js', tsx: 'js', mts: 'js', py: 'py', go: 'go', rb: 'rb', java: 'java', kt: 'java', cs: 'java', rs: 'rs', php: 'php', swift: 'swift' };
7
+ const MAX_BYTES = 400_000;
8
+
9
+ const RULES = {
10
+ js: [
11
+ [/^\s*export\s+(?:default\s+)?(?:async\s+)?function\*?\s+(\w+)\s*(\([^)]*\))?/, 'fn'],
12
+ [/^\s*(?:async\s+)?function\*?\s+(\w+)\s*(\([^)]*\))?/, 'fn'],
13
+ [/^\s*export\s+(?:default\s+)?(?:abstract\s+)?class\s+(\w+)/, 'class'],
14
+ [/^\s*class\s+(\w+)/, 'class'],
15
+ [/^\s*export\s+(?:const|let|var)\s+(\w+)\s*=\s*(?:async\s*)?(\([^)]*\)|\w+)\s*=>/, 'fn'],
16
+ [/^\s*export\s+(?:const|let|var)\s+(\w+)/, 'const'],
17
+ [/^\s*export\s+(?:interface|type|enum)\s+(\w+)/, 'type'],
18
+ [/^\s{2,4}(?:public\s+|private\s+|protected\s+|static\s+|async\s+)*(\w+)\s*(\([^)]*\))\s*\{/, 'method'],
19
+ ],
20
+ py: [[/^(?:async\s+)?def\s+(\w+)\s*(\([^)]*\))?/, 'fn'], [/^class\s+(\w+)/, 'class'], [/^\s{4}(?:async\s+)?def\s+(\w+)\s*(\([^)]*\))?/, 'method']],
21
+ go: [[/^func\s+(?:\([^)]*\)\s*)?(\w+)\s*(\([^)]*\))?/, 'fn'], [/^type\s+(\w+)\s+(struct|interface)/, 'type']],
22
+ rb: [[/^\s*def\s+(?:self\.)?(\w+[?!]?)/, 'fn'], [/^\s*(?:class|module)\s+(\w+)/, 'class']],
23
+ java: [[/^\s*(?:public|private|protected|internal)?\s*(?:static\s+)?(?:final\s+)?(?:class|interface|enum|record)\s+(\w+)/, 'class'], [/^\s+(?:public|private|protected)\s+[\w<>\[\],\s]+\s+(\w+)\s*(\([^)]*\))/, 'method']],
24
+ rs: [[/^\s*(?:pub\s+)?(?:async\s+)?fn\s+(\w+)\s*(\([^)]*\))?/, 'fn'], [/^\s*(?:pub\s+)?(?:struct|enum|trait)\s+(\w+)/, 'type']],
25
+ php: [[/^\s*(?:public\s+|private\s+|protected\s+|static\s+)*function\s+(\w+)\s*(\([^)]*\))?/, 'fn'], [/^\s*class\s+(\w+)/, 'class']],
26
+ swift: [[/^\s*(?:public\s+|private\s+)?func\s+(\w+)\s*(\([^)]*\))?/, 'fn'], [/^\s*(?:public\s+)?(?:class|struct|enum|protocol)\s+(\w+)/, 'type']],
27
+ };
28
+ const IMPORT = {
29
+ js: /(?:import\s[^'"]*?from\s*|import\s*|require\s*\(\s*)['"]([^'"]+)['"]/g,
30
+ py: /^\s*(?:from\s+([\w.]+)\s+import|import\s+([\w.]+))/gm,
31
+ go: /"([^"]+)"/g,
32
+ };
33
+ const KEYWORDS = new Set(['if', 'for', 'while', 'switch', 'catch', 'return', 'function', 'constructor']);
34
+
35
+ export function parseFile(text, lang) {
36
+ const symbols = [];
37
+ const lines = text.split('\n');
38
+ for (let i = 0; i < lines.length; i++) {
39
+ for (const [re, kind] of RULES[lang] || []) {
40
+ const m = lines[i].match(re);
41
+ if (m && !KEYWORDS.has(m[1])) { symbols.push({ n: m[1], k: kind, l: i + 1, s: m[2] ? m[2].replace(/\s+/g, ' ').slice(0, 80) : '' }); break; }
42
+ }
43
+ }
44
+ const imports = [];
45
+ const re = IMPORT[lang];
46
+ if (re) for (const m of text.matchAll(re)) imports.push(m[1] || m[2]);
47
+ return { symbols, imports: [...new Set(imports)].slice(0, 50) };
48
+ }
49
+
50
+ function resolveImport(from, spec, files) {
51
+ if (!spec.startsWith('.')) {
52
+ const dotted = spec.replaceAll('.', '/');
53
+ return [`${dotted}.py`, `${dotted}/__init__.py`].find((c) => files[c]);
54
+ }
55
+ const base = path.posix.normalize(path.posix.join(path.posix.dirname(from), spec));
56
+ const stem = base.replace(/\.(m|c)?js$/, '');
57
+ return [base, `${stem}.ts`, `${stem}.tsx`, `${stem}.js`, `${stem}.mjs`, `${stem}.jsx`, `${base}/index.ts`, `${base}/index.js`].find((c) => files[c]);
58
+ }
59
+
60
+ export function buildIndex(root) {
61
+ const p = path.join(tbDir(root), 'index.json');
62
+ const old = readJSON(p, { files: {} });
63
+ const files = {};
64
+ let parsed = 0;
65
+ for (const rel of walk(root)) {
66
+ const lang = LANG[rel.split('.').pop()];
67
+ if (!lang) continue;
68
+ let st; try { st = fs.statSync(path.join(root, rel)); } catch { continue; }
69
+ if (st.size > MAX_BYTES) continue;
70
+ const prev = old.files[rel];
71
+ if (prev && prev.m === st.mtimeMs && prev.z === st.size) { files[rel] = prev; continue; }
72
+ const text = fs.readFileSync(path.join(root, rel), 'utf8');
73
+ files[rel] = { m: st.mtimeMs, z: st.size, h: hash(text), lang, t: tokens(text), ...parseFile(text, lang) };
74
+ parsed++;
75
+ }
76
+ const inDeg = {};
77
+ for (const [f, v] of Object.entries(files)) {
78
+ v.deps = v.imports.map((s) => resolveImport(f, s, files)).filter(Boolean);
79
+ for (const d of v.deps) inDeg[d] = (inDeg[d] || 0) + 1;
80
+ }
81
+ for (const [f, v] of Object.entries(files)) v.rank = (inDeg[f] || 0) + v.symbols.length / 20;
82
+ const idx = { v: 1, root, built: Date.now(), files };
83
+ writeJSON(p, idx);
84
+ return { idx, parsed, total: Object.keys(files).length };
85
+ }
86
+
87
+ export function loadIndex(root) {
88
+ const idx = readJSON(path.join(tbDir(root), 'index.json'), null);
89
+ return idx || buildIndex(root).idx;
90
+ }
91
+
92
+ const sym = (s) => `${s.n}${s.s || ''}`;
93
+
94
+ // Compact repo map: directories, then files by importance with their top symbols, cut to budget.
95
+ export function repoMap(idx, budget = 2500) {
96
+ const entries = Object.entries(idx.files).sort((a, b) => b[1].rank - a[1].rank);
97
+ const out = [`# Repo map (${entries.length} files; most-imported first)`];
98
+ let used = tokens(out[0]);
99
+ for (const [f, v] of entries) {
100
+ const top = v.symbols.filter((s) => s.k !== 'method').slice(0, 6).map(sym).join(', ');
101
+ const line = `${f}${top ? ': ' + top : ''}`;
102
+ const t = tokens(line) + 1;
103
+ if (used + t > budget) { out.push(`… ${entries.length - out.length + 1} more files (use tb_find)`); break; }
104
+ out.push(line); used += t;
105
+ }
106
+ return out.join('\n');
107
+ }
108
+
109
+ const splitCamel = (s) => String(s).replace(/([a-z])([A-Z])/g, '$1 $2').toLowerCase();
110
+ const STOP = new Set(['the', 'and', 'for', 'with', 'add', 'fix', 'make', 'that', 'this', 'from', 'into', 'when', 'should', 'new', 'use', 'all', 'can', 'get', 'set']);
111
+ const words = (s) => (splitCamel(s).match(/[a-z][a-z0-9]{2,}/g) || []).filter((w) => !STOP.has(w));
112
+
113
+ function score(f, v, terms) {
114
+ const hay = splitCamel(f + ' ' + v.symbols.map((s) => s.n).join(' '));
115
+ let sc = 0;
116
+ for (const t of terms) { if (hay.includes(t)) sc += f.toLowerCase().includes(t) ? 3 : 1; }
117
+ return sc ? sc + Math.min(v.rank, 5) / 5 : 0;
118
+ }
119
+
120
+ // Symbol/file search: exact locations, so Claude reads lines instead of whole files.
121
+ export function find(idx, query, limit = 15) {
122
+ const q = query.toLowerCase();
123
+ const hits = [];
124
+ for (const [f, v] of Object.entries(idx.files)) {
125
+ for (const s of v.symbols) if (s.n.toLowerCase().includes(q)) hits.push({ f, ...s, exact: s.n.toLowerCase() === q });
126
+ if (f.toLowerCase().includes(q)) hits.push({ f, n: '(file)', k: 'file', l: 1, s: '', exact: false });
127
+ }
128
+ hits.sort((a, b) => b.exact - a.exact || a.f.length - b.f.length);
129
+ return hits.slice(0, limit).map((h) => `${h.f}:${h.l} ${h.k} ${h.n}${h.s || ''}`).join('\n') || 'No matches.';
130
+ }
131
+
132
+ // Context pack: the files a task most likely needs, as signatures + line numbers, under budget.
133
+ export function pack(idx, task, budget = 4000) {
134
+ const terms = [...new Set(words(task))].filter((w) => w.length > 2);
135
+ const ranked = Object.entries(idx.files).map(([f, v]) => [f, v, score(f, v, terms)]).filter((x) => x[2] > 0).sort((a, b) => b[2] - a[2]);
136
+ const chosen = new Map(ranked.slice(0, 8).map(([f, v]) => [f, v]));
137
+ for (const [f, v] of ranked.slice(0, 4)) for (const d of v.deps || []) if (!chosen.has(d) && chosen.size < 12) chosen.set(d, idx.files[d]);
138
+ const out = [`# Context pack: ${task.slice(0, 80)}`, 'Files likely involved (path:line symbol). Read only the line ranges you need.'];
139
+ let used = tokens(out.join('\n'));
140
+ for (const [f, v] of chosen) {
141
+ const head = `\n## ${f} (~${v.t} tok)`;
142
+ const body = v.symbols.slice(0, 25).map((s) => `${s.l}: ${s.k} ${sym(s)}`).join('\n');
143
+ const t = tokens(head + body);
144
+ if (used + t > budget) { out.push(`\n… more candidates omitted to stay under ${budget} tokens`); break; }
145
+ out.push(head, body); used += t;
146
+ }
147
+ if (chosen.size === 0) out.push('\nNo matching files. Try tb_find with a symbol name.');
148
+ return out.join('\n');
149
+ }
package/src/shrink.mjs ADDED
@@ -0,0 +1,133 @@
1
+ // Output Shrinker: runs a noisy command (tests, builds, installs, linters), saves the full log
2
+ // locally and prints only what Claude needs: errors with context, warnings, and the summary.
3
+ // Plain code, zero tokens to compute. The full log stays on disk for a targeted Read if needed.
4
+ import fs from 'node:fs';
5
+ import path from 'node:path';
6
+ import { spawn } from 'node:child_process';
7
+ import { tbDir, appendLedger, tokens, num } from './util.mjs';
8
+
9
+ export const BUDGET = 1500; // max tokens we print for a long log
10
+ const VERBATIM = 600; // logs at or under this many tokens are printed as-is (minus ANSI noise)
11
+ const KEEP_LOGS = 20;
12
+ const MAX_BYTES = 32 << 20; // keep at most 32 MB in memory; the middle is dropped beyond that
13
+
14
+ // Commands whose output is usually long and mostly noise. Matched against the first command
15
+ // of a line (after env assignments, `cd x &&`, and package-runner prefixes).
16
+ const NOISY = new RegExp('^(?:' + [
17
+ '(?:npm|pnpm|yarn|bun)\\s+(?:(?:run\\s+)?(?:test|build|lint|typecheck|check|ci)\\b|install|i|ci|add|update)\\b',
18
+ '(?:npx|pnpm\\s+exec|pnpm\\s+dlx|yarn\\s+dlx|bunx)\\s+(?:jest|vitest|mocha|tsc|eslint|playwright|cypress|next\\s+build|vite\\s+build|webpack|turbo|nx)\\b',
19
+ '(?:jest|vitest|mocha|tsc|eslint|playwright|turbo|nx)\\b',
20
+ 'node\\s+--test\\b',
21
+ '(?:python3?\\s+-m\\s+)?(?:pytest|unittest|tox|nox|mypy|pip\\s+install|pip3\\s+install|uv\\s+(?:sync|pip\\s+install)|poetry\\s+install)\\b',
22
+ 'go\\s+(?:test|build|vet|mod\\s+download|generate)\\b', 'cargo\\s+(?:test|build|check|clippy)\\b',
23
+ '(?:mvn|\\./mvnw|gradle|\\./gradlew)\\b', 'make\\b', 'cmake\\s+--build\\b', 'ctest\\b',
24
+ '(?:bundle\\s+(?:exec\\s+)?)?(?:rspec|rake|bundle\\s+install)\\b', 'swift\\s+(?:build|test)\\b', 'xcodebuild\\b',
25
+ 'dotnet\\s+(?:build|test|restore)\\b', 'docker\\s+(?:build|compose\\s+build)\\b', 'composer\\s+install\\b', '(?:php\\s+)?(?:vendor/bin/)?phpunit\\b',
26
+ ].join('|') + ')');
27
+
28
+ export function isNoisy(command) {
29
+ const c = String(command || '').trim();
30
+ if (!c || /(^|[\s;&|])\.?tokenbuild\/run\b|\btokenbuild\s+run\b/.test(c)) return false; // already wrapped
31
+ if (/\|\s*(?:head|tail|grep|rg|less|wc|jq)\b|>\s*\S|--watch\b|\s-w\b/.test(c)) return false; // already shaped, or interactive
32
+ const first = c.split(/\s*(?:&&|;)\s*/).find((p) => !/^cd\s/.test(p)) || '';
33
+ return NOISY.test(first.replace(/^(?:[A-Z_][A-Z0-9_]*=\S*\s+)+/, ''));
34
+ }
35
+
36
+ const ANSI = /\x1b\[[0-?]*[ -/]*[@-~]|\x1b\][^\x07\x1b]*(?:\x07|\x1b\\)|\x1b[@-Z\\-_]/g;
37
+ const ERR = /(?:\berror\b|\berrors?:|\bERR!|\bfail(?:ed|ure|s)?\b|\bFAIL\b|\bpanic(?:ked)?\b|\bexception\b|Traceback|\bassert(?:ion)?(?:Error)?\b|\bexpected\b|\breceived\b|\bfatal\b|\bundefined reference\b|\bcannot\b|\bcan't\b|\bunable to\b|\bnot found\b|\bdenied\b|\bsegmentation fault\b|\bunhandled\b|[✗✖×✘]|^\s*●|^\s*E\s{2,}|^not ok\b|-->\s+\S+:\d+)/i;
38
+ const WARN = /(?:\bwarn(?:ing)?s?\b|\bdeprecat)/i;
39
+ const NOISE = /^(?:\s*$|.*(?:\d{1,3}%\s*[|█▏▎▍▌▋▊▉#=\-]|[█▓▒░⣾⣽⣻⢿⡿⣟⣯⣷⠋⠙⠹⠸⠼⠴⠦⠧⠇⠏]|Downloading|Downloaded|Resolving|Fetching|Progress:|Using cached|Requirement already satisfied|Collecting\s|Already up[- ]to[- ]date)|\s*Compiling \S+ v\d|go: downloading|npm (?:notice|WARN deprecated)|> Task :|\s*(?:✓|✔|√|PASS\b|ok\s{2,}|test \S+ \.\.\. ok)|\s*\d+ passing\b|\S+\s+[.sFExX]+\s+\[\s*\d+%\]$)/i;
40
+ const SUMMARY = /\b(?:tests?|passed|failed|errors?|warnings?|suites?|total|duration|took|finished|built|done|summary)\b/i;
41
+
42
+ // Pure: condense a log. Returns { text, total, shown } (line counts).
43
+ export function shrink(raw, { exit = 0, budget = BUDGET } = {}) {
44
+ const clean = String(raw).replace(ANSI, '').split('\n').map((l) => { const i = l.lastIndexOf('\r', l.length - 2); return (i >= 0 ? l.slice(i + 1) : l).replace(/\s+$/, ''); });
45
+ while (clean.length && !clean.at(-1)) clean.pop();
46
+ const total = clean.length;
47
+ const plain = clean.join('\n');
48
+ if (tokens(plain) <= VERBATIM) return { text: plain, total, shown: total };
49
+
50
+ const keep = new Uint8Array(total); // 0 hide, 1 context, 2 important
51
+ const mark = (i, v = 1) => { if (i >= 0 && i < total && keep[i] < v) keep[i] = v; };
52
+ let errors = 0, warns = 0;
53
+ for (let i = 0; i < total; i++) {
54
+ const l = clean[i];
55
+ if (ERR.test(l) && !NOISE.test(l)) {
56
+ if (!/\b\d+\s+(?:failed|failures?|errors?)\b/i.test(l)) errors++; // "1 failed, 9 passed" is a summary
57
+ mark(i, 2);
58
+ for (let j = i - 1; j >= i - 2; j--) if (j >= 0 && clean[j] && !NOISE.test(clean[j])) mark(j);
59
+ // Follow the block that explains the error (indented lines, code frames, stack frames,
60
+ // diffs), across single blank lines, until the next unindented line.
61
+ for (let j = i + 1, n = 0; j < total && n < 16; j++, n++) {
62
+ const l = clean[j];
63
+ if (!l) { if (!clean[j + 1] || !/^\s/.test(clean[j + 1])) break; continue; }
64
+ if (!/^\s/.test(l) && n >= 2) break;
65
+ if (/^\s+at\s/.test(clean[j - 1] || '') && !/^\s+at\s/.test(l)) break; // stack trace ended
66
+ mark(j);
67
+ }
68
+ } else if (WARN.test(l) && !NOISE.test(l) && warns < 10) { warns++; mark(i, 2); }
69
+ }
70
+ for (let i = 0; i < Math.min(3, total); i++) if (!NOISE.test(clean[i])) mark(i);
71
+ // The tail is where runners print their summary: keep summary-looking lines near the end,
72
+ // and the last few lines whatever they are.
73
+ for (let i = total - 1, n = 0; i >= 0 && n < 4; i--) if (clean[i] && !NOISE.test(clean[i])) { mark(i); n++; }
74
+ for (let i = total - 1, n = 0; i >= 0 && n < 3; i--) if (clean[i]) { mark(i); n++; } // never show nothing
75
+ for (let i = Math.max(0, total - 40); i < total; i++) if (SUMMARY.test(clean[i]) && !NOISE.test(clean[i]) && /\d/.test(clean[i])) mark(i);
76
+
77
+ // Collapse runs of identical lines, render gaps, and stop at the budget.
78
+ const out = []; let used = 0, gap = 0, shown = 0, cut = false;
79
+ for (let i = 0; i < total; i++) {
80
+ if (!keep[i]) { if (clean[i]) gap++; continue; } // hidden blank lines aren't worth a marker
81
+ let rep = 1; while (i + rep < total && keep[i + rep] && clean[i + rep] === clean[i]) rep++;
82
+ const line = (gap ? ` … ${num(gap)} line${gap > 1 ? 's' : ''} hidden\n` : '') + clean[i].slice(0, 400) + (rep > 1 ? ` (×${rep})` : '');
83
+ if (used + tokens(line) > budget && i < total - 12) { // over budget: jump to the summary tail
84
+ cut = true; gap += (total - 12) - i; i = total - 13; continue;
85
+ }
86
+ out.push(line); used += tokens(line); shown += rep; gap = 0; i += rep - 1;
87
+ }
88
+ if (gap) out.push(` … ${num(gap)} line${gap > 1 ? 's' : ''} hidden`);
89
+ if (cut) out.push(' … more matches cut to stay under budget: grep the full log for them');
90
+ return { text: out.join('\n'), total, shown, errors };
91
+ }
92
+
93
+ const shq = (a) => (/^[\w@%+=:,./-]+$/.test(a) ? a : `'${a.replace(/'/g, `'\\''`)}'`);
94
+
95
+ // Run a command, save the full log, print the condensed view. Returns the exit code.
96
+ export async function run(argv, root, { write = (s) => process.stdout.write(s), passthrough = '' } = {}) {
97
+ const cmd = argv.length === 1 ? argv[0] : argv.map(shq).join(' ');
98
+ if (!cmd) { write('Usage: tokenbuild run <command>\n'); return 2; }
99
+ if (passthrough) { // no active plan: run the command exactly as if TokenBuild weren't there
100
+ const c = spawn(cmd, { cwd: process.cwd(), shell: process.env.SHELL || true, stdio: 'inherit' });
101
+ const code = await new Promise((ok) => { c.on('error', () => ok(127)); c.on('close', (x, sig) => ok(x ?? (sig ? 128 : 1))); });
102
+ process.stderr.write(`[${passthrough}]\n`);
103
+ return code;
104
+ }
105
+ // One pipe for both streams (via the shell) so errors stay next to the lines that caused them.
106
+ const child = spawn(`{ ${cmd}\n} 2>&1`, { cwd: process.cwd(), shell: process.env.SHELL || true, stdio: ['inherit', 'pipe', 'pipe'], env: { ...process.env, CI: process.env.CI || '1', FORCE_COLOR: '0', NO_COLOR: '1' } });
107
+ const chunks = []; let bytes = 0, dropped = 0;
108
+ const take = (c) => { chunks.push(c); bytes += c.length; while (bytes > MAX_BYTES && chunks.length > 2) { const x = chunks.splice(1, 1)[0]; bytes -= x.length; dropped += x.length; } };
109
+ child.stdout.on('data', take); child.stderr.on('data', take);
110
+ const stop = (sig) => () => { child.kill(sig); };
111
+ const handlers = { SIGINT: stop('SIGINT'), SIGTERM: stop('SIGTERM') };
112
+ for (const [s, h] of Object.entries(handlers)) process.on(s, h);
113
+ const exit = await new Promise((ok) => { child.on('error', () => ok(127)); child.on('close', (code, sig) => ok(code ?? (sig ? 128 : 1))); });
114
+ for (const [s, h] of Object.entries(handlers)) process.off(s, h);
115
+
116
+ let raw = Buffer.concat(chunks).toString('utf8');
117
+ if (dropped) raw = raw.replace(/\n/, `\n[… ${num(dropped)} bytes dropped from the middle …]\n`);
118
+ const r = shrink(raw, { exit });
119
+ const full = tokens(raw.replace(ANSI, ''));
120
+ if (r.shown === r.total) { write(r.text + (r.text ? '\n' : '')); if (exit) write(`[exit ${exit}]\n`); return exit; }
121
+
122
+ const dir = path.join(tbDir(root), 'logs');
123
+ fs.mkdirSync(dir, { recursive: true });
124
+ const name = `${new Date().toISOString().replace(/[-:]/g, '').replace('T', '-').slice(0, 15)}-${cmd.replace(/[^\w]+/g, '-').slice(0, 40).replace(/^-|-$/g, '')}.log`;
125
+ const logPath = path.join(dir, name);
126
+ fs.writeFileSync(logPath, raw);
127
+ try { const old = fs.readdirSync(dir).filter((f) => f.endsWith('.log')).sort(); for (const f of old.slice(0, -KEEP_LOGS)) fs.rmSync(path.join(dir, f)); } catch {}
128
+ const rel = path.relative(process.cwd(), logPath);
129
+ const head = `[tokenbuild] exit ${exit} · ${num(r.total)} lines → ${num(r.shown)} shown${r.errors ? ` · ${num(r.errors)} error line${r.errors > 1 ? 's' : ''}` : ''} · full log: ${rel}`;
130
+ write(`${head}\n${r.text}\n`);
131
+ appendLedger(root, { ev: 'shrink', tok: Math.max(0, full - tokens(head + r.text)), full });
132
+ return exit;
133
+ }
package/src/util.mjs ADDED
@@ -0,0 +1,62 @@
1
+ import fs from 'node:fs';
2
+ import path from 'node:path';
3
+ import crypto from 'node:crypto';
4
+
5
+ // ~4 chars per token for code and English; good enough for budgets.
6
+ export const tokens = (s) => Math.ceil((s ? String(s).length : 0) / 4);
7
+
8
+ export const JUNK = [
9
+ /(^|\/)node_modules\//, /(^|\/)\.git\//, /(^|\/)(dist|build|out|coverage|\.next|\.nuxt|\.turbo|\.cache|vendor|target|__pycache__|\.venv|venv)\//,
10
+ /(^|\/)(package-lock\.json|yarn\.lock|pnpm-lock\.yaml|bun\.lockb|Cargo\.lock|poetry\.lock|Gemfile\.lock|composer\.lock|go\.sum)$/,
11
+ /\.min\.(js|css)$/, /\.map$/, /\.(png|jpe?g|gif|webp|ico|pdf|zip|gz|woff2?|ttf|mp4|mov|so|dylib|dll|exe|bin)$/i,
12
+ ];
13
+ export const isJunk = (p) => JUNK.some((re) => re.test(p.replaceAll('\\', '/')));
14
+
15
+ const SKIP_DIRS = new Set(['node_modules', '.git', 'dist', 'build', 'out', 'coverage', '.next', '.nuxt', '.turbo', '.cache', 'vendor', 'target', '__pycache__', '.venv', 'venv', '.tokenbuild']);
16
+
17
+ function gitignore(root) {
18
+ try {
19
+ return fs.readFileSync(path.join(root, '.gitignore'), 'utf8').split('\n')
20
+ .map((l) => l.trim()).filter((l) => l && !l.startsWith('#') && !l.startsWith('!'))
21
+ .map((g) => new RegExp('(^|/)' + g.replace(/^\//, '').replace(/\/$/, '').replace(/[.+^${}()|[\]\\]/g, '\\$&').replace(/\*\*/g, '.*').replace(/\*/g, '[^/]*') + '(/|$)'));
22
+ } catch { return []; }
23
+ }
24
+
25
+ export function* walk(root, maxFiles = 20000) {
26
+ const ignore = gitignore(root);
27
+ const stack = [''];
28
+ let n = 0;
29
+ while (stack.length) {
30
+ const rel = stack.pop();
31
+ let entries;
32
+ try { entries = fs.readdirSync(path.join(root, rel), { withFileTypes: true }); } catch { continue; }
33
+ for (const e of entries) {
34
+ const r = rel ? `${rel}/${e.name}` : e.name;
35
+ if (ignore.some((re) => re.test(r))) continue;
36
+ if (e.isDirectory()) { if (!SKIP_DIRS.has(e.name) && !e.name.startsWith('.')) stack.push(r); }
37
+ else if (e.isFile() && !isJunk(r)) { if (++n > maxFiles) return; yield r; }
38
+ }
39
+ }
40
+ }
41
+
42
+ export const hash = (s) => crypto.createHash('sha1').update(s).digest('hex').slice(0, 12);
43
+
44
+ export function readJSON(p, fallback) {
45
+ try { return JSON.parse(fs.readFileSync(p, 'utf8')); } catch { return fallback; }
46
+ }
47
+
48
+ export function writeJSON(p, v) {
49
+ fs.mkdirSync(path.dirname(p), { recursive: true });
50
+ fs.writeFileSync(p, JSON.stringify(v));
51
+ }
52
+
53
+ export const tbDir = (root) => path.join(root, '.tokenbuild');
54
+
55
+ export function appendLedger(root, event) {
56
+ const p = path.join(tbDir(root), 'ledger.jsonl');
57
+ fs.mkdirSync(path.dirname(p), { recursive: true });
58
+ fs.appendFileSync(p, JSON.stringify({ t: Date.now(), ...event }) + '\n');
59
+ }
60
+
61
+ export const usd = (n) => '$' + (n >= 100 ? n.toFixed(0) : n.toFixed(2));
62
+ export const num = (n) => n >= 1e6 ? (n / 1e6).toFixed(1) + 'M' : n >= 1e3 ? (n / 1e3).toFixed(1) + 'K' : String(Math.round(n));