tokenlean 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +44 -1
- package/bin/tl-cache.mjs +168 -0
- package/bin/tl-entry.mjs +7 -1
- package/bin/tl-impact.mjs +10 -2
- package/bin/tl-related.mjs +10 -4
- package/bin/tl-search.mjs +10 -4
- package/bin/tl-todo.mjs +8 -1
- package/bin/tl-unused.mjs +29 -19
- package/package.json +2 -1
- package/src/cache.mjs +493 -0
- package/src/config.mjs +6 -0
package/README.md
CHANGED
|
@@ -18,7 +18,7 @@ the API surface.
|
|
|
18
18
|
|
|
19
19
|
## The Solution
|
|
20
20
|
|
|
21
|
-
tokenlean provides **
|
|
21
|
+
tokenlean provides **26 specialized CLI tools** that give you (or your AI agent) exactly the information needed - no
|
|
22
22
|
more, no less. Each tool is designed to answer a specific question about your codebase with minimal token overhead.
|
|
23
23
|
|
|
24
24
|
Instead of reading a 500-line file to understand its exports, run `tl-exports` (~50 tokens). Instead of reading all your
|
|
@@ -128,6 +128,7 @@ Search and discover code patterns.
|
|
|
128
128
|
|
|
129
129
|
| Tool | Description | Example |
|
|
130
130
|
|-------------|--------------------------------|-----------------------|
|
|
131
|
+
| `tl-cache` | Manage ripgrep result cache | `tl-cache stats` |
|
|
131
132
|
| `tl-config` | Show/manage configuration | `tl-config --init` |
|
|
132
133
|
| `tl-prompt` | Generate AI agent instructions | `tl-prompt --minimal` |
|
|
133
134
|
|
|
@@ -190,6 +191,48 @@ Project config overrides global config. Both are optional.
|
|
|
190
191
|
|
|
191
192
|
Config values extend built-in defaults (they don't replace them).
|
|
192
193
|
|
|
194
|
+
## Caching
|
|
195
|
+
|
|
196
|
+
tokenlean caches expensive ripgrep operations to speed up repeated searches. The cache uses **git-based invalidation** - it automatically invalidates when you make commits or modify files.
|
|
197
|
+
|
|
198
|
+
```bash
|
|
199
|
+
tl-cache stats # View cache statistics
|
|
200
|
+
tl-cache clear # Clear cache for current project
|
|
201
|
+
tl-cache clear-all # Clear all cached data
|
|
202
|
+
```
|
|
203
|
+
|
|
204
|
+
Cache is stored in `~/.tokenlean/cache/` and is enabled by default.
|
|
205
|
+
|
|
206
|
+
### Cache Configuration
|
|
207
|
+
|
|
208
|
+
```json
|
|
209
|
+
{
|
|
210
|
+
"cache": {
|
|
211
|
+
"enabled": true,
|
|
212
|
+
"ttl": 300,
|
|
213
|
+
"maxSize": "100MB",
|
|
214
|
+
"location": null
|
|
215
|
+
}
|
|
216
|
+
}
|
|
217
|
+
```
|
|
218
|
+
|
|
219
|
+
| Option | Default | Description |
|
|
220
|
+
|------------|--------------------|--------------------------------------------|
|
|
221
|
+
| `enabled` | `true` | Enable/disable caching |
|
|
222
|
+
| `ttl` | `300` | Max age in seconds (fallback for non-git) |
|
|
223
|
+
| `maxSize` | `"100MB"` | Max cache size per project |
|
|
224
|
+
| `location` | `null` | Override default `~/.tokenlean/cache` |
|
|
225
|
+
|
|
226
|
+
### Disable Caching
|
|
227
|
+
|
|
228
|
+
```bash
|
|
229
|
+
# Disable for a single command
|
|
230
|
+
TOKENLEAN_CACHE=0 tl-search hooks
|
|
231
|
+
|
|
232
|
+
# Disable in config
|
|
233
|
+
echo '{"cache":{"enabled":false}}' > .tokenleanrc.json
|
|
234
|
+
```
|
|
235
|
+
|
|
193
236
|
## Example Workflows
|
|
194
237
|
|
|
195
238
|
### Starting work on an unfamiliar codebase
|
package/bin/tl-cache.mjs
ADDED
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* tl-cache - Manage tokenlean cache
|
|
5
|
+
*
|
|
6
|
+
* View cache statistics and clear cached data.
|
|
7
|
+
*
|
|
8
|
+
* Usage: tl-cache [command]
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
// Prompt info for tl-prompt
|
|
12
|
+
if (process.argv.includes('--prompt')) {
|
|
13
|
+
console.log(JSON.stringify({
|
|
14
|
+
name: 'tl-cache',
|
|
15
|
+
desc: 'Manage tokenlean cache (stats, clear)',
|
|
16
|
+
when: 'maintenance',
|
|
17
|
+
example: 'tl-cache stats'
|
|
18
|
+
}));
|
|
19
|
+
process.exit(0);
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
import {
|
|
23
|
+
createOutput,
|
|
24
|
+
parseCommonArgs,
|
|
25
|
+
COMMON_OPTIONS_HELP
|
|
26
|
+
} from '../src/output.mjs';
|
|
27
|
+
import { findProjectRoot } from '../src/project.mjs';
|
|
28
|
+
import {
|
|
29
|
+
getCacheConfig,
|
|
30
|
+
getCacheStats,
|
|
31
|
+
clearCache,
|
|
32
|
+
getCacheDir
|
|
33
|
+
} from '../src/cache.mjs';
|
|
34
|
+
|
|
35
|
+
const HELP = `
|
|
36
|
+
tl-cache - Manage tokenlean cache
|
|
37
|
+
|
|
38
|
+
Usage: tl-cache <command> [options]
|
|
39
|
+
|
|
40
|
+
Commands:
|
|
41
|
+
stats Show cache statistics (default)
|
|
42
|
+
clear Clear cache for current project
|
|
43
|
+
clear-all Clear cache for all projects
|
|
44
|
+
|
|
45
|
+
${COMMON_OPTIONS_HELP}
|
|
46
|
+
|
|
47
|
+
Examples:
|
|
48
|
+
tl-cache # Show stats for current project
|
|
49
|
+
tl-cache stats # Same as above
|
|
50
|
+
tl-cache clear # Clear cache for this project
|
|
51
|
+
tl-cache clear-all # Clear all cached data
|
|
52
|
+
|
|
53
|
+
Configuration:
|
|
54
|
+
Cache can be configured in .tokenleanrc.json:
|
|
55
|
+
{
|
|
56
|
+
"cache": {
|
|
57
|
+
"enabled": true, // Enable/disable caching
|
|
58
|
+
"ttl": 300, // Max age in seconds (for non-git repos)
|
|
59
|
+
"maxSize": "100MB", // Max cache size per project
|
|
60
|
+
"location": null // Override ~/.tokenlean/cache
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
Environment:
|
|
65
|
+
TOKENLEAN_CACHE=0 Disable caching for this run
|
|
66
|
+
`;
|
|
67
|
+
|
|
68
|
+
// ─────────────────────────────────────────────────────────────
|
|
69
|
+
// Commands
|
|
70
|
+
// ─────────────────────────────────────────────────────────────
|
|
71
|
+
|
|
72
|
+
function showStats(out, projectRoot) {
|
|
73
|
+
const config = getCacheConfig();
|
|
74
|
+
const projectStats = getCacheStats(projectRoot);
|
|
75
|
+
const globalStats = getCacheStats(null);
|
|
76
|
+
|
|
77
|
+
out.header('Cache Configuration:');
|
|
78
|
+
out.add(` Enabled: ${config.enabled ? 'yes' : 'no'}`);
|
|
79
|
+
out.add(` Location: ${config.location}`);
|
|
80
|
+
out.add(` Max size: ${projectStats.maxSizeFormatted}`);
|
|
81
|
+
out.add(` TTL: ${config.ttl}s (fallback for non-git repos)`);
|
|
82
|
+
out.blank();
|
|
83
|
+
|
|
84
|
+
out.header('Current Project Cache:');
|
|
85
|
+
out.add(` Directory: ${projectStats.location}`);
|
|
86
|
+
out.add(` Entries: ${projectStats.entries}`);
|
|
87
|
+
out.add(` Size: ${projectStats.sizeFormatted}`);
|
|
88
|
+
out.blank();
|
|
89
|
+
|
|
90
|
+
out.header('Global Cache:');
|
|
91
|
+
out.add(` Projects: ${globalStats.projects}`);
|
|
92
|
+
out.add(` Entries: ${globalStats.totalEntries}`);
|
|
93
|
+
out.add(` Total size: ${globalStats.totalSizeFormatted}`);
|
|
94
|
+
out.blank();
|
|
95
|
+
|
|
96
|
+
// Set JSON data
|
|
97
|
+
out.setData('config', config);
|
|
98
|
+
out.setData('project', projectStats);
|
|
99
|
+
out.setData('global', globalStats);
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
function doClear(out, projectRoot) {
|
|
103
|
+
const before = getCacheStats(projectRoot);
|
|
104
|
+
clearCache(projectRoot);
|
|
105
|
+
const after = getCacheStats(projectRoot);
|
|
106
|
+
|
|
107
|
+
out.add(`Cleared ${before.entries} cache entries (${before.sizeFormatted})`);
|
|
108
|
+
out.blank();
|
|
109
|
+
|
|
110
|
+
out.setData('cleared', {
|
|
111
|
+
entries: before.entries,
|
|
112
|
+
size: before.size,
|
|
113
|
+
sizeFormatted: before.sizeFormatted
|
|
114
|
+
});
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
function doClearAll(out) {
|
|
118
|
+
const before = getCacheStats(null);
|
|
119
|
+
clearCache(null);
|
|
120
|
+
const after = getCacheStats(null);
|
|
121
|
+
|
|
122
|
+
out.add(`Cleared ${before.totalEntries} cache entries across ${before.projects} projects (${before.totalSizeFormatted})`);
|
|
123
|
+
out.blank();
|
|
124
|
+
|
|
125
|
+
out.setData('cleared', {
|
|
126
|
+
projects: before.projects,
|
|
127
|
+
entries: before.totalEntries,
|
|
128
|
+
size: before.totalSize,
|
|
129
|
+
sizeFormatted: before.totalSizeFormatted
|
|
130
|
+
});
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
// ─────────────────────────────────────────────────────────────
|
|
134
|
+
// Main
|
|
135
|
+
// ─────────────────────────────────────────────────────────────
|
|
136
|
+
|
|
137
|
+
const args = process.argv.slice(2);
|
|
138
|
+
const options = parseCommonArgs(args);
|
|
139
|
+
|
|
140
|
+
if (options.help) {
|
|
141
|
+
console.log(HELP);
|
|
142
|
+
process.exit(0);
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
const command = options.remaining.find(a => !a.startsWith('-')) || 'stats';
|
|
146
|
+
const projectRoot = findProjectRoot();
|
|
147
|
+
const out = createOutput(options);
|
|
148
|
+
|
|
149
|
+
switch (command) {
|
|
150
|
+
case 'stats':
|
|
151
|
+
showStats(out, projectRoot);
|
|
152
|
+
break;
|
|
153
|
+
|
|
154
|
+
case 'clear':
|
|
155
|
+
doClear(out, projectRoot);
|
|
156
|
+
break;
|
|
157
|
+
|
|
158
|
+
case 'clear-all':
|
|
159
|
+
doClearAll(out);
|
|
160
|
+
break;
|
|
161
|
+
|
|
162
|
+
default:
|
|
163
|
+
console.error(`Unknown command: ${command}`);
|
|
164
|
+
console.log('Use: stats, clear, or clear-all');
|
|
165
|
+
process.exit(1);
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
out.print();
|
package/bin/tl-entry.mjs
CHANGED
|
@@ -30,6 +30,7 @@ import {
|
|
|
30
30
|
COMMON_OPTIONS_HELP
|
|
31
31
|
} from '../src/output.mjs';
|
|
32
32
|
import { findProjectRoot } from '../src/project.mjs';
|
|
33
|
+
import { withCache } from '../src/cache.mjs';
|
|
33
34
|
|
|
34
35
|
const HELP = `
|
|
35
36
|
tl-entry - Find entry points in a codebase
|
|
@@ -130,7 +131,12 @@ function findEntryPoints(searchPath, projectRoot, filterType) {
|
|
|
130
131
|
for (const { pattern, desc } of config.patterns) {
|
|
131
132
|
try {
|
|
132
133
|
const cmd = `rg -n -g "*.{ts,tsx,js,jsx,mjs}" --no-heading -e "${shellEscape(pattern)}" "${shellEscape(searchPath)}" 2>/dev/null || true`;
|
|
133
|
-
const
|
|
134
|
+
const cacheKey = { op: 'rg-entry-pattern', pattern, path: searchPath };
|
|
135
|
+
const output = withCache(
|
|
136
|
+
cacheKey,
|
|
137
|
+
() => execSync(cmd, { encoding: 'utf-8', maxBuffer: 10 * 1024 * 1024 }),
|
|
138
|
+
{ projectRoot }
|
|
139
|
+
);
|
|
134
140
|
|
|
135
141
|
for (const line of output.trim().split('\n')) {
|
|
136
142
|
if (!line) continue;
|
package/bin/tl-impact.mjs
CHANGED
|
@@ -33,6 +33,7 @@ import {
|
|
|
33
33
|
COMMON_OPTIONS_HELP
|
|
34
34
|
} from '../src/output.mjs';
|
|
35
35
|
import { findProjectRoot, categorizeFile } from '../src/project.mjs';
|
|
36
|
+
import { withCache } from '../src/cache.mjs';
|
|
36
37
|
|
|
37
38
|
const HELP = `
|
|
38
39
|
tl-impact - Analyze the blast radius of changing a file
|
|
@@ -79,8 +80,15 @@ function findDirectImporters(filePath, projectRoot) {
|
|
|
79
80
|
const patterns = searchTerms.map(t => `-e "${rgEscape(t)}"`).join(' ');
|
|
80
81
|
|
|
81
82
|
try {
|
|
82
|
-
const
|
|
83
|
-
const result =
|
|
83
|
+
const cacheKey = { op: 'rg-find-candidates', terms: searchTerms.sort() };
|
|
84
|
+
const result = withCache(
|
|
85
|
+
cacheKey,
|
|
86
|
+
() => {
|
|
87
|
+
const rgCommand = `rg -l --type-add 'code:*.{js,jsx,ts,tsx,mjs,mts,cjs}' -t code ${patterns} "${projectRoot}" 2>/dev/null || true`;
|
|
88
|
+
return execSync(rgCommand, { encoding: 'utf-8', maxBuffer: 10 * 1024 * 1024 });
|
|
89
|
+
},
|
|
90
|
+
{ projectRoot }
|
|
91
|
+
);
|
|
84
92
|
const candidates = result.trim().split('\n').filter(Boolean);
|
|
85
93
|
|
|
86
94
|
for (const candidate of candidates) {
|
package/bin/tl-related.mjs
CHANGED
|
@@ -32,6 +32,7 @@ import {
|
|
|
32
32
|
COMMON_OPTIONS_HELP
|
|
33
33
|
} from '../src/output.mjs';
|
|
34
34
|
import { findProjectRoot } from '../src/project.mjs';
|
|
35
|
+
import { withCache } from '../src/cache.mjs';
|
|
35
36
|
|
|
36
37
|
const HELP = `
|
|
37
38
|
tl-related - Find related files (tests, types, usages)
|
|
@@ -120,11 +121,16 @@ function findImporters(filePath, projectRoot) {
|
|
|
120
121
|
const name = basename(filePath, extname(filePath));
|
|
121
122
|
const importers = new Set();
|
|
122
123
|
|
|
123
|
-
// Search for files that might import this module
|
|
124
|
+
// Search for files that might import this module (with caching)
|
|
124
125
|
try {
|
|
125
|
-
const
|
|
126
|
-
|
|
127
|
-
|
|
126
|
+
const cacheKey = { op: 'rg-find-importers', module: name, glob: '*.{js,mjs,ts,tsx,jsx}' };
|
|
127
|
+
const result = withCache(
|
|
128
|
+
cacheKey,
|
|
129
|
+
() => execSync(
|
|
130
|
+
`rg -l -g "*.{js,mjs,ts,tsx,jsx}" -e "${shellEscape(name)}" "${shellEscape(projectRoot)}" 2>/dev/null || true`,
|
|
131
|
+
{ encoding: 'utf-8', maxBuffer: 5 * 1024 * 1024 }
|
|
132
|
+
),
|
|
133
|
+
{ projectRoot }
|
|
128
134
|
);
|
|
129
135
|
|
|
130
136
|
for (const line of result.trim().split('\n')) {
|
package/bin/tl-search.mjs
CHANGED
|
@@ -28,6 +28,7 @@ import {
|
|
|
28
28
|
COMMON_OPTIONS_HELP
|
|
29
29
|
} from '../src/output.mjs';
|
|
30
30
|
import { loadConfig, CONFIG_FILENAME } from '../src/config.mjs';
|
|
31
|
+
import { withCache } from '../src/cache.mjs';
|
|
31
32
|
|
|
32
33
|
const HELP = `
|
|
33
34
|
tl-search - Run pre-defined search patterns
|
|
@@ -165,10 +166,15 @@ function runSearch(name, config, rootDir, jsonMode) {
|
|
|
165
166
|
|
|
166
167
|
if (jsonMode) {
|
|
167
168
|
try {
|
|
168
|
-
const
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
169
|
+
const cacheKey = { op: 'rg-search-pattern', name, pattern: config.pattern, glob: config.glob };
|
|
170
|
+
const result = withCache(
|
|
171
|
+
cacheKey,
|
|
172
|
+
() => execSync(`rg ${args.map(a => `"${a}"`).join(' ')} 2>/dev/null || true`, {
|
|
173
|
+
encoding: 'utf-8',
|
|
174
|
+
maxBuffer: 10 * 1024 * 1024
|
|
175
|
+
}),
|
|
176
|
+
{ projectRoot: rootDir }
|
|
177
|
+
);
|
|
172
178
|
const matches = result.trim().split('\n')
|
|
173
179
|
.filter(line => line.startsWith('{'))
|
|
174
180
|
.map(line => {
|
package/bin/tl-todo.mjs
CHANGED
|
@@ -32,6 +32,7 @@ import {
|
|
|
32
32
|
COMMON_OPTIONS_HELP
|
|
33
33
|
} from '../src/output.mjs';
|
|
34
34
|
import { findProjectRoot, shouldSkip, SKIP_DIRS } from '../src/project.mjs';
|
|
35
|
+
import { withCache } from '../src/cache.mjs';
|
|
35
36
|
|
|
36
37
|
const HELP = `
|
|
37
38
|
tl-todo - Extract TODOs, FIXMEs, and other markers from codebase
|
|
@@ -102,7 +103,13 @@ function findTodos(searchPath, projectRoot) {
|
|
|
102
103
|
// Require : or ( after marker to avoid false positives like "Todo Extraction"
|
|
103
104
|
const commentPattern = `(${commentPrefixes.join('|')})\\s*(${markerPattern})[:(]`;
|
|
104
105
|
const cmd = `rg -n --no-heading -i "${commentPattern}" "${shellEscape(searchPath)}" ${excludes} 2>/dev/null || true`;
|
|
105
|
-
|
|
106
|
+
|
|
107
|
+
const cacheKey = { op: 'rg-todo-markers', pattern: commentPattern, path: searchPath };
|
|
108
|
+
const output = withCache(
|
|
109
|
+
cacheKey,
|
|
110
|
+
() => execSync(cmd, { encoding: 'utf-8', maxBuffer: 50 * 1024 * 1024 }),
|
|
111
|
+
{ projectRoot }
|
|
112
|
+
);
|
|
106
113
|
|
|
107
114
|
if (!output.trim()) {
|
|
108
115
|
return todos;
|
package/bin/tl-unused.mjs
CHANGED
|
@@ -29,6 +29,7 @@ import {
|
|
|
29
29
|
COMMON_OPTIONS_HELP
|
|
30
30
|
} from '../src/output.mjs';
|
|
31
31
|
import { findProjectRoot, shouldSkip, isCodeFile } from '../src/project.mjs';
|
|
32
|
+
import { withCache } from '../src/cache.mjs';
|
|
32
33
|
|
|
33
34
|
const HELP = `
|
|
34
35
|
tl-unused - Find potentially unused exports and unreferenced files
|
|
@@ -221,26 +222,35 @@ function extractImports(content) {
|
|
|
221
222
|
}
|
|
222
223
|
|
|
223
224
|
function findReferencesWithGrep(name, projectRoot, excludeFile) {
|
|
224
|
-
// Use ripgrep for fast reference counting
|
|
225
|
-
const
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
225
|
+
// Use ripgrep for fast reference counting (with caching)
|
|
226
|
+
const cacheKey = { op: 'rg-ref-count', name, types: 'js,ts' };
|
|
227
|
+
|
|
228
|
+
const files = withCache(
|
|
229
|
+
cacheKey,
|
|
230
|
+
() => {
|
|
231
|
+
const args = [
|
|
232
|
+
'-l', // Files only
|
|
233
|
+
'--type', 'js',
|
|
234
|
+
'--type', 'ts',
|
|
235
|
+
'-w', // Word boundary
|
|
236
|
+
name,
|
|
237
|
+
'.'
|
|
238
|
+
];
|
|
239
|
+
|
|
240
|
+
const result = spawnSync('rg', args, {
|
|
241
|
+
cwd: projectRoot,
|
|
242
|
+
encoding: 'utf-8'
|
|
243
|
+
});
|
|
244
|
+
|
|
245
|
+
if (result.error || result.status !== 0) {
|
|
246
|
+
return [];
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
return result.stdout.trim().split('\n').filter(Boolean);
|
|
250
|
+
},
|
|
251
|
+
{ projectRoot }
|
|
252
|
+
);
|
|
242
253
|
|
|
243
|
-
const files = result.stdout.trim().split('\n').filter(Boolean);
|
|
244
254
|
// Exclude the file that exports it
|
|
245
255
|
const relExclude = relative(projectRoot, excludeFile);
|
|
246
256
|
const otherFiles = files.filter(f => f !== relExclude && !f.includes(relExclude));
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "tokenlean",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.3.0",
|
|
4
4
|
"description": "Lean CLI tools for AI agents and developers - reduce context, save tokens",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"engines": {
|
|
@@ -22,6 +22,7 @@
|
|
|
22
22
|
"bin": {
|
|
23
23
|
"tl-api": "./bin/tl-api.mjs",
|
|
24
24
|
"tl-blame": "./bin/tl-blame.mjs",
|
|
25
|
+
"tl-cache": "./bin/tl-cache.mjs",
|
|
25
26
|
"tl-config": "./bin/tl-config.mjs",
|
|
26
27
|
"tl-context": "./bin/tl-context.mjs",
|
|
27
28
|
"tl-coverage": "./bin/tl-coverage.mjs",
|
package/src/cache.mjs
ADDED
|
@@ -0,0 +1,493 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Shared caching system for tokenlean CLI tools
|
|
3
|
+
*
|
|
4
|
+
* Provides disk-based caching with git-based invalidation for expensive
|
|
5
|
+
* ripgrep operations. Falls back to TTL-based invalidation when not in a git repo.
|
|
6
|
+
*
|
|
7
|
+
* Cache storage: ~/.tokenlean/cache/<project-hash>/<key-hash>.json
|
|
8
|
+
*
|
|
9
|
+
* Usage:
|
|
10
|
+
* // High-level API (preferred)
|
|
11
|
+
* const result = withCache(
|
|
12
|
+
* { op: 'rg-search', pattern: 'useState', glob: '*.tsx' },
|
|
13
|
+
* () => execSync('rg ...'),
|
|
14
|
+
* { projectRoot }
|
|
15
|
+
* );
|
|
16
|
+
*
|
|
17
|
+
* // Low-level API
|
|
18
|
+
* let data = getCached(key, projectRoot);
|
|
19
|
+
* if (!data) {
|
|
20
|
+
* data = computeExpensiveResult();
|
|
21
|
+
* setCached(key, data, projectRoot);
|
|
22
|
+
* }
|
|
23
|
+
*/
|
|
24
|
+
|
|
25
|
+
import { existsSync, mkdirSync, readFileSync, writeFileSync, readdirSync, statSync, unlinkSync, rmSync } from 'fs';
|
|
26
|
+
import { join, dirname } from 'path';
|
|
27
|
+
import { homedir } from 'os';
|
|
28
|
+
import { execSync } from 'child_process';
|
|
29
|
+
import { createHash } from 'crypto';
|
|
30
|
+
import { loadConfig } from './config.mjs';
|
|
31
|
+
|
|
32
|
+
// ─────────────────────────────────────────────────────────────
|
|
33
|
+
// Constants
|
|
34
|
+
// ─────────────────────────────────────────────────────────────
|
|
35
|
+
|
|
36
|
+
const DEFAULT_CACHE_DIR = join(homedir(), '.tokenlean', 'cache');
|
|
37
|
+
const DEFAULT_TTL = 300; // 5 minutes fallback for non-git repos
|
|
38
|
+
const DEFAULT_MAX_SIZE = 100 * 1024 * 1024; // 100MB
|
|
39
|
+
|
|
40
|
+
// ─────────────────────────────────────────────────────────────
|
|
41
|
+
// Configuration
|
|
42
|
+
// ─────────────────────────────────────────────────────────────
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* Get cache configuration from config system
|
|
46
|
+
*/
|
|
47
|
+
export function getCacheConfig() {
|
|
48
|
+
const { config } = loadConfig();
|
|
49
|
+
const cacheConfig = config.cache || {};
|
|
50
|
+
|
|
51
|
+
return {
|
|
52
|
+
enabled: cacheConfig.enabled !== false && process.env.TOKENLEAN_CACHE !== '0',
|
|
53
|
+
ttl: cacheConfig.ttl ?? DEFAULT_TTL,
|
|
54
|
+
maxSize: parseSize(cacheConfig.maxSize) ?? DEFAULT_MAX_SIZE,
|
|
55
|
+
location: cacheConfig.location ?? DEFAULT_CACHE_DIR
|
|
56
|
+
};
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/**
|
|
60
|
+
* Parse size string like '100MB' to bytes
|
|
61
|
+
*/
|
|
62
|
+
function parseSize(size) {
|
|
63
|
+
if (typeof size === 'number') return size;
|
|
64
|
+
if (typeof size !== 'string') return null;
|
|
65
|
+
|
|
66
|
+
const match = size.match(/^(\d+(?:\.\d+)?)\s*(B|KB|MB|GB)?$/i);
|
|
67
|
+
if (!match) return null;
|
|
68
|
+
|
|
69
|
+
const num = parseFloat(match[1]);
|
|
70
|
+
const unit = (match[2] || 'B').toUpperCase();
|
|
71
|
+
|
|
72
|
+
const multipliers = {
|
|
73
|
+
'B': 1,
|
|
74
|
+
'KB': 1024,
|
|
75
|
+
'MB': 1024 * 1024,
|
|
76
|
+
'GB': 1024 * 1024 * 1024
|
|
77
|
+
};
|
|
78
|
+
|
|
79
|
+
return Math.floor(num * multipliers[unit]);
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
// ─────────────────────────────────────────────────────────────
|
|
83
|
+
// Hashing Utilities
|
|
84
|
+
// ─────────────────────────────────────────────────────────────
|
|
85
|
+
|
|
86
|
+
/**
|
|
87
|
+
* Create a short hash from any value
|
|
88
|
+
*/
|
|
89
|
+
function hash(value) {
|
|
90
|
+
const str = typeof value === 'string' ? value : JSON.stringify(value);
|
|
91
|
+
return createHash('sha256').update(str).digest('hex').slice(0, 16);
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
/**
|
|
95
|
+
* Get hash of project root path for cache directory
|
|
96
|
+
*/
|
|
97
|
+
function getProjectHash(projectRoot) {
|
|
98
|
+
return hash(projectRoot);
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
/**
|
|
102
|
+
* Get cache key hash from operation key object
|
|
103
|
+
*/
|
|
104
|
+
function getCacheKeyHash(key) {
|
|
105
|
+
return hash(key);
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
// ─────────────────────────────────────────────────────────────
|
|
109
|
+
// Git State Detection
|
|
110
|
+
// ─────────────────────────────────────────────────────────────
|
|
111
|
+
|
|
112
|
+
/**
|
|
113
|
+
* Check if directory is a git repository
|
|
114
|
+
*/
|
|
115
|
+
function isGitRepo(dir) {
|
|
116
|
+
try {
|
|
117
|
+
execSync('git rev-parse --git-dir', { cwd: dir, stdio: 'ignore' });
|
|
118
|
+
return true;
|
|
119
|
+
} catch {
|
|
120
|
+
return false;
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
/**
|
|
125
|
+
* Get current git state (HEAD commit + dirty files)
|
|
126
|
+
* Returns null if not in a git repo
|
|
127
|
+
*/
|
|
128
|
+
export function getGitState(projectRoot) {
|
|
129
|
+
if (!isGitRepo(projectRoot)) {
|
|
130
|
+
return null;
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
try {
|
|
134
|
+
// Get HEAD commit
|
|
135
|
+
const head = execSync('git rev-parse HEAD', {
|
|
136
|
+
cwd: projectRoot,
|
|
137
|
+
encoding: 'utf-8'
|
|
138
|
+
}).trim();
|
|
139
|
+
|
|
140
|
+
// Get list of modified/untracked files (sorted for consistency)
|
|
141
|
+
const status = execSync('git status --porcelain', {
|
|
142
|
+
cwd: projectRoot,
|
|
143
|
+
encoding: 'utf-8'
|
|
144
|
+
});
|
|
145
|
+
|
|
146
|
+
const dirtyFiles = status
|
|
147
|
+
.split('\n')
|
|
148
|
+
.filter(line => line.trim())
|
|
149
|
+
.map(line => line.slice(3)) // Remove status prefix
|
|
150
|
+
.sort();
|
|
151
|
+
|
|
152
|
+
return {
|
|
153
|
+
head,
|
|
154
|
+
dirtyFiles
|
|
155
|
+
};
|
|
156
|
+
} catch {
|
|
157
|
+
return null;
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
/**
|
|
162
|
+
* Check if git state matches stored state
|
|
163
|
+
*/
|
|
164
|
+
function gitStateMatches(stored, current) {
|
|
165
|
+
if (!stored || !current) return false;
|
|
166
|
+
if (stored.head !== current.head) return false;
|
|
167
|
+
if (stored.dirtyFiles.length !== current.dirtyFiles.length) return false;
|
|
168
|
+
|
|
169
|
+
for (let i = 0; i < stored.dirtyFiles.length; i++) {
|
|
170
|
+
if (stored.dirtyFiles[i] !== current.dirtyFiles[i]) return false;
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
return true;
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
// ─────────────────────────────────────────────────────────────
|
|
177
|
+
// Cache Directory Management
|
|
178
|
+
// ─────────────────────────────────────────────────────────────
|
|
179
|
+
|
|
180
|
+
/**
|
|
181
|
+
* Get or create cache directory for a project
|
|
182
|
+
*/
|
|
183
|
+
export function getCacheDir(projectRoot) {
|
|
184
|
+
const config = getCacheConfig();
|
|
185
|
+
const projectHash = getProjectHash(projectRoot);
|
|
186
|
+
const cacheDir = join(config.location, projectHash);
|
|
187
|
+
|
|
188
|
+
if (!existsSync(cacheDir)) {
|
|
189
|
+
mkdirSync(cacheDir, { recursive: true });
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
return cacheDir;
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
/**
|
|
196
|
+
* Get cache file path for a key
|
|
197
|
+
*/
|
|
198
|
+
function getCacheFilePath(key, projectRoot) {
|
|
199
|
+
const cacheDir = getCacheDir(projectRoot);
|
|
200
|
+
const keyHash = getCacheKeyHash(key);
|
|
201
|
+
return join(cacheDir, `${keyHash}.json`);
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
// ─────────────────────────────────────────────────────────────
|
|
205
|
+
// Cache Size Management
|
|
206
|
+
// ─────────────────────────────────────────────────────────────
|
|
207
|
+
|
|
208
|
+
/**
|
|
209
|
+
* Get total size of cache directory in bytes
|
|
210
|
+
*/
|
|
211
|
+
function getCacheDirSize(cacheDir) {
|
|
212
|
+
if (!existsSync(cacheDir)) return 0;
|
|
213
|
+
|
|
214
|
+
let total = 0;
|
|
215
|
+
try {
|
|
216
|
+
const files = readdirSync(cacheDir);
|
|
217
|
+
for (const file of files) {
|
|
218
|
+
try {
|
|
219
|
+
const stat = statSync(join(cacheDir, file));
|
|
220
|
+
if (stat.isFile()) {
|
|
221
|
+
total += stat.size;
|
|
222
|
+
}
|
|
223
|
+
} catch { /* skip unreadable files */ }
|
|
224
|
+
}
|
|
225
|
+
} catch { /* directory read error */ }
|
|
226
|
+
|
|
227
|
+
return total;
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
/**
|
|
231
|
+
* Get all cache entries with metadata
|
|
232
|
+
*/
|
|
233
|
+
function getCacheEntries(cacheDir) {
|
|
234
|
+
if (!existsSync(cacheDir)) return [];
|
|
235
|
+
|
|
236
|
+
const entries = [];
|
|
237
|
+
try {
|
|
238
|
+
const files = readdirSync(cacheDir).filter(f => f.endsWith('.json'));
|
|
239
|
+
for (const file of files) {
|
|
240
|
+
try {
|
|
241
|
+
const filePath = join(cacheDir, file);
|
|
242
|
+
const stat = statSync(filePath);
|
|
243
|
+
entries.push({
|
|
244
|
+
path: filePath,
|
|
245
|
+
size: stat.size,
|
|
246
|
+
mtime: stat.mtime.getTime()
|
|
247
|
+
});
|
|
248
|
+
} catch { /* skip unreadable files */ }
|
|
249
|
+
}
|
|
250
|
+
} catch { /* directory read error */ }
|
|
251
|
+
|
|
252
|
+
return entries;
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
/**
|
|
256
|
+
* Remove oldest cache entries until under maxSize
|
|
257
|
+
*/
|
|
258
|
+
function enforceMaxSize(projectRoot) {
|
|
259
|
+
const config = getCacheConfig();
|
|
260
|
+
const cacheDir = getCacheDir(projectRoot);
|
|
261
|
+
|
|
262
|
+
const entries = getCacheEntries(cacheDir);
|
|
263
|
+
let totalSize = entries.reduce((sum, e) => sum + e.size, 0);
|
|
264
|
+
|
|
265
|
+
if (totalSize <= config.maxSize) return;
|
|
266
|
+
|
|
267
|
+
// Sort by modification time (oldest first)
|
|
268
|
+
entries.sort((a, b) => a.mtime - b.mtime);
|
|
269
|
+
|
|
270
|
+
// Remove oldest entries until under limit
|
|
271
|
+
for (const entry of entries) {
|
|
272
|
+
if (totalSize <= config.maxSize) break;
|
|
273
|
+
|
|
274
|
+
try {
|
|
275
|
+
unlinkSync(entry.path);
|
|
276
|
+
totalSize -= entry.size;
|
|
277
|
+
} catch { /* skip if can't delete */ }
|
|
278
|
+
}
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
// ─────────────────────────────────────────────────────────────
|
|
282
|
+
// Low-Level Cache API
|
|
283
|
+
// ─────────────────────────────────────────────────────────────
|
|
284
|
+
|
|
285
|
+
/**
|
|
286
|
+
* Read from cache if valid
|
|
287
|
+
* Returns cached data or null if cache miss/invalid
|
|
288
|
+
*/
|
|
289
|
+
export function getCached(key, projectRoot) {
|
|
290
|
+
const config = getCacheConfig();
|
|
291
|
+
if (!config.enabled) return null;
|
|
292
|
+
|
|
293
|
+
const filePath = getCacheFilePath(key, projectRoot);
|
|
294
|
+
if (!existsSync(filePath)) return null;
|
|
295
|
+
|
|
296
|
+
try {
|
|
297
|
+
const cached = JSON.parse(readFileSync(filePath, 'utf-8'));
|
|
298
|
+
|
|
299
|
+
// Git-based invalidation
|
|
300
|
+
const currentGitState = getGitState(projectRoot);
|
|
301
|
+
if (currentGitState) {
|
|
302
|
+
// If we have git state, use it for validation
|
|
303
|
+
if (!gitStateMatches(cached.gitState, currentGitState)) {
|
|
304
|
+
return null;
|
|
305
|
+
}
|
|
306
|
+
} else {
|
|
307
|
+
// Fall back to TTL-based invalidation
|
|
308
|
+
const age = (Date.now() - cached.timestamp) / 1000;
|
|
309
|
+
if (age > config.ttl) {
|
|
310
|
+
return null;
|
|
311
|
+
}
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
return cached.data;
|
|
315
|
+
} catch {
|
|
316
|
+
return null;
|
|
317
|
+
}
|
|
318
|
+
}
|
|
319
|
+
|
|
320
|
+
/**
|
|
321
|
+
* Write to cache with git state
|
|
322
|
+
*/
|
|
323
|
+
export function setCached(key, data, projectRoot) {
|
|
324
|
+
const config = getCacheConfig();
|
|
325
|
+
if (!config.enabled) return;
|
|
326
|
+
|
|
327
|
+
const filePath = getCacheFilePath(key, projectRoot);
|
|
328
|
+
const gitState = getGitState(projectRoot);
|
|
329
|
+
|
|
330
|
+
const cacheEntry = {
|
|
331
|
+
data,
|
|
332
|
+
gitState,
|
|
333
|
+
timestamp: Date.now(),
|
|
334
|
+
key: typeof key === 'string' ? key : JSON.stringify(key)
|
|
335
|
+
};
|
|
336
|
+
|
|
337
|
+
try {
|
|
338
|
+
// Ensure directory exists
|
|
339
|
+
const dir = dirname(filePath);
|
|
340
|
+
if (!existsSync(dir)) {
|
|
341
|
+
mkdirSync(dir, { recursive: true });
|
|
342
|
+
}
|
|
343
|
+
|
|
344
|
+
writeFileSync(filePath, JSON.stringify(cacheEntry));
|
|
345
|
+
|
|
346
|
+
// Enforce size limit
|
|
347
|
+
enforceMaxSize(projectRoot);
|
|
348
|
+
} catch {
|
|
349
|
+
// Silently fail - caching is best-effort
|
|
350
|
+
}
|
|
351
|
+
}
|
|
352
|
+
|
|
353
|
+
// ─────────────────────────────────────────────────────────────
|
|
354
|
+
// High-Level Cache API
|
|
355
|
+
// ─────────────────────────────────────────────────────────────
|
|
356
|
+
|
|
357
|
+
/**
|
|
358
|
+
* Execute function with caching
|
|
359
|
+
* Preferred API for caching expensive operations
|
|
360
|
+
*
|
|
361
|
+
* @param {Object|string} key - Cache key (operation + args)
|
|
362
|
+
* @param {Function} fn - Function to execute if cache miss
|
|
363
|
+
* @param {Object} options - Options including projectRoot
|
|
364
|
+
* @returns {*} Cached or computed result
|
|
365
|
+
*/
|
|
366
|
+
export function withCache(key, fn, options = {}) {
|
|
367
|
+
const { projectRoot = process.cwd() } = options;
|
|
368
|
+
|
|
369
|
+
// Check cache first
|
|
370
|
+
const cached = getCached(key, projectRoot);
|
|
371
|
+
if (cached !== null) {
|
|
372
|
+
return cached;
|
|
373
|
+
}
|
|
374
|
+
|
|
375
|
+
// Execute function and cache result
|
|
376
|
+
const result = fn();
|
|
377
|
+
setCached(key, result, projectRoot);
|
|
378
|
+
|
|
379
|
+
return result;
|
|
380
|
+
}
|
|
381
|
+
|
|
382
|
+
// ─────────────────────────────────────────────────────────────
|
|
383
|
+
// Cache Management Utilities
|
|
384
|
+
// ─────────────────────────────────────────────────────────────
|
|
385
|
+
|
|
386
|
+
/**
|
|
387
|
+
* Clear cache for a project or all projects
|
|
388
|
+
* @param {string|null} projectRoot - Project to clear, or null for all
|
|
389
|
+
*/
|
|
390
|
+
export function clearCache(projectRoot = null) {
|
|
391
|
+
const config = getCacheConfig();
|
|
392
|
+
|
|
393
|
+
if (projectRoot) {
|
|
394
|
+
// Clear single project cache
|
|
395
|
+
const cacheDir = getCacheDir(projectRoot);
|
|
396
|
+
if (existsSync(cacheDir)) {
|
|
397
|
+
try {
|
|
398
|
+
rmSync(cacheDir, { recursive: true });
|
|
399
|
+
} catch { /* ignore errors */ }
|
|
400
|
+
}
|
|
401
|
+
} else {
|
|
402
|
+
// Clear all caches
|
|
403
|
+
if (existsSync(config.location)) {
|
|
404
|
+
try {
|
|
405
|
+
rmSync(config.location, { recursive: true });
|
|
406
|
+
} catch { /* ignore errors */ }
|
|
407
|
+
}
|
|
408
|
+
}
|
|
409
|
+
}
|
|
410
|
+
|
|
411
|
+
/**
|
|
412
|
+
* Get cache statistics
|
|
413
|
+
*/
|
|
414
|
+
export function getCacheStats(projectRoot = null) {
|
|
415
|
+
const config = getCacheConfig();
|
|
416
|
+
|
|
417
|
+
if (projectRoot) {
|
|
418
|
+
// Stats for single project
|
|
419
|
+
const cacheDir = getCacheDir(projectRoot);
|
|
420
|
+
const entries = getCacheEntries(cacheDir);
|
|
421
|
+
const totalSize = entries.reduce((sum, e) => sum + e.size, 0);
|
|
422
|
+
|
|
423
|
+
return {
|
|
424
|
+
enabled: config.enabled,
|
|
425
|
+
location: cacheDir,
|
|
426
|
+
entries: entries.length,
|
|
427
|
+
size: totalSize,
|
|
428
|
+
sizeFormatted: formatSize(totalSize),
|
|
429
|
+
maxSize: config.maxSize,
|
|
430
|
+
maxSizeFormatted: formatSize(config.maxSize)
|
|
431
|
+
};
|
|
432
|
+
}
|
|
433
|
+
|
|
434
|
+
// Stats for all projects
|
|
435
|
+
if (!existsSync(config.location)) {
|
|
436
|
+
return {
|
|
437
|
+
enabled: config.enabled,
|
|
438
|
+
location: config.location,
|
|
439
|
+
projects: 0,
|
|
440
|
+
totalEntries: 0,
|
|
441
|
+
totalSize: 0,
|
|
442
|
+
totalSizeFormatted: '0 B',
|
|
443
|
+
maxSize: config.maxSize,
|
|
444
|
+
maxSizeFormatted: formatSize(config.maxSize)
|
|
445
|
+
};
|
|
446
|
+
}
|
|
447
|
+
|
|
448
|
+
let totalEntries = 0;
|
|
449
|
+
let totalSize = 0;
|
|
450
|
+
let projects = 0;
|
|
451
|
+
|
|
452
|
+
try {
|
|
453
|
+
const projectDirs = readdirSync(config.location);
|
|
454
|
+
for (const dir of projectDirs) {
|
|
455
|
+
const projectDir = join(config.location, dir);
|
|
456
|
+
try {
|
|
457
|
+
if (statSync(projectDir).isDirectory()) {
|
|
458
|
+
projects++;
|
|
459
|
+
const entries = getCacheEntries(projectDir);
|
|
460
|
+
totalEntries += entries.length;
|
|
461
|
+
totalSize += entries.reduce((sum, e) => sum + e.size, 0);
|
|
462
|
+
}
|
|
463
|
+
} catch { /* skip */ }
|
|
464
|
+
}
|
|
465
|
+
} catch { /* location doesn't exist yet */ }
|
|
466
|
+
|
|
467
|
+
return {
|
|
468
|
+
enabled: config.enabled,
|
|
469
|
+
location: config.location,
|
|
470
|
+
projects,
|
|
471
|
+
totalEntries,
|
|
472
|
+
totalSize,
|
|
473
|
+
totalSizeFormatted: formatSize(totalSize),
|
|
474
|
+
maxSize: config.maxSize,
|
|
475
|
+
maxSizeFormatted: formatSize(config.maxSize)
|
|
476
|
+
};
|
|
477
|
+
}
|
|
478
|
+
|
|
479
|
+
/**
|
|
480
|
+
* Format bytes to human readable
|
|
481
|
+
*/
|
|
482
|
+
function formatSize(bytes) {
|
|
483
|
+
if (bytes >= 1024 * 1024 * 1024) {
|
|
484
|
+
return `${(bytes / (1024 * 1024 * 1024)).toFixed(1)} GB`;
|
|
485
|
+
}
|
|
486
|
+
if (bytes >= 1024 * 1024) {
|
|
487
|
+
return `${(bytes / (1024 * 1024)).toFixed(1)} MB`;
|
|
488
|
+
}
|
|
489
|
+
if (bytes >= 1024) {
|
|
490
|
+
return `${(bytes / 1024).toFixed(1)} KB`;
|
|
491
|
+
}
|
|
492
|
+
return `${bytes} B`;
|
|
493
|
+
}
|
package/src/config.mjs
CHANGED
|
@@ -81,6 +81,12 @@ const DEFAULT_CONFIG = {
|
|
|
81
81
|
},
|
|
82
82
|
impact: {
|
|
83
83
|
depth: 2
|
|
84
|
+
},
|
|
85
|
+
cache: {
|
|
86
|
+
enabled: true, // Enable/disable caching
|
|
87
|
+
ttl: 300, // Max age in seconds (fallback for non-git)
|
|
88
|
+
maxSize: '100MB', // Max cache directory size
|
|
89
|
+
location: null // Override ~/.tokenlean/cache
|
|
84
90
|
}
|
|
85
91
|
};
|
|
86
92
|
|