token-pilot 0.2.1 → 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/plugin.json +1 -1
- package/CHANGELOG.md +7 -0
- package/package.json +3 -2
- package/dist/core/context-window-tracker.d.ts +0 -89
- package/dist/core/context-window-tracker.d.ts.map +0 -1
- package/dist/core/context-window-tracker.js +0 -161
- package/dist/core/context-window-tracker.js.map +0 -1
- package/dist/core/context-window-tracker.test.d.ts +0 -2
- package/dist/core/context-window-tracker.test.d.ts.map +0 -1
- package/dist/core/context-window-tracker.test.js +0 -238
- package/dist/core/context-window-tracker.test.js.map +0 -1
- package/dist/core/diff-engine.d.ts +0 -64
- package/dist/core/diff-engine.d.ts.map +0 -1
- package/dist/core/diff-engine.js +0 -185
- package/dist/core/diff-engine.js.map +0 -1
- package/dist/core/diff-engine.test.d.ts +0 -2
- package/dist/core/diff-engine.test.d.ts.map +0 -1
- package/dist/core/diff-engine.test.js +0 -351
- package/dist/core/diff-engine.test.js.map +0 -1
- package/dist/core/persistent-cache.d.ts +0 -153
- package/dist/core/persistent-cache.d.ts.map +0 -1
- package/dist/core/persistent-cache.js +0 -555
- package/dist/core/persistent-cache.js.map +0 -1
- package/dist/core/persistent-cache.test.d.ts +0 -2
- package/dist/core/persistent-cache.test.d.ts.map +0 -1
- package/dist/core/persistent-cache.test.js +0 -251
- package/dist/core/persistent-cache.test.js.map +0 -1
- package/dist/core/real-token-estimator.d.ts +0 -49
- package/dist/core/real-token-estimator.d.ts.map +0 -1
- package/dist/core/real-token-estimator.js +0 -93
- package/dist/core/real-token-estimator.js.map +0 -1
- package/dist/formatters/context-markup.d.ts +0 -40
- package/dist/formatters/context-markup.d.ts.map +0 -1
- package/dist/formatters/context-markup.js +0 -55
- package/dist/formatters/context-markup.js.map +0 -1
- package/dist/formatters/smart-read-xml.d.ts +0 -20
- package/dist/formatters/smart-read-xml.d.ts.map +0 -1
- package/dist/formatters/smart-read-xml.js +0 -163
- package/dist/formatters/smart-read-xml.js.map +0 -1
|
@@ -1,153 +0,0 @@
|
|
|
1
|
-
import type { CacheEntry } from '../types';
|
|
2
|
-
/**
|
|
3
|
-
* Persistent File Cache with Two-Level Architecture
|
|
4
|
-
*
|
|
5
|
-
* Problem: Cold start takes 200s because each file needs AST parsing
|
|
6
|
-
* Solution: L1 (in-memory) + L2 (SQLite persistent) cache with LRU eviction
|
|
7
|
-
*
|
|
8
|
-
* L1 Cache (Memory):
|
|
9
|
-
* - Fast: <5ms lookups
|
|
10
|
-
* - Warm during session
|
|
11
|
-
* - Survives memory pressure via eviction
|
|
12
|
-
*
|
|
13
|
-
* L2 Cache (SQLite):
|
|
14
|
-
* - Persistent: survives restarts
|
|
15
|
-
* - Slower: ~100ms lookups
|
|
16
|
-
* - Enables 20x faster cold starts (200s → 10s)
|
|
17
|
-
*
|
|
18
|
-
* Strategy:
|
|
19
|
-
* - Always read from L1 first (fastest)
|
|
20
|
-
* - Miss in L1 → read from L2 (persistent)
|
|
21
|
-
* - Miss in L2 → parse file (slowest, but cached for next time)
|
|
22
|
-
* - Evict LRU when memory/disk limit exceeded
|
|
23
|
-
*/
|
|
24
|
-
export declare class PersistentFileCache {
|
|
25
|
-
private l1;
|
|
26
|
-
private db;
|
|
27
|
-
private maxSizeBytes;
|
|
28
|
-
private l1MaxSize;
|
|
29
|
-
private onSetCallback;
|
|
30
|
-
private smallFileThreshold;
|
|
31
|
-
private l1Only;
|
|
32
|
-
private l1Hits;
|
|
33
|
-
private l2Hits;
|
|
34
|
-
private misses;
|
|
35
|
-
constructor(projectRoot: string, maxSizeMB?: number, smallFileThreshold?: number);
|
|
36
|
-
/**
|
|
37
|
-
* Initialize database schema with file_cache table and indexes
|
|
38
|
-
*/
|
|
39
|
-
private initializeSchema;
|
|
40
|
-
/**
|
|
41
|
-
* Get cached entry from L1 or L2
|
|
42
|
-
* Returns from L1 first (fastest), then L2 (persistent)
|
|
43
|
-
*/
|
|
44
|
-
get(filePath: string): CacheEntry | null;
|
|
45
|
-
/**
|
|
46
|
-
* Set cache entry in L1 and L2
|
|
47
|
-
*/
|
|
48
|
-
set(filePath: string, entry: CacheEntry): void;
|
|
49
|
-
/**
|
|
50
|
-
* Get cached entry by content hash (semantic caching)
|
|
51
|
-
* Useful when same file content loaded from different paths
|
|
52
|
-
*/
|
|
53
|
-
getByHash(contentHash: string): CacheEntry | null;
|
|
54
|
-
/**
|
|
55
|
-
* Remove entry from both L1 and L2
|
|
56
|
-
*/
|
|
57
|
-
delete(filePath: string): void;
|
|
58
|
-
/**
|
|
59
|
-
* Clear all cache (useful for testing)
|
|
60
|
-
*/
|
|
61
|
-
clear(): void;
|
|
62
|
-
/**
|
|
63
|
-
* Get cache statistics
|
|
64
|
-
*/
|
|
65
|
-
getStats(): {
|
|
66
|
-
totalBytes: number;
|
|
67
|
-
entryCount: number;
|
|
68
|
-
l1Size: number;
|
|
69
|
-
l2Size: number;
|
|
70
|
-
maxBytes: number;
|
|
71
|
-
};
|
|
72
|
-
/**
|
|
73
|
-
* Get all cached entries (for iteration/analysis)
|
|
74
|
-
*/
|
|
75
|
-
private getAllEntries;
|
|
76
|
-
/**
|
|
77
|
-
* Evict LRU entries if cache exceeds size limit
|
|
78
|
-
*/
|
|
79
|
-
private evictIfNeeded;
|
|
80
|
-
/**
|
|
81
|
-
* Update lastAccess timestamp for file
|
|
82
|
-
*/
|
|
83
|
-
private updateLastAccess;
|
|
84
|
-
/**
|
|
85
|
-
* Update cache statistics
|
|
86
|
-
*/
|
|
87
|
-
private updateStats;
|
|
88
|
-
/**
|
|
89
|
-
* Estimate entry size in bytes
|
|
90
|
-
*/
|
|
91
|
-
private estimateSize;
|
|
92
|
-
/**
|
|
93
|
-
* Register callback for file writes (compatibility with FileCache)
|
|
94
|
-
*/
|
|
95
|
-
onSet(callback: (filePath: string) => void): void;
|
|
96
|
-
/**
|
|
97
|
-
* Invalidate cache entry (compatibility with FileCache)
|
|
98
|
-
*/
|
|
99
|
-
invalidate(filePath?: string): void;
|
|
100
|
-
/**
|
|
101
|
-
* Invalidate files by git diff (compatibility with FileCache)
|
|
102
|
-
*/
|
|
103
|
-
invalidateByGitDiff(changedFiles: string[]): Promise<void>;
|
|
104
|
-
/**
|
|
105
|
-
* Check if file is small (under threshold lines)
|
|
106
|
-
*/
|
|
107
|
-
isSmallFile(filePath: string): Promise<boolean>;
|
|
108
|
-
/**
|
|
109
|
-
* Check if cached entry is stale (mtime changed)
|
|
110
|
-
*/
|
|
111
|
-
isStale(filePath: string): Promise<boolean>;
|
|
112
|
-
/**
|
|
113
|
-
* Get small file threshold
|
|
114
|
-
*/
|
|
115
|
-
getSmallFileThreshold(): number;
|
|
116
|
-
/**
|
|
117
|
-
* Get all cached file paths
|
|
118
|
-
*/
|
|
119
|
-
cachedPaths(): string[];
|
|
120
|
-
/**
|
|
121
|
-
* Get cache statistics (compatibility with FileCache)
|
|
122
|
-
*/
|
|
123
|
-
stats(): {
|
|
124
|
-
entries: number;
|
|
125
|
-
sizeBytes: number;
|
|
126
|
-
hitRate: number;
|
|
127
|
-
};
|
|
128
|
-
/**
|
|
129
|
-
* Get detailed cache performance metrics
|
|
130
|
-
*/
|
|
131
|
-
getPerformanceMetrics(): {
|
|
132
|
-
l1Hits: number;
|
|
133
|
-
l2Hits: number;
|
|
134
|
-
misses: number;
|
|
135
|
-
totalRequests: number;
|
|
136
|
-
hitRate: number;
|
|
137
|
-
l1HitRate: number;
|
|
138
|
-
l2HitRate: number;
|
|
139
|
-
l1Only: boolean;
|
|
140
|
-
};
|
|
141
|
-
/**
|
|
142
|
-
* Trigger onSet callback after setting entry
|
|
143
|
-
*/
|
|
144
|
-
private triggerOnSet;
|
|
145
|
-
/**
|
|
146
|
-
* Close database connection
|
|
147
|
-
*/
|
|
148
|
-
close(): void;
|
|
149
|
-
}
|
|
150
|
-
export declare function initializePersistentCache(projectRoot: string, maxSizeMB?: number): PersistentFileCache;
|
|
151
|
-
export declare function getPersistentCache(): PersistentFileCache;
|
|
152
|
-
export declare function closePersistentCache(): void;
|
|
153
|
-
//# sourceMappingURL=persistent-cache.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"persistent-cache.d.ts","sourceRoot":"","sources":["../../src/core/persistent-cache.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,UAAU,EAAiB,MAAM,UAAU,CAAA;AAEzD;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,qBAAa,mBAAmB;IAC9B,OAAO,CAAC,EAAE,CAAqC;IAC/C,OAAO,CAAC,EAAE,CAAiC;IAC3C,OAAO,CAAC,YAAY,CAAQ;IAC5B,OAAO,CAAC,SAAS,CAAa;IAC9B,OAAO,CAAC,aAAa,CAA4C;IACjE,OAAO,CAAC,kBAAkB,CAAa;IACvC,OAAO,CAAC,MAAM,CAAiB;IAC/B,OAAO,CAAC,MAAM,CAAY;IAC1B,OAAO,CAAC,MAAM,CAAY;IAC1B,OAAO,CAAC,MAAM,CAAY;gBAEd,WAAW,EAAE,MAAM,EAAE,SAAS,GAAE,MAAa,EAAE,kBAAkB,GAAE,MAAW;IA4B1F;;OAEG;IACH,OAAO,CAAC,gBAAgB;IAqCxB;;;OAGG;IACH,GAAG,CAAC,QAAQ,EAAE,MAAM,GAAG,UAAU,GAAG,IAAI;IA0DxC;;OAEG;IACH,GAAG,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,UAAU,GAAG,IAAI;IAmD9C;;;OAGG;IACH,SAAS,CAAC,WAAW,EAAE,MAAM,GAAG,UAAU,GAAG,IAAI;IAkCjD;;OAEG;IACH,MAAM,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI;IAiB9B;;OAEG;IACH,KAAK,IAAI,IAAI;IAgBb;;OAEG;IACH,QAAQ,IAAI;QACV,UAAU,EAAE,MAAM,CAAA;QAClB,UAAU,EAAE,MAAM,CAAA;QAClB,MAAM,EAAE,MAAM,CAAA;QACd,MAAM,EAAE,MAAM,CAAA;QACd,QAAQ,EAAE,MAAM,CAAA;KACjB;IAmCD;;OAEG;IACH,OAAO,CAAC,aAAa;IAerB;;OAEG;IACH,OAAO,CAAC,aAAa;IAoCrB;;OAEG;IACH,OAAO,CAAC,gBAAgB;IAaxB;;OAEG;IACH,OAAO,CAAC,WAAW;IAyBnB;;OAEG;IACH,OAAO,CAAC,YAAY;IASpB;;OAEG;IACH,KAAK,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,IAAI,GAAG,IAAI;IAIjD;;OAEG;IACH,UAAU,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI;IAQnC;;OAEG;IACG,mBAAmB,CAAC,YAAY,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAMhE;;OAEG;IACG,WAAW,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IASrD;;OAEG;IACG,OAAO,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAYjD;;OAEG;IACH,qBAAqB,IAAI,MAAM;IAI/B;;OAEG;IACH,WAAW,IAAI,MAAM,EAAE;IAqBvB;;OAEG;IACH,KAAK,IAAI;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE;IAWhE;;OAEG;IACH,qBAAqB,IAAI;QACvB,MAAM,EAAE,MAAM,CAAA;QACd,MAAM,EAAE,MAAM,CAAA;QACd,MAAM,EAAE,MAAM,CAAA;QACd,aAAa,EAAE,MAAM,CAAA;QACrB,OAAO,EAAE,MAAM,CAAA;QACf,SAAS,EAAE,MAAM,CAAA;QACjB,SAAS,EAAE,MAAM,CAAA;QACjB,MAAM,EAAE,OAAO,CAAA;KAChB;IAmBD;;OAEG;IACH,OAAO,CAAC,YAAY;IAIpB;;OAEG;IACH,KAAK,IAAI,IAAI;CAYd;AAOD,wBAAgB,yBAAyB,CAAC,WAAW,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,mBAAmB,CAKtG;AAED,wBAAgB,kBAAkB,IAAI,mBAAmB,CAKxD;AAED,wBAAgB,oBAAoB,IAAI,IAAI,CAK3C"}
|
|
@@ -1,555 +0,0 @@
|
|
|
1
|
-
import Database from 'better-sqlite3';
|
|
2
|
-
import { join } from 'node:path';
|
|
3
|
-
import { mkdirSync, stat, readFile } from 'node:fs/promises';
|
|
4
|
-
/**
|
|
5
|
-
* Persistent File Cache with Two-Level Architecture
|
|
6
|
-
*
|
|
7
|
-
* Problem: Cold start takes 200s because each file needs AST parsing
|
|
8
|
-
* Solution: L1 (in-memory) + L2 (SQLite persistent) cache with LRU eviction
|
|
9
|
-
*
|
|
10
|
-
* L1 Cache (Memory):
|
|
11
|
-
* - Fast: <5ms lookups
|
|
12
|
-
* - Warm during session
|
|
13
|
-
* - Survives memory pressure via eviction
|
|
14
|
-
*
|
|
15
|
-
* L2 Cache (SQLite):
|
|
16
|
-
* - Persistent: survives restarts
|
|
17
|
-
* - Slower: ~100ms lookups
|
|
18
|
-
* - Enables 20x faster cold starts (200s → 10s)
|
|
19
|
-
*
|
|
20
|
-
* Strategy:
|
|
21
|
-
* - Always read from L1 first (fastest)
|
|
22
|
-
* - Miss in L1 → read from L2 (persistent)
|
|
23
|
-
* - Miss in L2 → parse file (slowest, but cached for next time)
|
|
24
|
-
* - Evict LRU when memory/disk limit exceeded
|
|
25
|
-
*/
|
|
26
|
-
export class PersistentFileCache {
|
|
27
|
-
l1 = new Map();
|
|
28
|
-
db = null;
|
|
29
|
-
maxSizeBytes;
|
|
30
|
-
l1MaxSize = 50; // max files in L1 memory
|
|
31
|
-
onSetCallback = null;
|
|
32
|
-
smallFileThreshold = 80;
|
|
33
|
-
l1Only = false; // graceful degradation mode
|
|
34
|
-
l1Hits = 0;
|
|
35
|
-
l2Hits = 0;
|
|
36
|
-
misses = 0;
|
|
37
|
-
constructor(projectRoot, maxSizeMB = 1000, smallFileThreshold = 80) {
|
|
38
|
-
this.maxSizeBytes = maxSizeMB * 1024 * 1024;
|
|
39
|
-
this.smallFileThreshold = smallFileThreshold;
|
|
40
|
-
try {
|
|
41
|
-
// Initialize SQLite database
|
|
42
|
-
const cacheDir = join(projectRoot, '.token-pilot');
|
|
43
|
-
mkdirSync(cacheDir, { recursive: true });
|
|
44
|
-
const dbPath = join(cacheDir, 'cache.db');
|
|
45
|
-
this.db = new Database(dbPath);
|
|
46
|
-
// Enable performance optimizations
|
|
47
|
-
this.db.pragma('journal_mode = WAL'); // Write-Ahead Logging for concurrency
|
|
48
|
-
this.db.pragma('synchronous = NORMAL'); // Balance safety & speed
|
|
49
|
-
this.db.pragma('cache_size = -64000'); // 64MB cache
|
|
50
|
-
// Create schema
|
|
51
|
-
this.initializeSchema();
|
|
52
|
-
}
|
|
53
|
-
catch (err) {
|
|
54
|
-
const message = err instanceof Error ? err.message : String(err);
|
|
55
|
-
console.error(`[token-pilot] PersistentFileCache init error: ${message}`);
|
|
56
|
-
console.error(`[token-pilot] Falling back to L1-only (memory) cache`);
|
|
57
|
-
this.db = null;
|
|
58
|
-
this.l1Only = true;
|
|
59
|
-
}
|
|
60
|
-
}
|
|
61
|
-
/**
|
|
62
|
-
* Initialize database schema with file_cache table and indexes
|
|
63
|
-
*/
|
|
64
|
-
initializeSchema() {
|
|
65
|
-
if (!this.db)
|
|
66
|
-
return;
|
|
67
|
-
try {
|
|
68
|
-
this.db.exec(`
|
|
69
|
-
CREATE TABLE IF NOT EXISTS file_cache (
|
|
70
|
-
path TEXT PRIMARY KEY,
|
|
71
|
-
structure TEXT NOT NULL,
|
|
72
|
-
content TEXT NOT NULL,
|
|
73
|
-
lines TEXT NOT NULL,
|
|
74
|
-
hash TEXT NOT NULL UNIQUE,
|
|
75
|
-
mtime INTEGER NOT NULL,
|
|
76
|
-
lastAccess INTEGER NOT NULL,
|
|
77
|
-
sizeBytes INTEGER NOT NULL
|
|
78
|
-
);
|
|
79
|
-
|
|
80
|
-
CREATE INDEX IF NOT EXISTS idx_hash ON file_cache(hash);
|
|
81
|
-
CREATE INDEX IF NOT EXISTS idx_lastAccess ON file_cache(lastAccess);
|
|
82
|
-
CREATE INDEX IF NOT EXISTS idx_mtime ON file_cache(mtime);
|
|
83
|
-
|
|
84
|
-
CREATE TABLE IF NOT EXISTS cache_stats (
|
|
85
|
-
id INTEGER PRIMARY KEY,
|
|
86
|
-
totalBytes INTEGER DEFAULT 0,
|
|
87
|
-
entryCount INTEGER DEFAULT 0,
|
|
88
|
-
lastEviction INTEGER DEFAULT 0
|
|
89
|
-
);
|
|
90
|
-
|
|
91
|
-
INSERT OR IGNORE INTO cache_stats (id, totalBytes, entryCount, lastEviction)
|
|
92
|
-
VALUES (1, 0, 0, 0);
|
|
93
|
-
`);
|
|
94
|
-
}
|
|
95
|
-
catch (err) {
|
|
96
|
-
const message = err instanceof Error ? err.message : String(err);
|
|
97
|
-
console.error(`[token-pilot] Failed to initialize cache schema: ${message}`);
|
|
98
|
-
throw err;
|
|
99
|
-
}
|
|
100
|
-
}
|
|
101
|
-
/**
|
|
102
|
-
* Get cached entry from L1 or L2
|
|
103
|
-
* Returns from L1 first (fastest), then L2 (persistent)
|
|
104
|
-
*/
|
|
105
|
-
get(filePath) {
|
|
106
|
-
// L1: In-memory cache (fastest)
|
|
107
|
-
const l1Entry = this.l1.get(filePath);
|
|
108
|
-
if (l1Entry) {
|
|
109
|
-
this.l1Hits++;
|
|
110
|
-
this.updateLastAccess(filePath);
|
|
111
|
-
return l1Entry;
|
|
112
|
-
}
|
|
113
|
-
// Skip L2 if in L1-only mode
|
|
114
|
-
if (this.l1Only || !this.db) {
|
|
115
|
-
this.misses++;
|
|
116
|
-
return null;
|
|
117
|
-
}
|
|
118
|
-
// L2: SQLite persistent cache
|
|
119
|
-
try {
|
|
120
|
-
const stmt = this.db.prepare(`
|
|
121
|
-
SELECT path, structure, content, lines, hash, mtime, lastAccess
|
|
122
|
-
FROM file_cache
|
|
123
|
-
WHERE path = ?
|
|
124
|
-
LIMIT 1
|
|
125
|
-
`);
|
|
126
|
-
const row = stmt.get(filePath);
|
|
127
|
-
if (row) {
|
|
128
|
-
this.l2Hits++;
|
|
129
|
-
const entry = {
|
|
130
|
-
structure: JSON.parse(row.structure),
|
|
131
|
-
content: row.content,
|
|
132
|
-
lines: JSON.parse(row.lines),
|
|
133
|
-
mtime: row.mtime,
|
|
134
|
-
hash: row.hash,
|
|
135
|
-
lastAccess: Date.now()
|
|
136
|
-
};
|
|
137
|
-
// Promote to L1 (capped at l1MaxSize)
|
|
138
|
-
if (this.l1.size >= this.l1MaxSize) {
|
|
139
|
-
const firstKey = this.l1.keys().next().value;
|
|
140
|
-
this.l1.delete(firstKey);
|
|
141
|
-
}
|
|
142
|
-
this.l1.set(filePath, entry);
|
|
143
|
-
// Update lastAccess in L2
|
|
144
|
-
this.updateLastAccess(filePath);
|
|
145
|
-
return entry;
|
|
146
|
-
}
|
|
147
|
-
this.misses++;
|
|
148
|
-
return null;
|
|
149
|
-
}
|
|
150
|
-
catch (err) {
|
|
151
|
-
const message = err instanceof Error ? err.message : String(err);
|
|
152
|
-
console.error(`[token-pilot] Cache get error for ${filePath}: ${message}`);
|
|
153
|
-
return null;
|
|
154
|
-
}
|
|
155
|
-
}
|
|
156
|
-
/**
|
|
157
|
-
* Set cache entry in L1 and L2
|
|
158
|
-
*/
|
|
159
|
-
set(filePath, entry) {
|
|
160
|
-
// Add to L1 (capped at l1MaxSize)
|
|
161
|
-
if (this.l1.size >= this.l1MaxSize) {
|
|
162
|
-
const firstKey = this.l1.keys().next().value;
|
|
163
|
-
this.l1.delete(firstKey);
|
|
164
|
-
}
|
|
165
|
-
this.l1.set(filePath, entry);
|
|
166
|
-
// Skip L2 if in L1-only mode
|
|
167
|
-
if (this.l1Only || !this.db) {
|
|
168
|
-
this.triggerOnSet(filePath);
|
|
169
|
-
return;
|
|
170
|
-
}
|
|
171
|
-
// Add to L2 (SQLite)
|
|
172
|
-
try {
|
|
173
|
-
const sizeBytes = this.estimateSize(entry);
|
|
174
|
-
const stmt = this.db.prepare(`
|
|
175
|
-
INSERT OR REPLACE INTO file_cache
|
|
176
|
-
(path, structure, content, lines, hash, mtime, lastAccess, sizeBytes)
|
|
177
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
|
178
|
-
`);
|
|
179
|
-
const now = Date.now();
|
|
180
|
-
stmt.run(filePath, JSON.stringify(entry.structure), entry.content, JSON.stringify(entry.lines), entry.hash, entry.mtime, now, sizeBytes);
|
|
181
|
-
// Update stats
|
|
182
|
-
this.updateStats();
|
|
183
|
-
// Check if eviction needed
|
|
184
|
-
this.evictIfNeeded();
|
|
185
|
-
}
|
|
186
|
-
catch (err) {
|
|
187
|
-
const message = err instanceof Error ? err.message : String(err);
|
|
188
|
-
console.error(`[token-pilot] Cache set error for ${filePath}: ${message}`);
|
|
189
|
-
// Continue anyway - we have L1 cache
|
|
190
|
-
}
|
|
191
|
-
// Trigger onSet callback
|
|
192
|
-
this.triggerOnSet(filePath);
|
|
193
|
-
}
|
|
194
|
-
/**
|
|
195
|
-
* Get cached entry by content hash (semantic caching)
|
|
196
|
-
* Useful when same file content loaded from different paths
|
|
197
|
-
*/
|
|
198
|
-
getByHash(contentHash) {
|
|
199
|
-
if (this.l1Only || !this.db) {
|
|
200
|
-
return null;
|
|
201
|
-
}
|
|
202
|
-
try {
|
|
203
|
-
const stmt = this.db.prepare(`
|
|
204
|
-
SELECT path, structure, content, lines, hash, mtime, lastAccess
|
|
205
|
-
FROM file_cache
|
|
206
|
-
WHERE hash = ?
|
|
207
|
-
LIMIT 1
|
|
208
|
-
`);
|
|
209
|
-
const row = stmt.get(contentHash);
|
|
210
|
-
if (row) {
|
|
211
|
-
const entry = {
|
|
212
|
-
structure: JSON.parse(row.structure),
|
|
213
|
-
content: row.content,
|
|
214
|
-
lines: JSON.parse(row.lines),
|
|
215
|
-
mtime: row.mtime,
|
|
216
|
-
hash: row.hash,
|
|
217
|
-
lastAccess: Date.now()
|
|
218
|
-
};
|
|
219
|
-
return entry;
|
|
220
|
-
}
|
|
221
|
-
return null;
|
|
222
|
-
}
|
|
223
|
-
catch (err) {
|
|
224
|
-
const message = err instanceof Error ? err.message : String(err);
|
|
225
|
-
console.error(`[token-pilot] Cache getByHash error: ${message}`);
|
|
226
|
-
return null;
|
|
227
|
-
}
|
|
228
|
-
}
|
|
229
|
-
/**
|
|
230
|
-
* Remove entry from both L1 and L2
|
|
231
|
-
*/
|
|
232
|
-
delete(filePath) {
|
|
233
|
-
this.l1.delete(filePath);
|
|
234
|
-
if (this.l1Only || !this.db) {
|
|
235
|
-
return;
|
|
236
|
-
}
|
|
237
|
-
try {
|
|
238
|
-
const stmt = this.db.prepare('DELETE FROM file_cache WHERE path = ?');
|
|
239
|
-
stmt.run(filePath);
|
|
240
|
-
this.updateStats();
|
|
241
|
-
}
|
|
242
|
-
catch (err) {
|
|
243
|
-
const message = err instanceof Error ? err.message : String(err);
|
|
244
|
-
console.error(`[token-pilot] Cache delete error: ${message}`);
|
|
245
|
-
}
|
|
246
|
-
}
|
|
247
|
-
/**
|
|
248
|
-
* Clear all cache (useful for testing)
|
|
249
|
-
*/
|
|
250
|
-
clear() {
|
|
251
|
-
this.l1.clear();
|
|
252
|
-
if (this.l1Only || !this.db) {
|
|
253
|
-
return;
|
|
254
|
-
}
|
|
255
|
-
try {
|
|
256
|
-
this.db.exec('DELETE FROM file_cache');
|
|
257
|
-
this.db.prepare('UPDATE cache_stats SET totalBytes = 0, entryCount = 0').run();
|
|
258
|
-
}
|
|
259
|
-
catch (err) {
|
|
260
|
-
const message = err instanceof Error ? err.message : String(err);
|
|
261
|
-
console.error(`[token-pilot] Cache clear error: ${message}`);
|
|
262
|
-
}
|
|
263
|
-
}
|
|
264
|
-
/**
|
|
265
|
-
* Get cache statistics
|
|
266
|
-
*/
|
|
267
|
-
getStats() {
|
|
268
|
-
if (this.l1Only || !this.db) {
|
|
269
|
-
return {
|
|
270
|
-
totalBytes: 0,
|
|
271
|
-
entryCount: 0,
|
|
272
|
-
l1Size: this.l1.size,
|
|
273
|
-
l2Size: 0,
|
|
274
|
-
maxBytes: this.maxSizeBytes
|
|
275
|
-
};
|
|
276
|
-
}
|
|
277
|
-
try {
|
|
278
|
-
const stmt = this.db.prepare('SELECT totalBytes, entryCount FROM cache_stats WHERE id = 1');
|
|
279
|
-
const row = stmt.get();
|
|
280
|
-
return {
|
|
281
|
-
totalBytes: row?.totalBytes || 0,
|
|
282
|
-
entryCount: row?.entryCount || 0,
|
|
283
|
-
l1Size: this.l1.size,
|
|
284
|
-
l2Size: this.getAllEntries().length,
|
|
285
|
-
maxBytes: this.maxSizeBytes
|
|
286
|
-
};
|
|
287
|
-
}
|
|
288
|
-
catch (err) {
|
|
289
|
-
const message = err instanceof Error ? err.message : String(err);
|
|
290
|
-
console.error(`[token-pilot] Cache getStats error: ${message}`);
|
|
291
|
-
return {
|
|
292
|
-
totalBytes: 0,
|
|
293
|
-
entryCount: 0,
|
|
294
|
-
l1Size: this.l1.size,
|
|
295
|
-
l2Size: 0,
|
|
296
|
-
maxBytes: this.maxSizeBytes
|
|
297
|
-
};
|
|
298
|
-
}
|
|
299
|
-
}
|
|
300
|
-
/**
|
|
301
|
-
* Get all cached entries (for iteration/analysis)
|
|
302
|
-
*/
|
|
303
|
-
getAllEntries() {
|
|
304
|
-
if (this.l1Only || !this.db) {
|
|
305
|
-
return [];
|
|
306
|
-
}
|
|
307
|
-
try {
|
|
308
|
-
const stmt = this.db.prepare('SELECT path, sizeBytes, lastAccess FROM file_cache');
|
|
309
|
-
return stmt.all();
|
|
310
|
-
}
|
|
311
|
-
catch (err) {
|
|
312
|
-
const message = err instanceof Error ? err.message : String(err);
|
|
313
|
-
console.error(`[token-pilot] Cache getAllEntries error: ${message}`);
|
|
314
|
-
return [];
|
|
315
|
-
}
|
|
316
|
-
}
|
|
317
|
-
/**
|
|
318
|
-
* Evict LRU entries if cache exceeds size limit
|
|
319
|
-
*/
|
|
320
|
-
evictIfNeeded() {
|
|
321
|
-
if (this.l1Only || !this.db) {
|
|
322
|
-
return;
|
|
323
|
-
}
|
|
324
|
-
try {
|
|
325
|
-
const stats = this.getStats();
|
|
326
|
-
if (stats.totalBytes <= this.maxSizeBytes) {
|
|
327
|
-
return;
|
|
328
|
-
}
|
|
329
|
-
// Need to evict: remove oldest entries until below 90% limit
|
|
330
|
-
const targetSize = this.maxSizeBytes * 0.9;
|
|
331
|
-
let currentSize = stats.totalBytes;
|
|
332
|
-
const stmt = this.db.prepare(`
|
|
333
|
-
SELECT path, sizeBytes FROM file_cache
|
|
334
|
-
ORDER BY lastAccess ASC
|
|
335
|
-
LIMIT 100
|
|
336
|
-
`);
|
|
337
|
-
const oldestEntries = stmt.all();
|
|
338
|
-
for (const entry of oldestEntries) {
|
|
339
|
-
if (currentSize <= targetSize)
|
|
340
|
-
break;
|
|
341
|
-
this.delete(entry.path);
|
|
342
|
-
currentSize -= entry.sizeBytes;
|
|
343
|
-
}
|
|
344
|
-
}
|
|
345
|
-
catch (err) {
|
|
346
|
-
const message = err instanceof Error ? err.message : String(err);
|
|
347
|
-
console.error(`[token-pilot] Cache evictIfNeeded error: ${message}`);
|
|
348
|
-
}
|
|
349
|
-
}
|
|
350
|
-
/**
|
|
351
|
-
* Update lastAccess timestamp for file
|
|
352
|
-
*/
|
|
353
|
-
updateLastAccess(filePath) {
|
|
354
|
-
if (this.l1Only || !this.db) {
|
|
355
|
-
return;
|
|
356
|
-
}
|
|
357
|
-
try {
|
|
358
|
-
const stmt = this.db.prepare('UPDATE file_cache SET lastAccess = ? WHERE path = ?');
|
|
359
|
-
stmt.run(Date.now(), filePath);
|
|
360
|
-
}
|
|
361
|
-
catch (err) {
|
|
362
|
-
// Silently ignore - not critical for operation
|
|
363
|
-
}
|
|
364
|
-
}
|
|
365
|
-
/**
|
|
366
|
-
* Update cache statistics
|
|
367
|
-
*/
|
|
368
|
-
updateStats() {
|
|
369
|
-
if (this.l1Only || !this.db) {
|
|
370
|
-
return;
|
|
371
|
-
}
|
|
372
|
-
try {
|
|
373
|
-
const stmt = this.db.prepare(`
|
|
374
|
-
SELECT COUNT(*) as count, COALESCE(SUM(sizeBytes), 0) as totalBytes
|
|
375
|
-
FROM file_cache
|
|
376
|
-
`);
|
|
377
|
-
const row = stmt.get();
|
|
378
|
-
const updateStmt = this.db.prepare(`
|
|
379
|
-
UPDATE cache_stats
|
|
380
|
-
SET entryCount = ?, totalBytes = ?, lastEviction = ?
|
|
381
|
-
WHERE id = 1
|
|
382
|
-
`);
|
|
383
|
-
updateStmt.run(row.count, row.totalBytes, Date.now());
|
|
384
|
-
}
|
|
385
|
-
catch (err) {
|
|
386
|
-
// Silently ignore - not critical for operation
|
|
387
|
-
}
|
|
388
|
-
}
|
|
389
|
-
/**
|
|
390
|
-
* Estimate entry size in bytes
|
|
391
|
-
*/
|
|
392
|
-
estimateSize(entry) {
|
|
393
|
-
return (entry.content.length +
|
|
394
|
-
JSON.stringify(entry.structure).length +
|
|
395
|
-
JSON.stringify(entry.lines).length +
|
|
396
|
-
entry.hash.length);
|
|
397
|
-
}
|
|
398
|
-
/**
|
|
399
|
-
* Register callback for file writes (compatibility with FileCache)
|
|
400
|
-
*/
|
|
401
|
-
onSet(callback) {
|
|
402
|
-
this.onSetCallback = callback;
|
|
403
|
-
}
|
|
404
|
-
/**
|
|
405
|
-
* Invalidate cache entry (compatibility with FileCache)
|
|
406
|
-
*/
|
|
407
|
-
invalidate(filePath) {
|
|
408
|
-
if (filePath) {
|
|
409
|
-
this.delete(filePath);
|
|
410
|
-
}
|
|
411
|
-
else {
|
|
412
|
-
this.clear();
|
|
413
|
-
}
|
|
414
|
-
}
|
|
415
|
-
/**
|
|
416
|
-
* Invalidate files by git diff (compatibility with FileCache)
|
|
417
|
-
*/
|
|
418
|
-
async invalidateByGitDiff(changedFiles) {
|
|
419
|
-
for (const file of changedFiles) {
|
|
420
|
-
this.invalidate(file);
|
|
421
|
-
}
|
|
422
|
-
}
|
|
423
|
-
/**
|
|
424
|
-
* Check if file is small (under threshold lines)
|
|
425
|
-
*/
|
|
426
|
-
async isSmallFile(filePath) {
|
|
427
|
-
try {
|
|
428
|
-
const content = await readFile(filePath, 'utf-8');
|
|
429
|
-
return content.split('\n').length <= this.smallFileThreshold;
|
|
430
|
-
}
|
|
431
|
-
catch {
|
|
432
|
-
return false;
|
|
433
|
-
}
|
|
434
|
-
}
|
|
435
|
-
/**
|
|
436
|
-
* Check if cached entry is stale (mtime changed)
|
|
437
|
-
*/
|
|
438
|
-
async isStale(filePath) {
|
|
439
|
-
const entry = this.get(filePath);
|
|
440
|
-
if (!entry)
|
|
441
|
-
return true;
|
|
442
|
-
try {
|
|
443
|
-
const fileStat = await stat(filePath);
|
|
444
|
-
return fileStat.mtimeMs !== entry.mtime;
|
|
445
|
-
}
|
|
446
|
-
catch {
|
|
447
|
-
return true;
|
|
448
|
-
}
|
|
449
|
-
}
|
|
450
|
-
/**
|
|
451
|
-
* Get small file threshold
|
|
452
|
-
*/
|
|
453
|
-
getSmallFileThreshold() {
|
|
454
|
-
return this.smallFileThreshold;
|
|
455
|
-
}
|
|
456
|
-
/**
|
|
457
|
-
* Get all cached file paths
|
|
458
|
-
*/
|
|
459
|
-
cachedPaths() {
|
|
460
|
-
// Return L1 paths + L2 paths
|
|
461
|
-
const l1Paths = Array.from(this.l1.keys());
|
|
462
|
-
if (this.l1Only || !this.db) {
|
|
463
|
-
return l1Paths;
|
|
464
|
-
}
|
|
465
|
-
try {
|
|
466
|
-
const stmt = this.db.prepare('SELECT path FROM file_cache');
|
|
467
|
-
const rows = stmt.all();
|
|
468
|
-
const l2Paths = rows.map(row => row.path);
|
|
469
|
-
// Combine and deduplicate
|
|
470
|
-
return Array.from(new Set([...l1Paths, ...l2Paths]));
|
|
471
|
-
}
|
|
472
|
-
catch (err) {
|
|
473
|
-
const message = err instanceof Error ? err.message : String(err);
|
|
474
|
-
console.error(`[token-pilot] Cache cachedPaths error: ${message}`);
|
|
475
|
-
return l1Paths;
|
|
476
|
-
}
|
|
477
|
-
}
|
|
478
|
-
/**
|
|
479
|
-
* Get cache statistics (compatibility with FileCache)
|
|
480
|
-
*/
|
|
481
|
-
stats() {
|
|
482
|
-
const cacheStats = this.getStats();
|
|
483
|
-
const total = this.l1Hits + this.l2Hits + this.misses;
|
|
484
|
-
const hitRate = total > 0 ? (this.l1Hits + this.l2Hits) / total : 0;
|
|
485
|
-
return {
|
|
486
|
-
entries: cacheStats.entryCount,
|
|
487
|
-
sizeBytes: cacheStats.totalBytes,
|
|
488
|
-
hitRate,
|
|
489
|
-
};
|
|
490
|
-
}
|
|
491
|
-
/**
|
|
492
|
-
* Get detailed cache performance metrics
|
|
493
|
-
*/
|
|
494
|
-
getPerformanceMetrics() {
|
|
495
|
-
const total = this.l1Hits + this.l2Hits + this.misses;
|
|
496
|
-
const hits = this.l1Hits + this.l2Hits;
|
|
497
|
-
const hitRate = total > 0 ? hits / total : 0;
|
|
498
|
-
const l1HitRate = this.l1Hits > 0 ? this.l1Hits / (this.l1Hits + this.l2Hits + this.misses) : 0;
|
|
499
|
-
const l2HitRate = this.l2Hits > 0 ? this.l2Hits / total : 0;
|
|
500
|
-
return {
|
|
501
|
-
l1Hits: this.l1Hits,
|
|
502
|
-
l2Hits: this.l2Hits,
|
|
503
|
-
misses: this.misses,
|
|
504
|
-
totalRequests: total,
|
|
505
|
-
hitRate,
|
|
506
|
-
l1HitRate,
|
|
507
|
-
l2HitRate,
|
|
508
|
-
l1Only: this.l1Only,
|
|
509
|
-
};
|
|
510
|
-
}
|
|
511
|
-
/**
|
|
512
|
-
* Trigger onSet callback after setting entry
|
|
513
|
-
*/
|
|
514
|
-
triggerOnSet(filePath) {
|
|
515
|
-
this.onSetCallback?.(filePath);
|
|
516
|
-
}
|
|
517
|
-
/**
|
|
518
|
-
* Close database connection
|
|
519
|
-
*/
|
|
520
|
-
close() {
|
|
521
|
-
if (!this.db) {
|
|
522
|
-
return;
|
|
523
|
-
}
|
|
524
|
-
try {
|
|
525
|
-
this.db.close();
|
|
526
|
-
}
|
|
527
|
-
catch (err) {
|
|
528
|
-
const message = err instanceof Error ? err.message : String(err);
|
|
529
|
-
console.error(`[token-pilot] Cache close error: ${message}`);
|
|
530
|
-
}
|
|
531
|
-
}
|
|
532
|
-
}
|
|
533
|
-
/**
|
|
534
|
-
* Singleton instance for global use
|
|
535
|
-
*/
|
|
536
|
-
let instance = null;
|
|
537
|
-
export function initializePersistentCache(projectRoot, maxSizeMB) {
|
|
538
|
-
if (!instance) {
|
|
539
|
-
instance = new PersistentFileCache(projectRoot, maxSizeMB);
|
|
540
|
-
}
|
|
541
|
-
return instance;
|
|
542
|
-
}
|
|
543
|
-
export function getPersistentCache() {
|
|
544
|
-
if (!instance) {
|
|
545
|
-
throw new Error('PersistentFileCache not initialized. Call initializePersistentCache first.');
|
|
546
|
-
}
|
|
547
|
-
return instance;
|
|
548
|
-
}
|
|
549
|
-
export function closePersistentCache() {
|
|
550
|
-
if (instance) {
|
|
551
|
-
instance.close();
|
|
552
|
-
instance = null;
|
|
553
|
-
}
|
|
554
|
-
}
|
|
555
|
-
//# sourceMappingURL=persistent-cache.js.map
|