specmem-hardwicksoftware 3.7.31 → 3.7.33
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +22 -0
- package/README.md +12 -0
- package/bin/specmem-cli.cjs +18 -33
- package/bootstrap.cjs +5 -1
- package/claude-hooks/agent-loading-hook.js +31 -47
- package/claude-hooks/bash-call-enforcer.cjs +140 -0
- package/claude-hooks/settings.json +33 -0
- package/claude-hooks/specmem-drilldown-hook.cjs +49 -2
- package/claude-hooks/specmem-drilldown-hook.js +49 -2
- package/claude-hooks/specmem-drilldown-hook.js.bak +495 -0
- package/claude-hooks/specmem-precompact.cjs +13 -36
- package/claude-hooks/specmem-precompact.js +3 -7
- package/claude-hooks/specmem-session-start.cjs +38 -50
- package/claude-hooks/specmem-session-start.js +19 -60
- package/claude-hooks/subagent-loading-hook.js +1 -1
- package/claude-hooks/team-comms-enforcer.cjs +112 -93
- package/dist/config/configSync.js +4 -1
- package/dist/index.js +44 -1
- package/dist/init/claudeConfigInjector.js +4 -1
- package/dist/installer/autoInstall.js +4 -1
- package/dist/mcp/compactionProxy.js +648 -73
- package/dist/mcp/compactionProxyDaemon.js +18 -4
- package/dist/mcp/specMemServer.js +8 -0
- package/dist/watcher/index.js +40 -0
- package/package.json +1 -1
- package/scripts/deploy-hooks.cjs +4 -1
- package/scripts/specmem-init.cjs +31 -35
- package/scripts/specmem-uninstall.cjs +4 -1
- package/specmem/model-config.json +4 -4
- package/specmem/supervisord.conf +1 -1
- package/svg-sections/readme-install.svg +94 -52
- package/svg-sections/readme-whats-new.svg +120 -114
|
@@ -15,7 +15,7 @@
|
|
|
15
15
|
* @website https://justcalljon.pro
|
|
16
16
|
*/
|
|
17
17
|
|
|
18
|
-
import { createServer, get as httpGet } from 'http';
|
|
18
|
+
import { createServer, get as httpGet, request as httpRequest } from 'http';
|
|
19
19
|
import { request as httpsRequest } from 'https';
|
|
20
20
|
import { appendFileSync, writeFileSync, readFileSync, mkdirSync, existsSync, copyFileSync, unlinkSync } from 'fs';
|
|
21
21
|
import { createConnection } from 'net';
|
|
@@ -58,16 +58,58 @@ const TOOL_USE_INPUT_PREVIEW_CHARS = 100;
|
|
|
58
58
|
|
|
59
59
|
// Old-message stripping config — applied to ALL requests (not just compaction)
|
|
60
60
|
// Only strips tool_results larger than this threshold in messages outside the recent window
|
|
61
|
-
const OLD_STRIP_THRESHOLD =
|
|
61
|
+
const OLD_STRIP_THRESHOLD = 100; // chars — only strip results bigger than this
|
|
62
62
|
const OLD_STRIP_PREVIEW_CHARS = 200; // chars — keep this much of the original
|
|
63
63
|
|
|
64
64
|
// Live neural MT compression config
|
|
65
|
-
const
|
|
65
|
+
const _TRANSLATE_SOCKET_FALLBACK = process.env.SPECMEM_TRANSLATE_SOCKET ||
|
|
66
66
|
join(process.env.SPECMEM_PROJECT_PATH || process.cwd(), 'specmem', 'run', 'translate.sock');
|
|
67
67
|
const PRESERVE_RECENT_MESSAGES = 3;
|
|
68
|
-
const MIN_TRANSLATE_LENGTH =
|
|
68
|
+
const MIN_TRANSLATE_LENGTH = 0; // No minimum — compress everything
|
|
69
69
|
const DONT_COMPRESS_FLAG = 'DONT_COMPRESS';
|
|
70
70
|
|
|
71
|
+
// ============================================================================
|
|
72
|
+
// Multi-Project Registry — tracks all registered projects for resource sharing
|
|
73
|
+
// ============================================================================
|
|
74
|
+
|
|
75
|
+
const projectRegistry = new Map(); // keyed by projectPath → { projectPath, pid, registeredAt, lastSeen }
|
|
76
|
+
|
|
77
|
+
// Auto-register the spawning project so daemon always has at least one
|
|
78
|
+
const _foundingProjectPath = process.env.SPECMEM_PROJECT_PATH || null;
|
|
79
|
+
if (_foundingProjectPath) {
|
|
80
|
+
projectRegistry.set(_foundingProjectPath, {
|
|
81
|
+
projectPath: _foundingProjectPath,
|
|
82
|
+
pid: process.ppid || 0,
|
|
83
|
+
registeredAt: Date.now(),
|
|
84
|
+
lastSeen: Date.now(),
|
|
85
|
+
});
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
/**
|
|
89
|
+
* Resolve translate socket dynamically across all registered projects.
|
|
90
|
+
* Iterates registry sorted by lastSeen desc, returns first socket that exists.
|
|
91
|
+
* Fallback: SPECMEM_TRANSLATE_SOCKET env var, then null.
|
|
92
|
+
*/
|
|
93
|
+
function resolveTranslateSocket() {
|
|
94
|
+
// Try registered projects, most recently seen first
|
|
95
|
+
const entries = [...projectRegistry.values()].sort((a, b) => b.lastSeen - a.lastSeen);
|
|
96
|
+
for (const entry of entries) {
|
|
97
|
+
const sockPath = join(entry.projectPath, 'specmem', 'run', 'translate.sock');
|
|
98
|
+
if (existsSync(sockPath)) return sockPath;
|
|
99
|
+
}
|
|
100
|
+
// Fallback to env var / founding project socket
|
|
101
|
+
if (existsSync(_TRANSLATE_SOCKET_FALLBACK)) return _TRANSLATE_SOCKET_FALLBACK;
|
|
102
|
+
return null;
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
/**
|
|
106
|
+
* Invalidate TM and synonym caches — called on registry changes.
|
|
107
|
+
*/
|
|
108
|
+
function _invalidateProjectCaches() {
|
|
109
|
+
_tmCache = null;
|
|
110
|
+
_synonyms = null;
|
|
111
|
+
}
|
|
112
|
+
|
|
71
113
|
// Preview ring buffer — stores last 5 compressed requests for TUI display
|
|
72
114
|
const PREVIEW_BUFFER_SIZE = 5;
|
|
73
115
|
const recentRequests = []; // { original, optimized, timestamp, type, savings }
|
|
@@ -160,6 +202,7 @@ let liveConfig = {
|
|
|
160
202
|
TRANSLATE_ENABLED: true, // toggle MT translation (requires translate.sock)
|
|
161
203
|
OLD_STRIP_ENABLED: true, // toggle old tool_result stripping
|
|
162
204
|
SYSTEM_PROMPT_COMPRESS: true, // toggle system prompt steno+translate compression
|
|
205
|
+
SYSTEM_REMINDER_STRIPPING: true, // toggle <system-reminder> stripping (keeps first, strips rest)
|
|
163
206
|
};
|
|
164
207
|
|
|
165
208
|
// Stats
|
|
@@ -171,6 +214,7 @@ const stats = {
|
|
|
171
214
|
tokensStripped: 0,
|
|
172
215
|
bytesStripped: 0,
|
|
173
216
|
liveCharsCompressed: 0,
|
|
217
|
+
systemRemindersStripped: 0,
|
|
174
218
|
liveBlocksCompressed: 0,
|
|
175
219
|
zhVerified: 0, // Chinese translations that passed loop-back verification
|
|
176
220
|
zhRejected: 0, // Chinese translations rejected (steno fallback)
|
|
@@ -200,18 +244,53 @@ const TM_MAX_ENTRIES = 10000;
|
|
|
200
244
|
let _tmCache = null; // { hash: { en, zh, sim, ts } }
|
|
201
245
|
let _synonyms = null; // { word: Set<synonym> } — learned from failed verifications
|
|
202
246
|
|
|
247
|
+
/**
|
|
248
|
+
* Get all TM file paths from registered projects.
|
|
249
|
+
*/
|
|
250
|
+
function _getAllTMPaths() {
|
|
251
|
+
const paths = [];
|
|
252
|
+
for (const entry of projectRegistry.values()) {
|
|
253
|
+
paths.push(join(entry.projectPath, 'specmem', 'run', 'translation-memory.json'));
|
|
254
|
+
}
|
|
255
|
+
// Always include founding project path as fallback
|
|
256
|
+
if (!paths.includes(TM_FILE)) paths.push(TM_FILE);
|
|
257
|
+
return paths;
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
/**
|
|
261
|
+
* Get all synonyms file paths from registered projects.
|
|
262
|
+
*/
|
|
263
|
+
function _getAllSynonymPaths() {
|
|
264
|
+
const paths = [];
|
|
265
|
+
for (const entry of projectRegistry.values()) {
|
|
266
|
+
paths.push(join(entry.projectPath, 'specmem', 'run', 'learned-synonyms.json'));
|
|
267
|
+
}
|
|
268
|
+
if (!paths.includes(SYNONYMS_FILE)) paths.push(SYNONYMS_FILE);
|
|
269
|
+
return paths;
|
|
270
|
+
}
|
|
271
|
+
|
|
203
272
|
function _tmHash(text) {
|
|
204
273
|
return createHash('md5').update(text.toLowerCase().trim()).digest('hex').slice(0, 16);
|
|
205
274
|
}
|
|
206
275
|
|
|
207
276
|
function _loadTM() {
|
|
208
277
|
if (_tmCache) return _tmCache;
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
278
|
+
_tmCache = {};
|
|
279
|
+
// Merge TM entries from all registered projects (later timestamp wins)
|
|
280
|
+
for (const tmPath of _getAllTMPaths()) {
|
|
281
|
+
try {
|
|
282
|
+
if (existsSync(tmPath)) {
|
|
283
|
+
const entries = JSON.parse(readFileSync(tmPath, 'utf8'));
|
|
284
|
+
if (entries && typeof entries === 'object') {
|
|
285
|
+
for (const [hash, entry] of Object.entries(entries)) {
|
|
286
|
+
if (!_tmCache[hash] || (entry.ts || 0) > (_tmCache[hash].ts || 0)) {
|
|
287
|
+
_tmCache[hash] = entry;
|
|
288
|
+
}
|
|
289
|
+
}
|
|
290
|
+
}
|
|
291
|
+
}
|
|
292
|
+
} catch { /* corrupt file, skip */ }
|
|
293
|
+
}
|
|
215
294
|
return _tmCache;
|
|
216
295
|
}
|
|
217
296
|
|
|
@@ -225,7 +304,14 @@ function _saveTM() {
|
|
|
225
304
|
const toRemove = sorted.slice(0, keys.length - TM_MAX_ENTRIES);
|
|
226
305
|
for (const k of toRemove) delete tm[k];
|
|
227
306
|
}
|
|
228
|
-
|
|
307
|
+
const data = JSON.stringify(tm);
|
|
308
|
+
// Fan-out write to ALL registered projects
|
|
309
|
+
for (const tmPath of _getAllTMPaths()) {
|
|
310
|
+
try {
|
|
311
|
+
mkdirSync(dirname(tmPath), { recursive: true });
|
|
312
|
+
writeFileSync(tmPath, data, 'utf8');
|
|
313
|
+
} catch { /* skip unreachable paths */ }
|
|
314
|
+
}
|
|
229
315
|
} catch (e) { log('warn', `TM save failed: ${e.message}`); }
|
|
230
316
|
}
|
|
231
317
|
|
|
@@ -253,29 +339,41 @@ let _tmSaveTimer = null;
|
|
|
253
339
|
// e.g., MT says "rebuild" but original had "refactor" — these are equivalent
|
|
254
340
|
function _loadSynonyms() {
|
|
255
341
|
if (_synonyms) return _synonyms;
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
342
|
+
_synonyms = {};
|
|
343
|
+
// Merge synonyms from all registered projects + package seed
|
|
344
|
+
const allPaths = [
|
|
345
|
+
..._getAllSynonymPaths(),
|
|
259
346
|
join(fileURLToPath(import.meta.url), '..', '..', '..', 'specmem', 'run', 'learned-synonyms.json'),
|
|
260
347
|
];
|
|
261
|
-
|
|
348
|
+
let loadedAny = false;
|
|
349
|
+
for (const synPath of allPaths) {
|
|
262
350
|
try {
|
|
263
351
|
if (existsSync(synPath)) {
|
|
264
352
|
const raw = JSON.parse(readFileSync(synPath, 'utf8'));
|
|
265
|
-
_synonyms = {};
|
|
266
353
|
for (const [word, syns] of Object.entries(raw)) {
|
|
267
|
-
_synonyms[word] = new Set(
|
|
354
|
+
if (!_synonyms[word]) _synonyms[word] = new Set();
|
|
355
|
+
const synList = Array.isArray(syns) ? syns : [...syns];
|
|
356
|
+
for (const s of synList) _synonyms[word].add(s);
|
|
268
357
|
}
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
358
|
+
loadedAny = true;
|
|
359
|
+
}
|
|
360
|
+
} catch { /* skip corrupt */ }
|
|
361
|
+
}
|
|
362
|
+
// If loaded from package seed and no project files exist, seed all projects
|
|
363
|
+
if (loadedAny && Object.keys(_synonyms).length > 0) {
|
|
364
|
+
for (const synPath of _getAllSynonymPaths()) {
|
|
365
|
+
if (!existsSync(synPath)) {
|
|
366
|
+
try { mkdirSync(dirname(synPath), { recursive: true }); } catch {}
|
|
367
|
+
try {
|
|
368
|
+
const serializable = {};
|
|
369
|
+
for (const [word, set] of Object.entries(_synonyms)) {
|
|
370
|
+
serializable[word] = [...set];
|
|
371
|
+
}
|
|
372
|
+
writeFileSync(synPath, JSON.stringify(serializable), 'utf8');
|
|
373
|
+
} catch {}
|
|
275
374
|
}
|
|
276
|
-
}
|
|
375
|
+
}
|
|
277
376
|
}
|
|
278
|
-
if (!_synonyms) _synonyms = {};
|
|
279
377
|
return _synonyms;
|
|
280
378
|
}
|
|
281
379
|
|
|
@@ -286,7 +384,14 @@ function _saveSynonyms() {
|
|
|
286
384
|
for (const [word, set] of Object.entries(syns)) {
|
|
287
385
|
serializable[word] = [...set];
|
|
288
386
|
}
|
|
289
|
-
|
|
387
|
+
const data = JSON.stringify(serializable);
|
|
388
|
+
// Fan-out write to ALL registered projects
|
|
389
|
+
for (const synPath of _getAllSynonymPaths()) {
|
|
390
|
+
try {
|
|
391
|
+
mkdirSync(dirname(synPath), { recursive: true });
|
|
392
|
+
writeFileSync(synPath, data, 'utf8');
|
|
393
|
+
} catch { /* skip unreachable paths */ }
|
|
394
|
+
}
|
|
290
395
|
} catch (e) { log('warn', `Synonyms save failed: ${e.message}`); }
|
|
291
396
|
}
|
|
292
397
|
|
|
@@ -380,6 +485,23 @@ function isCompactionRequest(body) {
|
|
|
380
485
|
return false;
|
|
381
486
|
}
|
|
382
487
|
|
|
488
|
+
/**
|
|
489
|
+
* Smart-strip Edit tool_use inputs to just - / + diff lines.
|
|
490
|
+
* Returns compact string or null if not an Edit block.
|
|
491
|
+
*/
|
|
492
|
+
function smartStripEdit(input) {
|
|
493
|
+
if (!input || !input.file_path || !input.old_string) return null;
|
|
494
|
+
const lines = [`Edit(${input.file_path})`];
|
|
495
|
+
// old_string → removed lines (prefix with -)
|
|
496
|
+
for (const l of input.old_string.split('\n')) lines.push(`- ${l}`);
|
|
497
|
+
// new_string → added lines (prefix with +)
|
|
498
|
+
if (input.new_string != null) {
|
|
499
|
+
for (const l of input.new_string.split('\n')) lines.push(`+ ${l}`);
|
|
500
|
+
}
|
|
501
|
+
if (input.replace_all) lines.push('(replace_all)');
|
|
502
|
+
return lines.join('\n');
|
|
503
|
+
}
|
|
504
|
+
|
|
383
505
|
function stripMessages(messages) {
|
|
384
506
|
if (!Array.isArray(messages)) return { strippedMessages: messages, strippingStats: { toolResultsStripped: 0, toolUsesStripped: 0, charsRemoved: 0 } };
|
|
385
507
|
|
|
@@ -422,6 +544,22 @@ function stripMessages(messages) {
|
|
|
422
544
|
const input = block.input;
|
|
423
545
|
if (!input) return block;
|
|
424
546
|
|
|
547
|
+
// Smart diff stripping for Edit tool — keep only - / + lines
|
|
548
|
+
const editDiff = (block.name === 'Edit') ? smartStripEdit(input) : null;
|
|
549
|
+
if (editDiff) {
|
|
550
|
+
const inputStr = JSON.stringify(input);
|
|
551
|
+
charsRemoved += inputStr.length - editDiff.length;
|
|
552
|
+
toolUsesStripped++;
|
|
553
|
+
const stripped = {
|
|
554
|
+
type: 'tool_use',
|
|
555
|
+
id: block.id,
|
|
556
|
+
name: block.name,
|
|
557
|
+
input: { _stripped: editDiff }
|
|
558
|
+
};
|
|
559
|
+
if (block.cache_control) stripped.cache_control = block.cache_control;
|
|
560
|
+
return stripped;
|
|
561
|
+
}
|
|
562
|
+
|
|
425
563
|
const inputStr = JSON.stringify(input);
|
|
426
564
|
if (inputStr.length <= TOOL_USE_INPUT_PREVIEW_CHARS * 2) return block;
|
|
427
565
|
|
|
@@ -513,6 +651,16 @@ function stripOldToolResults(messages) {
|
|
|
513
651
|
const input = block.input;
|
|
514
652
|
if (!input) return block;
|
|
515
653
|
|
|
654
|
+
// Smart Edit stripping — keep only - / + diff lines
|
|
655
|
+
const editDiff = smartStripEdit(block);
|
|
656
|
+
if (editDiff) {
|
|
657
|
+
const origLen = JSON.stringify(input).length;
|
|
658
|
+
const newLen = JSON.stringify(editDiff.input).length;
|
|
659
|
+
charsRemoved += origLen - newLen;
|
|
660
|
+
toolResultsStripped++;
|
|
661
|
+
return editDiff;
|
|
662
|
+
}
|
|
663
|
+
|
|
516
664
|
const inputStr = JSON.stringify(input);
|
|
517
665
|
if (inputStr.length <= liveConfig.OLD_STRIP_THRESHOLD) return block;
|
|
518
666
|
|
|
@@ -533,6 +681,36 @@ function stripOldToolResults(messages) {
|
|
|
533
681
|
return stripped;
|
|
534
682
|
}
|
|
535
683
|
|
|
684
|
+
// Strip specmem hook injection text blocks from old messages
|
|
685
|
+
if (block.type === 'text' && typeof block.text === 'string') {
|
|
686
|
+
const txt = block.text;
|
|
687
|
+
const isHookOutput = txt.includes('[SM-找]') || txt.includes('[SM-碼]') ||
|
|
688
|
+
txt.includes('[SM-檔改]') || txt.includes('[SPECMEM-SESSION]') ||
|
|
689
|
+
txt.includes('⚠️壓縮:繁中') || txt.includes('[SM-搜]');
|
|
690
|
+
if (isHookOutput && txt.length > liveConfig.OLD_STRIP_THRESHOLD) {
|
|
691
|
+
const removed = txt.length - OLD_STRIP_PREVIEW_CHARS;
|
|
692
|
+
charsRemoved += removed;
|
|
693
|
+
toolResultsStripped++;
|
|
694
|
+
return {
|
|
695
|
+
...block,
|
|
696
|
+
text: txt.slice(0, OLD_STRIP_PREVIEW_CHARS) + `...\n[HOOK-TRIMMED: ${txt.length} chars]`
|
|
697
|
+
};
|
|
698
|
+
}
|
|
699
|
+
|
|
700
|
+
// Strip old assistant text blocks — Claude's own output echoed back
|
|
701
|
+
// No point sending Claude its own words; keep first line as context anchor
|
|
702
|
+
if (msg.role === 'assistant' && txt.length > 120) {
|
|
703
|
+
const firstLine = txt.split('\n')[0].slice(0, 120);
|
|
704
|
+
const removed = txt.length - firstLine.length;
|
|
705
|
+
charsRemoved += removed;
|
|
706
|
+
toolResultsStripped++;
|
|
707
|
+
return {
|
|
708
|
+
...block,
|
|
709
|
+
text: `${firstLine}...\n[ASST-ECHO-STRIPPED: ${txt.length} chars → ${firstLine.length}]`
|
|
710
|
+
};
|
|
711
|
+
}
|
|
712
|
+
}
|
|
713
|
+
|
|
536
714
|
return block;
|
|
537
715
|
});
|
|
538
716
|
|
|
@@ -542,6 +720,81 @@ function stripOldToolResults(messages) {
|
|
|
542
720
|
return { messages: newMessages, strippingStats: { toolResultsStripped, charsRemoved } };
|
|
543
721
|
}
|
|
544
722
|
|
|
723
|
+
/**
|
|
724
|
+
* Strip <system-reminder> tags from messages.
|
|
725
|
+
* Keeps the FIRST system-reminder encountered (sets up style/context),
|
|
726
|
+
* strips all subsequent ones to save tokens.
|
|
727
|
+
*
|
|
728
|
+
* Returns { messages, remindersStripped, charsRemoved }.
|
|
729
|
+
*/
|
|
730
|
+
function stripSystemReminders(messages) {
|
|
731
|
+
if (!Array.isArray(messages)) return { messages, remindersStripped: 0, charsRemoved: 0 };
|
|
732
|
+
if (!liveConfig.SYSTEM_REMINDER_STRIPPING) return { messages, remindersStripped: 0, charsRemoved: 0 };
|
|
733
|
+
|
|
734
|
+
const SR_RE = /<system-reminder>[\s\S]*?<\/system-reminder>/g;
|
|
735
|
+
let firstSeen = false;
|
|
736
|
+
let remindersStripped = 0;
|
|
737
|
+
let charsRemoved = 0;
|
|
738
|
+
|
|
739
|
+
const newMessages = messages.map(msg => {
|
|
740
|
+
if (!msg) return msg;
|
|
741
|
+
|
|
742
|
+
// Handle string content
|
|
743
|
+
if (typeof msg.content === 'string') {
|
|
744
|
+
const matches = msg.content.match(SR_RE);
|
|
745
|
+
if (!matches) return msg;
|
|
746
|
+
|
|
747
|
+
let newText = msg.content;
|
|
748
|
+
for (const match of matches) {
|
|
749
|
+
if (!firstSeen) {
|
|
750
|
+
firstSeen = true; // keep the very first one
|
|
751
|
+
continue;
|
|
752
|
+
}
|
|
753
|
+
newText = newText.replace(match, '');
|
|
754
|
+
charsRemoved += match.length;
|
|
755
|
+
remindersStripped++;
|
|
756
|
+
}
|
|
757
|
+
return { ...msg, content: newText.replace(/\n{3,}/g, '\n\n').trim() };
|
|
758
|
+
}
|
|
759
|
+
|
|
760
|
+
// Handle array content blocks
|
|
761
|
+
if (!Array.isArray(msg.content)) return msg;
|
|
762
|
+
|
|
763
|
+
const newContent = msg.content.map(block => {
|
|
764
|
+
if (block.type !== 'text' || typeof block.text !== 'string') return block;
|
|
765
|
+
|
|
766
|
+
const matches = block.text.match(SR_RE);
|
|
767
|
+
if (!matches) return block;
|
|
768
|
+
|
|
769
|
+
let newText = block.text;
|
|
770
|
+
for (const match of matches) {
|
|
771
|
+
if (!firstSeen) {
|
|
772
|
+
firstSeen = true; // keep the very first one
|
|
773
|
+
continue;
|
|
774
|
+
}
|
|
775
|
+
newText = newText.replace(match, '');
|
|
776
|
+
charsRemoved += match.length;
|
|
777
|
+
remindersStripped++;
|
|
778
|
+
}
|
|
779
|
+
const cleaned = newText.replace(/\n{3,}/g, '\n\n').trim();
|
|
780
|
+
|
|
781
|
+
// If block is now empty after stripping, remove it entirely
|
|
782
|
+
if (!cleaned) return null;
|
|
783
|
+
|
|
784
|
+
return { ...block, text: cleaned };
|
|
785
|
+
}).filter(Boolean);
|
|
786
|
+
|
|
787
|
+
// If all content blocks were stripped, keep a minimal marker
|
|
788
|
+
if (newContent.length === 0) {
|
|
789
|
+
return { ...msg, content: [{ type: 'text', text: '[context]' }] };
|
|
790
|
+
}
|
|
791
|
+
|
|
792
|
+
return { ...msg, content: newContent };
|
|
793
|
+
});
|
|
794
|
+
|
|
795
|
+
return { messages: newMessages, remindersStripped, charsRemoved };
|
|
796
|
+
}
|
|
797
|
+
|
|
545
798
|
// ============================================================================
|
|
546
799
|
// Live Compression — Stenography + Neural MT with Loop-back Verification
|
|
547
800
|
// ============================================================================
|
|
@@ -789,6 +1042,9 @@ async function compressMessagesLive(messages) {
|
|
|
789
1042
|
|
|
790
1043
|
// Standalone text blocks (user/assistant conversational text) → steno only
|
|
791
1044
|
if (block.type === 'text' && typeof block.text === 'string') {
|
|
1045
|
+
// Skip specmem hook injection text — already partially compressed Chinese, steno would garble
|
|
1046
|
+
const _htxt = block.text;
|
|
1047
|
+
if (_htxt.includes('[SM-') || _htxt.includes('[SPECMEM-SESSION]') || _htxt.includes('⚠️壓縮:繁中')) continue;
|
|
792
1048
|
if (looksLikeNaturalLanguage(block.text)) {
|
|
793
1049
|
originals.push(block.text);
|
|
794
1050
|
textLocations.push({ msgIdx: i, blockIdx: j, mtEligible: false });
|
|
@@ -829,15 +1085,60 @@ async function compressMessagesLive(messages) {
|
|
|
829
1085
|
let stenoOnlyCount = 0;
|
|
830
1086
|
let tmHits = 0;
|
|
831
1087
|
|
|
832
|
-
const
|
|
1088
|
+
const _resolvedSocket = resolveTranslateSocket();
|
|
1089
|
+
const translateAvailable = liveConfig.TRANSLATE_ENABLED && !!_resolvedSocket;
|
|
1090
|
+
|
|
1091
|
+
// ── Pre-translation filter: detect "translation-hostile" content ──
|
|
1092
|
+
// File paths, IDs, hashes, permissions, IPs, URLs, code identifiers, etc.
|
|
1093
|
+
// These will ALWAYS fail backtranslation verification, so skip MT entirely.
|
|
1094
|
+
const _PATH_RE = /(?:^|\s)[.~]?\/[\w./-]{3,}/; // /foo/bar.js, ./rel, ~/home
|
|
1095
|
+
const _UUID_RE = /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/i;
|
|
1096
|
+
const _HEX_RE = /\b[0-9a-f]{7,64}\b/i; // git hashes, hex IDs
|
|
1097
|
+
const _PERM_RE = /[dl-][r-][w-][xsStT-][r-][w-][xsStT-][r-][w-][xsStT-]/; // drwxr-xr-x
|
|
1098
|
+
const _IP_RE = /\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b/; // IPv4
|
|
1099
|
+
const _URL_RE = /https?:\/\/\S{6,}/; // URLs
|
|
1100
|
+
const _PKG_RE = /[\w@][\w./-]*@\d+\.\d+/; // pkg@1.2.3
|
|
1101
|
+
const _CODE_ID = /\b[a-z][a-zA-Z0-9]{2,}\.[a-zA-Z]{1,4}\b/; // file.ext patterns
|
|
1102
|
+
const _CAMEL = /\b[a-z]+[A-Z][a-zA-Z]+\b/; // camelCase identifiers
|
|
1103
|
+
const _SNAKE = /\b[a-z]+_[a-z_]+\b/; // snake_case identifiers
|
|
1104
|
+
|
|
1105
|
+
function isTranslationHostile(text) {
|
|
1106
|
+
if (!text || text.length < 20) return true; // too short to compress meaningfully
|
|
1107
|
+
// Count how many "technical markers" appear
|
|
1108
|
+
const markers = [_PATH_RE, _UUID_RE, _HEX_RE, _PERM_RE, _IP_RE, _URL_RE, _PKG_RE, _CODE_ID];
|
|
1109
|
+
let hits = 0;
|
|
1110
|
+
for (const re of markers) if (re.test(text)) hits++;
|
|
1111
|
+
if (hits >= 3) return true; // 3+ different marker types = definitely technical
|
|
1112
|
+
|
|
1113
|
+
// Check density: what fraction of words are code/path-like?
|
|
1114
|
+
const words = text.split(/\s+/).filter(w => w.length > 1);
|
|
1115
|
+
if (words.length === 0) return true;
|
|
1116
|
+
let techWords = 0;
|
|
1117
|
+
for (const w of words) {
|
|
1118
|
+
if (_PATH_RE.test(w) || _HEX_RE.test(w) || _UUID_RE.test(w) ||
|
|
1119
|
+
_PERM_RE.test(w) || _IP_RE.test(w) || _URL_RE.test(w) ||
|
|
1120
|
+
_CAMEL.test(w) || _SNAKE.test(w) || _PKG_RE.test(w) ||
|
|
1121
|
+
/^\d+$/.test(w) || /^[A-Z_]{3,}$/.test(w)) {
|
|
1122
|
+
techWords++;
|
|
1123
|
+
}
|
|
1124
|
+
}
|
|
1125
|
+
return (techWords / words.length) > 0.40; // >40% technical tokens = skip MT
|
|
1126
|
+
}
|
|
833
1127
|
|
|
834
1128
|
// Step 1.5: Check Translation Memory cache first
|
|
835
1129
|
const needMT = []; // indices that need neural MT (only mtEligible blocks)
|
|
1130
|
+
let mtSkippedTech = 0;
|
|
836
1131
|
for (let i = 0; i < stenoTexts.length; i++) {
|
|
837
1132
|
// Only tool_result content is eligible for Chinese MT
|
|
838
1133
|
// User/assistant text stays steno-only to prevent Claude responding in Chinese
|
|
839
1134
|
if (!textLocations[i].mtEligible) continue;
|
|
840
1135
|
|
|
1136
|
+
// Skip translation-hostile content (file paths, IDs, code, etc.)
|
|
1137
|
+
if (isTranslationHostile(stenoTexts[i])) {
|
|
1138
|
+
mtSkippedTech++;
|
|
1139
|
+
continue; // stays steno-only — finalTexts[i] already has steno text
|
|
1140
|
+
}
|
|
1141
|
+
|
|
841
1142
|
const cached = tmLookup(stenoTexts[i]);
|
|
842
1143
|
if (cached && cached.length < stenoTexts[i].length) {
|
|
843
1144
|
finalTexts[i] = cached;
|
|
@@ -847,12 +1148,15 @@ async function compressMessagesLive(messages) {
|
|
|
847
1148
|
needMT.push(i);
|
|
848
1149
|
}
|
|
849
1150
|
}
|
|
1151
|
+
if (mtSkippedTech > 0) {
|
|
1152
|
+
log('info', `MT-SKIP: ${mtSkippedTech} blocks skipped (translation-hostile content)`);
|
|
1153
|
+
}
|
|
850
1154
|
|
|
851
1155
|
if (translateAvailable && needMT.length > 0) {
|
|
852
1156
|
try {
|
|
853
1157
|
// Step 2: Translate uncached steno English → Chinese (zt)
|
|
854
1158
|
const mtInputs = needMT.map(i => stenoTexts[i]);
|
|
855
|
-
const chineseTexts = await translateBatch(mtInputs,
|
|
1159
|
+
const chineseTexts = await translateBatch(mtInputs, _resolvedSocket, 'en', 'zh');
|
|
856
1160
|
|
|
857
1161
|
// Filter: only verify texts that actually changed and got shorter
|
|
858
1162
|
const needVerify = [];
|
|
@@ -868,7 +1172,7 @@ async function compressMessagesLive(messages) {
|
|
|
868
1172
|
|
|
869
1173
|
if (needVerify.length > 0) {
|
|
870
1174
|
// Step 3: Translate Chinese → English (loop-back)
|
|
871
|
-
const backTranslated = await translateBatch(needVerify,
|
|
1175
|
+
const backTranslated = await translateBatch(needVerify, _resolvedSocket, 'zh', 'en');
|
|
872
1176
|
|
|
873
1177
|
// Step 4: Verify each — compare back-translated with original
|
|
874
1178
|
for (let v = 0; v < verifyIndices.length; v++) {
|
|
@@ -1011,13 +1315,14 @@ async function compressSystemPrompt(system) {
|
|
|
1011
1315
|
: [...textBlocks];
|
|
1012
1316
|
|
|
1013
1317
|
// Step 2: Apply Chinese translation to eligible sections
|
|
1014
|
-
|
|
1318
|
+
const _sysSocket = resolveTranslateSocket();
|
|
1319
|
+
if (liveConfig.TRANSLATE_ENABLED && _sysSocket) {
|
|
1015
1320
|
try {
|
|
1016
1321
|
// Translate entire steno'd text via Hardwick Translate
|
|
1017
|
-
const translations = await translateBatch(compressed,
|
|
1322
|
+
const translations = await translateBatch(compressed, _sysSocket, 'en', 'zh');
|
|
1018
1323
|
if (translations && translations.length === compressed.length) {
|
|
1019
1324
|
// Verify each translation round-trips correctly
|
|
1020
|
-
const backTranslated = await translateBatch(translations,
|
|
1325
|
+
const backTranslated = await translateBatch(translations, _sysSocket, 'zh', 'en');
|
|
1021
1326
|
|
|
1022
1327
|
for (let i = 0; i < compressed.length; i++) {
|
|
1023
1328
|
if (!translations[i] || !backTranslated) continue;
|
|
@@ -1133,7 +1438,7 @@ async function handleRequest(req, res) {
|
|
|
1133
1438
|
const tmSize = Object.keys(tm).length;
|
|
1134
1439
|
const synSize = Object.keys(syns).length;
|
|
1135
1440
|
res.writeHead(200, { 'Content-Type': 'application/json' });
|
|
1136
|
-
res.end(JSON.stringify({ status: 'ok', paused: proxyPaused, uptime: Math.round((Date.now() - stats.startTime) / 1000), ...stats, tmEntries: tmSize, synonymPairs: synSize, envBaseUrlSet: !!process.env.ANTHROPIC_BASE_URL, envBaseUrl: process.env.ANTHROPIC_BASE_URL || '(not set)' }, null, 2));
|
|
1441
|
+
res.end(JSON.stringify({ status: 'ok', paused: proxyPaused, uptime: Math.round((Date.now() - stats.startTime) / 1000), ...stats, tmEntries: tmSize, synonymPairs: synSize, registeredProjects: projectRegistry.size, activeTranslateSocket: resolveTranslateSocket(), envBaseUrlSet: !!process.env.ANTHROPIC_BASE_URL, envBaseUrl: process.env.ANTHROPIC_BASE_URL || '(not set)' }, null, 2));
|
|
1137
1442
|
return;
|
|
1138
1443
|
}
|
|
1139
1444
|
|
|
@@ -1156,13 +1461,14 @@ async function handleRequest(req, res) {
|
|
|
1156
1461
|
try {
|
|
1157
1462
|
const body = JSON.parse((await collectBody(req)).toString('utf8'));
|
|
1158
1463
|
if (body.PRESERVE_RECENT_MESSAGES != null) liveConfig.PRESERVE_RECENT_MESSAGES = Math.max(1, Math.min(20, parseInt(body.PRESERVE_RECENT_MESSAGES) || 3));
|
|
1159
|
-
if (body.OLD_STRIP_THRESHOLD != null) liveConfig.OLD_STRIP_THRESHOLD = Math.max(
|
|
1160
|
-
if (body.MIN_TRANSLATE_LENGTH != null) liveConfig.MIN_TRANSLATE_LENGTH = Math.max(
|
|
1464
|
+
if (body.OLD_STRIP_THRESHOLD != null) liveConfig.OLD_STRIP_THRESHOLD = Math.max(0, Math.min(2000, parseInt(body.OLD_STRIP_THRESHOLD) || 0));
|
|
1465
|
+
if (body.MIN_TRANSLATE_LENGTH != null) liveConfig.MIN_TRANSLATE_LENGTH = Math.max(0, Math.min(200, parseInt(body.MIN_TRANSLATE_LENGTH) || 0));
|
|
1161
1466
|
if (body.TOOL_RESULT_PREVIEW_CHARS != null) liveConfig.TOOL_RESULT_PREVIEW_CHARS = Math.max(50, Math.min(500, parseInt(body.TOOL_RESULT_PREVIEW_CHARS) || 200));
|
|
1162
1467
|
if (body.STENO_ENABLED != null) liveConfig.STENO_ENABLED = !!body.STENO_ENABLED;
|
|
1163
1468
|
if (body.TRANSLATE_ENABLED != null) liveConfig.TRANSLATE_ENABLED = !!body.TRANSLATE_ENABLED;
|
|
1164
1469
|
if (body.OLD_STRIP_ENABLED != null) liveConfig.OLD_STRIP_ENABLED = !!body.OLD_STRIP_ENABLED;
|
|
1165
1470
|
if (body.SYSTEM_PROMPT_COMPRESS != null) liveConfig.SYSTEM_PROMPT_COMPRESS = !!body.SYSTEM_PROMPT_COMPRESS;
|
|
1471
|
+
if (body.SYSTEM_REMINDER_STRIPPING != null) liveConfig.SYSTEM_REMINDER_STRIPPING = !!body.SYSTEM_REMINDER_STRIPPING;
|
|
1166
1472
|
res.writeHead(200, { 'Content-Type': 'application/json' });
|
|
1167
1473
|
res.end(JSON.stringify({ ok: true, ...liveConfig }));
|
|
1168
1474
|
} catch (e) {
|
|
@@ -1204,11 +1510,97 @@ async function handleRequest(req, res) {
|
|
|
1204
1510
|
return;
|
|
1205
1511
|
}
|
|
1206
1512
|
|
|
1513
|
+
// Explicit pause/resume endpoints — no guessing with toggle
|
|
1514
|
+
if (req.url === '/pause' && req.method === 'POST') {
|
|
1515
|
+
proxyPaused = true;
|
|
1516
|
+
log('info', 'Proxy PAUSED via /pause');
|
|
1517
|
+
res.writeHead(200, { 'Content-Type': 'application/json' });
|
|
1518
|
+
res.end(JSON.stringify({ paused: true }));
|
|
1519
|
+
return;
|
|
1520
|
+
}
|
|
1521
|
+
if (req.url === '/resume' && req.method === 'POST') {
|
|
1522
|
+
proxyPaused = false;
|
|
1523
|
+
log('info', 'Proxy RESUMED via /resume');
|
|
1524
|
+
res.writeHead(200, { 'Content-Type': 'application/json' });
|
|
1525
|
+
res.end(JSON.stringify({ paused: false }));
|
|
1526
|
+
return;
|
|
1527
|
+
}
|
|
1528
|
+
|
|
1529
|
+
// --- Multi-Project Registry Endpoints ---
|
|
1530
|
+
|
|
1531
|
+
// POST /register — register a project with the daemon
|
|
1532
|
+
if (req.url === '/register' && req.method === 'POST') {
|
|
1533
|
+
try {
|
|
1534
|
+
const body = JSON.parse((await collectBody(req)).toString('utf8'));
|
|
1535
|
+
if (!body.projectPath) {
|
|
1536
|
+
res.writeHead(400, { 'Content-Type': 'application/json' });
|
|
1537
|
+
res.end(JSON.stringify({ error: 'projectPath required' }));
|
|
1538
|
+
return;
|
|
1539
|
+
}
|
|
1540
|
+
projectRegistry.set(body.projectPath, {
|
|
1541
|
+
projectPath: body.projectPath,
|
|
1542
|
+
pid: body.pid || 0,
|
|
1543
|
+
registeredAt: projectRegistry.get(body.projectPath)?.registeredAt || Date.now(),
|
|
1544
|
+
lastSeen: Date.now(),
|
|
1545
|
+
});
|
|
1546
|
+
_invalidateProjectCaches();
|
|
1547
|
+
log('info', `REGISTRY: registered project "${body.projectPath}" (pid=${body.pid || 0}, total=${projectRegistry.size})`);
|
|
1548
|
+
res.writeHead(200, { 'Content-Type': 'application/json' });
|
|
1549
|
+
res.end(JSON.stringify({ ok: true, registered: projectRegistry.size }));
|
|
1550
|
+
} catch (e) {
|
|
1551
|
+
res.writeHead(400, { 'Content-Type': 'application/json' });
|
|
1552
|
+
res.end(JSON.stringify({ error: e.message }));
|
|
1553
|
+
}
|
|
1554
|
+
return;
|
|
1555
|
+
}
|
|
1556
|
+
|
|
1557
|
+
// POST /deregister — remove a project from the daemon
|
|
1558
|
+
if (req.url === '/deregister' && req.method === 'POST') {
|
|
1559
|
+
try {
|
|
1560
|
+
const body = JSON.parse((await collectBody(req)).toString('utf8'));
|
|
1561
|
+
if (body.projectPath && projectRegistry.has(body.projectPath)) {
|
|
1562
|
+
projectRegistry.delete(body.projectPath);
|
|
1563
|
+
_invalidateProjectCaches();
|
|
1564
|
+
log('info', `REGISTRY: deregistered project "${body.projectPath}" (remaining=${projectRegistry.size})`);
|
|
1565
|
+
}
|
|
1566
|
+
res.writeHead(200, { 'Content-Type': 'application/json' });
|
|
1567
|
+
res.end(JSON.stringify({ ok: true, registered: projectRegistry.size }));
|
|
1568
|
+
} catch (e) {
|
|
1569
|
+
res.writeHead(400, { 'Content-Type': 'application/json' });
|
|
1570
|
+
res.end(JSON.stringify({ error: e.message }));
|
|
1571
|
+
}
|
|
1572
|
+
return;
|
|
1573
|
+
}
|
|
1574
|
+
|
|
1575
|
+
// GET /registry — diagnostic: list all registered projects
|
|
1576
|
+
if (req.url === '/registry' && req.method === 'GET') {
|
|
1577
|
+
const projects = [];
|
|
1578
|
+
for (const [path, entry] of projectRegistry.entries()) {
|
|
1579
|
+
let alive = false;
|
|
1580
|
+
if (entry.pid > 0) {
|
|
1581
|
+
try { process.kill(entry.pid, 0); alive = true; } catch {}
|
|
1582
|
+
}
|
|
1583
|
+
const sockPath = join(path, 'specmem', 'run', 'translate.sock');
|
|
1584
|
+
projects.push({
|
|
1585
|
+
projectPath: path,
|
|
1586
|
+
pid: entry.pid,
|
|
1587
|
+
alive,
|
|
1588
|
+
registeredAt: new Date(entry.registeredAt).toISOString(),
|
|
1589
|
+
lastSeen: new Date(entry.lastSeen).toISOString(),
|
|
1590
|
+
translateSocket: existsSync(sockPath) ? sockPath : null,
|
|
1591
|
+
});
|
|
1592
|
+
}
|
|
1593
|
+
res.writeHead(200, { 'Content-Type': 'application/json' });
|
|
1594
|
+
res.end(JSON.stringify({ projects, total: projects.length }, null, 2));
|
|
1595
|
+
return;
|
|
1596
|
+
}
|
|
1597
|
+
|
|
1207
1598
|
// Count only real API traffic, not management endpoint polls
|
|
1208
1599
|
stats.totalRequests++;
|
|
1209
1600
|
|
|
1601
|
+
let rawBody;
|
|
1210
1602
|
try {
|
|
1211
|
-
|
|
1603
|
+
rawBody = await collectBody(req);
|
|
1212
1604
|
|
|
1213
1605
|
// Paused — pass everything through untouched
|
|
1214
1606
|
if (proxyPaused) {
|
|
@@ -1255,7 +1647,8 @@ async function handleRequest(req, res) {
|
|
|
1255
1647
|
pushEvent('info', `POST /v1/messages model=${body.model || '?'} msgs=${messageCount} size=${(originalSize / 1024).toFixed(0)}KB`);
|
|
1256
1648
|
|
|
1257
1649
|
const isCompaction = isCompactionRequest(body);
|
|
1258
|
-
|
|
1650
|
+
// No passthrough — always process everything (system-reminder strip + steno + translate)
|
|
1651
|
+
const isPassthrough = false;
|
|
1259
1652
|
let sysPromptModified = false;
|
|
1260
1653
|
|
|
1261
1654
|
// === SYSTEM PROMPT COMPRESSION ===
|
|
@@ -1266,7 +1659,7 @@ async function handleRequest(req, res) {
|
|
|
1266
1659
|
const _sysKey = typeof body.system === 'string' ? body.system
|
|
1267
1660
|
: Array.isArray(body.system) ? body.system.map(b => typeof b === 'string' ? b : (b?.text || '')).join('')
|
|
1268
1661
|
: JSON.stringify(body.system);
|
|
1269
|
-
const _sysHash =
|
|
1662
|
+
const _sysHash = createHash('md5').update(_sysKey).digest('hex');
|
|
1270
1663
|
const _sysCached = _sysPromptCache.get(_sysHash);
|
|
1271
1664
|
|
|
1272
1665
|
if (_sysCached) {
|
|
@@ -1313,8 +1706,26 @@ async function handleRequest(req, res) {
|
|
|
1313
1706
|
log('info', `=== ${logMsg} ===`);
|
|
1314
1707
|
pushEvent('compaction', logMsg);
|
|
1315
1708
|
|
|
1316
|
-
|
|
1317
|
-
|
|
1709
|
+
// Strip redundant system-reminders first (before tool stripping)
|
|
1710
|
+
try {
|
|
1711
|
+
const srResult = stripSystemReminders(body.messages);
|
|
1712
|
+
if (srResult.remindersStripped > 0) {
|
|
1713
|
+
body.messages = srResult.messages;
|
|
1714
|
+
log('info', `[compaction] Stripped ${srResult.remindersStripped} system-reminders (${srResult.charsRemoved} chars)`);
|
|
1715
|
+
}
|
|
1716
|
+
} catch (srErr) {
|
|
1717
|
+
log('warn', `[compaction] stripSystemReminders failed: ${srErr.message}`);
|
|
1718
|
+
}
|
|
1719
|
+
|
|
1720
|
+
let strippingStats = { toolResultsStripped: 0, toolUsesStripped: 0, charsRemoved: 0 };
|
|
1721
|
+
try {
|
|
1722
|
+
const result = stripMessages(body.messages);
|
|
1723
|
+
body.messages = result.strippedMessages;
|
|
1724
|
+
strippingStats = result.strippingStats;
|
|
1725
|
+
} catch (stripErr) {
|
|
1726
|
+
log('warn', `stripMessages failed, using raw messages: ${stripErr.message}`);
|
|
1727
|
+
// Keep body.messages as-is
|
|
1728
|
+
}
|
|
1318
1729
|
|
|
1319
1730
|
// Run steno+MT compression in parallel (independent of strip)
|
|
1320
1731
|
if (!dontCompress) {
|
|
@@ -1349,24 +1760,38 @@ async function handleRequest(req, res) {
|
|
|
1349
1760
|
return;
|
|
1350
1761
|
}
|
|
1351
1762
|
|
|
1352
|
-
// ===
|
|
1353
|
-
|
|
1354
|
-
|
|
1355
|
-
|
|
1356
|
-
|
|
1357
|
-
|
|
1358
|
-
|
|
1359
|
-
|
|
1360
|
-
|
|
1361
|
-
|
|
1763
|
+
// === NO PASSTHROUGH — always process all requests ===
|
|
1764
|
+
// (passthrough mode removed — every request gets stripped + compressed)
|
|
1765
|
+
|
|
1766
|
+
// Step 0: Strip redundant <system-reminder> tags
|
|
1767
|
+
let srCharsRemoved = 0;
|
|
1768
|
+
let srRemindersStripped = 0;
|
|
1769
|
+
try {
|
|
1770
|
+
const srResult = stripSystemReminders(body.messages);
|
|
1771
|
+
if (srResult.remindersStripped > 0) {
|
|
1772
|
+
body.messages = srResult.messages;
|
|
1773
|
+
srCharsRemoved = srResult.charsRemoved;
|
|
1774
|
+
srRemindersStripped = srResult.remindersStripped;
|
|
1775
|
+
log('info', `Stripped ${srResult.remindersStripped} system-reminders (${srResult.charsRemoved} chars)`);
|
|
1776
|
+
}
|
|
1777
|
+
} catch (srErr) {
|
|
1778
|
+
log('warn', `stripSystemReminders failed, skipping: ${srErr.message}`);
|
|
1362
1779
|
}
|
|
1363
1780
|
|
|
1364
1781
|
// Step 1: Strip large tool_results from old messages (biggest savings)
|
|
1365
|
-
|
|
1366
|
-
|
|
1367
|
-
|
|
1782
|
+
let oldStrip = { toolResultsStripped: 0, charsRemoved: 0 };
|
|
1783
|
+
try {
|
|
1784
|
+
if (liveConfig.OLD_STRIP_ENABLED) {
|
|
1785
|
+
const result = stripOldToolResults(body.messages);
|
|
1786
|
+
if (result.strippingStats.charsRemoved > 0) {
|
|
1787
|
+
body.messages = result.messages;
|
|
1788
|
+
oldStrip = result.strippingStats;
|
|
1789
|
+
}
|
|
1790
|
+
}
|
|
1791
|
+
} catch (stripErr) {
|
|
1792
|
+
log('warn', `stripOldToolResults failed, skipping: ${stripErr.message}`);
|
|
1793
|
+
}
|
|
1368
1794
|
if (oldStrip.charsRemoved > 0) {
|
|
1369
|
-
body.messages = preStripped;
|
|
1370
1795
|
stats.oldStripped += oldStrip.toolResultsStripped;
|
|
1371
1796
|
stats.oldCharsRemoved += oldStrip.charsRemoved;
|
|
1372
1797
|
stats.tokensStripped += Math.floor(oldStrip.charsRemoved / 4);
|
|
@@ -1377,13 +1802,18 @@ async function handleRequest(req, res) {
|
|
|
1377
1802
|
// Step 2: Steno+MT compression on remaining text
|
|
1378
1803
|
const { messages: compressed, blocksCompressed, charsCompressed, verifiedCount = 0, stenoOnlyCount = 0, tmHits: hits = 0, samples: liveSamples = [] } = await compressMessagesLive(body.messages);
|
|
1379
1804
|
|
|
1380
|
-
|
|
1805
|
+
// srCharsRemoved already tracked from Step 0 above
|
|
1806
|
+
|
|
1807
|
+
if (blocksCompressed === 0 && oldStrip.charsRemoved === 0 && srCharsRemoved === 0) {
|
|
1381
1808
|
stats.passthrough++;
|
|
1382
1809
|
pushEvent('pass', `msgs=${messageCount} (nothing to compress)`);
|
|
1383
1810
|
forwardRequest(req, res, rawBody);
|
|
1384
1811
|
return;
|
|
1385
1812
|
}
|
|
1386
1813
|
|
|
1814
|
+
// If anything was stripped/compressed, body was mutated — must re-serialize
|
|
1815
|
+
body.messages = compressed;
|
|
1816
|
+
|
|
1387
1817
|
// Store translation samples for preview
|
|
1388
1818
|
if (liveSamples.length > 0) stats._lastSamples = liveSamples;
|
|
1389
1819
|
|
|
@@ -1397,8 +1827,8 @@ async function handleRequest(req, res) {
|
|
|
1397
1827
|
stats.stenoOnly += (blocksCompressed - verifiedCount - stenoOnlyCount);
|
|
1398
1828
|
stats.tmHits += hits;
|
|
1399
1829
|
pushEvent('compress', `LIVE: ${blocksCompressed} blocks, ${charsCompressed} chars (${verifiedCount} zh, ${stenoOnlyCount} steno, ${hits} TM)`);
|
|
1400
|
-
} else if (oldStrip.charsRemoved > 0) {
|
|
1401
|
-
// Only stripping, no compression — still send modified body
|
|
1830
|
+
} else if (oldStrip.charsRemoved > 0 || srCharsRemoved > 0) {
|
|
1831
|
+
// Only stripping (tool results or system-reminders), no compression — still send modified body
|
|
1402
1832
|
stats.liveCompressed++;
|
|
1403
1833
|
}
|
|
1404
1834
|
stats.lastLiveCompress = new Date().toISOString();
|
|
@@ -1408,7 +1838,8 @@ async function handleRequest(req, res) {
|
|
|
1408
1838
|
const savedPercent = originalSize > 0 ? ((savedBytes / originalSize) * 100).toFixed(1) : '0';
|
|
1409
1839
|
|
|
1410
1840
|
const stripInfo = oldStrip.charsRemoved > 0 ? ` [STRIP: ${oldStrip.toolResultsStripped} results, ${(oldStrip.charsRemoved / 1024).toFixed(0)}KB]` : '';
|
|
1411
|
-
|
|
1841
|
+
const srInfo = srCharsRemoved > 0 ? ` [SR: ${srRemindersStripped} reminders, ${(srCharsRemoved / 1024).toFixed(1)}KB]` : '';
|
|
1842
|
+
log('info', `LIVE: ${blocksCompressed} blocks | ${(originalSize / 1024).toFixed(0)}KB → ${(compressedBody.length / 1024).toFixed(0)}KB (-${savedPercent}%)${stripInfo}${srInfo}`);
|
|
1412
1843
|
pushEvent('saved', `${(originalSize / 1024).toFixed(0)}KB → ${(compressedBody.length / 1024).toFixed(0)}KB (-${savedPercent}%) ~${Math.round(savedBytes / 3.5)} tokens`);
|
|
1413
1844
|
|
|
1414
1845
|
pushPreview(rawBody.toString('utf8'), compressedBody.toString('utf8'), 'live', liveSamples);
|
|
@@ -1417,11 +1848,35 @@ async function handleRequest(req, res) {
|
|
|
1417
1848
|
|
|
1418
1849
|
} catch (err) {
|
|
1419
1850
|
log('error', `Handler error: ${err.message}`);
|
|
1851
|
+
log('error', `Handler stack: ${err.stack || '(no stack)'}`);
|
|
1420
1852
|
pushEvent('error', err.message);
|
|
1421
1853
|
stats.errors++;
|
|
1854
|
+
// CRITICAL: Never return 502 — fall back to forwarding raw body.
|
|
1855
|
+
// The proxy must NEVER break the API call. Compression is optional;
|
|
1856
|
+
// if it fails, just passthrough the original request.
|
|
1422
1857
|
if (!res.headersSent) {
|
|
1423
|
-
|
|
1424
|
-
|
|
1858
|
+
try {
|
|
1859
|
+
const fallbackBody = (typeof rawBody !== 'undefined' && rawBody && rawBody.length > 0) ? rawBody : null;
|
|
1860
|
+
if (fallbackBody) {
|
|
1861
|
+
log('warn', `Falling back to passthrough after handler error: ${err.message}`);
|
|
1862
|
+
stats.passthrough++;
|
|
1863
|
+
forwardRequest(req, res, fallbackBody);
|
|
1864
|
+
} else {
|
|
1865
|
+
// rawBody unavailable — re-collect from request if possible, else auto-pause and passthrough next time
|
|
1866
|
+
log('warn', `No rawBody available — auto-pausing proxy to prevent further 502s: ${err.message}`);
|
|
1867
|
+
proxyPaused = true;
|
|
1868
|
+
// Return a retriable error so Claude Code retries (and next attempt hits passthrough path)
|
|
1869
|
+
res.writeHead(503, { 'Content-Type': 'application/json', 'Retry-After': '1' });
|
|
1870
|
+
res.end(JSON.stringify({ type: 'error', error: { type: 'overloaded_error', message: 'Proxy auto-paused due to handler error, retry immediately' } }));
|
|
1871
|
+
}
|
|
1872
|
+
} catch (fallbackErr) {
|
|
1873
|
+
log('error', `Fallback also failed: ${fallbackErr.message}`);
|
|
1874
|
+
if (!res.headersSent) {
|
|
1875
|
+
proxyPaused = true;
|
|
1876
|
+
res.writeHead(503, { 'Content-Type': 'application/json', 'Retry-After': '1' });
|
|
1877
|
+
res.end(JSON.stringify({ type: 'error', error: { type: 'overloaded_error', message: 'Proxy auto-paused, retry immediately' } }));
|
|
1878
|
+
}
|
|
1879
|
+
}
|
|
1425
1880
|
}
|
|
1426
1881
|
}
|
|
1427
1882
|
}
|
|
@@ -1439,13 +1894,9 @@ let proxyServer = null;
|
|
|
1439
1894
|
* Returns { started: boolean, port: number }
|
|
1440
1895
|
*/
|
|
1441
1896
|
export async function startCompactionProxy() {
|
|
1442
|
-
//
|
|
1443
|
-
|
|
1444
|
-
|
|
1445
|
-
cleanupPortFile();
|
|
1446
|
-
cleanupPidFile();
|
|
1447
|
-
return { started: false, port: PROXY_PORT, reason: 'disabled_by_user' };
|
|
1448
|
-
}
|
|
1897
|
+
// Disabled flag = start in passthrough mode (proxy always in path for seamless toggling).
|
|
1898
|
+
// The daemon itself reads the flag and starts paused.
|
|
1899
|
+
const disabledByUser = existsSync(DISABLED_FILE);
|
|
1449
1900
|
|
|
1450
1901
|
// Check PID file — is daemon already running?
|
|
1451
1902
|
if (existsSync(PID_FILE)) {
|
|
@@ -1458,8 +1909,11 @@ export async function startCompactionProxy() {
|
|
|
1458
1909
|
// Process exists, verify it responds to health check
|
|
1459
1910
|
const healthy = await checkDaemonHealth();
|
|
1460
1911
|
if (healthy) {
|
|
1461
|
-
log('info', `Daemon already running (PID ${pid}), skipping spawn`);
|
|
1462
|
-
|
|
1912
|
+
log('info', `Daemon already running (PID ${pid})${disabledByUser ? ' (passthrough mode)' : ''}, skipping spawn`);
|
|
1913
|
+
// If disabled flag changed since daemon started, sync pause state via SIGUSR1
|
|
1914
|
+
// (daemon checks disabled file on toggle, but we can hint via stats check)
|
|
1915
|
+
registerWithDaemon(process.env.SPECMEM_PROJECT_PATH, process.pid);
|
|
1916
|
+
return { started: false, port: PROXY_PORT, paused: disabledByUser, reason: 'daemon_already_running' };
|
|
1463
1917
|
}
|
|
1464
1918
|
// Process alive but not responding — stale, kill it
|
|
1465
1919
|
log('warn', `Daemon PID ${pid} alive but not healthy, killing stale process`);
|
|
@@ -1495,14 +1949,16 @@ export async function startCompactionProxy() {
|
|
|
1495
1949
|
const deadline = Date.now() + 3000;
|
|
1496
1950
|
while (Date.now() < deadline) {
|
|
1497
1951
|
if (existsSync(PORT_FILE)) {
|
|
1498
|
-
log('info', `Daemon ready (PID ${child.pid}), port file appeared`);
|
|
1499
|
-
|
|
1952
|
+
log('info', `Daemon ready (PID ${child.pid})${disabledByUser ? ' in passthrough mode' : ''}, port file appeared`);
|
|
1953
|
+
registerWithDaemon(process.env.SPECMEM_PROJECT_PATH, process.pid);
|
|
1954
|
+
return { started: true, port: PROXY_PORT, paused: disabledByUser };
|
|
1500
1955
|
}
|
|
1501
1956
|
await new Promise(r => setTimeout(r, 100));
|
|
1502
1957
|
}
|
|
1503
1958
|
|
|
1504
1959
|
log('warn', 'Daemon spawned but port file did not appear within 3s');
|
|
1505
|
-
|
|
1960
|
+
registerWithDaemon(process.env.SPECMEM_PROJECT_PATH, process.pid);
|
|
1961
|
+
return { started: true, port: PROXY_PORT, paused: disabledByUser, reason: 'timeout_waiting_for_port' };
|
|
1506
1962
|
}
|
|
1507
1963
|
|
|
1508
1964
|
/**
|
|
@@ -1539,6 +1995,12 @@ function togglePause() {
|
|
|
1539
1995
|
return proxyPaused;
|
|
1540
1996
|
}
|
|
1541
1997
|
|
|
1998
|
+
function setPaused(state) {
|
|
1999
|
+
proxyPaused = !!state;
|
|
2000
|
+
log('info', `Proxy pause state set to ${proxyPaused ? 'PAUSED' : 'ACTIVE'}`);
|
|
2001
|
+
return proxyPaused;
|
|
2002
|
+
}
|
|
2003
|
+
|
|
1542
2004
|
/**
|
|
1543
2005
|
* Check if the daemon is responding on its health endpoint.
|
|
1544
2006
|
*/
|
|
@@ -1555,11 +2017,12 @@ function checkDaemonHealth() {
|
|
|
1555
2017
|
}
|
|
1556
2018
|
|
|
1557
2019
|
/**
|
|
1558
|
-
* Stop the proxy —
|
|
2020
|
+
* Stop the proxy — deregisters this project from the daemon.
|
|
1559
2021
|
* The daemon persists independently of the MCP process.
|
|
1560
2022
|
* Use killCompactionProxy() to explicitly terminate the daemon.
|
|
1561
2023
|
*/
|
|
1562
2024
|
export function stopCompactionProxy() {
|
|
2025
|
+
deregisterFromDaemon(process.env.SPECMEM_PROJECT_PATH);
|
|
1563
2026
|
return Promise.resolve();
|
|
1564
2027
|
}
|
|
1565
2028
|
|
|
@@ -1647,6 +2110,115 @@ export function getCompactionProxyStats() {
|
|
|
1647
2110
|
return { running, daemonPid, port: PROXY_PORT };
|
|
1648
2111
|
}
|
|
1649
2112
|
|
|
2113
|
+
// ============================================================================
|
|
2114
|
+
// Registration helpers — fire-and-forget HTTP to the daemon
|
|
2115
|
+
// ============================================================================
|
|
2116
|
+
|
|
2117
|
+
/**
|
|
2118
|
+
* Register this project with the running daemon.
|
|
2119
|
+
* Fire-and-forget — if daemon isn't running yet, fails silently.
|
|
2120
|
+
*/
|
|
2121
|
+
function registerWithDaemon(projectPath, pid) {
|
|
2122
|
+
if (!projectPath) return;
|
|
2123
|
+
const body = JSON.stringify({ projectPath, pid: pid || process.pid });
|
|
2124
|
+
try {
|
|
2125
|
+
const req = httpRequest({
|
|
2126
|
+
hostname: '127.0.0.1',
|
|
2127
|
+
port: PROXY_PORT,
|
|
2128
|
+
path: '/register',
|
|
2129
|
+
method: 'POST',
|
|
2130
|
+
headers: { 'Content-Type': 'application/json', 'Content-Length': Buffer.byteLength(body) },
|
|
2131
|
+
timeout: 2000,
|
|
2132
|
+
});
|
|
2133
|
+
req.on('error', () => {}); // fire-and-forget
|
|
2134
|
+
req.on('timeout', () => req.destroy());
|
|
2135
|
+
req.write(body);
|
|
2136
|
+
req.end();
|
|
2137
|
+
log('info', `Sent /register for "${projectPath}" (pid=${pid || process.pid})`);
|
|
2138
|
+
} catch { /* fire-and-forget */ }
|
|
2139
|
+
}
|
|
2140
|
+
|
|
2141
|
+
/**
|
|
2142
|
+
* Deregister this project from the running daemon.
|
|
2143
|
+
* Fire-and-forget — best effort cleanup on shutdown.
|
|
2144
|
+
*/
|
|
2145
|
+
function deregisterFromDaemon(projectPath) {
|
|
2146
|
+
if (!projectPath) return;
|
|
2147
|
+
const body = JSON.stringify({ projectPath });
|
|
2148
|
+
try {
|
|
2149
|
+
const req = httpRequest({
|
|
2150
|
+
hostname: '127.0.0.1',
|
|
2151
|
+
port: PROXY_PORT,
|
|
2152
|
+
path: '/deregister',
|
|
2153
|
+
method: 'POST',
|
|
2154
|
+
headers: { 'Content-Type': 'application/json', 'Content-Length': Buffer.byteLength(body) },
|
|
2155
|
+
timeout: 2000,
|
|
2156
|
+
});
|
|
2157
|
+
req.on('error', () => {}); // fire-and-forget
|
|
2158
|
+
req.on('timeout', () => req.destroy());
|
|
2159
|
+
req.write(body);
|
|
2160
|
+
req.end();
|
|
2161
|
+
log('info', `Sent /deregister for "${projectPath}"`);
|
|
2162
|
+
} catch { /* fire-and-forget */ }
|
|
2163
|
+
}
|
|
2164
|
+
|
|
2165
|
+
// ============================================================================
|
|
2166
|
+
// Stale Project Reaper — removes dead projects from the registry
|
|
2167
|
+
// ============================================================================
|
|
2168
|
+
|
|
2169
|
+
/**
|
|
2170
|
+
* Check if a PID is alive using signal 0.
|
|
2171
|
+
*/
|
|
2172
|
+
function isPidAlive(pid) {
|
|
2173
|
+
if (!pid || pid <= 0) return false;
|
|
2174
|
+
try {
|
|
2175
|
+
process.kill(pid, 0);
|
|
2176
|
+
return true;
|
|
2177
|
+
} catch {
|
|
2178
|
+
return false;
|
|
2179
|
+
}
|
|
2180
|
+
}
|
|
2181
|
+
|
|
2182
|
+
/**
|
|
2183
|
+
* Reap stale projects — remove entries with dead PIDs.
|
|
2184
|
+
*/
|
|
2185
|
+
function reapStaleProjects() {
|
|
2186
|
+
let reaped = 0;
|
|
2187
|
+
for (const [path, entry] of projectRegistry.entries()) {
|
|
2188
|
+
if (entry.pid > 0 && !isPidAlive(entry.pid)) {
|
|
2189
|
+
projectRegistry.delete(path);
|
|
2190
|
+
reaped++;
|
|
2191
|
+
log('info', `REAPER: removed stale project "${path}" (dead pid=${entry.pid})`);
|
|
2192
|
+
}
|
|
2193
|
+
}
|
|
2194
|
+
if (reaped > 0) {
|
|
2195
|
+
_invalidateProjectCaches();
|
|
2196
|
+
log('info', `REAPER: reaped ${reaped} stale projects, ${projectRegistry.size} remaining`);
|
|
2197
|
+
}
|
|
2198
|
+
}
|
|
2199
|
+
|
|
2200
|
+
let _reaperTimer = null;
|
|
2201
|
+
|
|
2202
|
+
/**
|
|
2203
|
+
* Start the stale project reaper (30s interval, unref'd).
|
|
2204
|
+
*/
|
|
2205
|
+
function startReaper() {
|
|
2206
|
+
if (_reaperTimer) return;
|
|
2207
|
+
_reaperTimer = setInterval(reapStaleProjects, 30000);
|
|
2208
|
+
_reaperTimer.unref();
|
|
2209
|
+
log('info', 'REAPER: started (30s interval)');
|
|
2210
|
+
}
|
|
2211
|
+
|
|
2212
|
+
/**
|
|
2213
|
+
* Check if any registered projects have live PIDs.
|
|
2214
|
+
*/
|
|
2215
|
+
function hasLiveProjects() {
|
|
2216
|
+
for (const entry of projectRegistry.values()) {
|
|
2217
|
+
if (entry.pid > 0 && isPidAlive(entry.pid)) return true;
|
|
2218
|
+
}
|
|
2219
|
+
return false;
|
|
2220
|
+
}
|
|
2221
|
+
|
|
1650
2222
|
// ============================================================================
|
|
1651
2223
|
// Daemon Internals — exported for compactionProxyDaemon.js
|
|
1652
2224
|
// ============================================================================
|
|
@@ -1655,6 +2227,9 @@ export {
|
|
|
1655
2227
|
handleRequest,
|
|
1656
2228
|
log,
|
|
1657
2229
|
togglePause,
|
|
2230
|
+
setPaused,
|
|
2231
|
+
startReaper,
|
|
2232
|
+
hasLiveProjects,
|
|
1658
2233
|
PROXY_PORT,
|
|
1659
2234
|
PORT_FILE,
|
|
1660
2235
|
PID_FILE,
|