specmem-hardwicksoftware 3.7.32 → 3.7.34
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +32 -0
- package/README.md +4 -2
- package/bin/specmem-cli.cjs +18 -33
- package/bootstrap.cjs +5 -1
- package/claude-hooks/agent-loading-hook.js +31 -47
- package/claude-hooks/subagent-loading-hook.js +1 -1
- package/claude-hooks/team-comms-enforcer.cjs +112 -93
- package/dist/config/configSync.js +4 -1
- package/dist/index.js +44 -1
- package/dist/init/claudeConfigInjector.js +4 -1
- package/dist/installer/autoInstall.js +4 -1
- package/dist/mcp/compactionProxy.js +634 -72
- package/dist/mcp/compactionProxyDaemon.js +18 -4
- package/dist/mcp/specMemServer.js +8 -0
- package/dist/watcher/index.js +16 -2
- package/package.json +1 -1
- package/scripts/deploy-hooks.cjs +4 -1
- package/scripts/specmem-init.cjs +31 -35
- package/scripts/specmem-uninstall.cjs +4 -1
- package/specmem/model-config.json +3 -3
- package/specmem/supervisord.conf +1 -1
- package/svg-sections/readme-how-to-install.svg +145 -0
- package/svg-sections/readme-install.svg +89 -53
|
@@ -15,7 +15,7 @@
|
|
|
15
15
|
* @website https://justcalljon.pro
|
|
16
16
|
*/
|
|
17
17
|
|
|
18
|
-
import { createServer, get as httpGet } from 'http';
|
|
18
|
+
import { createServer, get as httpGet, request as httpRequest } from 'http';
|
|
19
19
|
import { request as httpsRequest } from 'https';
|
|
20
20
|
import { appendFileSync, writeFileSync, readFileSync, mkdirSync, existsSync, copyFileSync, unlinkSync } from 'fs';
|
|
21
21
|
import { createConnection } from 'net';
|
|
@@ -62,12 +62,54 @@ const OLD_STRIP_THRESHOLD = 100; // chars — only strip results bigger
|
|
|
62
62
|
const OLD_STRIP_PREVIEW_CHARS = 200; // chars — keep this much of the original
|
|
63
63
|
|
|
64
64
|
// Live neural MT compression config
|
|
65
|
-
const
|
|
65
|
+
const _TRANSLATE_SOCKET_FALLBACK = process.env.SPECMEM_TRANSLATE_SOCKET ||
|
|
66
66
|
join(process.env.SPECMEM_PROJECT_PATH || process.cwd(), 'specmem', 'run', 'translate.sock');
|
|
67
67
|
const PRESERVE_RECENT_MESSAGES = 3;
|
|
68
|
-
const MIN_TRANSLATE_LENGTH =
|
|
68
|
+
const MIN_TRANSLATE_LENGTH = 0; // No minimum — compress everything
|
|
69
69
|
const DONT_COMPRESS_FLAG = 'DONT_COMPRESS';
|
|
70
70
|
|
|
71
|
+
// ============================================================================
|
|
72
|
+
// Multi-Project Registry — tracks all registered projects for resource sharing
|
|
73
|
+
// ============================================================================
|
|
74
|
+
|
|
75
|
+
const projectRegistry = new Map(); // keyed by projectPath → { projectPath, pid, registeredAt, lastSeen }
|
|
76
|
+
|
|
77
|
+
// Auto-register the spawning project so daemon always has at least one
|
|
78
|
+
const _foundingProjectPath = process.env.SPECMEM_PROJECT_PATH || null;
|
|
79
|
+
if (_foundingProjectPath) {
|
|
80
|
+
projectRegistry.set(_foundingProjectPath, {
|
|
81
|
+
projectPath: _foundingProjectPath,
|
|
82
|
+
pid: process.ppid || 0,
|
|
83
|
+
registeredAt: Date.now(),
|
|
84
|
+
lastSeen: Date.now(),
|
|
85
|
+
});
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
/**
|
|
89
|
+
* Resolve translate socket dynamically across all registered projects.
|
|
90
|
+
* Iterates registry sorted by lastSeen desc, returns first socket that exists.
|
|
91
|
+
* Fallback: SPECMEM_TRANSLATE_SOCKET env var, then null.
|
|
92
|
+
*/
|
|
93
|
+
function resolveTranslateSocket() {
|
|
94
|
+
// Try registered projects, most recently seen first
|
|
95
|
+
const entries = [...projectRegistry.values()].sort((a, b) => b.lastSeen - a.lastSeen);
|
|
96
|
+
for (const entry of entries) {
|
|
97
|
+
const sockPath = join(entry.projectPath, 'specmem', 'run', 'translate.sock');
|
|
98
|
+
if (existsSync(sockPath)) return sockPath;
|
|
99
|
+
}
|
|
100
|
+
// Fallback to env var / founding project socket
|
|
101
|
+
if (existsSync(_TRANSLATE_SOCKET_FALLBACK)) return _TRANSLATE_SOCKET_FALLBACK;
|
|
102
|
+
return null;
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
/**
|
|
106
|
+
* Invalidate TM and synonym caches — called on registry changes.
|
|
107
|
+
*/
|
|
108
|
+
function _invalidateProjectCaches() {
|
|
109
|
+
_tmCache = null;
|
|
110
|
+
_synonyms = null;
|
|
111
|
+
}
|
|
112
|
+
|
|
71
113
|
// Preview ring buffer — stores last 5 compressed requests for TUI display
|
|
72
114
|
const PREVIEW_BUFFER_SIZE = 5;
|
|
73
115
|
const recentRequests = []; // { original, optimized, timestamp, type, savings }
|
|
@@ -160,6 +202,7 @@ let liveConfig = {
|
|
|
160
202
|
TRANSLATE_ENABLED: true, // toggle MT translation (requires translate.sock)
|
|
161
203
|
OLD_STRIP_ENABLED: true, // toggle old tool_result stripping
|
|
162
204
|
SYSTEM_PROMPT_COMPRESS: true, // toggle system prompt steno+translate compression
|
|
205
|
+
SYSTEM_REMINDER_STRIPPING: true, // toggle <system-reminder> stripping (keeps first, strips rest)
|
|
163
206
|
};
|
|
164
207
|
|
|
165
208
|
// Stats
|
|
@@ -171,6 +214,7 @@ const stats = {
|
|
|
171
214
|
tokensStripped: 0,
|
|
172
215
|
bytesStripped: 0,
|
|
173
216
|
liveCharsCompressed: 0,
|
|
217
|
+
systemRemindersStripped: 0,
|
|
174
218
|
liveBlocksCompressed: 0,
|
|
175
219
|
zhVerified: 0, // Chinese translations that passed loop-back verification
|
|
176
220
|
zhRejected: 0, // Chinese translations rejected (steno fallback)
|
|
@@ -200,18 +244,53 @@ const TM_MAX_ENTRIES = 10000;
|
|
|
200
244
|
let _tmCache = null; // { hash: { en, zh, sim, ts } }
|
|
201
245
|
let _synonyms = null; // { word: Set<synonym> } — learned from failed verifications
|
|
202
246
|
|
|
247
|
+
/**
|
|
248
|
+
* Get all TM file paths from registered projects.
|
|
249
|
+
*/
|
|
250
|
+
function _getAllTMPaths() {
|
|
251
|
+
const paths = [];
|
|
252
|
+
for (const entry of projectRegistry.values()) {
|
|
253
|
+
paths.push(join(entry.projectPath, 'specmem', 'run', 'translation-memory.json'));
|
|
254
|
+
}
|
|
255
|
+
// Always include founding project path as fallback
|
|
256
|
+
if (!paths.includes(TM_FILE)) paths.push(TM_FILE);
|
|
257
|
+
return paths;
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
/**
|
|
261
|
+
* Get all synonyms file paths from registered projects.
|
|
262
|
+
*/
|
|
263
|
+
function _getAllSynonymPaths() {
|
|
264
|
+
const paths = [];
|
|
265
|
+
for (const entry of projectRegistry.values()) {
|
|
266
|
+
paths.push(join(entry.projectPath, 'specmem', 'run', 'learned-synonyms.json'));
|
|
267
|
+
}
|
|
268
|
+
if (!paths.includes(SYNONYMS_FILE)) paths.push(SYNONYMS_FILE);
|
|
269
|
+
return paths;
|
|
270
|
+
}
|
|
271
|
+
|
|
203
272
|
function _tmHash(text) {
|
|
204
273
|
return createHash('md5').update(text.toLowerCase().trim()).digest('hex').slice(0, 16);
|
|
205
274
|
}
|
|
206
275
|
|
|
207
276
|
function _loadTM() {
|
|
208
277
|
if (_tmCache) return _tmCache;
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
278
|
+
_tmCache = {};
|
|
279
|
+
// Merge TM entries from all registered projects (later timestamp wins)
|
|
280
|
+
for (const tmPath of _getAllTMPaths()) {
|
|
281
|
+
try {
|
|
282
|
+
if (existsSync(tmPath)) {
|
|
283
|
+
const entries = JSON.parse(readFileSync(tmPath, 'utf8'));
|
|
284
|
+
if (entries && typeof entries === 'object') {
|
|
285
|
+
for (const [hash, entry] of Object.entries(entries)) {
|
|
286
|
+
if (!_tmCache[hash] || (entry.ts || 0) > (_tmCache[hash].ts || 0)) {
|
|
287
|
+
_tmCache[hash] = entry;
|
|
288
|
+
}
|
|
289
|
+
}
|
|
290
|
+
}
|
|
291
|
+
}
|
|
292
|
+
} catch { /* corrupt file, skip */ }
|
|
293
|
+
}
|
|
215
294
|
return _tmCache;
|
|
216
295
|
}
|
|
217
296
|
|
|
@@ -225,7 +304,14 @@ function _saveTM() {
|
|
|
225
304
|
const toRemove = sorted.slice(0, keys.length - TM_MAX_ENTRIES);
|
|
226
305
|
for (const k of toRemove) delete tm[k];
|
|
227
306
|
}
|
|
228
|
-
|
|
307
|
+
const data = JSON.stringify(tm);
|
|
308
|
+
// Fan-out write to ALL registered projects
|
|
309
|
+
for (const tmPath of _getAllTMPaths()) {
|
|
310
|
+
try {
|
|
311
|
+
mkdirSync(dirname(tmPath), { recursive: true });
|
|
312
|
+
writeFileSync(tmPath, data, 'utf8');
|
|
313
|
+
} catch { /* skip unreachable paths */ }
|
|
314
|
+
}
|
|
229
315
|
} catch (e) { log('warn', `TM save failed: ${e.message}`); }
|
|
230
316
|
}
|
|
231
317
|
|
|
@@ -253,29 +339,41 @@ let _tmSaveTimer = null;
|
|
|
253
339
|
// e.g., MT says "rebuild" but original had "refactor" — these are equivalent
|
|
254
340
|
function _loadSynonyms() {
|
|
255
341
|
if (_synonyms) return _synonyms;
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
342
|
+
_synonyms = {};
|
|
343
|
+
// Merge synonyms from all registered projects + package seed
|
|
344
|
+
const allPaths = [
|
|
345
|
+
..._getAllSynonymPaths(),
|
|
259
346
|
join(fileURLToPath(import.meta.url), '..', '..', '..', 'specmem', 'run', 'learned-synonyms.json'),
|
|
260
347
|
];
|
|
261
|
-
|
|
348
|
+
let loadedAny = false;
|
|
349
|
+
for (const synPath of allPaths) {
|
|
262
350
|
try {
|
|
263
351
|
if (existsSync(synPath)) {
|
|
264
352
|
const raw = JSON.parse(readFileSync(synPath, 'utf8'));
|
|
265
|
-
_synonyms = {};
|
|
266
353
|
for (const [word, syns] of Object.entries(raw)) {
|
|
267
|
-
_synonyms[word] = new Set(
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
if (synPath !== SYNONYMS_FILE && _synonyms && Object.keys(_synonyms).length > 0) {
|
|
271
|
-
try { mkdirSync(dirname(SYNONYMS_FILE), { recursive: true }); } catch {}
|
|
272
|
-
try { copyFileSync(synPath, SYNONYMS_FILE); } catch {}
|
|
354
|
+
if (!_synonyms[word]) _synonyms[word] = new Set();
|
|
355
|
+
const synList = Array.isArray(syns) ? syns : [...syns];
|
|
356
|
+
for (const s of synList) _synonyms[word].add(s);
|
|
273
357
|
}
|
|
274
|
-
|
|
358
|
+
loadedAny = true;
|
|
275
359
|
}
|
|
276
|
-
} catch { /*
|
|
360
|
+
} catch { /* skip corrupt */ }
|
|
361
|
+
}
|
|
362
|
+
// If loaded from package seed and no project files exist, seed all projects
|
|
363
|
+
if (loadedAny && Object.keys(_synonyms).length > 0) {
|
|
364
|
+
for (const synPath of _getAllSynonymPaths()) {
|
|
365
|
+
if (!existsSync(synPath)) {
|
|
366
|
+
try { mkdirSync(dirname(synPath), { recursive: true }); } catch {}
|
|
367
|
+
try {
|
|
368
|
+
const serializable = {};
|
|
369
|
+
for (const [word, set] of Object.entries(_synonyms)) {
|
|
370
|
+
serializable[word] = [...set];
|
|
371
|
+
}
|
|
372
|
+
writeFileSync(synPath, JSON.stringify(serializable), 'utf8');
|
|
373
|
+
} catch {}
|
|
374
|
+
}
|
|
375
|
+
}
|
|
277
376
|
}
|
|
278
|
-
if (!_synonyms) _synonyms = {};
|
|
279
377
|
return _synonyms;
|
|
280
378
|
}
|
|
281
379
|
|
|
@@ -286,7 +384,14 @@ function _saveSynonyms() {
|
|
|
286
384
|
for (const [word, set] of Object.entries(syns)) {
|
|
287
385
|
serializable[word] = [...set];
|
|
288
386
|
}
|
|
289
|
-
|
|
387
|
+
const data = JSON.stringify(serializable);
|
|
388
|
+
// Fan-out write to ALL registered projects
|
|
389
|
+
for (const synPath of _getAllSynonymPaths()) {
|
|
390
|
+
try {
|
|
391
|
+
mkdirSync(dirname(synPath), { recursive: true });
|
|
392
|
+
writeFileSync(synPath, data, 'utf8');
|
|
393
|
+
} catch { /* skip unreachable paths */ }
|
|
394
|
+
}
|
|
290
395
|
} catch (e) { log('warn', `Synonyms save failed: ${e.message}`); }
|
|
291
396
|
}
|
|
292
397
|
|
|
@@ -380,6 +485,23 @@ function isCompactionRequest(body) {
|
|
|
380
485
|
return false;
|
|
381
486
|
}
|
|
382
487
|
|
|
488
|
+
/**
|
|
489
|
+
* Smart-strip Edit tool_use inputs to just - / + diff lines.
|
|
490
|
+
* Returns compact string or null if not an Edit block.
|
|
491
|
+
*/
|
|
492
|
+
function smartStripEdit(input) {
|
|
493
|
+
if (!input || !input.file_path || !input.old_string) return null;
|
|
494
|
+
const lines = [`Edit(${input.file_path})`];
|
|
495
|
+
// old_string → removed lines (prefix with -)
|
|
496
|
+
for (const l of input.old_string.split('\n')) lines.push(`- ${l}`);
|
|
497
|
+
// new_string → added lines (prefix with +)
|
|
498
|
+
if (input.new_string != null) {
|
|
499
|
+
for (const l of input.new_string.split('\n')) lines.push(`+ ${l}`);
|
|
500
|
+
}
|
|
501
|
+
if (input.replace_all) lines.push('(replace_all)');
|
|
502
|
+
return lines.join('\n');
|
|
503
|
+
}
|
|
504
|
+
|
|
383
505
|
function stripMessages(messages) {
|
|
384
506
|
if (!Array.isArray(messages)) return { strippedMessages: messages, strippingStats: { toolResultsStripped: 0, toolUsesStripped: 0, charsRemoved: 0 } };
|
|
385
507
|
|
|
@@ -422,6 +544,22 @@ function stripMessages(messages) {
|
|
|
422
544
|
const input = block.input;
|
|
423
545
|
if (!input) return block;
|
|
424
546
|
|
|
547
|
+
// Smart diff stripping for Edit tool — keep only - / + lines
|
|
548
|
+
const editDiff = (block.name === 'Edit') ? smartStripEdit(input) : null;
|
|
549
|
+
if (editDiff) {
|
|
550
|
+
const inputStr = JSON.stringify(input);
|
|
551
|
+
charsRemoved += inputStr.length - editDiff.length;
|
|
552
|
+
toolUsesStripped++;
|
|
553
|
+
const stripped = {
|
|
554
|
+
type: 'tool_use',
|
|
555
|
+
id: block.id,
|
|
556
|
+
name: block.name,
|
|
557
|
+
input: { _stripped: editDiff }
|
|
558
|
+
};
|
|
559
|
+
if (block.cache_control) stripped.cache_control = block.cache_control;
|
|
560
|
+
return stripped;
|
|
561
|
+
}
|
|
562
|
+
|
|
425
563
|
const inputStr = JSON.stringify(input);
|
|
426
564
|
if (inputStr.length <= TOOL_USE_INPUT_PREVIEW_CHARS * 2) return block;
|
|
427
565
|
|
|
@@ -513,6 +651,16 @@ function stripOldToolResults(messages) {
|
|
|
513
651
|
const input = block.input;
|
|
514
652
|
if (!input) return block;
|
|
515
653
|
|
|
654
|
+
// Smart Edit stripping — keep only - / + diff lines
|
|
655
|
+
const editDiff = smartStripEdit(input);
|
|
656
|
+
if (editDiff) {
|
|
657
|
+
const origLen = JSON.stringify(input).length;
|
|
658
|
+
const newLen = JSON.stringify(editDiff.input).length;
|
|
659
|
+
charsRemoved += origLen - newLen;
|
|
660
|
+
toolResultsStripped++;
|
|
661
|
+
return editDiff;
|
|
662
|
+
}
|
|
663
|
+
|
|
516
664
|
const inputStr = JSON.stringify(input);
|
|
517
665
|
if (inputStr.length <= liveConfig.OLD_STRIP_THRESHOLD) return block;
|
|
518
666
|
|
|
@@ -548,6 +696,19 @@ function stripOldToolResults(messages) {
|
|
|
548
696
|
text: txt.slice(0, OLD_STRIP_PREVIEW_CHARS) + `...\n[HOOK-TRIMMED: ${txt.length} chars]`
|
|
549
697
|
};
|
|
550
698
|
}
|
|
699
|
+
|
|
700
|
+
// Strip old assistant text blocks — Claude's own output echoed back
|
|
701
|
+
// No point sending Claude its own words; keep first line as context anchor
|
|
702
|
+
if (msg.role === 'assistant' && txt.length > 120) {
|
|
703
|
+
const firstLine = txt.split('\n')[0].slice(0, 120);
|
|
704
|
+
const removed = txt.length - firstLine.length;
|
|
705
|
+
charsRemoved += removed;
|
|
706
|
+
toolResultsStripped++;
|
|
707
|
+
return {
|
|
708
|
+
...block,
|
|
709
|
+
text: `${firstLine}...\n[ASST-ECHO-STRIPPED: ${txt.length} chars → ${firstLine.length}]`
|
|
710
|
+
};
|
|
711
|
+
}
|
|
551
712
|
}
|
|
552
713
|
|
|
553
714
|
return block;
|
|
@@ -559,6 +720,88 @@ function stripOldToolResults(messages) {
|
|
|
559
720
|
return { messages: newMessages, strippingStats: { toolResultsStripped, charsRemoved } };
|
|
560
721
|
}
|
|
561
722
|
|
|
723
|
+
/**
|
|
724
|
+
* Strip <system-reminder> tags from messages.
|
|
725
|
+
* Keeps the FIRST system-reminder encountered (sets up style/context),
|
|
726
|
+
* strips all subsequent ones to save tokens.
|
|
727
|
+
*
|
|
728
|
+
* Returns { messages, remindersStripped, charsRemoved }.
|
|
729
|
+
*/
|
|
730
|
+
function stripSystemReminders(messages) {
|
|
731
|
+
if (!Array.isArray(messages)) return { messages, remindersStripped: 0, charsRemoved: 0 };
|
|
732
|
+
if (!liveConfig.SYSTEM_REMINDER_STRIPPING) return { messages, remindersStripped: 0, charsRemoved: 0 };
|
|
733
|
+
|
|
734
|
+
const SR_RE = /<system-reminder>[\s\S]*?<\/system-reminder>/g;
|
|
735
|
+
let firstSeen = false;
|
|
736
|
+
let remindersStripped = 0;
|
|
737
|
+
let charsRemoved = 0;
|
|
738
|
+
|
|
739
|
+
const newMessages = messages.map(msg => {
|
|
740
|
+
if (!msg) return msg;
|
|
741
|
+
|
|
742
|
+
// Handle string content
|
|
743
|
+
if (typeof msg.content === 'string') {
|
|
744
|
+
const matches = msg.content.match(SR_RE);
|
|
745
|
+
if (!matches) return msg;
|
|
746
|
+
|
|
747
|
+
let newText = msg.content;
|
|
748
|
+
for (const match of matches) {
|
|
749
|
+
if (!firstSeen) {
|
|
750
|
+
firstSeen = true; // keep the very first one
|
|
751
|
+
continue;
|
|
752
|
+
}
|
|
753
|
+
// replaceAll to nuke ALL occurrences of this exact match in the string
|
|
754
|
+
// .replace(string, '') only kills the first occurrence — duplicates slip through
|
|
755
|
+
while (newText.includes(match)) {
|
|
756
|
+
newText = newText.replace(match, '');
|
|
757
|
+
charsRemoved += match.length;
|
|
758
|
+
remindersStripped++;
|
|
759
|
+
}
|
|
760
|
+
}
|
|
761
|
+
return { ...msg, content: newText.replace(/\n{3,}/g, '\n\n').trim() };
|
|
762
|
+
}
|
|
763
|
+
|
|
764
|
+
// Handle array content blocks
|
|
765
|
+
if (!Array.isArray(msg.content)) return msg;
|
|
766
|
+
|
|
767
|
+
const newContent = msg.content.map(block => {
|
|
768
|
+
if (block.type !== 'text' || typeof block.text !== 'string') return block;
|
|
769
|
+
|
|
770
|
+
const matches = block.text.match(SR_RE);
|
|
771
|
+
if (!matches) return block;
|
|
772
|
+
|
|
773
|
+
let newText = block.text;
|
|
774
|
+
for (const match of matches) {
|
|
775
|
+
if (!firstSeen) {
|
|
776
|
+
firstSeen = true; // keep the very first one
|
|
777
|
+
continue;
|
|
778
|
+
}
|
|
779
|
+
// replaceAll to nuke ALL occurrences of this exact match in the block
|
|
780
|
+
while (newText.includes(match)) {
|
|
781
|
+
newText = newText.replace(match, '');
|
|
782
|
+
charsRemoved += match.length;
|
|
783
|
+
remindersStripped++;
|
|
784
|
+
}
|
|
785
|
+
}
|
|
786
|
+
const cleaned = newText.replace(/\n{3,}/g, '\n\n').trim();
|
|
787
|
+
|
|
788
|
+
// If block is now empty after stripping, remove it entirely
|
|
789
|
+
if (!cleaned) return null;
|
|
790
|
+
|
|
791
|
+
return { ...block, text: cleaned };
|
|
792
|
+
}).filter(Boolean);
|
|
793
|
+
|
|
794
|
+
// If all content blocks were stripped, keep a minimal marker
|
|
795
|
+
if (newContent.length === 0) {
|
|
796
|
+
return { ...msg, content: [{ type: 'text', text: '[context]' }] };
|
|
797
|
+
}
|
|
798
|
+
|
|
799
|
+
return { ...msg, content: newContent };
|
|
800
|
+
});
|
|
801
|
+
|
|
802
|
+
return { messages: newMessages, remindersStripped, charsRemoved };
|
|
803
|
+
}
|
|
804
|
+
|
|
562
805
|
// ============================================================================
|
|
563
806
|
// Live Compression — Stenography + Neural MT with Loop-back Verification
|
|
564
807
|
// ============================================================================
|
|
@@ -849,15 +1092,60 @@ async function compressMessagesLive(messages) {
|
|
|
849
1092
|
let stenoOnlyCount = 0;
|
|
850
1093
|
let tmHits = 0;
|
|
851
1094
|
|
|
852
|
-
const
|
|
1095
|
+
const _resolvedSocket = resolveTranslateSocket();
|
|
1096
|
+
const translateAvailable = liveConfig.TRANSLATE_ENABLED && !!_resolvedSocket;
|
|
1097
|
+
|
|
1098
|
+
// ── Pre-translation filter: detect "translation-hostile" content ──
|
|
1099
|
+
// File paths, IDs, hashes, permissions, IPs, URLs, code identifiers, etc.
|
|
1100
|
+
// These will ALWAYS fail backtranslation verification, so skip MT entirely.
|
|
1101
|
+
const _PATH_RE = /(?:^|\s)[.~]?\/[\w./-]{3,}/; // /foo/bar.js, ./rel, ~/home
|
|
1102
|
+
const _UUID_RE = /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/i;
|
|
1103
|
+
const _HEX_RE = /\b[0-9a-f]{7,64}\b/i; // git hashes, hex IDs
|
|
1104
|
+
const _PERM_RE = /[dl-][r-][w-][xsStT-][r-][w-][xsStT-][r-][w-][xsStT-]/; // drwxr-xr-x
|
|
1105
|
+
const _IP_RE = /\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b/; // IPv4
|
|
1106
|
+
const _URL_RE = /https?:\/\/\S{6,}/; // URLs
|
|
1107
|
+
const _PKG_RE = /[\w@][\w./-]*@\d+\.\d+/; // pkg@1.2.3
|
|
1108
|
+
const _CODE_ID = /\b[a-z][a-zA-Z0-9]{2,}\.[a-zA-Z]{1,4}\b/; // file.ext patterns
|
|
1109
|
+
const _CAMEL = /\b[a-z]+[A-Z][a-zA-Z]+\b/; // camelCase identifiers
|
|
1110
|
+
const _SNAKE = /\b[a-z]+_[a-z_]+\b/; // snake_case identifiers
|
|
1111
|
+
|
|
1112
|
+
function isTranslationHostile(text) {
|
|
1113
|
+
if (!text || text.length < 20) return true; // too short to compress meaningfully
|
|
1114
|
+
// Count how many "technical markers" appear
|
|
1115
|
+
const markers = [_PATH_RE, _UUID_RE, _HEX_RE, _PERM_RE, _IP_RE, _URL_RE, _PKG_RE, _CODE_ID];
|
|
1116
|
+
let hits = 0;
|
|
1117
|
+
for (const re of markers) if (re.test(text)) hits++;
|
|
1118
|
+
if (hits >= 3) return true; // 3+ different marker types = definitely technical
|
|
1119
|
+
|
|
1120
|
+
// Check density: what fraction of words are code/path-like?
|
|
1121
|
+
const words = text.split(/\s+/).filter(w => w.length > 1);
|
|
1122
|
+
if (words.length === 0) return true;
|
|
1123
|
+
let techWords = 0;
|
|
1124
|
+
for (const w of words) {
|
|
1125
|
+
if (_PATH_RE.test(w) || _HEX_RE.test(w) || _UUID_RE.test(w) ||
|
|
1126
|
+
_PERM_RE.test(w) || _IP_RE.test(w) || _URL_RE.test(w) ||
|
|
1127
|
+
_CAMEL.test(w) || _SNAKE.test(w) || _PKG_RE.test(w) ||
|
|
1128
|
+
/^\d+$/.test(w) || /^[A-Z_]{3,}$/.test(w)) {
|
|
1129
|
+
techWords++;
|
|
1130
|
+
}
|
|
1131
|
+
}
|
|
1132
|
+
return (techWords / words.length) > 0.40; // >40% technical tokens = skip MT
|
|
1133
|
+
}
|
|
853
1134
|
|
|
854
1135
|
// Step 1.5: Check Translation Memory cache first
|
|
855
1136
|
const needMT = []; // indices that need neural MT (only mtEligible blocks)
|
|
1137
|
+
let mtSkippedTech = 0;
|
|
856
1138
|
for (let i = 0; i < stenoTexts.length; i++) {
|
|
857
1139
|
// Only tool_result content is eligible for Chinese MT
|
|
858
1140
|
// User/assistant text stays steno-only to prevent Claude responding in Chinese
|
|
859
1141
|
if (!textLocations[i].mtEligible) continue;
|
|
860
1142
|
|
|
1143
|
+
// Skip translation-hostile content (file paths, IDs, code, etc.)
|
|
1144
|
+
if (isTranslationHostile(stenoTexts[i])) {
|
|
1145
|
+
mtSkippedTech++;
|
|
1146
|
+
continue; // stays steno-only — finalTexts[i] already has steno text
|
|
1147
|
+
}
|
|
1148
|
+
|
|
861
1149
|
const cached = tmLookup(stenoTexts[i]);
|
|
862
1150
|
if (cached && cached.length < stenoTexts[i].length) {
|
|
863
1151
|
finalTexts[i] = cached;
|
|
@@ -867,12 +1155,15 @@ async function compressMessagesLive(messages) {
|
|
|
867
1155
|
needMT.push(i);
|
|
868
1156
|
}
|
|
869
1157
|
}
|
|
1158
|
+
if (mtSkippedTech > 0) {
|
|
1159
|
+
log('info', `MT-SKIP: ${mtSkippedTech} blocks skipped (translation-hostile content)`);
|
|
1160
|
+
}
|
|
870
1161
|
|
|
871
1162
|
if (translateAvailable && needMT.length > 0) {
|
|
872
1163
|
try {
|
|
873
1164
|
// Step 2: Translate uncached steno English → Chinese (zt)
|
|
874
1165
|
const mtInputs = needMT.map(i => stenoTexts[i]);
|
|
875
|
-
const chineseTexts = await translateBatch(mtInputs,
|
|
1166
|
+
const chineseTexts = await translateBatch(mtInputs, _resolvedSocket, 'en', 'zh');
|
|
876
1167
|
|
|
877
1168
|
// Filter: only verify texts that actually changed and got shorter
|
|
878
1169
|
const needVerify = [];
|
|
@@ -888,7 +1179,7 @@ async function compressMessagesLive(messages) {
|
|
|
888
1179
|
|
|
889
1180
|
if (needVerify.length > 0) {
|
|
890
1181
|
// Step 3: Translate Chinese → English (loop-back)
|
|
891
|
-
const backTranslated = await translateBatch(needVerify,
|
|
1182
|
+
const backTranslated = await translateBatch(needVerify, _resolvedSocket, 'zh', 'en');
|
|
892
1183
|
|
|
893
1184
|
// Step 4: Verify each — compare back-translated with original
|
|
894
1185
|
for (let v = 0; v < verifyIndices.length; v++) {
|
|
@@ -1031,13 +1322,14 @@ async function compressSystemPrompt(system) {
|
|
|
1031
1322
|
: [...textBlocks];
|
|
1032
1323
|
|
|
1033
1324
|
// Step 2: Apply Chinese translation to eligible sections
|
|
1034
|
-
|
|
1325
|
+
const _sysSocket = resolveTranslateSocket();
|
|
1326
|
+
if (liveConfig.TRANSLATE_ENABLED && _sysSocket) {
|
|
1035
1327
|
try {
|
|
1036
1328
|
// Translate entire steno'd text via Hardwick Translate
|
|
1037
|
-
const translations = await translateBatch(compressed,
|
|
1329
|
+
const translations = await translateBatch(compressed, _sysSocket, 'en', 'zh');
|
|
1038
1330
|
if (translations && translations.length === compressed.length) {
|
|
1039
1331
|
// Verify each translation round-trips correctly
|
|
1040
|
-
const backTranslated = await translateBatch(translations,
|
|
1332
|
+
const backTranslated = await translateBatch(translations, _sysSocket, 'zh', 'en');
|
|
1041
1333
|
|
|
1042
1334
|
for (let i = 0; i < compressed.length; i++) {
|
|
1043
1335
|
if (!translations[i] || !backTranslated) continue;
|
|
@@ -1153,7 +1445,7 @@ async function handleRequest(req, res) {
|
|
|
1153
1445
|
const tmSize = Object.keys(tm).length;
|
|
1154
1446
|
const synSize = Object.keys(syns).length;
|
|
1155
1447
|
res.writeHead(200, { 'Content-Type': 'application/json' });
|
|
1156
|
-
res.end(JSON.stringify({ status: 'ok', paused: proxyPaused, uptime: Math.round((Date.now() - stats.startTime) / 1000), ...stats, tmEntries: tmSize, synonymPairs: synSize, envBaseUrlSet: !!process.env.ANTHROPIC_BASE_URL, envBaseUrl: process.env.ANTHROPIC_BASE_URL || '(not set)' }, null, 2));
|
|
1448
|
+
res.end(JSON.stringify({ status: 'ok', paused: proxyPaused, uptime: Math.round((Date.now() - stats.startTime) / 1000), ...stats, tmEntries: tmSize, synonymPairs: synSize, registeredProjects: projectRegistry.size, activeTranslateSocket: resolveTranslateSocket(), envBaseUrlSet: !!process.env.ANTHROPIC_BASE_URL, envBaseUrl: process.env.ANTHROPIC_BASE_URL || '(not set)' }, null, 2));
|
|
1157
1449
|
return;
|
|
1158
1450
|
}
|
|
1159
1451
|
|
|
@@ -1176,13 +1468,14 @@ async function handleRequest(req, res) {
|
|
|
1176
1468
|
try {
|
|
1177
1469
|
const body = JSON.parse((await collectBody(req)).toString('utf8'));
|
|
1178
1470
|
if (body.PRESERVE_RECENT_MESSAGES != null) liveConfig.PRESERVE_RECENT_MESSAGES = Math.max(1, Math.min(20, parseInt(body.PRESERVE_RECENT_MESSAGES) || 3));
|
|
1179
|
-
if (body.OLD_STRIP_THRESHOLD != null) liveConfig.OLD_STRIP_THRESHOLD = Math.max(
|
|
1180
|
-
if (body.MIN_TRANSLATE_LENGTH != null) liveConfig.MIN_TRANSLATE_LENGTH = Math.max(
|
|
1471
|
+
if (body.OLD_STRIP_THRESHOLD != null) liveConfig.OLD_STRIP_THRESHOLD = Math.max(0, Math.min(2000, parseInt(body.OLD_STRIP_THRESHOLD) || 0));
|
|
1472
|
+
if (body.MIN_TRANSLATE_LENGTH != null) liveConfig.MIN_TRANSLATE_LENGTH = Math.max(0, Math.min(200, parseInt(body.MIN_TRANSLATE_LENGTH) || 0));
|
|
1181
1473
|
if (body.TOOL_RESULT_PREVIEW_CHARS != null) liveConfig.TOOL_RESULT_PREVIEW_CHARS = Math.max(50, Math.min(500, parseInt(body.TOOL_RESULT_PREVIEW_CHARS) || 200));
|
|
1182
1474
|
if (body.STENO_ENABLED != null) liveConfig.STENO_ENABLED = !!body.STENO_ENABLED;
|
|
1183
1475
|
if (body.TRANSLATE_ENABLED != null) liveConfig.TRANSLATE_ENABLED = !!body.TRANSLATE_ENABLED;
|
|
1184
1476
|
if (body.OLD_STRIP_ENABLED != null) liveConfig.OLD_STRIP_ENABLED = !!body.OLD_STRIP_ENABLED;
|
|
1185
1477
|
if (body.SYSTEM_PROMPT_COMPRESS != null) liveConfig.SYSTEM_PROMPT_COMPRESS = !!body.SYSTEM_PROMPT_COMPRESS;
|
|
1478
|
+
if (body.SYSTEM_REMINDER_STRIPPING != null) liveConfig.SYSTEM_REMINDER_STRIPPING = !!body.SYSTEM_REMINDER_STRIPPING;
|
|
1186
1479
|
res.writeHead(200, { 'Content-Type': 'application/json' });
|
|
1187
1480
|
res.end(JSON.stringify({ ok: true, ...liveConfig }));
|
|
1188
1481
|
} catch (e) {
|
|
@@ -1224,11 +1517,97 @@ async function handleRequest(req, res) {
|
|
|
1224
1517
|
return;
|
|
1225
1518
|
}
|
|
1226
1519
|
|
|
1520
|
+
// Explicit pause/resume endpoints — no guessing with toggle
|
|
1521
|
+
if (req.url === '/pause' && req.method === 'POST') {
|
|
1522
|
+
proxyPaused = true;
|
|
1523
|
+
log('info', 'Proxy PAUSED via /pause');
|
|
1524
|
+
res.writeHead(200, { 'Content-Type': 'application/json' });
|
|
1525
|
+
res.end(JSON.stringify({ paused: true }));
|
|
1526
|
+
return;
|
|
1527
|
+
}
|
|
1528
|
+
if (req.url === '/resume' && req.method === 'POST') {
|
|
1529
|
+
proxyPaused = false;
|
|
1530
|
+
log('info', 'Proxy RESUMED via /resume');
|
|
1531
|
+
res.writeHead(200, { 'Content-Type': 'application/json' });
|
|
1532
|
+
res.end(JSON.stringify({ paused: false }));
|
|
1533
|
+
return;
|
|
1534
|
+
}
|
|
1535
|
+
|
|
1536
|
+
// --- Multi-Project Registry Endpoints ---
|
|
1537
|
+
|
|
1538
|
+
// POST /register — register a project with the daemon
|
|
1539
|
+
if (req.url === '/register' && req.method === 'POST') {
|
|
1540
|
+
try {
|
|
1541
|
+
const body = JSON.parse((await collectBody(req)).toString('utf8'));
|
|
1542
|
+
if (!body.projectPath) {
|
|
1543
|
+
res.writeHead(400, { 'Content-Type': 'application/json' });
|
|
1544
|
+
res.end(JSON.stringify({ error: 'projectPath required' }));
|
|
1545
|
+
return;
|
|
1546
|
+
}
|
|
1547
|
+
projectRegistry.set(body.projectPath, {
|
|
1548
|
+
projectPath: body.projectPath,
|
|
1549
|
+
pid: body.pid || 0,
|
|
1550
|
+
registeredAt: projectRegistry.get(body.projectPath)?.registeredAt || Date.now(),
|
|
1551
|
+
lastSeen: Date.now(),
|
|
1552
|
+
});
|
|
1553
|
+
_invalidateProjectCaches();
|
|
1554
|
+
log('info', `REGISTRY: registered project "${body.projectPath}" (pid=${body.pid || 0}, total=${projectRegistry.size})`);
|
|
1555
|
+
res.writeHead(200, { 'Content-Type': 'application/json' });
|
|
1556
|
+
res.end(JSON.stringify({ ok: true, registered: projectRegistry.size }));
|
|
1557
|
+
} catch (e) {
|
|
1558
|
+
res.writeHead(400, { 'Content-Type': 'application/json' });
|
|
1559
|
+
res.end(JSON.stringify({ error: e.message }));
|
|
1560
|
+
}
|
|
1561
|
+
return;
|
|
1562
|
+
}
|
|
1563
|
+
|
|
1564
|
+
// POST /deregister — remove a project from the daemon
|
|
1565
|
+
if (req.url === '/deregister' && req.method === 'POST') {
|
|
1566
|
+
try {
|
|
1567
|
+
const body = JSON.parse((await collectBody(req)).toString('utf8'));
|
|
1568
|
+
if (body.projectPath && projectRegistry.has(body.projectPath)) {
|
|
1569
|
+
projectRegistry.delete(body.projectPath);
|
|
1570
|
+
_invalidateProjectCaches();
|
|
1571
|
+
log('info', `REGISTRY: deregistered project "${body.projectPath}" (remaining=${projectRegistry.size})`);
|
|
1572
|
+
}
|
|
1573
|
+
res.writeHead(200, { 'Content-Type': 'application/json' });
|
|
1574
|
+
res.end(JSON.stringify({ ok: true, registered: projectRegistry.size }));
|
|
1575
|
+
} catch (e) {
|
|
1576
|
+
res.writeHead(400, { 'Content-Type': 'application/json' });
|
|
1577
|
+
res.end(JSON.stringify({ error: e.message }));
|
|
1578
|
+
}
|
|
1579
|
+
return;
|
|
1580
|
+
}
|
|
1581
|
+
|
|
1582
|
+
// GET /registry — diagnostic: list all registered projects
|
|
1583
|
+
if (req.url === '/registry' && req.method === 'GET') {
|
|
1584
|
+
const projects = [];
|
|
1585
|
+
for (const [path, entry] of projectRegistry.entries()) {
|
|
1586
|
+
let alive = false;
|
|
1587
|
+
if (entry.pid > 0) {
|
|
1588
|
+
try { process.kill(entry.pid, 0); alive = true; } catch {}
|
|
1589
|
+
}
|
|
1590
|
+
const sockPath = join(path, 'specmem', 'run', 'translate.sock');
|
|
1591
|
+
projects.push({
|
|
1592
|
+
projectPath: path,
|
|
1593
|
+
pid: entry.pid,
|
|
1594
|
+
alive,
|
|
1595
|
+
registeredAt: new Date(entry.registeredAt).toISOString(),
|
|
1596
|
+
lastSeen: new Date(entry.lastSeen).toISOString(),
|
|
1597
|
+
translateSocket: existsSync(sockPath) ? sockPath : null,
|
|
1598
|
+
});
|
|
1599
|
+
}
|
|
1600
|
+
res.writeHead(200, { 'Content-Type': 'application/json' });
|
|
1601
|
+
res.end(JSON.stringify({ projects, total: projects.length }, null, 2));
|
|
1602
|
+
return;
|
|
1603
|
+
}
|
|
1604
|
+
|
|
1227
1605
|
// Count only real API traffic, not management endpoint polls
|
|
1228
1606
|
stats.totalRequests++;
|
|
1229
1607
|
|
|
1608
|
+
let rawBody;
|
|
1230
1609
|
try {
|
|
1231
|
-
|
|
1610
|
+
rawBody = await collectBody(req);
|
|
1232
1611
|
|
|
1233
1612
|
// Paused — pass everything through untouched
|
|
1234
1613
|
if (proxyPaused) {
|
|
@@ -1275,7 +1654,8 @@ async function handleRequest(req, res) {
|
|
|
1275
1654
|
pushEvent('info', `POST /v1/messages model=${body.model || '?'} msgs=${messageCount} size=${(originalSize / 1024).toFixed(0)}KB`);
|
|
1276
1655
|
|
|
1277
1656
|
const isCompaction = isCompactionRequest(body);
|
|
1278
|
-
|
|
1657
|
+
// No passthrough — always process everything (system-reminder strip + steno + translate)
|
|
1658
|
+
const isPassthrough = false;
|
|
1279
1659
|
let sysPromptModified = false;
|
|
1280
1660
|
|
|
1281
1661
|
// === SYSTEM PROMPT COMPRESSION ===
|
|
@@ -1286,7 +1666,7 @@ async function handleRequest(req, res) {
|
|
|
1286
1666
|
const _sysKey = typeof body.system === 'string' ? body.system
|
|
1287
1667
|
: Array.isArray(body.system) ? body.system.map(b => typeof b === 'string' ? b : (b?.text || '')).join('')
|
|
1288
1668
|
: JSON.stringify(body.system);
|
|
1289
|
-
const _sysHash =
|
|
1669
|
+
const _sysHash = createHash('md5').update(_sysKey).digest('hex');
|
|
1290
1670
|
const _sysCached = _sysPromptCache.get(_sysHash);
|
|
1291
1671
|
|
|
1292
1672
|
if (_sysCached) {
|
|
@@ -1333,8 +1713,26 @@ async function handleRequest(req, res) {
|
|
|
1333
1713
|
log('info', `=== ${logMsg} ===`);
|
|
1334
1714
|
pushEvent('compaction', logMsg);
|
|
1335
1715
|
|
|
1336
|
-
|
|
1337
|
-
|
|
1716
|
+
// Strip redundant system-reminders first (before tool stripping)
|
|
1717
|
+
try {
|
|
1718
|
+
const srResult = stripSystemReminders(body.messages);
|
|
1719
|
+
if (srResult.remindersStripped > 0) {
|
|
1720
|
+
body.messages = srResult.messages;
|
|
1721
|
+
log('info', `[compaction] Stripped ${srResult.remindersStripped} system-reminders (${srResult.charsRemoved} chars)`);
|
|
1722
|
+
}
|
|
1723
|
+
} catch (srErr) {
|
|
1724
|
+
log('warn', `[compaction] stripSystemReminders failed: ${srErr.message}`);
|
|
1725
|
+
}
|
|
1726
|
+
|
|
1727
|
+
let strippingStats = { toolResultsStripped: 0, toolUsesStripped: 0, charsRemoved: 0 };
|
|
1728
|
+
try {
|
|
1729
|
+
const result = stripMessages(body.messages);
|
|
1730
|
+
body.messages = result.strippedMessages;
|
|
1731
|
+
strippingStats = result.strippingStats;
|
|
1732
|
+
} catch (stripErr) {
|
|
1733
|
+
log('warn', `stripMessages failed, using raw messages: ${stripErr.message}`);
|
|
1734
|
+
// Keep body.messages as-is
|
|
1735
|
+
}
|
|
1338
1736
|
|
|
1339
1737
|
// Run steno+MT compression in parallel (independent of strip)
|
|
1340
1738
|
if (!dontCompress) {
|
|
@@ -1369,24 +1767,38 @@ async function handleRequest(req, res) {
|
|
|
1369
1767
|
return;
|
|
1370
1768
|
}
|
|
1371
1769
|
|
|
1372
|
-
// ===
|
|
1373
|
-
|
|
1374
|
-
|
|
1375
|
-
|
|
1376
|
-
|
|
1377
|
-
|
|
1378
|
-
|
|
1379
|
-
|
|
1380
|
-
|
|
1381
|
-
|
|
1770
|
+
// === NO PASSTHROUGH — always process all requests ===
|
|
1771
|
+
// (passthrough mode removed — every request gets stripped + compressed)
|
|
1772
|
+
|
|
1773
|
+
// Step 0: Strip redundant <system-reminder> tags
|
|
1774
|
+
let srCharsRemoved = 0;
|
|
1775
|
+
let srRemindersStripped = 0;
|
|
1776
|
+
try {
|
|
1777
|
+
const srResult = stripSystemReminders(body.messages);
|
|
1778
|
+
if (srResult.remindersStripped > 0) {
|
|
1779
|
+
body.messages = srResult.messages;
|
|
1780
|
+
srCharsRemoved = srResult.charsRemoved;
|
|
1781
|
+
srRemindersStripped = srResult.remindersStripped;
|
|
1782
|
+
log('info', `Stripped ${srResult.remindersStripped} system-reminders (${srResult.charsRemoved} chars)`);
|
|
1783
|
+
}
|
|
1784
|
+
} catch (srErr) {
|
|
1785
|
+
log('warn', `stripSystemReminders failed, skipping: ${srErr.message}`);
|
|
1382
1786
|
}
|
|
1383
1787
|
|
|
1384
1788
|
// Step 1: Strip large tool_results from old messages (biggest savings)
|
|
1385
|
-
|
|
1386
|
-
|
|
1387
|
-
|
|
1789
|
+
let oldStrip = { toolResultsStripped: 0, charsRemoved: 0 };
|
|
1790
|
+
try {
|
|
1791
|
+
if (liveConfig.OLD_STRIP_ENABLED) {
|
|
1792
|
+
const result = stripOldToolResults(body.messages);
|
|
1793
|
+
if (result.strippingStats.charsRemoved > 0) {
|
|
1794
|
+
body.messages = result.messages;
|
|
1795
|
+
oldStrip = result.strippingStats;
|
|
1796
|
+
}
|
|
1797
|
+
}
|
|
1798
|
+
} catch (stripErr) {
|
|
1799
|
+
log('warn', `stripOldToolResults failed, skipping: ${stripErr.message}`);
|
|
1800
|
+
}
|
|
1388
1801
|
if (oldStrip.charsRemoved > 0) {
|
|
1389
|
-
body.messages = preStripped;
|
|
1390
1802
|
stats.oldStripped += oldStrip.toolResultsStripped;
|
|
1391
1803
|
stats.oldCharsRemoved += oldStrip.charsRemoved;
|
|
1392
1804
|
stats.tokensStripped += Math.floor(oldStrip.charsRemoved / 4);
|
|
@@ -1397,13 +1809,18 @@ async function handleRequest(req, res) {
|
|
|
1397
1809
|
// Step 2: Steno+MT compression on remaining text
|
|
1398
1810
|
const { messages: compressed, blocksCompressed, charsCompressed, verifiedCount = 0, stenoOnlyCount = 0, tmHits: hits = 0, samples: liveSamples = [] } = await compressMessagesLive(body.messages);
|
|
1399
1811
|
|
|
1400
|
-
|
|
1812
|
+
// srCharsRemoved already tracked from Step 0 above
|
|
1813
|
+
|
|
1814
|
+
if (blocksCompressed === 0 && oldStrip.charsRemoved === 0 && srCharsRemoved === 0) {
|
|
1401
1815
|
stats.passthrough++;
|
|
1402
1816
|
pushEvent('pass', `msgs=${messageCount} (nothing to compress)`);
|
|
1403
1817
|
forwardRequest(req, res, rawBody);
|
|
1404
1818
|
return;
|
|
1405
1819
|
}
|
|
1406
1820
|
|
|
1821
|
+
// If anything was stripped/compressed, body was mutated — must re-serialize
|
|
1822
|
+
body.messages = compressed;
|
|
1823
|
+
|
|
1407
1824
|
// Store translation samples for preview
|
|
1408
1825
|
if (liveSamples.length > 0) stats._lastSamples = liveSamples;
|
|
1409
1826
|
|
|
@@ -1417,8 +1834,8 @@ async function handleRequest(req, res) {
|
|
|
1417
1834
|
stats.stenoOnly += (blocksCompressed - verifiedCount - stenoOnlyCount);
|
|
1418
1835
|
stats.tmHits += hits;
|
|
1419
1836
|
pushEvent('compress', `LIVE: ${blocksCompressed} blocks, ${charsCompressed} chars (${verifiedCount} zh, ${stenoOnlyCount} steno, ${hits} TM)`);
|
|
1420
|
-
} else if (oldStrip.charsRemoved > 0) {
|
|
1421
|
-
// Only stripping, no compression — still send modified body
|
|
1837
|
+
} else if (oldStrip.charsRemoved > 0 || srCharsRemoved > 0) {
|
|
1838
|
+
// Only stripping (tool results or system-reminders), no compression — still send modified body
|
|
1422
1839
|
stats.liveCompressed++;
|
|
1423
1840
|
}
|
|
1424
1841
|
stats.lastLiveCompress = new Date().toISOString();
|
|
@@ -1428,7 +1845,8 @@ async function handleRequest(req, res) {
|
|
|
1428
1845
|
const savedPercent = originalSize > 0 ? ((savedBytes / originalSize) * 100).toFixed(1) : '0';
|
|
1429
1846
|
|
|
1430
1847
|
const stripInfo = oldStrip.charsRemoved > 0 ? ` [STRIP: ${oldStrip.toolResultsStripped} results, ${(oldStrip.charsRemoved / 1024).toFixed(0)}KB]` : '';
|
|
1431
|
-
|
|
1848
|
+
const srInfo = srCharsRemoved > 0 ? ` [SR: ${srRemindersStripped} reminders, ${(srCharsRemoved / 1024).toFixed(1)}KB]` : '';
|
|
1849
|
+
log('info', `LIVE: ${blocksCompressed} blocks | ${(originalSize / 1024).toFixed(0)}KB → ${(compressedBody.length / 1024).toFixed(0)}KB (-${savedPercent}%)${stripInfo}${srInfo}`);
|
|
1432
1850
|
pushEvent('saved', `${(originalSize / 1024).toFixed(0)}KB → ${(compressedBody.length / 1024).toFixed(0)}KB (-${savedPercent}%) ~${Math.round(savedBytes / 3.5)} tokens`);
|
|
1433
1851
|
|
|
1434
1852
|
pushPreview(rawBody.toString('utf8'), compressedBody.toString('utf8'), 'live', liveSamples);
|
|
@@ -1437,11 +1855,35 @@ async function handleRequest(req, res) {
|
|
|
1437
1855
|
|
|
1438
1856
|
} catch (err) {
|
|
1439
1857
|
log('error', `Handler error: ${err.message}`);
|
|
1858
|
+
log('error', `Handler stack: ${err.stack || '(no stack)'}`);
|
|
1440
1859
|
pushEvent('error', err.message);
|
|
1441
1860
|
stats.errors++;
|
|
1861
|
+
// CRITICAL: Never return 502 — fall back to forwarding raw body.
|
|
1862
|
+
// The proxy must NEVER break the API call. Compression is optional;
|
|
1863
|
+
// if it fails, just passthrough the original request.
|
|
1442
1864
|
if (!res.headersSent) {
|
|
1443
|
-
|
|
1444
|
-
|
|
1865
|
+
try {
|
|
1866
|
+
const fallbackBody = (typeof rawBody !== 'undefined' && rawBody && rawBody.length > 0) ? rawBody : null;
|
|
1867
|
+
if (fallbackBody) {
|
|
1868
|
+
log('warn', `Falling back to passthrough after handler error: ${err.message}`);
|
|
1869
|
+
stats.passthrough++;
|
|
1870
|
+
forwardRequest(req, res, fallbackBody);
|
|
1871
|
+
} else {
|
|
1872
|
+
// rawBody unavailable — re-collect from request if possible, else auto-pause and passthrough next time
|
|
1873
|
+
log('warn', `No rawBody available — auto-pausing proxy to prevent further 502s: ${err.message}`);
|
|
1874
|
+
proxyPaused = true;
|
|
1875
|
+
// Return a retriable error so Claude Code retries (and next attempt hits passthrough path)
|
|
1876
|
+
res.writeHead(503, { 'Content-Type': 'application/json', 'Retry-After': '1' });
|
|
1877
|
+
res.end(JSON.stringify({ type: 'error', error: { type: 'overloaded_error', message: 'Proxy auto-paused due to handler error, retry immediately' } }));
|
|
1878
|
+
}
|
|
1879
|
+
} catch (fallbackErr) {
|
|
1880
|
+
log('error', `Fallback also failed: ${fallbackErr.message}`);
|
|
1881
|
+
if (!res.headersSent) {
|
|
1882
|
+
proxyPaused = true;
|
|
1883
|
+
res.writeHead(503, { 'Content-Type': 'application/json', 'Retry-After': '1' });
|
|
1884
|
+
res.end(JSON.stringify({ type: 'error', error: { type: 'overloaded_error', message: 'Proxy auto-paused, retry immediately' } }));
|
|
1885
|
+
}
|
|
1886
|
+
}
|
|
1445
1887
|
}
|
|
1446
1888
|
}
|
|
1447
1889
|
}
|
|
@@ -1459,13 +1901,9 @@ let proxyServer = null;
|
|
|
1459
1901
|
* Returns { started: boolean, port: number }
|
|
1460
1902
|
*/
|
|
1461
1903
|
export async function startCompactionProxy() {
|
|
1462
|
-
//
|
|
1463
|
-
|
|
1464
|
-
|
|
1465
|
-
cleanupPortFile();
|
|
1466
|
-
cleanupPidFile();
|
|
1467
|
-
return { started: false, port: PROXY_PORT, reason: 'disabled_by_user' };
|
|
1468
|
-
}
|
|
1904
|
+
// Disabled flag = start in passthrough mode (proxy always in path for seamless toggling).
|
|
1905
|
+
// The daemon itself reads the flag and starts paused.
|
|
1906
|
+
const disabledByUser = existsSync(DISABLED_FILE);
|
|
1469
1907
|
|
|
1470
1908
|
// Check PID file — is daemon already running?
|
|
1471
1909
|
if (existsSync(PID_FILE)) {
|
|
@@ -1478,8 +1916,11 @@ export async function startCompactionProxy() {
|
|
|
1478
1916
|
// Process exists, verify it responds to health check
|
|
1479
1917
|
const healthy = await checkDaemonHealth();
|
|
1480
1918
|
if (healthy) {
|
|
1481
|
-
log('info', `Daemon already running (PID ${pid}), skipping spawn`);
|
|
1482
|
-
|
|
1919
|
+
log('info', `Daemon already running (PID ${pid})${disabledByUser ? ' (passthrough mode)' : ''}, skipping spawn`);
|
|
1920
|
+
// If disabled flag changed since daemon started, sync pause state via SIGUSR1
|
|
1921
|
+
// (daemon checks disabled file on toggle, but we can hint via stats check)
|
|
1922
|
+
registerWithDaemon(process.env.SPECMEM_PROJECT_PATH, process.pid);
|
|
1923
|
+
return { started: false, port: PROXY_PORT, paused: disabledByUser, reason: 'daemon_already_running' };
|
|
1483
1924
|
}
|
|
1484
1925
|
// Process alive but not responding — stale, kill it
|
|
1485
1926
|
log('warn', `Daemon PID ${pid} alive but not healthy, killing stale process`);
|
|
@@ -1515,14 +1956,16 @@ export async function startCompactionProxy() {
|
|
|
1515
1956
|
const deadline = Date.now() + 3000;
|
|
1516
1957
|
while (Date.now() < deadline) {
|
|
1517
1958
|
if (existsSync(PORT_FILE)) {
|
|
1518
|
-
log('info', `Daemon ready (PID ${child.pid}), port file appeared`);
|
|
1519
|
-
|
|
1959
|
+
log('info', `Daemon ready (PID ${child.pid})${disabledByUser ? ' in passthrough mode' : ''}, port file appeared`);
|
|
1960
|
+
registerWithDaemon(process.env.SPECMEM_PROJECT_PATH, process.pid);
|
|
1961
|
+
return { started: true, port: PROXY_PORT, paused: disabledByUser };
|
|
1520
1962
|
}
|
|
1521
1963
|
await new Promise(r => setTimeout(r, 100));
|
|
1522
1964
|
}
|
|
1523
1965
|
|
|
1524
1966
|
log('warn', 'Daemon spawned but port file did not appear within 3s');
|
|
1525
|
-
|
|
1967
|
+
registerWithDaemon(process.env.SPECMEM_PROJECT_PATH, process.pid);
|
|
1968
|
+
return { started: true, port: PROXY_PORT, paused: disabledByUser, reason: 'timeout_waiting_for_port' };
|
|
1526
1969
|
}
|
|
1527
1970
|
|
|
1528
1971
|
/**
|
|
@@ -1559,6 +2002,12 @@ function togglePause() {
|
|
|
1559
2002
|
return proxyPaused;
|
|
1560
2003
|
}
|
|
1561
2004
|
|
|
2005
|
+
function setPaused(state) {
|
|
2006
|
+
proxyPaused = !!state;
|
|
2007
|
+
log('info', `Proxy pause state set to ${proxyPaused ? 'PAUSED' : 'ACTIVE'}`);
|
|
2008
|
+
return proxyPaused;
|
|
2009
|
+
}
|
|
2010
|
+
|
|
1562
2011
|
/**
|
|
1563
2012
|
* Check if the daemon is responding on its health endpoint.
|
|
1564
2013
|
*/
|
|
@@ -1575,11 +2024,12 @@ function checkDaemonHealth() {
|
|
|
1575
2024
|
}
|
|
1576
2025
|
|
|
1577
2026
|
/**
|
|
1578
|
-
* Stop the proxy —
|
|
2027
|
+
* Stop the proxy — deregisters this project from the daemon.
|
|
1579
2028
|
* The daemon persists independently of the MCP process.
|
|
1580
2029
|
* Use killCompactionProxy() to explicitly terminate the daemon.
|
|
1581
2030
|
*/
|
|
1582
2031
|
export function stopCompactionProxy() {
|
|
2032
|
+
deregisterFromDaemon(process.env.SPECMEM_PROJECT_PATH);
|
|
1583
2033
|
return Promise.resolve();
|
|
1584
2034
|
}
|
|
1585
2035
|
|
|
@@ -1667,6 +2117,115 @@ export function getCompactionProxyStats() {
|
|
|
1667
2117
|
return { running, daemonPid, port: PROXY_PORT };
|
|
1668
2118
|
}
|
|
1669
2119
|
|
|
2120
|
+
// ============================================================================
|
|
2121
|
+
// Registration helpers — fire-and-forget HTTP to the daemon
|
|
2122
|
+
// ============================================================================
|
|
2123
|
+
|
|
2124
|
+
/**
|
|
2125
|
+
* Register this project with the running daemon.
|
|
2126
|
+
* Fire-and-forget — if daemon isn't running yet, fails silently.
|
|
2127
|
+
*/
|
|
2128
|
+
function registerWithDaemon(projectPath, pid) {
|
|
2129
|
+
if (!projectPath) return;
|
|
2130
|
+
const body = JSON.stringify({ projectPath, pid: pid || process.pid });
|
|
2131
|
+
try {
|
|
2132
|
+
const req = httpRequest({
|
|
2133
|
+
hostname: '127.0.0.1',
|
|
2134
|
+
port: PROXY_PORT,
|
|
2135
|
+
path: '/register',
|
|
2136
|
+
method: 'POST',
|
|
2137
|
+
headers: { 'Content-Type': 'application/json', 'Content-Length': Buffer.byteLength(body) },
|
|
2138
|
+
timeout: 2000,
|
|
2139
|
+
});
|
|
2140
|
+
req.on('error', () => {}); // fire-and-forget
|
|
2141
|
+
req.on('timeout', () => req.destroy());
|
|
2142
|
+
req.write(body);
|
|
2143
|
+
req.end();
|
|
2144
|
+
log('info', `Sent /register for "${projectPath}" (pid=${pid || process.pid})`);
|
|
2145
|
+
} catch { /* fire-and-forget */ }
|
|
2146
|
+
}
|
|
2147
|
+
|
|
2148
|
+
/**
|
|
2149
|
+
* Deregister this project from the running daemon.
|
|
2150
|
+
* Fire-and-forget — best effort cleanup on shutdown.
|
|
2151
|
+
*/
|
|
2152
|
+
function deregisterFromDaemon(projectPath) {
|
|
2153
|
+
if (!projectPath) return;
|
|
2154
|
+
const body = JSON.stringify({ projectPath });
|
|
2155
|
+
try {
|
|
2156
|
+
const req = httpRequest({
|
|
2157
|
+
hostname: '127.0.0.1',
|
|
2158
|
+
port: PROXY_PORT,
|
|
2159
|
+
path: '/deregister',
|
|
2160
|
+
method: 'POST',
|
|
2161
|
+
headers: { 'Content-Type': 'application/json', 'Content-Length': Buffer.byteLength(body) },
|
|
2162
|
+
timeout: 2000,
|
|
2163
|
+
});
|
|
2164
|
+
req.on('error', () => {}); // fire-and-forget
|
|
2165
|
+
req.on('timeout', () => req.destroy());
|
|
2166
|
+
req.write(body);
|
|
2167
|
+
req.end();
|
|
2168
|
+
log('info', `Sent /deregister for "${projectPath}"`);
|
|
2169
|
+
} catch { /* fire-and-forget */ }
|
|
2170
|
+
}
|
|
2171
|
+
|
|
2172
|
+
// ============================================================================
|
|
2173
|
+
// Stale Project Reaper — removes dead projects from the registry
|
|
2174
|
+
// ============================================================================
|
|
2175
|
+
|
|
2176
|
+
/**
|
|
2177
|
+
* Check if a PID is alive using signal 0.
|
|
2178
|
+
*/
|
|
2179
|
+
function isPidAlive(pid) {
|
|
2180
|
+
if (!pid || pid <= 0) return false;
|
|
2181
|
+
try {
|
|
2182
|
+
process.kill(pid, 0);
|
|
2183
|
+
return true;
|
|
2184
|
+
} catch {
|
|
2185
|
+
return false;
|
|
2186
|
+
}
|
|
2187
|
+
}
|
|
2188
|
+
|
|
2189
|
+
/**
|
|
2190
|
+
* Reap stale projects — remove entries with dead PIDs.
|
|
2191
|
+
*/
|
|
2192
|
+
function reapStaleProjects() {
|
|
2193
|
+
let reaped = 0;
|
|
2194
|
+
for (const [path, entry] of projectRegistry.entries()) {
|
|
2195
|
+
if (entry.pid > 0 && !isPidAlive(entry.pid)) {
|
|
2196
|
+
projectRegistry.delete(path);
|
|
2197
|
+
reaped++;
|
|
2198
|
+
log('info', `REAPER: removed stale project "${path}" (dead pid=${entry.pid})`);
|
|
2199
|
+
}
|
|
2200
|
+
}
|
|
2201
|
+
if (reaped > 0) {
|
|
2202
|
+
_invalidateProjectCaches();
|
|
2203
|
+
log('info', `REAPER: reaped ${reaped} stale projects, ${projectRegistry.size} remaining`);
|
|
2204
|
+
}
|
|
2205
|
+
}
|
|
2206
|
+
|
|
2207
|
+
let _reaperTimer = null;
|
|
2208
|
+
|
|
2209
|
+
/**
|
|
2210
|
+
* Start the stale project reaper (30s interval, unref'd).
|
|
2211
|
+
*/
|
|
2212
|
+
function startReaper() {
|
|
2213
|
+
if (_reaperTimer) return;
|
|
2214
|
+
_reaperTimer = setInterval(reapStaleProjects, 30000);
|
|
2215
|
+
_reaperTimer.unref();
|
|
2216
|
+
log('info', 'REAPER: started (30s interval)');
|
|
2217
|
+
}
|
|
2218
|
+
|
|
2219
|
+
/**
|
|
2220
|
+
* Check if any registered projects have live PIDs.
|
|
2221
|
+
*/
|
|
2222
|
+
function hasLiveProjects() {
|
|
2223
|
+
for (const entry of projectRegistry.values()) {
|
|
2224
|
+
if (entry.pid > 0 && isPidAlive(entry.pid)) return true;
|
|
2225
|
+
}
|
|
2226
|
+
return false;
|
|
2227
|
+
}
|
|
2228
|
+
|
|
1670
2229
|
// ============================================================================
|
|
1671
2230
|
// Daemon Internals — exported for compactionProxyDaemon.js
|
|
1672
2231
|
// ============================================================================
|
|
@@ -1675,6 +2234,9 @@ export {
|
|
|
1675
2234
|
handleRequest,
|
|
1676
2235
|
log,
|
|
1677
2236
|
togglePause,
|
|
2237
|
+
setPaused,
|
|
2238
|
+
startReaper,
|
|
2239
|
+
hasLiveProjects,
|
|
1678
2240
|
PROXY_PORT,
|
|
1679
2241
|
PORT_FILE,
|
|
1680
2242
|
PID_FILE,
|