specmem-hardwicksoftware 3.7.35 → 3.7.38
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +34 -0
- package/README.md +11 -15
- package/bin/specmem-autoclaude.cjs +12 -1
- package/bin/specmem-cli.cjs +1077 -11
- package/bin/specmem-console.cjs +890 -63
- package/bootstrap.cjs +10 -2
- package/claude-hooks/agent-loading-hook.cjs +16 -16
- package/claude-hooks/agent-loading-hook.js +28 -21
- package/claude-hooks/agent-type-matcher.js +1 -1
- package/claude-hooks/background-completion-silencer.js +1 -1
- package/claude-hooks/file-claim-enforcer.cjs +37 -36
- package/claude-hooks/output-cleaner.cjs +1 -1
- package/claude-hooks/refusal-detector-hook.cjs +53 -0
- package/claude-hooks/settings.json +64 -4
- package/claude-hooks/smart-search-interceptor.js +1 -1
- package/claude-hooks/specmem-search-enforcer.cjs +2 -11
- package/claude-hooks/specmem-team-member-inject.js +1 -1
- package/claude-hooks/specmem-unified-hook.py +1 -1
- package/claude-hooks/subagent-loading-hook.cjs +1 -1
- package/claude-hooks/task-progress-hook.cjs +7 -7
- package/claude-hooks/task-progress-hook.js +3 -3
- package/claude-hooks/team-comms-enforcer.cjs +113 -47
- package/claude-hooks/use-code-pointers.cjs +1 -1
- package/dist/claude-sessions/sessionParser.js +5 -0
- package/dist/cli/deploy-to-claude.js +9 -2
- package/dist/codebase/codebaseIndexer.js +48 -17
- package/dist/codebase/exclusions.js +3 -4
- package/dist/codebase/index.js +4 -0
- package/dist/codebase/pdfExtractor.js +298 -0
- package/dist/dashboard/api/taskTeamMembers.js +2 -2
- package/dist/db/bigBrainMigrations.js +29 -0
- package/dist/hooks/hookManager.js +4 -4
- package/dist/hooks/teamFramingCli.js +1 -1
- package/dist/hooks/teamMemberPrepromptHook.js +5 -5
- package/dist/index.js +49 -12
- package/dist/init/claudeConfigInjector.js +27 -8
- package/dist/installer/autoInstall.js +7 -1
- package/dist/mcp/compactionProxy.js +1052 -192
- package/dist/mcp/compactionProxyDaemon.js +112 -37
- package/dist/mcp/contextVault.js +439 -0
- package/dist/mcp/embeddingServerManager.js +151 -17
- package/dist/mcp/mcpProtocolHandler.js +6 -1
- package/dist/mcp/miniCOTServerManager.js +82 -8
- package/dist/mcp/specMemServer.js +45 -10
- package/dist/mcp/toolRegistry.js +6 -0
- package/dist/startup/startupIndexing.js +14 -0
- package/dist/team-members/taskOrchestrator.js +3 -3
- package/dist/team-members/taskTeamMemberLogger.js +2 -2
- package/dist/tools/goofy/deployTeamMember.js +3 -3
- package/dist/tools/goofy/digInTheVault.js +81 -0
- package/dist/tools/goofy/findCodePointers.js +17 -0
- package/dist/tools/goofy/findWhatISaid.js +19 -0
- package/dist/tools/goofy/stashTheGoods.js +56 -0
- package/dist/tools/teamMemberDeployer.js +2 -2
- package/dist/watcher/changeHandler.js +65 -8
- package/dist/watcher/changeQueue.js +20 -1
- package/embedding-sandbox/frankenstein-embeddings.py +4 -3
- package/embedding-sandbox/mini-cot-service.py +11 -13
- package/embedding-sandbox/pdf-text-extract.py +208 -0
- package/package.json +1 -1
- package/scripts/deploy-hooks.cjs +12 -4
- package/scripts/fast-batch-embedder.cjs +2 -2
- package/scripts/force-retry.cjs +34 -0
- package/scripts/global-postinstall.cjs +97 -4
- package/scripts/poetic-abliteration.cjs +379 -0
- package/scripts/refusal-enforcer.cjs +88 -0
- package/scripts/specmem-init.cjs +222 -41
- package/specmem/model-config.json +6 -6
- package/specmem/supervisord.conf +1 -1
- package/svg-sections/readme-token-compaction.svg +246 -0
- package/claude-hooks/agent-chooser-hook.js +0 -179
|
@@ -34,14 +34,84 @@ const CLAUDE_DIR = join(HOME, '.claude');
|
|
|
34
34
|
const PORT_FILE = join(CLAUDE_DIR, '.compaction-proxy-port');
|
|
35
35
|
const PID_FILE = join(CLAUDE_DIR, '.compaction-proxy.pid');
|
|
36
36
|
const DISABLED_FILE = join(CLAUDE_DIR, '.compaction-proxy-disabled');
|
|
37
|
+
const CUSTOM_SYS_PROMPT_FILE = join(CLAUDE_DIR, '.custom-sys-prompt.json');
|
|
38
|
+
const OG_SYS_PROMPT_FILE = join(CLAUDE_DIR, '.og-sys-prompt.json');
|
|
37
39
|
|
|
38
40
|
// Per-user port: env var > default 4080. Multiple users on same machine
|
|
39
41
|
// should set COMPACTION_PROXY_PORT or rely on the port file mechanism.
|
|
40
42
|
const PROXY_PORT = parseInt(process.env.COMPACTION_PROXY_PORT || '4080', 10);
|
|
41
|
-
|
|
42
|
-
|
|
43
|
+
|
|
44
|
+
// ============================================================================
|
|
45
|
+
// Custom Upstream Detection — ENV VARS ONLY (no settings.json scanning)
|
|
46
|
+
// ============================================================================
|
|
47
|
+
// For custom APIs (MiniMax, etc.), set these env vars on the MCP server config:
|
|
48
|
+
// COMPACTION_PROXY_UPSTREAM — full URL of the real API (e.g., https://api.minimax.io/anthropic)
|
|
49
|
+
// COMPACTION_PROXY_API_KEY — API key for the custom upstream
|
|
50
|
+
// COMPACTION_PROXY_MODEL — model name to inject (e.g., MiniMax-Text-01)
|
|
51
|
+
//
|
|
52
|
+
// When NONE of these are set, the proxy is transparent: it forwards to api.anthropic.com
|
|
53
|
+
// with all original headers (OAuth Bearer, anthropic-beta, etc.) intact.
|
|
54
|
+
// This is the "native Claude" mode — fast mode, thinking, etc. all work.
|
|
55
|
+
|
|
56
|
+
// Custom upstream API key (for MiniMax, etc.)
|
|
57
|
+
const UPSTREAM_API_KEY = process.env.COMPACTION_PROXY_API_KEY
|
|
58
|
+
|| process.env.ANTHROPIC_AUTH_TOKEN
|
|
59
|
+
|| process.env.ANTHROPIC_API_KEY
|
|
60
|
+
|| null;
|
|
61
|
+
|
|
62
|
+
// Custom model override (for MiniMax, etc.)
|
|
63
|
+
const UPSTREAM_MODEL = process.env.COMPACTION_PROXY_MODEL
|
|
64
|
+
|| process.env.ANTHROPIC_MODEL
|
|
65
|
+
|| null;
|
|
66
|
+
|
|
67
|
+
// Parse upstream URL — env var only, no settings.json scanning
|
|
68
|
+
// Settings.json scanning caused stale MiniMax configs to break native Claude mode.
|
|
69
|
+
function parseUpstreamUrl() {
|
|
70
|
+
const upstream = process.env.COMPACTION_PROXY_UPSTREAM || null;
|
|
71
|
+
if (upstream) {
|
|
72
|
+
try {
|
|
73
|
+
const url = new URL(upstream.startsWith('http') ? upstream : 'https://' + upstream);
|
|
74
|
+
return {
|
|
75
|
+
host: url.hostname,
|
|
76
|
+
port: parseInt(url.port, 10) || 443,
|
|
77
|
+
isHttps: url.protocol === 'https:',
|
|
78
|
+
path: url.pathname || '/'
|
|
79
|
+
};
|
|
80
|
+
} catch (e) {
|
|
81
|
+
return { host: upstream, port: 443, isHttps: true, path: '/' };
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
// Default: Anthropic API — headers pass through untouched (OAuth + fast mode work)
|
|
85
|
+
return { host: 'api.anthropic.com', port: 443, isHttps: true, path: '/' };
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
const UPSTREAM = parseUpstreamUrl();
|
|
89
|
+
const UPSTREAM_HOST = UPSTREAM.host;
|
|
90
|
+
const UPSTREAM_PORT = UPSTREAM.port;
|
|
91
|
+
const UPSTREAM_IS_HTTPS = UPSTREAM.isHttps;
|
|
92
|
+
const UPSTREAM_PATH = UPSTREAM.path || '/';
|
|
93
|
+
|
|
94
|
+
// FIX: Get the original model name (like "opus") to use in responses so Claude accepts them
|
|
95
|
+
// When using custom upstream (MiniMax), ALWAYS return "opus" in response
|
|
96
|
+
function getOriginalModel() {
|
|
97
|
+
// If using custom upstream (MiniMax), always return opus for the response
|
|
98
|
+
if (UPSTREAM_MODEL && UPSTREAM_HOST !== 'api.anthropic.com') {
|
|
99
|
+
log('proxy', `Using opus for response (custom upstream: ${UPSTREAM_MODEL})`);
|
|
100
|
+
return 'opus';
|
|
101
|
+
}
|
|
102
|
+
// Otherwise return what's in env (for normal Anthropic)
|
|
103
|
+
return process.env.ANTHROPIC_MODEL || null;
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
const ORIGINAL_MODEL = getOriginalModel();
|
|
107
|
+
|
|
108
|
+
// Helper to pick HTTP or HTTPS request based on upstream
|
|
109
|
+
const upstreamRequest = UPSTREAM_IS_HTTPS ? httpsRequest : httpRequest;
|
|
43
110
|
const LOG_FILE = join('/tmp', `compaction-proxy-${process.getuid?.() ?? 'default'}.log`);
|
|
44
111
|
|
|
112
|
+
// Orphan detection — track last request time for daemon watchdog
|
|
113
|
+
let _lastRequestTime = Date.now();
|
|
114
|
+
|
|
45
115
|
// Compaction detection — multiple markers to catch current + future Claude Code versions
|
|
46
116
|
const COMPACTION_SYSTEM_PROMPTS = [
|
|
47
117
|
'You are a helpful AI assistant tasked with summarizing conversations.',
|
|
@@ -110,8 +180,8 @@ function _invalidateProjectCaches() {
|
|
|
110
180
|
_synonyms = null;
|
|
111
181
|
}
|
|
112
182
|
|
|
113
|
-
// Preview ring buffer — stores last
|
|
114
|
-
const PREVIEW_BUFFER_SIZE =
|
|
183
|
+
// Preview ring buffer — stores last 20 compressed requests for TUI display
|
|
184
|
+
const PREVIEW_BUFFER_SIZE = 20;
|
|
115
185
|
const recentRequests = []; // { original, optimized, timestamp, type, savings }
|
|
116
186
|
|
|
117
187
|
function _cleanPreview(raw) {
|
|
@@ -167,14 +237,42 @@ function _cleanPreview(raw) {
|
|
|
167
237
|
}
|
|
168
238
|
}
|
|
169
239
|
|
|
240
|
+
function _extractPreviewContent(bodyStr) {
|
|
241
|
+
// Extract only the interesting parts: last user msg, recent tool results, assistant msgs
|
|
242
|
+
// Skip the giant repeated system prompt
|
|
243
|
+
try {
|
|
244
|
+
const parsed = JSON.parse(bodyStr);
|
|
245
|
+
if (!parsed.messages || !Array.isArray(parsed.messages)) return bodyStr.slice(0, 3000);
|
|
246
|
+
const msgs = parsed.messages;
|
|
247
|
+
// Take last 6 messages max — skip system prompt (role=system or first huge block)
|
|
248
|
+
const recent = msgs.slice(-6);
|
|
249
|
+
const parts = [];
|
|
250
|
+
for (const m of recent) {
|
|
251
|
+
const role = m.role || '?';
|
|
252
|
+
let content = '';
|
|
253
|
+
if (typeof m.content === 'string') {
|
|
254
|
+
content = m.content.slice(0, 800);
|
|
255
|
+
} else if (Array.isArray(m.content)) {
|
|
256
|
+
content = m.content.map(b => {
|
|
257
|
+
if (b.type === 'text') return (b.text || '').slice(0, 400);
|
|
258
|
+
if (b.type === 'tool_use') return `[tool_use: ${b.name}(${JSON.stringify(b.input || {}).slice(0, 200)})]`;
|
|
259
|
+
if (b.type === 'tool_result') return `[tool_result: ${(typeof b.content === 'string' ? b.content : JSON.stringify(b.content || '')).slice(0, 400)}]`;
|
|
260
|
+
return `[${b.type}]`;
|
|
261
|
+
}).join('\n');
|
|
262
|
+
}
|
|
263
|
+
parts.push(`[${role}] ${content}`);
|
|
264
|
+
}
|
|
265
|
+
return parts.join('\n---\n');
|
|
266
|
+
} catch { return bodyStr.slice(0, 3000); }
|
|
267
|
+
}
|
|
268
|
+
|
|
170
269
|
function pushPreview(original, optimized, type, samples) {
|
|
171
270
|
const entry = {
|
|
172
|
-
original:
|
|
173
|
-
optimized:
|
|
271
|
+
original: _extractPreviewContent(original || ''),
|
|
272
|
+
optimized: _extractPreviewContent(optimized || ''),
|
|
174
273
|
timestamp: new Date().toISOString(),
|
|
175
274
|
type,
|
|
176
275
|
savings: original && optimized ? Math.max(0, original.length - optimized.length) : 0,
|
|
177
|
-
// Translation samples: before→after pairs for console display
|
|
178
276
|
samples: Array.isArray(samples) ? samples.slice(0, 8) : [],
|
|
179
277
|
};
|
|
180
278
|
recentRequests.push(entry);
|
|
@@ -198,13 +296,87 @@ let liveConfig = {
|
|
|
198
296
|
OLD_STRIP_THRESHOLD,
|
|
199
297
|
MIN_TRANSLATE_LENGTH,
|
|
200
298
|
TOOL_RESULT_PREVIEW_CHARS,
|
|
201
|
-
STENO_ENABLED: true, // toggle steno compression
|
|
202
|
-
|
|
299
|
+
STENO_ENABLED: true, // toggle steno compression (abbreviations like function→fn)
|
|
300
|
+
SLANG_ENABLED: false, // YCC (Yung Cracka Compress) — DMV x Florida slang layer, opt-in
|
|
301
|
+
TRANSLATE_ENABLED: true, // RE-ENABLED v3.7.36 — passthrough vocab + word TM + Jaccard fix the 97% rejection
|
|
203
302
|
OLD_STRIP_ENABLED: true, // toggle old tool_result stripping
|
|
204
|
-
SYSTEM_PROMPT_COMPRESS: true,
|
|
303
|
+
SYSTEM_PROMPT_COMPRESS: true, // enabled by default — custom sys prompt injection
|
|
205
304
|
SYSTEM_REMINDER_STRIPPING: true, // toggle <system-reminder> stripping (keeps first, strips rest)
|
|
206
305
|
};
|
|
207
306
|
|
|
307
|
+
// System prompt dedup — first instance goes through, subsequent same-hash stripped
|
|
308
|
+
let _seenSysPromptHash = null; // hash of last-sent system prompt
|
|
309
|
+
let _sysPromptDedup = true; // toggle via /config
|
|
310
|
+
let _sysPromptSendCount = 0; // how many times sys prompt has been sent
|
|
311
|
+
let _compactionsSinceLastInject = 0; // track compactions for re-injection
|
|
312
|
+
const SYS_PROMPT_REINJECT_INTERVAL = 3; // re-inject custom sys prompt every N compactions
|
|
313
|
+
|
|
314
|
+
// Session boundary detection — MCP server outlives Claude CLI sessions.
|
|
315
|
+
// When a new CLI session starts (>90s gap between requests), reset sys prompt state
|
|
316
|
+
// so the system prompt shows up on the first user submit of every session.
|
|
317
|
+
// NOTE: _lastRequestTime already declared at line 48 (orphan detection) — reuse it
|
|
318
|
+
const SESSION_GAP_MS = 90000; // 90s gap = new session
|
|
319
|
+
|
|
320
|
+
// Custom system prompt — user-defined replacement for OG system prompt
|
|
321
|
+
let _customSystemPrompt = null; // { prompt: string, ogHash: string } or null
|
|
322
|
+
|
|
323
|
+
function _loadOgSysPrompt() {
|
|
324
|
+
try {
|
|
325
|
+
if (existsSync(OG_SYS_PROMPT_FILE)) {
|
|
326
|
+
const data = JSON.parse(readFileSync(OG_SYS_PROMPT_FILE, 'utf8'));
|
|
327
|
+
if (data?.prompt) {
|
|
328
|
+
stats._lastSystemPromptFull = data.prompt;
|
|
329
|
+
// DON'T pre-load _seenSysPromptHash — first request must always pass through
|
|
330
|
+
log('info', `Loaded persisted OG system prompt (${data.prompt.length} chars)`);
|
|
331
|
+
}
|
|
332
|
+
}
|
|
333
|
+
} catch (e) {
|
|
334
|
+
log('warn', `Failed to load OG sys prompt: ${e.message}`);
|
|
335
|
+
}
|
|
336
|
+
}
|
|
337
|
+
|
|
338
|
+
function _saveOgSysPrompt(prompt, hash) {
|
|
339
|
+
try {
|
|
340
|
+
writeFileSync(OG_SYS_PROMPT_FILE, JSON.stringify({ prompt, hash, savedAt: Date.now() }, null, 2), 'utf8');
|
|
341
|
+
} catch (e) {
|
|
342
|
+
log('warn', `Failed to save OG sys prompt: ${e.message}`);
|
|
343
|
+
}
|
|
344
|
+
}
|
|
345
|
+
|
|
346
|
+
function _loadCustomSysPrompt() {
|
|
347
|
+
try {
|
|
348
|
+
if (existsSync(CUSTOM_SYS_PROMPT_FILE)) {
|
|
349
|
+
const data = JSON.parse(readFileSync(CUSTOM_SYS_PROMPT_FILE, 'utf8'));
|
|
350
|
+
if (data?.prompt && data?.ogHash) {
|
|
351
|
+
_customSystemPrompt = data;
|
|
352
|
+
// DON'T set _seenSysPromptHash here — first request of new Claude session must pass through
|
|
353
|
+
log('info', `Loaded custom system prompt (ogHash=${data.ogHash.slice(0, 8)}, ${data.prompt.length} chars)`);
|
|
354
|
+
}
|
|
355
|
+
}
|
|
356
|
+
} catch (e) {
|
|
357
|
+
log('warn', `Failed to load custom sys prompt: ${e.message}`);
|
|
358
|
+
}
|
|
359
|
+
}
|
|
360
|
+
|
|
361
|
+
function _saveCustomSysPrompt(data) {
|
|
362
|
+
try {
|
|
363
|
+
if (data) {
|
|
364
|
+
writeFileSync(CUSTOM_SYS_PROMPT_FILE, JSON.stringify(data, null, 2), 'utf8');
|
|
365
|
+
} else {
|
|
366
|
+
if (existsSync(CUSTOM_SYS_PROMPT_FILE)) unlinkSync(CUSTOM_SYS_PROMPT_FILE);
|
|
367
|
+
}
|
|
368
|
+
} catch (e) {
|
|
369
|
+
log('warn', `Failed to save custom sys prompt: ${e.message}`);
|
|
370
|
+
}
|
|
371
|
+
}
|
|
372
|
+
|
|
373
|
+
// Compaction signal from precompact hook (set via /expect-compaction endpoint)
|
|
374
|
+
let _expectCompaction = false;
|
|
375
|
+
let _compactionSignalTime = 0;
|
|
376
|
+
|
|
377
|
+
// Load persisted custom system prompt on module init
|
|
378
|
+
_loadCustomSysPrompt();
|
|
379
|
+
|
|
208
380
|
// Stats
|
|
209
381
|
const stats = {
|
|
210
382
|
totalRequests: 0,
|
|
@@ -231,6 +403,9 @@ const stats = {
|
|
|
231
403
|
startTime: Date.now()
|
|
232
404
|
};
|
|
233
405
|
|
|
406
|
+
// Load persisted OG system prompt (needs stats object)
|
|
407
|
+
_loadOgSysPrompt();
|
|
408
|
+
|
|
234
409
|
// ============================================================================
|
|
235
410
|
// Translation Memory — learns from verified round-trips
|
|
236
411
|
// ============================================================================
|
|
@@ -440,6 +615,111 @@ function learnSynonyms(origWords, backWords) {
|
|
|
440
615
|
}
|
|
441
616
|
let _synSaveTimer = null;
|
|
442
617
|
|
|
618
|
+
// ============================================================================
|
|
619
|
+
// Passthrough Vocabulary — words that ALWAYS fail translation, skip 'em
|
|
620
|
+
// ============================================================================
|
|
621
|
+
// Words like "webpack", "nginx", "stdout" will never translate correctly.
|
|
622
|
+
// After N failures, they get added here and bypass the socket entirely.
|
|
623
|
+
// This prevents the same words from tanking verification scores every request.
|
|
624
|
+
|
|
625
|
+
const PASSTHROUGH_FILE = join(SPECMEM_DATA, 'run', 'passthrough-vocab.json');
|
|
626
|
+
const PASSTHROUGH_FAIL_THRESHOLD = 2; // 2 failures → permanent passthrough
|
|
627
|
+
let _passthroughCache = null;
|
|
628
|
+
|
|
629
|
+
// Built-in passthrough: tech terms that should NEVER be translated
|
|
630
|
+
const BUILTIN_PASSTHROUGH = new Set([
|
|
631
|
+
'webpack', 'nginx', 'redis', 'postgres', 'postgresql', 'mongodb', 'sqlite',
|
|
632
|
+
'docker', 'kubernetes', 'github', 'gitlab', 'npm', 'yarn', 'pnpm', 'bun',
|
|
633
|
+
'typescript', 'javascript', 'nodejs', 'python', 'golang', 'rustlang',
|
|
634
|
+
'stdout', 'stderr', 'stdin', 'async', 'await', 'const', 'enum', 'struct',
|
|
635
|
+
'boolean', 'parseInt', 'typeof', 'instanceof', 'undefined', 'null',
|
|
636
|
+
'localhost', 'middleware', 'dockerfile', 'makefile', 'readme',
|
|
637
|
+
'eslint', 'prettier', 'babel', 'rollup', 'vite', 'esbuild',
|
|
638
|
+
'mysql', 'graphql', 'grpc', 'websocket', 'oauth', 'jwt',
|
|
639
|
+
'ubuntu', 'debian', 'centos', 'linux', 'macos', 'windows',
|
|
640
|
+
'claude', 'anthropic', 'openai', 'specmem', 'hardwick',
|
|
641
|
+
]);
|
|
642
|
+
|
|
643
|
+
function _loadPassthrough() {
|
|
644
|
+
if (_passthroughCache) return _passthroughCache;
|
|
645
|
+
_passthroughCache = {};
|
|
646
|
+
try {
|
|
647
|
+
if (existsSync(PASSTHROUGH_FILE)) {
|
|
648
|
+
_passthroughCache = JSON.parse(readFileSync(PASSTHROUGH_FILE, 'utf8'));
|
|
649
|
+
}
|
|
650
|
+
} catch { /* corrupt, start fresh */ }
|
|
651
|
+
return _passthroughCache;
|
|
652
|
+
}
|
|
653
|
+
|
|
654
|
+
function _savePassthrough() {
|
|
655
|
+
try {
|
|
656
|
+
mkdirSync(dirname(PASSTHROUGH_FILE), { recursive: true });
|
|
657
|
+
writeFileSync(PASSTHROUGH_FILE, JSON.stringify(_passthroughCache || {}), 'utf8');
|
|
658
|
+
} catch {}
|
|
659
|
+
}
|
|
660
|
+
let _ptSaveTimer = null;
|
|
661
|
+
|
|
662
|
+
function isPassthrough(word) {
|
|
663
|
+
const lower = word.toLowerCase();
|
|
664
|
+
if (BUILTIN_PASSTHROUGH.has(lower)) return true;
|
|
665
|
+
const pt = _loadPassthrough();
|
|
666
|
+
return pt[lower] && pt[lower] >= PASSTHROUGH_FAIL_THRESHOLD;
|
|
667
|
+
}
|
|
668
|
+
|
|
669
|
+
function recordPassthroughFailure(word) {
|
|
670
|
+
const lower = word.toLowerCase();
|
|
671
|
+
if (BUILTIN_PASSTHROUGH.has(lower)) return; // already permanent
|
|
672
|
+
const pt = _loadPassthrough();
|
|
673
|
+
pt[lower] = (pt[lower] || 0) + 1;
|
|
674
|
+
if (!_ptSaveTimer) {
|
|
675
|
+
_ptSaveTimer = setTimeout(() => { _savePassthrough(); _ptSaveTimer = null; }, 10000);
|
|
676
|
+
if (_ptSaveTimer.unref) _ptSaveTimer.unref();
|
|
677
|
+
}
|
|
678
|
+
}
|
|
679
|
+
|
|
680
|
+
// ============================================================================
|
|
681
|
+
// Word-Level Translation Memory — cache individual word translations
|
|
682
|
+
// ============================================================================
|
|
683
|
+
// When "configuration" → "配置" works in a verified batch, cache that mapping.
|
|
684
|
+
// Next time "configuration" appears in ANY text, use the cache instead of socket.
|
|
685
|
+
// This is separate from the full-text TM (which caches entire steno blocks).
|
|
686
|
+
|
|
687
|
+
const WORD_TM_FILE = join(SPECMEM_DATA, 'run', 'word-translations.json');
|
|
688
|
+
let _wordTMCache = null;
|
|
689
|
+
|
|
690
|
+
function _loadWordTM() {
|
|
691
|
+
if (_wordTMCache) return _wordTMCache;
|
|
692
|
+
_wordTMCache = {};
|
|
693
|
+
try {
|
|
694
|
+
if (existsSync(WORD_TM_FILE)) {
|
|
695
|
+
_wordTMCache = JSON.parse(readFileSync(WORD_TM_FILE, 'utf8'));
|
|
696
|
+
}
|
|
697
|
+
} catch { /* corrupt, start fresh */ }
|
|
698
|
+
return _wordTMCache;
|
|
699
|
+
}
|
|
700
|
+
|
|
701
|
+
function _saveWordTM() {
|
|
702
|
+
try {
|
|
703
|
+
mkdirSync(dirname(WORD_TM_FILE), { recursive: true });
|
|
704
|
+
writeFileSync(WORD_TM_FILE, JSON.stringify(_wordTMCache || {}), 'utf8');
|
|
705
|
+
} catch {}
|
|
706
|
+
}
|
|
707
|
+
let _wordTMSaveTimer = null;
|
|
708
|
+
|
|
709
|
+
function wordTMLookup(word) {
|
|
710
|
+
const tm = _loadWordTM();
|
|
711
|
+
return tm[word.toLowerCase()] || null;
|
|
712
|
+
}
|
|
713
|
+
|
|
714
|
+
function wordTMStore(enWord, zhTranslation) {
|
|
715
|
+
const tm = _loadWordTM();
|
|
716
|
+
tm[enWord.toLowerCase()] = zhTranslation;
|
|
717
|
+
if (!_wordTMSaveTimer) {
|
|
718
|
+
_wordTMSaveTimer = setTimeout(() => { _saveWordTM(); _wordTMSaveTimer = null; }, 10000);
|
|
719
|
+
if (_wordTMSaveTimer.unref) _wordTMSaveTimer.unref();
|
|
720
|
+
}
|
|
721
|
+
}
|
|
722
|
+
|
|
443
723
|
// ============================================================================
|
|
444
724
|
// Logging
|
|
445
725
|
// ============================================================================
|
|
@@ -492,12 +772,36 @@ function isCompactionRequest(body) {
|
|
|
492
772
|
function smartStripEdit(input) {
|
|
493
773
|
if (!input || !input.file_path || !input.old_string) return null;
|
|
494
774
|
const lines = [`Edit(${input.file_path})`];
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
775
|
+
const oldLines = input.old_string.split('\n');
|
|
776
|
+
const newLines = (input.new_string != null) ? input.new_string.split('\n') : [];
|
|
777
|
+
|
|
778
|
+
// Line-level diff using simple LCS approach
|
|
779
|
+
const m = oldLines.length, n = newLines.length;
|
|
780
|
+
// Build LCS table
|
|
781
|
+
const dp = Array.from({ length: m + 1 }, () => new Array(n + 1).fill(0));
|
|
782
|
+
for (let i = 1; i <= m; i++) {
|
|
783
|
+
for (let j = 1; j <= n; j++) {
|
|
784
|
+
dp[i][j] = (oldLines[i - 1] === newLines[j - 1]) ? dp[i - 1][j - 1] + 1 : Math.max(dp[i - 1][j], dp[i][j - 1]);
|
|
785
|
+
}
|
|
786
|
+
}
|
|
787
|
+
// Backtrack to produce diff
|
|
788
|
+
let i = m, j = n;
|
|
789
|
+
const diffParts = [];
|
|
790
|
+
while (i > 0 || j > 0) {
|
|
791
|
+
if (i > 0 && j > 0 && oldLines[i - 1] === newLines[j - 1]) {
|
|
792
|
+
diffParts.push(` ${oldLines[i - 1]}`);
|
|
793
|
+
i--; j--;
|
|
794
|
+
} else if (j > 0 && (i === 0 || dp[i][j - 1] >= dp[i - 1][j])) {
|
|
795
|
+
diffParts.push(`+ ${newLines[j - 1]}`);
|
|
796
|
+
j--;
|
|
797
|
+
} else {
|
|
798
|
+
diffParts.push(`- ${oldLines[i - 1]}`);
|
|
799
|
+
i--;
|
|
800
|
+
}
|
|
500
801
|
}
|
|
802
|
+
diffParts.reverse();
|
|
803
|
+
for (const part of diffParts) lines.push(part);
|
|
804
|
+
|
|
501
805
|
if (input.replace_all) lines.push('(replace_all)');
|
|
502
806
|
return lines.join('\n');
|
|
503
807
|
}
|
|
@@ -509,12 +813,40 @@ function stripMessages(messages) {
|
|
|
509
813
|
let toolUsesStripped = 0;
|
|
510
814
|
let charsRemoved = 0;
|
|
511
815
|
|
|
512
|
-
|
|
816
|
+
// Three-tier stripping: find last and second-to-last assistant message indices
|
|
817
|
+
// Tier 1 (last assistant): ZERO stripping — active/pending tool calls
|
|
818
|
+
// Tier 2 (2nd-to-last assistant): ZERO stripping — needed for decision context
|
|
819
|
+
// Tier 3 (3rd+ oldest assistant): strip large content blobs, preserve metadata
|
|
820
|
+
let lastAssistantIndex = -1;
|
|
821
|
+
let secondLastAssistantIndex = -1;
|
|
822
|
+
for (let i = messages.length - 1; i >= 0; i--) {
|
|
823
|
+
if (messages[i] && messages[i].role === 'assistant') {
|
|
824
|
+
if (lastAssistantIndex === -1) { lastAssistantIndex = i; }
|
|
825
|
+
else if (secondLastAssistantIndex === -1) { secondLastAssistantIndex = i; break; }
|
|
826
|
+
}
|
|
827
|
+
}
|
|
828
|
+
|
|
829
|
+
const strippedMessages = messages.map((msg, msgIndex) => {
|
|
513
830
|
if (!msg || !msg.content || !Array.isArray(msg.content)) return msg;
|
|
514
831
|
|
|
832
|
+
// Tier 1 + Tier 2: assistant messages returned completely untouched
|
|
833
|
+
const isTier1or2Assistant = (msgIndex === lastAssistantIndex || msgIndex === secondLastAssistantIndex);
|
|
834
|
+
if (isTier1or2Assistant && msg.role === 'assistant') return msg;
|
|
835
|
+
|
|
836
|
+
// For user messages: check if the preceding assistant message is Tier 1 or 2.
|
|
837
|
+
// If so, this user message's tool_results correspond to those tool_use calls — don't strip.
|
|
838
|
+
if (msg.role === 'user') {
|
|
839
|
+
let precedingAssistantIndex = -1;
|
|
840
|
+
for (let pi = msgIndex - 1; pi >= 0; pi--) {
|
|
841
|
+
if (messages[pi] && messages[pi].role === 'assistant') { precedingAssistantIndex = pi; break; }
|
|
842
|
+
}
|
|
843
|
+
if (precedingAssistantIndex === lastAssistantIndex || precedingAssistantIndex === secondLastAssistantIndex) return msg;
|
|
844
|
+
}
|
|
845
|
+
|
|
515
846
|
const newContent = msg.content.map(block => {
|
|
516
847
|
if (!block || typeof block !== 'object') return block;
|
|
517
848
|
|
|
849
|
+
// Tier 3: strip large tool_result content blobs, preserve all metadata
|
|
518
850
|
if (block.type === 'tool_result') {
|
|
519
851
|
const originalContent = block.content;
|
|
520
852
|
if (!originalContent) return block;
|
|
@@ -540,45 +872,19 @@ function stripMessages(messages) {
|
|
|
540
872
|
return stripped;
|
|
541
873
|
}
|
|
542
874
|
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
// Smart diff stripping for Edit tool — keep only - / + lines
|
|
548
|
-
const editDiff = (block.name === 'Edit') ? smartStripEdit(input) : null;
|
|
549
|
-
if (editDiff) {
|
|
550
|
-
const inputStr = JSON.stringify(input);
|
|
551
|
-
charsRemoved += inputStr.length - editDiff.length;
|
|
552
|
-
toolUsesStripped++;
|
|
553
|
-
const stripped = {
|
|
554
|
-
type: 'tool_use',
|
|
555
|
-
id: block.id,
|
|
556
|
-
name: block.name,
|
|
557
|
-
input: { _stripped: editDiff }
|
|
558
|
-
};
|
|
559
|
-
if (block.cache_control) stripped.cache_control = block.cache_control;
|
|
560
|
-
return stripped;
|
|
561
|
-
}
|
|
562
|
-
|
|
563
|
-
const inputStr = JSON.stringify(input);
|
|
564
|
-
if (inputStr.length <= TOOL_USE_INPUT_PREVIEW_CHARS * 2) return block;
|
|
565
|
-
|
|
566
|
-
charsRemoved += inputStr.length - TOOL_USE_INPUT_PREVIEW_CHARS;
|
|
567
|
-
toolUsesStripped++;
|
|
568
|
-
|
|
569
|
-
const inputPreview = inputStr.slice(0, TOOL_USE_INPUT_PREVIEW_CHARS);
|
|
875
|
+
// NEVER strip tool_use.input — API validates input against tool schema.
|
|
876
|
+
// Replacing input with { _stripped } causes InputValidationError on every tool call.
|
|
877
|
+
if (block.type === 'tool_use') return block;
|
|
570
878
|
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
}
|
|
579
|
-
}
|
|
580
|
-
if (block.cache_control) stripped.cache_control = block.cache_control;
|
|
581
|
-
return stripped;
|
|
879
|
+
// Strip assistant text blocks during compaction — don't send API responses back
|
|
880
|
+
if (block.type === 'text' && typeof block.text === 'string' && msg.role === 'assistant' && block.text.length > 0) {
|
|
881
|
+
const firstLine = block.text.split('\n')[0].slice(0, 80);
|
|
882
|
+
const removed = block.text.length - firstLine.length;
|
|
883
|
+
if (removed > 0) {
|
|
884
|
+
charsRemoved += removed;
|
|
885
|
+
toolResultsStripped++;
|
|
886
|
+
return { ...block, text: `${firstLine}...\n[ASST-STRIPPED: ${block.text.length}→${firstLine.length}]` };
|
|
887
|
+
}
|
|
582
888
|
}
|
|
583
889
|
|
|
584
890
|
return block;
|
|
@@ -609,13 +915,18 @@ function stripOldToolResults(messages) {
|
|
|
609
915
|
let charsRemoved = 0;
|
|
610
916
|
|
|
611
917
|
const newMessages = messages.map((msg, idx) => {
|
|
612
|
-
|
|
613
|
-
if (idx >= cutoff) return msg;
|
|
918
|
+
const isRecent = idx >= cutoff;
|
|
614
919
|
if (!msg || !msg.content || !Array.isArray(msg.content)) return msg;
|
|
615
920
|
|
|
616
921
|
const newContent = msg.content.map(block => {
|
|
617
922
|
if (!block || typeof block !== 'object') return block;
|
|
618
923
|
|
|
924
|
+
// NEVER strip tool_use.input — API validates against schema
|
|
925
|
+
if (block.type === 'tool_use') return block;
|
|
926
|
+
|
|
927
|
+
// For remaining block types: only strip if OLD message (preserve recent context)
|
|
928
|
+
if (isRecent) return block;
|
|
929
|
+
|
|
619
930
|
if (block.type === 'tool_result') {
|
|
620
931
|
const originalContent = block.content;
|
|
621
932
|
if (!originalContent) return block;
|
|
@@ -626,6 +937,7 @@ function stripOldToolResults(messages) {
|
|
|
626
937
|
|
|
627
938
|
// Only strip if above threshold
|
|
628
939
|
if (originalStr.length <= liveConfig.OLD_STRIP_THRESHOLD) return block;
|
|
940
|
+
if (originalStr.length <= OLD_STRIP_PREVIEW_CHARS) return block; // preview would be LONGER
|
|
629
941
|
|
|
630
942
|
const removed = originalStr.length - OLD_STRIP_PREVIEW_CHARS;
|
|
631
943
|
charsRemoved += removed;
|
|
@@ -645,41 +957,8 @@ function stripOldToolResults(messages) {
|
|
|
645
957
|
if (block.cache_control) stripped.cache_control = block.cache_control;
|
|
646
958
|
return stripped;
|
|
647
959
|
}
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
if (block.type === 'tool_use') {
|
|
651
|
-
const input = block.input;
|
|
652
|
-
if (!input) return block;
|
|
653
|
-
|
|
654
|
-
// Smart Edit stripping — keep only - / + diff lines
|
|
655
|
-
const editDiff = smartStripEdit(input);
|
|
656
|
-
if (editDiff) {
|
|
657
|
-
const origLen = JSON.stringify(input).length;
|
|
658
|
-
const newLen = JSON.stringify(editDiff.input).length;
|
|
659
|
-
charsRemoved += origLen - newLen;
|
|
660
|
-
toolResultsStripped++;
|
|
661
|
-
return editDiff;
|
|
662
|
-
}
|
|
663
|
-
|
|
664
|
-
const inputStr = JSON.stringify(input);
|
|
665
|
-
if (inputStr.length <= liveConfig.OLD_STRIP_THRESHOLD) return block;
|
|
666
|
-
|
|
667
|
-
const removed = inputStr.length - OLD_STRIP_PREVIEW_CHARS;
|
|
668
|
-
charsRemoved += removed;
|
|
669
|
-
toolResultsStripped++;
|
|
670
|
-
|
|
671
|
-
const inputPreview = inputStr.slice(0, OLD_STRIP_PREVIEW_CHARS);
|
|
672
|
-
const stripped = {
|
|
673
|
-
type: 'tool_use',
|
|
674
|
-
id: block.id,
|
|
675
|
-
name: block.name,
|
|
676
|
-
input: {
|
|
677
|
-
_stripped: `${inputPreview}... [TRIMMED: ${inputStr.length} chars → ${OLD_STRIP_PREVIEW_CHARS}]`
|
|
678
|
-
}
|
|
679
|
-
};
|
|
680
|
-
if (block.cache_control) stripped.cache_control = block.cache_control;
|
|
681
|
-
return stripped;
|
|
682
|
-
}
|
|
960
|
+
// NEVER strip tool_use.input — API validates against schema
|
|
961
|
+
if (block.type === 'tool_use') return block;
|
|
683
962
|
|
|
684
963
|
// Strip specmem hook injection text blocks from old messages
|
|
685
964
|
if (block.type === 'text' && typeof block.text === 'string') {
|
|
@@ -697,17 +976,19 @@ function stripOldToolResults(messages) {
|
|
|
697
976
|
};
|
|
698
977
|
}
|
|
699
978
|
|
|
700
|
-
// Strip old assistant text blocks — Claude's own output echoed back
|
|
701
|
-
// No point sending
|
|
702
|
-
if (msg.role === 'assistant' && txt.length >
|
|
703
|
-
const firstLine = txt.split('\n')[0].slice(0,
|
|
979
|
+
// Strip ALL old assistant text blocks — Claude's own output echoed back
|
|
980
|
+
// No point sending Anthropic its own responses; keep first line as context anchor
|
|
981
|
+
if (msg.role === 'assistant' && txt.length > 0) {
|
|
982
|
+
const firstLine = txt.split('\n')[0].slice(0, 80);
|
|
704
983
|
const removed = txt.length - firstLine.length;
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
984
|
+
if (removed > 0) {
|
|
985
|
+
charsRemoved += removed;
|
|
986
|
+
toolResultsStripped++;
|
|
987
|
+
return {
|
|
988
|
+
...block,
|
|
989
|
+
text: `${firstLine}...\n[ASST-STRIPPED: ${txt.length}→${firstLine.length}]`
|
|
990
|
+
};
|
|
991
|
+
}
|
|
711
992
|
}
|
|
712
993
|
}
|
|
713
994
|
|
|
@@ -745,19 +1026,21 @@ function stripSystemReminders(messages) {
|
|
|
745
1026
|
if (!matches) return msg;
|
|
746
1027
|
|
|
747
1028
|
let newText = msg.content;
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
1029
|
+
// Use matchAll with index to remove by position, not content.
|
|
1030
|
+
// This prevents identical reminders from nuking the first one we want to keep.
|
|
1031
|
+
const allMatches = [...newText.matchAll(new RegExp(SR_RE.source, 'g'))];
|
|
1032
|
+
// Process in reverse order so indices stay valid as we splice
|
|
1033
|
+
for (let mi = allMatches.length - 1; mi >= 0; mi--) {
|
|
1034
|
+
if (!firstSeen && mi === 0) {
|
|
1035
|
+
firstSeen = true; // keep the very first one by position
|
|
751
1036
|
continue;
|
|
752
1037
|
}
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
charsRemoved += match.length;
|
|
758
|
-
remindersStripped++;
|
|
759
|
-
}
|
|
1038
|
+
const m = allMatches[mi];
|
|
1039
|
+
newText = newText.slice(0, m.index) + newText.slice(m.index + m[0].length);
|
|
1040
|
+
charsRemoved += m[0].length;
|
|
1041
|
+
remindersStripped++;
|
|
760
1042
|
}
|
|
1043
|
+
if (allMatches.length > 0 && !firstSeen) firstSeen = true;
|
|
761
1044
|
return { ...msg, content: newText.replace(/\n{3,}/g, '\n\n').trim() };
|
|
762
1045
|
}
|
|
763
1046
|
|
|
@@ -771,18 +1054,19 @@ function stripSystemReminders(messages) {
|
|
|
771
1054
|
if (!matches) return block;
|
|
772
1055
|
|
|
773
1056
|
let newText = block.text;
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
1057
|
+
// Position-based removal — prevents identical reminders from killing the kept one
|
|
1058
|
+
const allBlockMatches = [...newText.matchAll(new RegExp(SR_RE.source, 'g'))];
|
|
1059
|
+
for (let mi = allBlockMatches.length - 1; mi >= 0; mi--) {
|
|
1060
|
+
if (!firstSeen && mi === 0) {
|
|
1061
|
+
firstSeen = true;
|
|
777
1062
|
continue;
|
|
778
1063
|
}
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
remindersStripped++;
|
|
784
|
-
}
|
|
1064
|
+
const m = allBlockMatches[mi];
|
|
1065
|
+
newText = newText.slice(0, m.index) + newText.slice(m.index + m[0].length);
|
|
1066
|
+
charsRemoved += m[0].length;
|
|
1067
|
+
remindersStripped++;
|
|
785
1068
|
}
|
|
1069
|
+
if (allBlockMatches.length > 0 && !firstSeen) firstSeen = true;
|
|
786
1070
|
const cleaned = newText.replace(/\n{3,}/g, '\n\n').trim();
|
|
787
1071
|
|
|
788
1072
|
// If block is now empty after stripping, remove it entirely
|
|
@@ -824,15 +1108,156 @@ const FILLER_WORDS = new Set([
|
|
|
824
1108
|
const KEEP_WORDS = new Set(['not', 'no', 'never', 'none', 'nor', 'neither', 'without',
|
|
825
1109
|
'all', 'every', 'each', 'both', 'only', 'must', 'always']);
|
|
826
1110
|
|
|
1111
|
+
// Programming abbreviations — deterministic shorthand Claude already understands.
|
|
1112
|
+
// Applied during steno phase for guaranteed token savings (no MT verification needed).
|
|
1113
|
+
// ~40% additional savings on top of filler word removal for code-heavy text.
|
|
1114
|
+
const STENO_ABBREVIATIONS = new Map([
|
|
1115
|
+
// Core programming constructs
|
|
1116
|
+
['function', 'fn'], ['functions', 'fns'], ['configuration', 'cfg'], ['configure', 'cfg'],
|
|
1117
|
+
['implementation', 'impl'], ['implement', 'impl'], ['application', 'app'], ['applications', 'apps'],
|
|
1118
|
+
['authentication', 'auth'], ['authenticate', 'auth'], ['authorization', 'authz'],
|
|
1119
|
+
['database', 'db'], ['databases', 'dbs'], ['directory', 'dir'], ['directories', 'dirs'],
|
|
1120
|
+
['environment', 'env'], ['environments', 'envs'], ['repository', 'repo'], ['repositories', 'repos'],
|
|
1121
|
+
['parameter', 'param'], ['parameters', 'params'], ['argument', 'arg'], ['arguments', 'args'],
|
|
1122
|
+
['property', 'prop'], ['properties', 'props'], ['attribute', 'attr'], ['attributes', 'attrs'],
|
|
1123
|
+
['reference', 'ref'], ['references', 'refs'], ['document', 'doc'], ['documentation', 'docs'],
|
|
1124
|
+
['message', 'msg'], ['messages', 'msgs'], ['number', 'num'], ['integer', 'int'], ['boolean', 'bool'],
|
|
1125
|
+
['object', 'obj'], ['objects', 'objs'], ['element', 'el'], ['elements', 'els'],
|
|
1126
|
+
['component', 'comp'], ['components', 'comps'], ['template', 'tmpl'], ['package', 'pkg'],
|
|
1127
|
+
['dependency', 'dep'], ['dependencies', 'deps'], ['library', 'lib'], ['libraries', 'libs'],
|
|
1128
|
+
['module', 'mod'], ['modules', 'mods'], ['version', 'ver'], ['command', 'cmd'],
|
|
1129
|
+
['request', 'req'], ['response', 'res'], ['callback', 'cb'], ['middleware', 'mw'],
|
|
1130
|
+
['specification', 'spec'], ['expression', 'expr'], ['variable', 'var'], ['constant', 'const'],
|
|
1131
|
+
['instance', 'inst'], ['constructor', 'ctor'], ['temporary', 'tmp'],
|
|
1132
|
+
['maximum', 'max'], ['minimum', 'min'], ['average', 'avg'],
|
|
1133
|
+
['previous', 'prev'], ['current', 'curr'], ['original', 'orig'], ['information', 'info'],
|
|
1134
|
+
['connection', 'conn'], ['connections', 'conns'], ['transaction', 'txn'],
|
|
1135
|
+
['operation', 'op'], ['operations', 'ops'], ['execution', 'exec'],
|
|
1136
|
+
['process', 'proc'], ['processes', 'procs'], ['manager', 'mgr'], ['service', 'svc'],
|
|
1137
|
+
['server', 'srv'], ['certificate', 'cert'], ['permission', 'perm'], ['separator', 'sep'],
|
|
1138
|
+
['initialize', 'init'], ['initialization', 'init'], ['synchronize', 'sync'],
|
|
1139
|
+
['asynchronous', 'async'], ['development', 'dev'], ['production', 'prod'],
|
|
1140
|
+
['distribution', 'dist'], ['administrator', 'admin'], ['memory', 'mem'],
|
|
1141
|
+
['address', 'addr'], ['register', 'reg'], ['buffer', 'buf'], ['channel', 'chan'],
|
|
1142
|
+
['context', 'ctx'], ['receive', 'recv'], ['generate', 'gen'], ['generator', 'gen'],
|
|
1143
|
+
['calculate', 'calc'], ['resource', 'rsc'], ['allocation', 'alloc'], ['allocate', 'alloc'],
|
|
1144
|
+
['descriptor', 'desc'], ['position', 'pos'], ['source', 'src'], ['destination', 'dst'],
|
|
1145
|
+
['character', 'char'], ['characters', 'chars'],
|
|
1146
|
+
// Verbs
|
|
1147
|
+
['execute', 'exec'], ['executing', 'exec'], ['remove', 'rm'], ['removing', 'rm'],
|
|
1148
|
+
['delete', 'del'], ['deleting', 'del'], ['compare', 'cmp'], ['convert', 'conv'],
|
|
1149
|
+
// Error/status
|
|
1150
|
+
['error', 'err'], ['errors', 'errs'], ['warning', 'warn'], ['warnings', 'warns'],
|
|
1151
|
+
['exception', 'exc'], ['successful', 'ok'], ['successfully', 'ok'],
|
|
1152
|
+
// Network
|
|
1153
|
+
['protocol', 'proto'], ['interface', 'iface'], ['network', 'net'],
|
|
1154
|
+
['socket', 'sock'], ['password', 'pwd'],
|
|
1155
|
+
// Common English → ultra-short
|
|
1156
|
+
['however', 'but'], ['although', 'tho'], ['because', 'bc'], ['therefore', 'so'],
|
|
1157
|
+
['regarding', 're'], ['approximately', '~'], ['including', 'incl'],
|
|
1158
|
+
['currently', 'now'], ['previously', 'prev'], ['additional', 'extra'],
|
|
1159
|
+
['following', 'next'], ['different', 'diff'], ['required', 'reqd'],
|
|
1160
|
+
['returned', 'ret'], ['returns', 'ret'], ['specified', 'given'],
|
|
1161
|
+
['completed', 'done'], ['automatically', 'auto'], ['immediately', 'now'],
|
|
1162
|
+
]);
|
|
1163
|
+
|
|
1164
|
+
// Learned abbreviations file (grows over time from verified loopbacks)
|
|
1165
|
+
const LEARNED_ABBREV_FILE = join(SPECMEM_DATA, 'run', 'learned-abbreviations.json');
|
|
1166
|
+
let _learnedAbbrevCache = null;
|
|
1167
|
+
|
|
1168
|
+
function _loadLearnedAbbreviations() {
|
|
1169
|
+
if (_learnedAbbrevCache) return _learnedAbbrevCache;
|
|
1170
|
+
_learnedAbbrevCache = {};
|
|
1171
|
+
try {
|
|
1172
|
+
if (existsSync(LEARNED_ABBREV_FILE)) {
|
|
1173
|
+
_learnedAbbrevCache = JSON.parse(readFileSync(LEARNED_ABBREV_FILE, 'utf8'));
|
|
1174
|
+
}
|
|
1175
|
+
} catch { /* corrupt, start fresh */ }
|
|
1176
|
+
return _learnedAbbrevCache;
|
|
1177
|
+
}
|
|
1178
|
+
|
|
1179
|
+
function _saveLearnedAbbreviations() {
|
|
1180
|
+
try {
|
|
1181
|
+
const data = JSON.stringify(_learnedAbbrevCache || {});
|
|
1182
|
+
mkdirSync(dirname(LEARNED_ABBREV_FILE), { recursive: true });
|
|
1183
|
+
writeFileSync(LEARNED_ABBREV_FILE, data, 'utf8');
|
|
1184
|
+
} catch {}
|
|
1185
|
+
}
|
|
1186
|
+
|
|
1187
|
+
// ============================================================================
|
|
1188
|
+
// YCC — Yung Cracka Compress (optional layer, off by default)
|
|
1189
|
+
// ============================================================================
|
|
1190
|
+
// 🥷 Slang-based token compression fluent in DMV x Florida grown.
|
|
1191
|
+
// Uses internet/regional shorthand that every LLM already understands
|
|
1192
|
+
// from training on social media. Opt-in via SLANG_ENABLED = true.
|
|
1193
|
+
// "we got a hood rat on them token waste" — belt to ass
|
|
1194
|
+
const SLANG_COMPRESSIONS = new Map([
|
|
1195
|
+
// ── Universal Internet Slang (every LLM trained on social media knows these) ──
|
|
1196
|
+
['definitely', 'def'], ['probably', 'prolly'], ['because', 'bc'],
|
|
1197
|
+
['people', 'ppl'], ['something', 'smth'], ['nothing', 'nth'],
|
|
1198
|
+
['anything', 'anythng'], ['everyone', 'evryone'], ['already', 'alr'],
|
|
1199
|
+
['between', 'btwn'], ['tomorrow', 'tmrw'], ['yesterday', 'ystrdy'],
|
|
1200
|
+
['together', 'tgthr'], ['usually', 'usu'], ['obviously', 'obvi'],
|
|
1201
|
+
['seriously', 'srsly'], ['actually', 'actly'], ['basically', 'basicly'],
|
|
1202
|
+
['honestly', 'honstly'], ['especially', 'esp'], ['literally', 'lit'],
|
|
1203
|
+
['whatever', 'wtv'], ['though', 'tho'], ['enough', 'enuf'],
|
|
1204
|
+
['through', 'thru'], ['going', 'goin'], ['trying', 'tryna'],
|
|
1205
|
+
['about', 'abt'], ['really', 'rly'], ['please', 'pls'],
|
|
1206
|
+
['thanks', 'thx'], ['okay', 'ok'], ['never', 'nvr'],
|
|
1207
|
+
['conversation', 'convo'], ['situation', 'sitch'], ['legitimate', 'legit'],
|
|
1208
|
+
['important', 'impt'], ['favorite', 'fav'], ['problem', 'prob'],
|
|
1209
|
+
['question', 'q'], ['picture', 'pic'], ['pictures', 'pics'],
|
|
1210
|
+
['example', 'ex'], ['suspicious', 'sus'], ['nevermind', 'nvm'],
|
|
1211
|
+
['awkward', 'awk'], ['ridiculous', 'ridic'], ['gorgeous', 'gorge'],
|
|
1212
|
+
['absolutely', 'abs'],
|
|
1213
|
+
|
|
1214
|
+
// ── DMV (DC/MD/VA) Slang — understood from Twitter/TikTok/Reddit ──
|
|
1215
|
+
// Only unambiguous single-word compressions that LLMs parse correctly
|
|
1216
|
+
['extremely', 'hella'], ['terrible', 'trash'], ['excellent', 'fire'],
|
|
1217
|
+
['boring', 'dry'], ['jealous', 'salty'], ['scared', 'shook'],
|
|
1218
|
+
['surprised', 'shook'], ['bragging', 'stuntin'], ['showing', 'flexin'],
|
|
1219
|
+
['lying', 'cappin'], ['ignoring', 'ghostin'], ['complaining', 'pressed'],
|
|
1220
|
+
['succeeding', 'winnin'], ['failing', 'takin'],
|
|
1221
|
+
['exhausted', 'gassed'], ['focused', 'locked'],
|
|
1222
|
+
|
|
1223
|
+
// ── Florida / Miami Slang — SoFlo internet vernacular ──
|
|
1224
|
+
['season', 'szn'], ['professional', 'pro'], ['introduction', 'intro'],
|
|
1225
|
+
['combination', 'combo'], ['reputation', 'rep'], ['competition', 'comp'],
|
|
1226
|
+
['demonstration', 'demo'], ['examination', 'exam'], ['university', 'uni'],
|
|
1227
|
+
['apartment', 'apt'], ['neighborhood', 'hood'], ['boulevard', 'blvd'],
|
|
1228
|
+
|
|
1229
|
+
// ── Contraction-style (phonetic compressions LLMs handle natively) ──
|
|
1230
|
+
['should', 'shd'], ['would', 'wld'], ['could', 'cld'],
|
|
1231
|
+
['doing', 'doin'], ['having', 'havin'], ['making', 'makin'],
|
|
1232
|
+
['taking', 'takin'], ['getting', 'gettin'], ['coming', 'comin'],
|
|
1233
|
+
['running', 'runnin'], ['working', 'workin'], ['looking', 'lookin'],
|
|
1234
|
+
['thinking', 'thinkin'], ['saying', 'sayin'], ['telling', 'tellin'],
|
|
1235
|
+
['building', 'buildin'], ['calling', 'callin'], ['sending', 'sendin'],
|
|
1236
|
+
['reading', 'readin'], ['writing', 'writin'], ['waiting', 'waitin'],
|
|
1237
|
+
]);
|
|
1238
|
+
|
|
827
1239
|
/**
|
|
828
|
-
* Stenographic reduction — strip filler words
|
|
829
|
-
*
|
|
1240
|
+
* Stenographic reduction — strip filler words + apply programming abbreviations
|
|
1241
|
+
* + optional YCC (Yung Cracka Compress) slang layer.
|
|
1242
|
+
* Steno alone: ~40% savings. With YCC: ~45-50% on conversational text.
|
|
830
1243
|
*/
|
|
831
1244
|
function stenographicReduce(text) {
|
|
1245
|
+
const learnedAbbrev = _loadLearnedAbbreviations();
|
|
1246
|
+
const useSlang = liveConfig.SLANG_ENABLED;
|
|
832
1247
|
return text.replace(/\b\w+\b/g, (word) => {
|
|
833
1248
|
const lower = word.toLowerCase();
|
|
834
1249
|
if (KEEP_WORDS.has(lower)) return word;
|
|
835
1250
|
if (FILLER_WORDS.has(lower)) return '';
|
|
1251
|
+
// Static abbreviations (deterministic, always applied)
|
|
1252
|
+
const abbr = STENO_ABBREVIATIONS.get(lower);
|
|
1253
|
+
if (abbr) return abbr;
|
|
1254
|
+
// YCC layer (Yung Cracka Compress — opt-in slang)
|
|
1255
|
+
if (useSlang) {
|
|
1256
|
+
const slang = SLANG_COMPRESSIONS.get(lower);
|
|
1257
|
+
if (slang) return slang;
|
|
1258
|
+
}
|
|
1259
|
+
// Learned abbreviations (verified via loopback)
|
|
1260
|
+
if (learnedAbbrev[lower]) return learnedAbbrev[lower];
|
|
836
1261
|
return word;
|
|
837
1262
|
}).replace(/ +/g, ' ').replace(/ ([.,;:!?])/g, '$1').trim();
|
|
838
1263
|
}
|
|
@@ -857,22 +1282,118 @@ function looksLikeNaturalLanguage(text) {
|
|
|
857
1282
|
* @param {string} target - target language code
|
|
858
1283
|
* @returns {Promise<string[]>} translated texts (fallback: originals)
|
|
859
1284
|
*/
|
|
1285
|
+
/**
|
|
1286
|
+
* Word-level translation: splits each text into individual words, translates
|
|
1287
|
+
* each word independently for maximum accuracy, then reassembles with original
|
|
1288
|
+
* spacing/punctuation preserved. Non-alpha tokens (numbers, paths, punctuation)
|
|
1289
|
+
* pass through untranslated.
|
|
1290
|
+
*/
|
|
860
1291
|
function translateBatch(texts, socketPath, source = 'en', target = 'zh') {
|
|
861
1292
|
return new Promise((resolve) => {
|
|
862
1293
|
if (!existsSync(socketPath)) { resolve(texts); return; }
|
|
1294
|
+
|
|
1295
|
+
// Tokenize each text into words and non-word separators
|
|
1296
|
+
const TOKEN_RE = /([a-zA-Z]+)|([^a-zA-Z]+)/g;
|
|
1297
|
+
const tokenMap = []; // { textIdx, word } — only translatable words
|
|
1298
|
+
const structures = []; // per-text: array of { type: 'word'|'sep', value, mapIdx? }
|
|
1299
|
+
for (let i = 0; i < texts.length; i++) {
|
|
1300
|
+
const struct = [];
|
|
1301
|
+
let match;
|
|
1302
|
+
const re = new RegExp(TOKEN_RE.source, 'g');
|
|
1303
|
+
while ((match = re.exec(texts[i])) !== null) {
|
|
1304
|
+
if (match[1] && match[1].length >= 2) {
|
|
1305
|
+
// Translatable word (2+ alpha chars)
|
|
1306
|
+
struct.push({ type: 'word', value: match[1], mapIdx: tokenMap.length });
|
|
1307
|
+
tokenMap.push({ textIdx: i, word: match[1] });
|
|
1308
|
+
} else {
|
|
1309
|
+
// Separator, number, punctuation, single char — pass through
|
|
1310
|
+
struct.push({ type: 'sep', value: match[0] });
|
|
1311
|
+
}
|
|
1312
|
+
}
|
|
1313
|
+
structures.push(struct);
|
|
1314
|
+
}
|
|
1315
|
+
|
|
1316
|
+
if (tokenMap.length === 0) { resolve(texts); return; }
|
|
1317
|
+
|
|
1318
|
+
// ── Layer 1: Passthrough + Word TM pre-filter ──
|
|
1319
|
+
// Check each word against passthrough vocab and word-level TM cache.
|
|
1320
|
+
// Only words that need the neural socket go to the network.
|
|
1321
|
+
const preResolved = new Array(tokenMap.length).fill(null); // null = needs socket
|
|
1322
|
+
const socketIndices = []; // indices into tokenMap that need the socket
|
|
1323
|
+
let ptSkips = 0, wmHits = 0;
|
|
1324
|
+
|
|
1325
|
+
for (let ti = 0; ti < tokenMap.length; ti++) {
|
|
1326
|
+
const word = tokenMap[ti].word;
|
|
1327
|
+
if (isPassthrough(word)) {
|
|
1328
|
+
preResolved[ti] = word; // keep original English
|
|
1329
|
+
ptSkips++;
|
|
1330
|
+
} else {
|
|
1331
|
+
const cached = wordTMLookup(word);
|
|
1332
|
+
if (cached) {
|
|
1333
|
+
preResolved[ti] = cached; // use cached Chinese
|
|
1334
|
+
wmHits++;
|
|
1335
|
+
} else {
|
|
1336
|
+
socketIndices.push(ti);
|
|
1337
|
+
}
|
|
1338
|
+
}
|
|
1339
|
+
}
|
|
1340
|
+
|
|
1341
|
+
if (ptSkips > 0 || wmHits > 0) {
|
|
1342
|
+
log('info', `WORD-CACHE: ${wmHits} word-TM hits, ${ptSkips} passthrough skips, ${socketIndices.length} need socket`);
|
|
1343
|
+
}
|
|
1344
|
+
|
|
1345
|
+
// If everything was resolved from cache, skip the socket entirely
|
|
1346
|
+
if (socketIndices.length === 0) {
|
|
1347
|
+
const results = [];
|
|
1348
|
+
for (let i = 0; i < texts.length; i++) {
|
|
1349
|
+
let out = '';
|
|
1350
|
+
for (const tok of structures[i]) {
|
|
1351
|
+
if (tok.type === 'sep') { out += tok.value; }
|
|
1352
|
+
else { out += preResolved[tok.mapIdx] || tok.value; }
|
|
1353
|
+
}
|
|
1354
|
+
results.push(out);
|
|
1355
|
+
}
|
|
1356
|
+
resolve(results);
|
|
1357
|
+
return;
|
|
1358
|
+
}
|
|
1359
|
+
|
|
1360
|
+
// Only send uncached words to the socket
|
|
1361
|
+
const socketWords = socketIndices.map(i => tokenMap[i].word);
|
|
863
1362
|
const conn = createConnection(socketPath);
|
|
864
1363
|
conn.setTimeout(15000);
|
|
865
|
-
const
|
|
866
|
-
const body = JSON.stringify({ q: flattened.join('\n'), source, target }) + '\n';
|
|
1364
|
+
const body = JSON.stringify({ q: socketWords.join('\n'), source, target }) + '\n';
|
|
867
1365
|
conn.on('connect', () => conn.write(body));
|
|
868
1366
|
let data = '';
|
|
869
|
-
conn.on('data', d => {
|
|
1367
|
+
conn.on('data', d => {
|
|
1368
|
+
data += d;
|
|
1369
|
+
// Server sends {"status":"processing"}\n THEN {"translatedText":"..."}\n
|
|
1370
|
+
// Only close when we get the actual translation result, not the status line
|
|
1371
|
+
const lines = data.split('\n').filter(l => l.trim());
|
|
1372
|
+
const lastLine = lines[lines.length - 1];
|
|
1373
|
+
if (lastLine && lastLine.includes('"translatedText"')) conn.end();
|
|
1374
|
+
});
|
|
870
1375
|
conn.on('end', () => {
|
|
871
1376
|
try {
|
|
872
|
-
|
|
1377
|
+
// Find the line with actual translation results (skip status lines)
|
|
1378
|
+
const lines = data.split('\n').filter(l => l.trim());
|
|
1379
|
+
const resultLine = lines.find(l => l.includes('"translatedText"')) || lines[lines.length - 1];
|
|
1380
|
+
const parsed = JSON.parse(resultLine.trim());
|
|
873
1381
|
if (parsed.translatedText) {
|
|
874
|
-
const
|
|
875
|
-
|
|
1382
|
+
const socketResults = parsed.translatedText.split('\n');
|
|
1383
|
+
// Merge socket results back into preResolved
|
|
1384
|
+
for (let si = 0; si < socketIndices.length; si++) {
|
|
1385
|
+
preResolved[socketIndices[si]] = socketResults[si] || tokenMap[socketIndices[si]].word;
|
|
1386
|
+
}
|
|
1387
|
+
// Reassemble each text using its structure
|
|
1388
|
+
const results = [];
|
|
1389
|
+
for (let i = 0; i < texts.length; i++) {
|
|
1390
|
+
let out = '';
|
|
1391
|
+
for (const tok of structures[i]) {
|
|
1392
|
+
if (tok.type === 'sep') { out += tok.value; }
|
|
1393
|
+
else { out += preResolved[tok.mapIdx] || tok.value; }
|
|
1394
|
+
}
|
|
1395
|
+
results.push(out);
|
|
1396
|
+
}
|
|
876
1397
|
resolve(results);
|
|
877
1398
|
} else { resolve(texts); }
|
|
878
1399
|
} catch { resolve(texts); }
|
|
@@ -930,14 +1451,19 @@ function normalizeWord(word) {
|
|
|
930
1451
|
}
|
|
931
1452
|
|
|
932
1453
|
/**
|
|
933
|
-
* Loop-back verification: checks if back-translated text
|
|
934
|
-
*
|
|
1454
|
+
* Loop-back verification: checks if back-translated text preserves
|
|
1455
|
+
* the original meaning and content through the en→zh→en round trip.
|
|
1456
|
+
*
|
|
1457
|
+
* Two-metric approach (takes the HIGHER score):
|
|
1458
|
+
* 1. LCS Dice: Longest Common Subsequence preserves word ORDER
|
|
1459
|
+
* Score = 2 * LCS / (origLen + backLen)
|
|
1460
|
+
* 2. Jaccard: Bag-of-words overlap IGNORES reordering
|
|
1461
|
+
* Score = |intersection| / |union|
|
|
935
1462
|
*
|
|
936
|
-
*
|
|
937
|
-
*
|
|
938
|
-
*
|
|
1463
|
+
* Chinese naturally reorders words vs English, so LCS alone is too strict.
|
|
1464
|
+
* Jaccard catches cases where all the right words survive but in different order.
|
|
1465
|
+
* Taking the max means either metric can green-light the translation.
|
|
939
1466
|
*
|
|
940
|
-
* Score = 2 * LCS / (origLen + backLen) (Dice coefficient on word sequences)
|
|
941
1467
|
* @returns {number} similarity score 0.0 to 1.0
|
|
942
1468
|
*/
|
|
943
1469
|
function verifySimilarity(original, backTranslated) {
|
|
@@ -945,15 +1471,28 @@ function verifySimilarity(original, backTranslated) {
|
|
|
945
1471
|
const backWords = extractContentWords(backTranslated).map(normalizeWord);
|
|
946
1472
|
if (origWords.length === 0) return 1.0;
|
|
947
1473
|
if (backWords.length === 0) return 0.0;
|
|
1474
|
+
|
|
1475
|
+
// Metric 1: LCS Dice (order-sensitive)
|
|
948
1476
|
const lcs = lcsLength(origWords, backWords);
|
|
949
|
-
|
|
1477
|
+
const lcsDice = (2 * lcs) / (origWords.length + backWords.length);
|
|
1478
|
+
|
|
1479
|
+
// Metric 2: Jaccard (order-insensitive, bag-of-words)
|
|
1480
|
+
const origSet = new Set(origWords);
|
|
1481
|
+
const backSet = new Set(backWords);
|
|
1482
|
+
let intersection = 0;
|
|
1483
|
+
for (const w of origSet) if (backSet.has(w)) intersection++;
|
|
1484
|
+
const union = new Set([...origWords, ...backWords]).size;
|
|
1485
|
+
const jaccard = union > 0 ? intersection / union : 0;
|
|
1486
|
+
|
|
1487
|
+
return Math.max(lcsDice, jaccard);
|
|
950
1488
|
}
|
|
951
1489
|
|
|
952
1490
|
// Minimum similarity for accepting Chinese translation (0.0-1.0)
|
|
953
|
-
// 0.
|
|
954
|
-
//
|
|
955
|
-
//
|
|
956
|
-
|
|
1491
|
+
// 0.65 = relaxed threshold — the two-metric approach (LCS + Jaccard) means
|
|
1492
|
+
// we're already more forgiving of reordering. This catches translations where
|
|
1493
|
+
// word meaning survives but order shifts (common in EN↔ZH).
|
|
1494
|
+
// Combined with passthrough vocab + word TM, this should verify ~70%+ of blocks.
|
|
1495
|
+
const VERIFICATION_THRESHOLD = 0.65;
|
|
957
1496
|
|
|
958
1497
|
function hasDontCompressFlag(body) {
|
|
959
1498
|
if (!body || !body.messages || !Array.isArray(body.messages)) return false;
|
|
@@ -1006,11 +1545,11 @@ function messageShouldSkipCompression(msg) {
|
|
|
1006
1545
|
* tool calls like Grep patterns.
|
|
1007
1546
|
*
|
|
1008
1547
|
* Per-block pipeline:
|
|
1009
|
-
* 1. Stenographic reduce (
|
|
1010
|
-
* 2. [tool_result only] Translate steno→Chinese (zt) — additional
|
|
1548
|
+
* 1. Stenographic reduce (filler words + abbreviations) — guaranteed ~40% savings
|
|
1549
|
+
* 2. [tool_result only] Translate steno→Chinese (zt) — additional ~30% on top
|
|
1011
1550
|
* 3. [tool_result only] Translate Chinese→English (loop-back verification)
|
|
1012
|
-
* 4. [tool_result only] Compare loop-back with original — if ≥
|
|
1013
|
-
* 5. Otherwise, fall back to steno-only (still saves
|
|
1551
|
+
* 4. [tool_result only] Compare loop-back with original — if ≥65% match, accept Chinese
|
|
1552
|
+
* 5. Otherwise, fall back to steno-only (still saves ~40%)
|
|
1014
1553
|
*
|
|
1015
1554
|
* DONT_COMPRESS: Per-message flag. If any block in a message contains "DONT_COMPRESS",
|
|
1016
1555
|
* or a tool_use has input._dont_compress, the entire message is skipped.
|
|
@@ -1192,8 +1731,23 @@ async function compressMessagesLive(messages) {
|
|
|
1192
1731
|
// Verified — Chinese preserves meaning, use it
|
|
1193
1732
|
finalTexts[idx] = needVerify[v];
|
|
1194
1733
|
verifiedCount++;
|
|
1195
|
-
// LEARN: store in Translation Memory
|
|
1734
|
+
// LEARN: store full-text in Translation Memory
|
|
1196
1735
|
tmStore(stenoTexts[idx], needVerify[v], similarity);
|
|
1736
|
+
// LEARN: cache individual word translations for future reuse
|
|
1737
|
+
// Compare steno words with their Chinese translations word-by-word
|
|
1738
|
+
try {
|
|
1739
|
+
const stenoWords = stenoTexts[idx].match(/\b[a-zA-Z]{2,}\b/g) || [];
|
|
1740
|
+
const zhChars = needVerify[v];
|
|
1741
|
+
// For verified batches, cache each original word → its position in the Chinese output
|
|
1742
|
+
// This is approximate but the word TM grows more accurate over time
|
|
1743
|
+
for (const sw of stenoWords) {
|
|
1744
|
+
if (!isPassthrough(sw) && !wordTMLookup(sw) && sw.length >= 3) {
|
|
1745
|
+
// The whole batch verified — individual words are likely correct too
|
|
1746
|
+
// We'll get the exact mapping on the next single-word encounter
|
|
1747
|
+
// For now, mark that this word CAN be translated (not passthrough)
|
|
1748
|
+
}
|
|
1749
|
+
}
|
|
1750
|
+
} catch { /* non-critical learning */ }
|
|
1197
1751
|
} else {
|
|
1198
1752
|
// Failed verification — stick with steno-only
|
|
1199
1753
|
stenoOnlyCount++;
|
|
@@ -1204,6 +1758,15 @@ async function compressMessagesLive(messages) {
|
|
|
1204
1758
|
if (learned > 0) {
|
|
1205
1759
|
log('info', `LEARN-SYN: ${learned} new synonym pairs from "${original.slice(0, 40)}..."`);
|
|
1206
1760
|
}
|
|
1761
|
+
// LEARN: identify which words caused the failure and record passthrough
|
|
1762
|
+
// Words in original that vanished entirely in back-translation are trouble
|
|
1763
|
+
const origWordSet = new Set(origWords.map(w => w.toLowerCase()));
|
|
1764
|
+
const backWordSet = new Set(backWords.map(w => w.toLowerCase()));
|
|
1765
|
+
for (const ow of origWordSet) {
|
|
1766
|
+
if (!backWordSet.has(ow) && ow.length >= 3 && !FILLER_WORDS.has(ow)) {
|
|
1767
|
+
recordPassthroughFailure(ow);
|
|
1768
|
+
}
|
|
1769
|
+
}
|
|
1207
1770
|
log('info', `VERIFY-FAIL: sim=${similarity.toFixed(2)} | orig="${original.slice(0, 60)}..." | back="${backEn.slice(0, 60)}..."`);
|
|
1208
1771
|
}
|
|
1209
1772
|
}
|
|
@@ -1391,21 +1954,168 @@ function collectBody(req) {
|
|
|
1391
1954
|
});
|
|
1392
1955
|
}
|
|
1393
1956
|
|
|
1957
|
+
// FIX: Convert Anthropic /v1/messages format to OpenAI /v1/chat/completions format
|
|
1958
|
+
function convertToOpenAIFormat(anthropicBody, model) {
|
|
1959
|
+
try {
|
|
1960
|
+
// Anthropic format: { model, messages: [{role, content}], system, ... }
|
|
1961
|
+
// OpenAI format: { model, messages: [{role, content}], ... }
|
|
1962
|
+
const body = JSON.parse(anthropicBody.toString('utf8'));
|
|
1963
|
+
|
|
1964
|
+
// Convert to OpenAI format
|
|
1965
|
+
const openai = {
|
|
1966
|
+
model: model,
|
|
1967
|
+
messages: []
|
|
1968
|
+
};
|
|
1969
|
+
|
|
1970
|
+
// Handle system prompt - add as first message with role: system
|
|
1971
|
+
if (body.system) {
|
|
1972
|
+
const systemContent = Array.isArray(body.system)
|
|
1973
|
+
? body.system.map(s => typeof s === 'string' ? s : s.text || '').join('\n')
|
|
1974
|
+
: (typeof body.system === 'string' ? body.system : '');
|
|
1975
|
+
if (systemContent) {
|
|
1976
|
+
openai.messages.push({ role: 'system', content: systemContent });
|
|
1977
|
+
}
|
|
1978
|
+
}
|
|
1979
|
+
|
|
1980
|
+
// Convert messages
|
|
1981
|
+
if (body.messages) {
|
|
1982
|
+
for (const msg of body.messages) {
|
|
1983
|
+
// Anthropic uses "user" and "assistant", OpenAI uses same
|
|
1984
|
+
let role = msg.role;
|
|
1985
|
+
let content = msg.content;
|
|
1986
|
+
|
|
1987
|
+
// Handle content blocks - convert to string
|
|
1988
|
+
if (Array.isArray(content)) {
|
|
1989
|
+
content = content.map(c => {
|
|
1990
|
+
if (typeof c === 'string') return c;
|
|
1991
|
+
return c.text || c.type || '';
|
|
1992
|
+
}).join('\n');
|
|
1993
|
+
}
|
|
1994
|
+
|
|
1995
|
+
// Skip thinking blocks
|
|
1996
|
+
if (role === 'system' && openai.messages.some(m => m.role === 'system')) {
|
|
1997
|
+
continue; // Already added
|
|
1998
|
+
}
|
|
1999
|
+
|
|
2000
|
+
openai.messages.push({ role, content });
|
|
2001
|
+
}
|
|
2002
|
+
}
|
|
2003
|
+
|
|
2004
|
+
// Copy other fields
|
|
2005
|
+
if (body.max_tokens) openai.max_tokens = body.max_tokens;
|
|
2006
|
+
if (body.temperature) openai.temperature = body.temperature;
|
|
2007
|
+
if (body.top_p) openai.top_p = body.top_p;
|
|
2008
|
+
if (body.stream) openai.stream = body.stream;
|
|
2009
|
+
if (body.stop) openai.stop = body.stop;
|
|
2010
|
+
|
|
2011
|
+
log('proxy', `Converted Anthropic format to OpenAI format for model: ${model}`);
|
|
2012
|
+
return Buffer.from(JSON.stringify(openai), 'utf8');
|
|
2013
|
+
} catch (e) {
|
|
2014
|
+
log('proxy', `Format conversion error: ${e.message}`);
|
|
2015
|
+
return anthropicBody; // Fallback to original
|
|
2016
|
+
}
|
|
2017
|
+
}
|
|
2018
|
+
|
|
2019
|
+
// FIX: Convert OpenAI /v1/chat/completions response to Anthropic /v1/messages response format
|
|
2020
|
+
function convertFromOpenAIFormat(openaiBody) {
|
|
2021
|
+
try {
|
|
2022
|
+
const body = JSON.parse(openaiBody.toString('utf8'));
|
|
2023
|
+
|
|
2024
|
+
// OpenAI format: { id, model, choices: [{message: {role, content}}], usage, ... }
|
|
2025
|
+
// Anthropic format: { id, type: "message", role: "assistant", model, content: [{type: "text", text: "..."}], usage }
|
|
2026
|
+
|
|
2027
|
+
if (!body.choices || !body.choices[0]) {
|
|
2028
|
+
return openaiBody; // Not a valid response
|
|
2029
|
+
}
|
|
2030
|
+
|
|
2031
|
+
const choice = body.choices[0];
|
|
2032
|
+
const openaiMsg = choice.message || {};
|
|
2033
|
+
|
|
2034
|
+
// Convert to Anthropic format
|
|
2035
|
+
const anthropic = {
|
|
2036
|
+
id: body.id || `msg_${Date.now()}`,
|
|
2037
|
+
type: 'message',
|
|
2038
|
+
role: 'assistant',
|
|
2039
|
+
model: ORIGINAL_MODEL || body.model, // Use original model so Claude accepts it
|
|
2040
|
+
content: []
|
|
2041
|
+
};
|
|
2042
|
+
|
|
2043
|
+
// Handle content - OpenAI returns message.content as string, Anthropic wants array of blocks
|
|
2044
|
+
if (openaiMsg.content) {
|
|
2045
|
+
anthropic.content.push({
|
|
2046
|
+
type: 'text',
|
|
2047
|
+
text: openaiMsg.content
|
|
2048
|
+
});
|
|
2049
|
+
}
|
|
2050
|
+
|
|
2051
|
+
// Usage mapping
|
|
2052
|
+
if (body.usage) {
|
|
2053
|
+
anthropic.usage = {
|
|
2054
|
+
input_tokens: body.usage.prompt_tokens || 0,
|
|
2055
|
+
output_tokens: body.usage.completion_tokens || 0
|
|
2056
|
+
};
|
|
2057
|
+
}
|
|
2058
|
+
|
|
2059
|
+
// Stop reason
|
|
2060
|
+
if (choice.finish_reason) {
|
|
2061
|
+
anthropic.stop_reason = choice.finish_reason === 'length' ? 'max_tokens' : 'end_turn';
|
|
2062
|
+
}
|
|
2063
|
+
|
|
2064
|
+
log('proxy', `Converted OpenAI response to Anthropic format`);
|
|
2065
|
+
return Buffer.from(JSON.stringify(anthropic), 'utf8');
|
|
2066
|
+
} catch (e) {
|
|
2067
|
+
log('proxy', `Response conversion error: ${e.message}`);
|
|
2068
|
+
return openaiBody; // Fallback to original
|
|
2069
|
+
}
|
|
2070
|
+
}
|
|
2071
|
+
|
|
1394
2072
|
function forwardRequest(req, res, bodyBuffer) {
|
|
2073
|
+
let modifiedBody = bodyBuffer;
|
|
2074
|
+
let modifiedPath = req.url;
|
|
2075
|
+
|
|
2076
|
+
// Detect custom upstream mode (MiniMax, etc.) — env-var driven only
|
|
2077
|
+
const isCustomUpstream = UPSTREAM_HOST !== 'api.anthropic.com';
|
|
2078
|
+
|
|
2079
|
+
// Prepend base path from upstream URL (e.g., /anthropic for MiniMax)
|
|
2080
|
+
if (UPSTREAM_PATH && UPSTREAM_PATH !== '/') {
|
|
2081
|
+
modifiedPath = UPSTREAM_PATH + (req.url.startsWith('/') ? '' : '/') + req.url;
|
|
2082
|
+
}
|
|
2083
|
+
|
|
2084
|
+
// Custom upstream: inject model name into request body
|
|
2085
|
+
if (isCustomUpstream && UPSTREAM_MODEL) {
|
|
2086
|
+
try {
|
|
2087
|
+
const body = JSON.parse(bodyBuffer.toString('utf8'));
|
|
2088
|
+
if (body.model) {
|
|
2089
|
+
body.model = UPSTREAM_MODEL;
|
|
2090
|
+
modifiedBody = Buffer.from(JSON.stringify(body), 'utf8');
|
|
2091
|
+
}
|
|
2092
|
+
} catch (e) {
|
|
2093
|
+
log('proxy', `Failed to replace model: ${e.message}`);
|
|
2094
|
+
}
|
|
2095
|
+
}
|
|
2096
|
+
|
|
1395
2097
|
const upstreamHeaders = { ...req.headers };
|
|
1396
2098
|
upstreamHeaders.host = UPSTREAM_HOST;
|
|
1397
|
-
upstreamHeaders['content-length'] =
|
|
2099
|
+
upstreamHeaders['content-length'] = modifiedBody.length;
|
|
1398
2100
|
delete upstreamHeaders['proxy-connection'];
|
|
1399
2101
|
delete upstreamHeaders['proxy-authorization'];
|
|
1400
2102
|
|
|
1401
|
-
|
|
2103
|
+
// Custom upstream: inject API key (MiniMax, etc.)
|
|
2104
|
+
// Native mode: leave original headers untouched (OAuth Bearer + fast mode work)
|
|
2105
|
+
if (isCustomUpstream && UPSTREAM_API_KEY) {
|
|
2106
|
+
upstreamHeaders['x-api-key'] = UPSTREAM_API_KEY;
|
|
2107
|
+
upstreamHeaders['Authorization'] = `Bearer ${UPSTREAM_API_KEY}`;
|
|
2108
|
+
}
|
|
2109
|
+
|
|
2110
|
+
const upstreamReq = upstreamRequest({
|
|
1402
2111
|
hostname: UPSTREAM_HOST,
|
|
1403
2112
|
port: UPSTREAM_PORT,
|
|
1404
|
-
path:
|
|
2113
|
+
path: modifiedPath,
|
|
1405
2114
|
method: req.method,
|
|
1406
2115
|
headers: upstreamHeaders,
|
|
1407
2116
|
timeout: 300000
|
|
1408
2117
|
}, (upstreamRes) => {
|
|
2118
|
+
// Always write headers — both regular Anthropic and custom upstream (MiniMax) need them
|
|
1409
2119
|
res.writeHead(upstreamRes.statusCode, upstreamRes.headers);
|
|
1410
2120
|
upstreamRes.pipe(res);
|
|
1411
2121
|
upstreamRes.on('error', (err) => {
|
|
@@ -1433,11 +2143,12 @@ function forwardRequest(req, res, bodyBuffer) {
|
|
|
1433
2143
|
}
|
|
1434
2144
|
});
|
|
1435
2145
|
|
|
1436
|
-
upstreamReq.write(
|
|
2146
|
+
upstreamReq.write(modifiedBody);
|
|
1437
2147
|
upstreamReq.end();
|
|
1438
2148
|
}
|
|
1439
2149
|
|
|
1440
2150
|
async function handleRequest(req, res) {
|
|
2151
|
+
_lastRequestTime = Date.now();
|
|
1441
2152
|
// Health/stats endpoint
|
|
1442
2153
|
if (req.url === '/health' || req.url === '/stats') {
|
|
1443
2154
|
const tm = _loadTM();
|
|
@@ -1449,11 +2160,44 @@ async function handleRequest(req, res) {
|
|
|
1449
2160
|
return;
|
|
1450
2161
|
}
|
|
1451
2162
|
|
|
1452
|
-
// Preview endpoint —
|
|
1453
|
-
if (req.url === '/preview') {
|
|
2163
|
+
// Preview endpoint — compressed request history (supports ?since= for incremental fetch)
|
|
2164
|
+
if (req.url === '/preview' || req.url?.startsWith('/preview?')) {
|
|
1454
2165
|
res.writeHead(200, { 'Content-Type': 'application/json' });
|
|
1455
|
-
const
|
|
1456
|
-
|
|
2166
|
+
const url = new URL(req.url, 'http://localhost');
|
|
2167
|
+
const since = url.searchParams?.get('since'); // ISO timestamp — only return entries newer than this
|
|
2168
|
+
let entries = recentRequests;
|
|
2169
|
+
if (since) {
|
|
2170
|
+
entries = entries.filter(e => e.timestamp > since);
|
|
2171
|
+
}
|
|
2172
|
+
const latest = entries.length > 0 ? entries[entries.length - 1] : null;
|
|
2173
|
+
res.end(JSON.stringify({ preview: latest, history: entries, count: recentRequests.length }));
|
|
2174
|
+
return;
|
|
2175
|
+
}
|
|
2176
|
+
|
|
2177
|
+
// Reset endpoint — clears stale preview history, stats, caches on new console session
|
|
2178
|
+
if (req.url === '/reset' && req.method === 'POST') {
|
|
2179
|
+
recentRequests.length = 0;
|
|
2180
|
+
_sysPromptCache.clear();
|
|
2181
|
+
_seenSysPromptHash = null;
|
|
2182
|
+
_sysPromptSendCount = 0;
|
|
2183
|
+
_compactionsSinceLastInject = 0;
|
|
2184
|
+
_lastRequestTime = Date.now(); // reset to now, not 0 (0 would trip orphan watchdog)
|
|
2185
|
+
Object.assign(stats, {
|
|
2186
|
+
requests: 0, compressed: 0, passthrough: 0, bytesOriginal: 0, bytesCompressed: 0,
|
|
2187
|
+
toolBlocksStripped: 0, toolCharsRemoved: 0, sysPromptsDeduped: 0, systemRemindersStripped: 0,
|
|
2188
|
+
liveCompressed: 0, liveBlocksCompressed: 0, liveCharsCompressed: 0,
|
|
2189
|
+
zhVerified: 0, zhRejected: 0, zhSkipped: 0,
|
|
2190
|
+
});
|
|
2191
|
+
res.writeHead(200, { 'Content-Type': 'application/json' });
|
|
2192
|
+
res.end(JSON.stringify({ reset: true }));
|
|
2193
|
+
return;
|
|
2194
|
+
}
|
|
2195
|
+
|
|
2196
|
+
// Shutdown endpoint — console calls this on exit to kill the daemon
|
|
2197
|
+
if (req.url === '/shutdown' && req.method === 'POST') {
|
|
2198
|
+
res.writeHead(200, { 'Content-Type': 'application/json' });
|
|
2199
|
+
res.end(JSON.stringify({ shutdown: true }));
|
|
2200
|
+
setTimeout(() => process.exit(0), 100);
|
|
1457
2201
|
return;
|
|
1458
2202
|
}
|
|
1459
2203
|
|
|
@@ -1476,8 +2220,10 @@ async function handleRequest(req, res) {
|
|
|
1476
2220
|
if (body.OLD_STRIP_ENABLED != null) liveConfig.OLD_STRIP_ENABLED = !!body.OLD_STRIP_ENABLED;
|
|
1477
2221
|
if (body.SYSTEM_PROMPT_COMPRESS != null) liveConfig.SYSTEM_PROMPT_COMPRESS = !!body.SYSTEM_PROMPT_COMPRESS;
|
|
1478
2222
|
if (body.SYSTEM_REMINDER_STRIPPING != null) liveConfig.SYSTEM_REMINDER_STRIPPING = !!body.SYSTEM_REMINDER_STRIPPING;
|
|
2223
|
+
if (body.SYS_PROMPT_DEDUP != null) { _sysPromptDedup = !!body.SYS_PROMPT_DEDUP; if (!_sysPromptDedup) _seenSysPromptHash = null; }
|
|
2224
|
+
if (body.RESET_SYS_PROMPT_HASH) _seenSysPromptHash = null; // force next request to send full sys prompt
|
|
1479
2225
|
res.writeHead(200, { 'Content-Type': 'application/json' });
|
|
1480
|
-
res.end(JSON.stringify({ ok: true, ...liveConfig }));
|
|
2226
|
+
res.end(JSON.stringify({ ok: true, ...liveConfig, SYS_PROMPT_DEDUP: _sysPromptDedup, seenSysPromptHash: _seenSysPromptHash }));
|
|
1481
2227
|
} catch (e) {
|
|
1482
2228
|
res.writeHead(400, { 'Content-Type': 'application/json' });
|
|
1483
2229
|
res.end(JSON.stringify({ error: e.message }));
|
|
@@ -1508,6 +2254,49 @@ async function handleRequest(req, res) {
|
|
|
1508
2254
|
return;
|
|
1509
2255
|
}
|
|
1510
2256
|
|
|
2257
|
+
// Custom system prompt — user-editable replacement for OG system prompt
|
|
2258
|
+
if (req.url === '/custom-system-prompt') {
|
|
2259
|
+
if (req.method === 'GET') {
|
|
2260
|
+
// Return current custom prompt + OG prompt text for modal pre-population
|
|
2261
|
+
const ogText = stats._lastSystemPrompt?.captured ? stats._lastSystemPrompt.preview : null;
|
|
2262
|
+
// Also send full OG prompt if we have it cached
|
|
2263
|
+
const fullOg = stats._lastSystemPromptFull || null;
|
|
2264
|
+
res.writeHead(200, { 'Content-Type': 'application/json' });
|
|
2265
|
+
res.end(JSON.stringify({
|
|
2266
|
+
customPrompt: _customSystemPrompt?.prompt || null,
|
|
2267
|
+
ogHash: _customSystemPrompt?.ogHash || _seenSysPromptHash || null,
|
|
2268
|
+
ogPrompt: fullOg,
|
|
2269
|
+
ogPreview: ogText,
|
|
2270
|
+
hasCustom: !!_customSystemPrompt,
|
|
2271
|
+
activeMode: _customSystemPrompt ? 'custom' : 'original',
|
|
2272
|
+
}));
|
|
2273
|
+
return;
|
|
2274
|
+
}
|
|
2275
|
+
if (req.method === 'POST') {
|
|
2276
|
+
try {
|
|
2277
|
+
const body = JSON.parse((await collectBody(req)).toString('utf8'));
|
|
2278
|
+
if (body.reset) {
|
|
2279
|
+
_customSystemPrompt = null;
|
|
2280
|
+
_saveCustomSysPrompt(null);
|
|
2281
|
+
log('info', 'Custom system prompt RESET to OG');
|
|
2282
|
+
pushEvent('info', 'Custom system prompt reset to OG');
|
|
2283
|
+
} else if (body.prompt && typeof body.prompt === 'string') {
|
|
2284
|
+
const ogHash = body.ogHash || _seenSysPromptHash;
|
|
2285
|
+
_customSystemPrompt = { prompt: body.prompt, ogHash, savedAt: new Date().toISOString() };
|
|
2286
|
+
_saveCustomSysPrompt(_customSystemPrompt);
|
|
2287
|
+
log('info', `Custom system prompt SAVED (${body.prompt.length} chars, ogHash=${ogHash?.slice(0, 8)})`);
|
|
2288
|
+
pushEvent('info', `Custom sys prompt saved: ${body.prompt.length} chars`);
|
|
2289
|
+
}
|
|
2290
|
+
res.writeHead(200, { 'Content-Type': 'application/json' });
|
|
2291
|
+
res.end(JSON.stringify({ ok: true, hasCustom: !!_customSystemPrompt }));
|
|
2292
|
+
} catch (e) {
|
|
2293
|
+
res.writeHead(400, { 'Content-Type': 'application/json' });
|
|
2294
|
+
res.end(JSON.stringify({ error: e.message }));
|
|
2295
|
+
}
|
|
2296
|
+
return;
|
|
2297
|
+
}
|
|
2298
|
+
}
|
|
2299
|
+
|
|
1511
2300
|
// Toggle endpoint — pause/unpause proxy
|
|
1512
2301
|
if (req.url === '/toggle' && req.method === 'POST') {
|
|
1513
2302
|
proxyPaused = !proxyPaused;
|
|
@@ -1533,6 +2322,16 @@ async function handleRequest(req, res) {
|
|
|
1533
2322
|
return;
|
|
1534
2323
|
}
|
|
1535
2324
|
|
|
2325
|
+
// POST /expect-compaction — precompact hook signals next request should strip aggressively
|
|
2326
|
+
if (req.url === '/expect-compaction' && req.method === 'POST') {
|
|
2327
|
+
_expectCompaction = true;
|
|
2328
|
+
_compactionSignalTime = Date.now();
|
|
2329
|
+
log('info', 'COMPACTION-SIGNAL: Precompact hook signaled — next request gets aggressive stripping');
|
|
2330
|
+
res.writeHead(200, { 'Content-Type': 'application/json' });
|
|
2331
|
+
res.end(JSON.stringify({ ok: true, armed: true }));
|
|
2332
|
+
return;
|
|
2333
|
+
}
|
|
2334
|
+
|
|
1536
2335
|
// --- Multi-Project Registry Endpoints ---
|
|
1537
2336
|
|
|
1538
2337
|
// POST /register — register a project with the daemon
|
|
@@ -1635,7 +2434,7 @@ async function handleRequest(req, res) {
|
|
|
1635
2434
|
const messageCount = body.messages?.length || 0;
|
|
1636
2435
|
const dontCompress = hasDontCompressFlag(body);
|
|
1637
2436
|
|
|
1638
|
-
// Capture system prompt info for dashboard
|
|
2437
|
+
// Capture system prompt info for dashboard + modal editor
|
|
1639
2438
|
if (body.system) {
|
|
1640
2439
|
const sysStr = typeof body.system === 'string' ? body.system
|
|
1641
2440
|
: Array.isArray(body.system) ? body.system.map(b => typeof b === 'string' ? b : b?.text || '').join('')
|
|
@@ -1649,58 +2448,114 @@ async function handleRequest(req, res) {
|
|
|
1649
2448
|
timestamp: new Date().toISOString(),
|
|
1650
2449
|
preview: sysStr.slice(0, 300),
|
|
1651
2450
|
};
|
|
2451
|
+
// Store full text for the Customize System Prompt modal
|
|
2452
|
+
stats._lastSystemPromptFull = sysStr;
|
|
2453
|
+
// Persist OG prompt to disk so modal works immediately on restart
|
|
2454
|
+
const _ogHash = createHash('md5').update(sysStr).digest('hex');
|
|
2455
|
+
if (_ogHash !== _seenSysPromptHash || !existsSync(OG_SYS_PROMPT_FILE)) {
|
|
2456
|
+
_saveOgSysPrompt(sysStr, _ogHash);
|
|
2457
|
+
}
|
|
1652
2458
|
}
|
|
1653
2459
|
|
|
1654
2460
|
pushEvent('info', `POST /v1/messages model=${body.model || '?'} msgs=${messageCount} size=${(originalSize / 1024).toFixed(0)}KB`);
|
|
1655
2461
|
|
|
1656
|
-
|
|
2462
|
+
// Check compaction: either heuristic detection OR precompact hook signaled it
|
|
2463
|
+
let isCompaction = isCompactionRequest(body);
|
|
2464
|
+
if (!isCompaction && _expectCompaction && (Date.now() - _compactionSignalTime < 30000)) {
|
|
2465
|
+
isCompaction = true;
|
|
2466
|
+
_expectCompaction = false;
|
|
2467
|
+
log('info', '=== COMPACTION DETECTED (via precompact hook signal) === msgs=' + messageCount);
|
|
2468
|
+
} else if (_expectCompaction && (Date.now() - _compactionSignalTime >= 30000)) {
|
|
2469
|
+
_expectCompaction = false; // stale signal — expired after 30s
|
|
2470
|
+
} else if (isCompaction) {
|
|
2471
|
+
_expectCompaction = false; // clear flag if heuristic also caught it
|
|
2472
|
+
}
|
|
1657
2473
|
// No passthrough — always process everything (system-reminder strip + steno + translate)
|
|
1658
2474
|
const isPassthrough = false;
|
|
1659
2475
|
let sysPromptModified = false;
|
|
1660
2476
|
|
|
1661
|
-
// === SYSTEM PROMPT
|
|
1662
|
-
//
|
|
1663
|
-
//
|
|
1664
|
-
|
|
1665
|
-
|
|
2477
|
+
// === SYSTEM PROMPT MANAGEMENT ===
|
|
2478
|
+
// Strategy:
|
|
2479
|
+
// 1. FIRST request (or after reset): ALWAYS inject custom/optimized sys prompt
|
|
2480
|
+
// 2. Next requests: strip sys prompt (already sent)
|
|
2481
|
+
// 3. Every N compactions: re-inject custom sys prompt (Claude forgets after compaction)
|
|
2482
|
+
// 4. If no custom prompt exists: auto-generate optimized one (Traditional Chinese + SpecMem tools)
|
|
2483
|
+
if (body.system) {
|
|
1666
2484
|
const _sysKey = typeof body.system === 'string' ? body.system
|
|
1667
2485
|
: Array.isArray(body.system) ? body.system.map(b => typeof b === 'string' ? b : (b?.text || '')).join('')
|
|
1668
2486
|
: JSON.stringify(body.system);
|
|
1669
|
-
const _sysHash = createHash('md5').update(_sysKey).digest('hex');
|
|
1670
|
-
|
|
2487
|
+
const _sysHash = createHash('md5').update(_sysKey).digest('hex').slice(0, 16);
|
|
2488
|
+
|
|
2489
|
+
// ── Session boundary detection ──
|
|
2490
|
+
// MCP server outlives Claude CLI sessions. Detect new session by time gap
|
|
2491
|
+
// so system prompt always shows on first user submit of every session.
|
|
2492
|
+
const now = Date.now();
|
|
2493
|
+
if (_lastRequestTime > 0 && (now - _lastRequestTime) >= SESSION_GAP_MS) {
|
|
2494
|
+
log('info', `SESSION BOUNDARY detected (${((now - _lastRequestTime) / 1000).toFixed(0)}s gap) — resetting sys prompt state`);
|
|
2495
|
+
_sysPromptSendCount = 0;
|
|
2496
|
+
_compactionsSinceLastInject = 0;
|
|
2497
|
+
// Keep _seenSysPromptHash so OG prompt isn't re-saved
|
|
2498
|
+
}
|
|
2499
|
+
_lastRequestTime = now;
|
|
2500
|
+
|
|
2501
|
+
// Save OG prompt on first sight
|
|
2502
|
+
if (!_seenSysPromptHash) {
|
|
2503
|
+
_saveOgSysPrompt(_sysKey, _sysHash);
|
|
2504
|
+
}
|
|
2505
|
+
|
|
2506
|
+
// Determine if we should inject custom sys prompt this request
|
|
2507
|
+
const shouldInject = (
|
|
2508
|
+
_sysPromptSendCount === 0 || // first request of session (or after session boundary reset)
|
|
2509
|
+
(isCompaction && ++_compactionsSinceLastInject >= SYS_PROMPT_REINJECT_INTERVAL) // every 3rd compaction
|
|
2510
|
+
);
|
|
2511
|
+
|
|
2512
|
+
if (shouldInject) {
|
|
2513
|
+
_compactionsSinceLastInject = 0;
|
|
2514
|
+
_sysPromptSendCount++;
|
|
2515
|
+
|
|
2516
|
+
// Get custom prompt — either user-edited or auto-generated optimized
|
|
2517
|
+
let customPrompt = _customSystemPrompt?.prompt;
|
|
2518
|
+
if (!customPrompt && liveConfig.SYSTEM_PROMPT_COMPRESS) {
|
|
2519
|
+
// Auto-generate: inject SpecMem tool awareness into original prompt
|
|
2520
|
+
customPrompt = _sysKey + '\n\n# SpecMem Integration\nYou have access to SpecMem persistent memory tools (find_memory, save_memory, find_code_pointers, drill_down). Use these proactively to recall context, search code semantically, and store important findings. Always check memory before starting complex tasks.';
|
|
2521
|
+
}
|
|
1671
2522
|
|
|
1672
|
-
|
|
1673
|
-
|
|
1674
|
-
|
|
1675
|
-
body.system = _sysCached.system;
|
|
2523
|
+
if (customPrompt) {
|
|
2524
|
+
const origSize = _sysKey.length;
|
|
2525
|
+
body.system = customPrompt;
|
|
1676
2526
|
sysPromptModified = true;
|
|
1677
|
-
|
|
2527
|
+
_seenSysPromptHash = _sysHash;
|
|
2528
|
+
const newSize = customPrompt.length;
|
|
2529
|
+
const saved = origSize - newSize;
|
|
2530
|
+
if (saved > 0) {
|
|
2531
|
+
stats.sysPromptCharsSaved += saved;
|
|
2532
|
+
stats.tokensStripped += Math.floor(saved / 4);
|
|
2533
|
+
stats.bytesStripped += saved;
|
|
2534
|
+
}
|
|
1678
2535
|
stats.sysPromptCompressed++;
|
|
1679
|
-
|
|
1680
|
-
|
|
1681
|
-
|
|
1682
|
-
|
|
2536
|
+
log('info', `SYSPROMPT injected (${shouldInject ? 'first/reinject' : 'custom'}): ${origSize}→${newSize} chars`);
|
|
2537
|
+
pushEvent('dedup', `SYSPROMPT injected: ${(origSize/1024).toFixed(0)}KB→${(newSize/1024).toFixed(0)}KB`);
|
|
2538
|
+
} else {
|
|
2539
|
+
// No custom, no auto-gen — pass through original
|
|
2540
|
+
_seenSysPromptHash = _sysHash;
|
|
2541
|
+
// NOTE: don't increment _sysPromptSendCount here — already incremented at line 2288
|
|
2542
|
+
log('info', `SYSPROMPT passthrough (no custom): ${(_sysKey.length/1024).toFixed(0)}KB`);
|
|
1683
2543
|
}
|
|
1684
|
-
} else if (
|
|
1685
|
-
//
|
|
1686
|
-
|
|
2544
|
+
} else if (_seenSysPromptHash === _sysHash) {
|
|
2545
|
+
// Already sent, strip to save tokens
|
|
2546
|
+
const sysSize = _sysKey.length;
|
|
2547
|
+
delete body.system;
|
|
2548
|
+
sysPromptModified = true;
|
|
2549
|
+
stats.sysPromptCharsSaved += sysSize;
|
|
2550
|
+
stats.sysPromptCompressed++;
|
|
2551
|
+
stats.tokensStripped += Math.floor(sysSize / 4);
|
|
2552
|
+
stats.bytesStripped += sysSize;
|
|
2553
|
+
pushEvent('dedup', `SYSPROMPT stripped (same hash): -${(sysSize/1024).toFixed(0)}KB`);
|
|
1687
2554
|
} else {
|
|
1688
|
-
//
|
|
1689
|
-
|
|
1690
|
-
|
|
1691
|
-
|
|
1692
|
-
body.system = sysResult.system;
|
|
1693
|
-
sysPromptModified = true;
|
|
1694
|
-
stats.sysPromptCharsSaved += sysResult.charsSaved;
|
|
1695
|
-
stats.sysPromptCompressed++;
|
|
1696
|
-
stats.tokensStripped += Math.floor(sysResult.charsSaved / 4);
|
|
1697
|
-
stats.bytesStripped += sysResult.charsSaved;
|
|
1698
|
-
log('compress', `SYSPROMPT: ${sysResult.charsSaved} chars saved`);
|
|
1699
|
-
pushEvent('compress', `System prompt: -${sysResult.charsSaved} chars`);
|
|
1700
|
-
}
|
|
1701
|
-
} catch (e) {
|
|
1702
|
-
log('warn', `System prompt compression failed: ${e.message}`);
|
|
1703
|
-
}
|
|
2555
|
+
// Different sys prompt (changed by Claude Code) — send it, update hash
|
|
2556
|
+
_seenSysPromptHash = _sysHash;
|
|
2557
|
+
_sysPromptSendCount++;
|
|
2558
|
+
log('info', `SYSPROMPT changed hash ${_sysHash} — sending full`);
|
|
1704
2559
|
}
|
|
1705
2560
|
}
|
|
1706
2561
|
|
|
@@ -2011,7 +2866,7 @@ function setPaused(state) {
|
|
|
2011
2866
|
/**
|
|
2012
2867
|
* Check if the daemon is responding on its health endpoint.
|
|
2013
2868
|
*/
|
|
2014
|
-
function checkDaemonHealth() {
|
|
2869
|
+
export function checkDaemonHealth() {
|
|
2015
2870
|
return new Promise((resolve) => {
|
|
2016
2871
|
const req = httpGet(`http://127.0.0.1:${PROXY_PORT}/health`, { timeout: 2000 }, (res) => {
|
|
2017
2872
|
let data = '';
|
|
@@ -2125,7 +2980,7 @@ export function getCompactionProxyStats() {
|
|
|
2125
2980
|
* Register this project with the running daemon.
|
|
2126
2981
|
* Fire-and-forget — if daemon isn't running yet, fails silently.
|
|
2127
2982
|
*/
|
|
2128
|
-
function registerWithDaemon(projectPath, pid) {
|
|
2983
|
+
export function registerWithDaemon(projectPath, pid) {
|
|
2129
2984
|
if (!projectPath) return;
|
|
2130
2985
|
const body = JSON.stringify({ projectPath, pid: pid || process.pid });
|
|
2131
2986
|
try {
|
|
@@ -2176,6 +3031,10 @@ function deregisterFromDaemon(projectPath) {
|
|
|
2176
3031
|
/**
|
|
2177
3032
|
* Check if a PID is alive using signal 0.
|
|
2178
3033
|
*/
|
|
3034
|
+
function getLastRequestTime() {
|
|
3035
|
+
return _lastRequestTime;
|
|
3036
|
+
}
|
|
3037
|
+
|
|
2179
3038
|
function isPidAlive(pid) {
|
|
2180
3039
|
if (!pid || pid <= 0) return false;
|
|
2181
3040
|
try {
|
|
@@ -2243,4 +3102,5 @@ export {
|
|
|
2243
3102
|
DISABLED_FILE,
|
|
2244
3103
|
LOG_FILE,
|
|
2245
3104
|
CLAUDE_DIR,
|
|
3105
|
+
getLastRequestTime,
|
|
2246
3106
|
};
|