squeezr-ai 1.81.2 → 1.99.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (69) hide show
  1. package/bin/squeezr.js +82 -1
  2. package/dist/__tests__/bench.test.d.ts +1 -0
  3. package/dist/__tests__/bench.test.js +55 -0
  4. package/dist/__tests__/cacheAligner.test.d.ts +1 -0
  5. package/dist/__tests__/cacheAligner.test.js +75 -0
  6. package/dist/__tests__/codeStructure.test.d.ts +1 -0
  7. package/dist/__tests__/codeStructure.test.js +74 -0
  8. package/dist/__tests__/compressor.test.js +5 -1
  9. package/dist/__tests__/contentRouter.test.d.ts +1 -0
  10. package/dist/__tests__/contentRouter.test.js +76 -0
  11. package/dist/__tests__/controlEndpointGuard.test.d.ts +1 -0
  12. package/dist/__tests__/controlEndpointGuard.test.js +61 -0
  13. package/dist/__tests__/deterministic.test.js +13 -0
  14. package/dist/__tests__/doctor.test.d.ts +1 -0
  15. package/dist/__tests__/doctor.test.js +86 -0
  16. package/dist/__tests__/jsonCrush.test.d.ts +1 -0
  17. package/dist/__tests__/jsonCrush.test.js +168 -0
  18. package/dist/__tests__/learn.test.d.ts +1 -0
  19. package/dist/__tests__/learn.test.js +159 -0
  20. package/dist/__tests__/outputSavings.test.d.ts +1 -0
  21. package/dist/__tests__/outputSavings.test.js +49 -0
  22. package/dist/__tests__/outputShaper.test.d.ts +1 -0
  23. package/dist/__tests__/outputShaper.test.js +216 -0
  24. package/dist/__tests__/relevance.test.d.ts +1 -0
  25. package/dist/__tests__/relevance.test.js +66 -0
  26. package/dist/__tests__/secureKeyFile.test.d.ts +1 -0
  27. package/dist/__tests__/secureKeyFile.test.js +27 -0
  28. package/dist/__tests__/serverBinding.test.d.ts +1 -0
  29. package/dist/__tests__/serverBinding.test.js +18 -0
  30. package/dist/__tests__/statsOutput.test.d.ts +1 -0
  31. package/dist/__tests__/statsOutput.test.js +46 -0
  32. package/dist/__tests__/textCrusher.test.d.ts +1 -0
  33. package/dist/__tests__/textCrusher.test.js +133 -0
  34. package/dist/bench.d.ts +45 -0
  35. package/dist/bench.js +114 -0
  36. package/dist/cacheAligner.d.ts +34 -0
  37. package/dist/cacheAligner.js +73 -0
  38. package/dist/codexMitm.d.ts +1 -0
  39. package/dist/codexMitm.js +26 -2
  40. package/dist/compressor.js +42 -4
  41. package/dist/config.d.ts +6 -0
  42. package/dist/config.js +25 -0
  43. package/dist/contentRouter.d.ts +30 -0
  44. package/dist/contentRouter.js +112 -0
  45. package/dist/dashboard.d.ts +1 -1
  46. package/dist/dashboard.js +26 -7
  47. package/dist/deterministic.d.ts +4 -1
  48. package/dist/deterministic.js +43 -10
  49. package/dist/doctor.d.ts +38 -0
  50. package/dist/doctor.js +113 -0
  51. package/dist/index.js +4 -1
  52. package/dist/jsonCrush.d.ts +34 -0
  53. package/dist/jsonCrush.js +177 -0
  54. package/dist/learn.d.ts +65 -0
  55. package/dist/learn.js +244 -0
  56. package/dist/outputSavings.d.ts +20 -0
  57. package/dist/outputSavings.js +52 -0
  58. package/dist/outputShaper.d.ts +71 -0
  59. package/dist/outputShaper.js +155 -0
  60. package/dist/relevance.d.ts +27 -0
  61. package/dist/relevance.js +78 -0
  62. package/dist/server.js +106 -11
  63. package/dist/stats.d.ts +17 -0
  64. package/dist/stats.js +55 -0
  65. package/dist/systemPrompt.js +3 -2
  66. package/dist/textCrusher.d.ts +35 -0
  67. package/dist/textCrusher.js +160 -0
  68. package/package.json +69 -69
  69. package/squeezr.toml +15 -1
package/dist/server.js CHANGED
@@ -13,6 +13,29 @@ import { isAiCompressionEnabled, setAiCompression, toggleAiCompression } from '.
13
13
  import { circuitBreaker } from './circuitBreaker.js';
14
14
  import { injectExpandToolAnthropic, injectExpandToolOpenAI, injectExpandDirectiveAnthropic, injectExpandDirectiveOpenAI, handleAnthropicExpandCall, handleOpenAIExpandCall, retrieveOriginal, expandStoreSize, EXPAND_TOOL_ANTHROPIC_CHARS, } from './expand.js';
15
15
  import { compressSystemPrompt } from './systemPrompt.js';
16
+ import { shapeRequest } from './outputShaper.js';
17
+ import { warnIfVolatile } from './cacheAligner.js';
18
+ import { echoRatio, extractAssistantTextFromSse, extractAssistantTextFromContent } from './outputSavings.js';
19
+ // Concatenated text the model saw (capped), used as the echo-measurement context.
20
+ function contextTextForEcho(messages) {
21
+ let out = '';
22
+ for (const m of messages) {
23
+ const c = m?.content;
24
+ if (typeof c === 'string')
25
+ out += c + '\n';
26
+ else if (Array.isArray(c)) {
27
+ for (const b of c) {
28
+ if (typeof b?.text === 'string')
29
+ out += b.text + '\n';
30
+ else if (typeof b?.content === 'string')
31
+ out += b.content + '\n';
32
+ }
33
+ }
34
+ if (out.length > 40000)
35
+ break;
36
+ }
37
+ return out;
38
+ }
16
39
  import { captureRequest } from './requestCapture.js';
17
40
  import { dedupSkillBlocks } from './skillDedup.js';
18
41
  import { collapseStaleTurns } from './staleTurns.js';
@@ -199,20 +222,48 @@ async function proxyStream(upstream, body, headers, params) {
199
222
  });
200
223
  }
201
224
  export const app = new Hono();
202
- // ── CORS middleware (required for Cursor IDE and browser-based tools) ─────────
203
- // Cursor's Electron renderer sends OPTIONS preflight before every POST.
204
- // Without this the request is blocked and Cursor shows a network error.
225
+ // ── CORS + CSRF guard middleware ─────────────────────────────────────────────
226
+ // Cursor's Electron renderer sends OPTIONS preflight before every POST, so the
227
+ // proxy endpoints (/v1, /v1beta) stay permissive for browser-based IDE tooling.
228
+ // The control endpoints (/squeezr/*) are different: they change state and the
229
+ // process holds the user's API keys, so they must NOT be reachable cross-origin.
230
+ // Combined with the loopback bind, this kills the CSRF/DoS vector where a page
231
+ // open in the user's browser POSTs to http://localhost:<port>/squeezr/control/*.
232
+ // Only browser origins on loopback may touch the control endpoints.
233
+ const LOOPBACK_ORIGIN = /^https?:\/\/(localhost|127\.0\.0\.1|\[::1\])(:\d+)?$/i;
234
+ const isLoopbackOrigin = (origin) => origin !== undefined && LOOPBACK_ORIGIN.test(origin);
205
235
  app.use('*', async (c, next) => {
236
+ const origin = c.req.header('origin');
237
+ const isControl = c.req.path.startsWith('/squeezr/');
238
+ const isMutation = !['GET', 'HEAD', 'OPTIONS'].includes(c.req.method);
239
+ // CSRF guard: a state-changing request to a control endpoint that carries a
240
+ // non-loopback browser Origin is a cross-site attack — reject it. Requests
241
+ // without an Origin (curl, MCP, native clients over loopback) are allowed;
242
+ // the loopback bind already limits those to processes on this machine.
243
+ if (isControl && isMutation && origin !== undefined && !isLoopbackOrigin(origin)) {
244
+ return c.json({ error: 'forbidden: cross-origin control request rejected' }, 403);
245
+ }
246
+ // CORS: control endpoints reflect ONLY loopback origins (never wildcard, so a
247
+ // malicious page cannot read control/state responses). Proxy endpoints keep
248
+ // the permissive wildcard that Cursor and other browser tools need.
249
+ const allowOrigin = isControl ? (isLoopbackOrigin(origin) ? origin : '') : '*';
206
250
  if (c.req.method === 'OPTIONS') {
207
- return c.body(null, 204, {
208
- 'Access-Control-Allow-Origin': '*',
251
+ const headers = {
209
252
  'Access-Control-Allow-Methods': 'GET, POST, PUT, DELETE, PATCH, OPTIONS',
210
253
  'Access-Control-Allow-Headers': '*',
211
254
  'Access-Control-Max-Age': '86400',
212
- });
255
+ };
256
+ if (allowOrigin)
257
+ headers['Access-Control-Allow-Origin'] = allowOrigin;
258
+ if (isControl)
259
+ headers['Vary'] = 'Origin';
260
+ return c.body(null, 204, headers);
213
261
  }
214
262
  await next();
215
- c.res.headers.set('Access-Control-Allow-Origin', '*');
263
+ if (allowOrigin)
264
+ c.res.headers.set('Access-Control-Allow-Origin', allowOrigin);
265
+ if (isControl)
266
+ c.res.headers.set('Vary', 'Origin');
216
267
  c.res.headers.set('Access-Control-Allow-Methods', 'GET, POST, PUT, DELETE, PATCH, OPTIONS');
217
268
  c.res.headers.set('Access-Control-Allow-Headers', '*');
218
269
  });
@@ -289,6 +340,10 @@ app.post('/v1/messages', async (c) => {
289
340
  }
290
341
  // Extract project name BEFORE compressing system prompt (compression destroys <cwd> tags)
291
342
  const project = extractProjectName(body);
343
+ // Cache-aligner (detector-only): if the CLIENT's system prompt embeds volatile tokens
344
+ // (UUID/timestamp/JWT/hash) they may bust Anthropic's prompt cache. Warns once per
345
+ // distinct signature; never mutates the prompt.
346
+ warnIfVolatile(body.system);
292
347
  const messages = (body.messages ?? []);
293
348
  // Measure FULL request (messages + tools + system) before ANY compression
294
349
  const originalRequestChars = estimateFullRequestChars(body);
@@ -309,10 +364,12 @@ app.post('/v1/messages', async (c) => {
309
364
  savings,
310
365
  });
311
366
  }
312
- // Bypass mode: skip all compression, still record request stats
367
+ // Bypass mode: skip all compression. Do NOT record processed/saved/cost stats — Squeezr
368
+ // did nothing to this traffic, and counting it (as it did before) inflated "processed"
369
+ // and dragged the savings ratio / cost to ~0 after a day spent bypassed. Track it in a
370
+ // separate bypassed-only counter instead.
313
371
  if (isBypassed()) {
314
- stats.recordWithProject(project, originalRequestChars, originalRequestChars, emptySavings(), undefined, clientId, modelId);
315
- recordRequest(project, 0, 0, [], originalRequestChars);
372
+ stats.recordBypassed();
316
373
  storeKey('anthropic', apiKey);
317
374
  const fwdHeaders = forwardHeaders(c.req.raw.headers);
318
375
  if (body.stream) {
@@ -455,6 +512,23 @@ app.post('/v1/messages', async (c) => {
455
512
  // the highest-weight channel (tool description alone was observed to be ignored).
456
513
  // Cache-safe: appends a new trailing block, never mutates cache_control blocks.
457
514
  injectExpandDirectiveAnthropic(body);
515
+ // Output-side token reduction (opt-in). Verbosity steering appends a byte-stable
516
+ // block to the system tail (same cache-safe append as the expand directive);
517
+ // effort routing lowers an EXISTING thinking budget on mechanical continuations.
518
+ // Runs LAST so the trailing steering block is deterministic per request.
519
+ if (config.outputShaperEnabled) {
520
+ const shaped = shapeRequest(body, {
521
+ enabled: true,
522
+ verbositySteering: config.outputVerbositySteering,
523
+ level: config.outputLevel,
524
+ effortRouting: config.outputEffortRouting,
525
+ mechanicalThinkingFloor: config.outputMechanicalThinkingFloor,
526
+ });
527
+ stats.recordShaping(shaped.steered, shaped.effortLowered);
528
+ if (shaped.effortLowered || shaped.steered) {
529
+ console.log(`[squeezr/output-shaper] turn=${shaped.turn} steered=${shaped.steered} effort-lowered=${shaped.effortLowered}`);
530
+ }
531
+ }
458
532
  // Attach per-feature savings to the savings object for accurate breakdown reporting
459
533
  savings.toolDescSavedChars = toolDescSaved;
460
534
  savings.mcpFilterSavedChars = mcpFilterSaved;
@@ -486,6 +560,8 @@ app.post('/v1/messages', async (c) => {
486
560
  if (!SKIP_RESP_HEADERS.has(k.toLowerCase()))
487
561
  c.header(k, v);
488
562
  }
563
+ const measureEcho = config.outputShaperEnabled;
564
+ let sseBuf = '';
489
565
  return stream(c, async (s) => {
490
566
  const reader = upstream.body.getReader();
491
567
  const decoder = new TextDecoder();
@@ -495,7 +571,18 @@ app.post('/v1/messages', async (c) => {
495
571
  if (done)
496
572
  break;
497
573
  await s.write(value);
498
- sseParser(decoder.decode(value, { stream: true }));
574
+ const chunk = decoder.decode(value, { stream: true });
575
+ sseParser(chunk);
576
+ if (measureEcho)
577
+ sseBuf += chunk;
578
+ }
579
+ if (measureEcho) {
580
+ const outText = extractAssistantTextFromSse(sseBuf);
581
+ if (outText.length > 40) {
582
+ const echo = echoRatio(outText, contextTextForEcho(body.messages));
583
+ stats.recordEcho(echo);
584
+ console.log(`[squeezr/output] echo=${Math.round(echo * 100)}% of assistant output restated existing context (${outText.length} chars)`);
585
+ }
499
586
  }
500
587
  });
501
588
  }
@@ -511,6 +598,14 @@ app.post('/v1/messages', async (c) => {
511
598
  const u = respBody.usage;
512
599
  addAnthropicUsage(u.input_tokens ?? 0, u.output_tokens ?? 0, u.cache_creation_input_tokens ?? 0, u.cache_read_input_tokens ?? 0);
513
600
  }
601
+ if (config.outputShaperEnabled) {
602
+ const outText = extractAssistantTextFromContent(respBody.content);
603
+ if (outText.length > 40) {
604
+ const echo = echoRatio(outText, contextTextForEcho(body.messages));
605
+ stats.recordEcho(echo);
606
+ console.log(`[squeezr/output] echo=${Math.round(echo * 100)}% of assistant output restated existing context (${outText.length} chars)`);
607
+ }
608
+ }
514
609
  // Handle expand() call if model requested one (track expand rate)
515
610
  const expandCall = handleAnthropicExpandCall(respBody);
516
611
  if (expandCall) {
package/dist/stats.d.ts CHANGED
@@ -35,6 +35,16 @@ export declare class Stats {
35
35
  private expandHits;
36
36
  private expandMisses;
37
37
  private expandPartial;
38
+ private outputEchoSamples;
39
+ private outputEchoSum;
40
+ private outputSteered;
41
+ private outputEffortLowered;
42
+ /** Record one measured assistant response's echo ratio (0..1). */
43
+ recordEcho(echo: number): void;
44
+ /** Record one shaped request: whether verbosity steering was applied / effort was lowered. */
45
+ recordShaping(steered: boolean, effortLowered: boolean): void;
46
+ private bypassedRequests;
47
+ recordBypassed(): void;
38
48
  record(originalChars: number, compressedChars: number, savings: Savings, latency?: LatencyInfo, client?: string, model?: string): void;
39
49
  /** @deprecated Pass savings.syspromptSavedChars instead */
40
50
  recordSystemPromptSaved(originalLen: number, compressedLen: number): void;
@@ -65,6 +75,7 @@ export declare class Stats {
65
75
  current_project: string;
66
76
  last_original_chars: number;
67
77
  last_compressed_chars: number;
78
+ bypassed_requests: number;
68
79
  breakdown: {
69
80
  tool_results_det: number;
70
81
  tool_results_ai: number;
@@ -103,6 +114,12 @@ export declare class Stats {
103
114
  last: number;
104
115
  };
105
116
  };
117
+ output: {
118
+ echo_samples: number;
119
+ avg_echo_pct: number;
120
+ steered: number;
121
+ effort_lowered: number;
122
+ };
106
123
  expand: {
107
124
  calls: number;
108
125
  hits: number;
package/dist/stats.js CHANGED
@@ -75,6 +75,46 @@ export class Stats {
75
75
  expandMisses = Stats.persistedNum('expand_misses');
76
76
  // Partial expands = squeezr_expand("<id>~i") (recover one segment) vs whole-block.
77
77
  expandPartial = Stats.persistedNum('expand_partial');
78
+ // Output-side metrics (output shaper + echo). Persisted via the main persist() snapshot
79
+ // (no per-call disk write). echoSum is the running sum of echo ratios (0..1); avg = sum/samples.
80
+ outputEchoSamples = Stats.persistedNum('output_echo_samples');
81
+ outputEchoSum = Stats.persistedNum('output_echo_sum');
82
+ outputSteered = Stats.persistedNum('output_steered');
83
+ outputEffortLowered = Stats.persistedNum('output_effort_lowered');
84
+ /** Record one measured assistant response's echo ratio (0..1). */
85
+ recordEcho(echo) {
86
+ this.outputEchoSamples++;
87
+ this.outputEchoSum += echo;
88
+ }
89
+ /** Record one shaped request: whether verbosity steering was applied / effort was lowered. */
90
+ recordShaping(steered, effortLowered) {
91
+ if (steered)
92
+ this.outputSteered++;
93
+ if (effortLowered)
94
+ this.outputEffortLowered++;
95
+ }
96
+ // Requests that passed through while BYPASSED. Tracked separately and NEVER added to
97
+ // processed/saved/cost totals — bypass means Squeezr did nothing, so counting that
98
+ // traffic as "processed, 0 saved" would drag the ratio to ~0 and inflate cost (the
99
+ // "$7048 processed / $0.55 saved after a bypass day" confusion). Throttled persist to
100
+ // avoid disk thrash under heavy bypass traffic.
101
+ bypassedRequests = Stats.persistedNum('bypassed_requests');
102
+ recordBypassed() {
103
+ this.bypassedRequests++;
104
+ if (this.bypassedRequests % 25 === 0) {
105
+ try {
106
+ const dir = join(homedir(), '.squeezr');
107
+ if (!existsSync(dir))
108
+ mkdirSync(dir, { recursive: true });
109
+ const existing = Stats.loadGlobal();
110
+ existing.bypassed_requests = this.bypassedRequests;
111
+ const tmp = STATS_FILE + '.tmp';
112
+ writeFileSync(tmp, JSON.stringify(existing));
113
+ renameSync(tmp, STATS_FILE);
114
+ }
115
+ catch { /* ignore */ }
116
+ }
117
+ }
78
118
  record(originalChars, compressedChars, savings, latency, client, model) {
79
119
  this.requests++;
80
120
  this.totalOriginalChars += originalChars;
@@ -218,6 +258,7 @@ export class Stats {
218
258
  current_project: this.currentProject,
219
259
  last_original_chars: this.lastOriginalChars,
220
260
  last_compressed_chars: this.lastCompressedChars,
261
+ bypassed_requests: this.bypassedRequests,
221
262
  // Savings breakdown for honest dashboard reporting
222
263
  breakdown: {
223
264
  tool_results_det: this.totalDetSaved,
@@ -237,6 +278,16 @@ export class Stats {
237
278
  deterministic: this.latencyDet.summary(),
238
279
  ai: this.latencyAi.summary(),
239
280
  },
281
+ // Output-side metrics (output shaper). avg_echo_pct = mean % of assistant output
282
+ // that merely restated existing context (lower is better; steering targets it).
283
+ output: {
284
+ echo_samples: this.outputEchoSamples,
285
+ avg_echo_pct: this.outputEchoSamples > 0
286
+ ? Math.round((this.outputEchoSum / this.outputEchoSamples) * 1000) / 10
287
+ : 0,
288
+ steered: this.outputSteered,
289
+ effort_lowered: this.outputEffortLowered,
290
+ },
240
291
  // Expand rate — key quality metric (high = compression too aggressive)
241
292
  expand: {
242
293
  calls: this.expandCalls,
@@ -318,6 +369,10 @@ export class Stats {
318
369
  existing.expand_hits = this.expandHits;
319
370
  existing.expand_misses = this.expandMisses;
320
371
  existing.expand_partial = this.expandPartial;
372
+ existing.output_echo_samples = this.outputEchoSamples;
373
+ existing.output_echo_sum = this.outputEchoSum;
374
+ existing.output_steered = this.outputSteered;
375
+ existing.output_effort_lowered = this.outputEffortLowered;
321
376
  // Local AI calls (Zest/Ollama) — persisted separately, not added to cost counters
322
377
  if (savings.localAiCalls != null && savings.localAiCalls > 0) {
323
378
  existing.local_ai_calls = (existing.local_ai_calls ?? 0) + savings.localAiCalls;
@@ -81,10 +81,11 @@ export async function compressSystemPrompt(prompt, apiKey, backend) {
81
81
  compressed = resp.choices[0].message.content ?? prompt;
82
82
  }
83
83
  else if (backend === 'gemini-flash') {
84
- const url = `https://generativelanguage.googleapis.com/v1beta/models/gemini-1.5-flash-8b:generateContent?key=${apiKey}`;
84
+ // Key in the x-goog-api-key header, never in the URL query (URLs leak to logs).
85
+ const url = `https://generativelanguage.googleapis.com/v1beta/models/gemini-1.5-flash-8b:generateContent`;
85
86
  const resp = await fetch(url, {
86
87
  method: 'POST',
87
- headers: { 'content-type': 'application/json' },
88
+ headers: { 'content-type': 'application/json', 'x-goog-api-key': apiKey },
88
89
  body: JSON.stringify({ contents: [{ role: 'user', parts: [{ text: input }] }] }),
89
90
  });
90
91
  const data = (await resp.json());
@@ -0,0 +1,35 @@
1
+ /**
2
+ * TextCrusher — deterministic, cache-safe extractive compression for large PROSE/LOG
3
+ * blocks. Squeezr's take on headroom's TextCrusher (their fast, no-model alternative to
4
+ * the ML compressor).
5
+ *
6
+ * Exact line-dedup (already in the base pipeline) only catches byte-identical repeats.
7
+ * A huge share of log spam is NOT byte-identical — it's the same line with a different
8
+ * number, timestamp or hex id ("processed record 1 in 2ms", "…record 2 in 4ms", …).
9
+ * TextCrusher collapses those NEAR-duplicates by normalizing volatile tokens, while:
10
+ *
11
+ * - always keeping high-signal lines (error/warn/fail/exception/…),
12
+ * - always keeping head + tail anchor lines (context),
13
+ * - preserving original order,
14
+ * - staying fully deterministic → byte-stable → prompt-cache safe.
15
+ *
16
+ * It never drops content irrecoverably: callers wrap the result with makeRecoverable so
17
+ * the full original is one squeezr_expand away.
18
+ */
19
+ export interface CrushTextOpts {
20
+ maxLines: number;
21
+ headKeep: number;
22
+ tailKeep: number;
23
+ query: string;
24
+ }
25
+ /** Collapse volatile tokens so near-duplicate lines share a key. */
26
+ export declare function normalizeLineKey(line: string): string;
27
+ /** Word 3-gram shingles of a line (falls back to the word set for short lines). */
28
+ export declare function wordShingles(line: string, k?: number): Set<string>;
29
+ export declare function jaccard(a: Set<string>, b: Set<string>): number;
30
+ export interface CrushResult {
31
+ text: string;
32
+ kept: number;
33
+ dropped: number;
34
+ }
35
+ export declare function crushText(text: string, opts?: Partial<CrushTextOpts>): CrushResult;
@@ -0,0 +1,160 @@
1
+ /**
2
+ * TextCrusher — deterministic, cache-safe extractive compression for large PROSE/LOG
3
+ * blocks. Squeezr's take on headroom's TextCrusher (their fast, no-model alternative to
4
+ * the ML compressor).
5
+ *
6
+ * Exact line-dedup (already in the base pipeline) only catches byte-identical repeats.
7
+ * A huge share of log spam is NOT byte-identical — it's the same line with a different
8
+ * number, timestamp or hex id ("processed record 1 in 2ms", "…record 2 in 4ms", …).
9
+ * TextCrusher collapses those NEAR-duplicates by normalizing volatile tokens, while:
10
+ *
11
+ * - always keeping high-signal lines (error/warn/fail/exception/…),
12
+ * - always keeping head + tail anchor lines (context),
13
+ * - preserving original order,
14
+ * - staying fully deterministic → byte-stable → prompt-cache safe.
15
+ *
16
+ * It never drops content irrecoverably: callers wrap the result with makeRecoverable so
17
+ * the full original is one squeezr_expand away.
18
+ */
19
+ const DEFAULTS = { maxLines: 60, headKeep: 5, tailKeep: 5, query: '' };
20
+ const SIGNAL_RE = /\b(error|err|warn|warning|fail|failed|failure|exception|fatal|panic|traceback|denied|refused|timeout|timed out|cannot|unable|missing|undefined|null pointer|segfault|assert)\b/i;
21
+ /** Collapse volatile tokens so near-duplicate lines share a key. */
22
+ export function normalizeLineKey(line) {
23
+ return line
24
+ .replace(/\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2}(?:\.\d+)?Z?/g, '') // ISO timestamps
25
+ .replace(/\b\d{1,2}:\d{2}:\d{2}\b/g, '') // clock times
26
+ .replace(/0x[0-9a-fA-F]+/g, '#') // hex
27
+ .replace(/\b\d+\b/g, '#') // numbers
28
+ .replace(/\s+/g, ' ')
29
+ .trim()
30
+ .toLowerCase();
31
+ }
32
+ function isSignal(line) {
33
+ return SIGNAL_RE.test(line);
34
+ }
35
+ // ── Shingle-based near-duplicate detection (catches REWORDED dups) ────────────
36
+ // Bigram shingles (k=2) + a 0.6 Jaccard threshold: catches a 1-word rewording in a
37
+ // typical log line while leaving genuinely different lines (Jaccard ~0) untouched.
38
+ // (Sentence-level 0.85 like headroom is too strict for short log lines.)
39
+ const SHINGLE_K = 2;
40
+ const NEAR_DUP_JACCARD = 0.6;
41
+ /** Word 3-gram shingles of a line (falls back to the word set for short lines). */
42
+ export function wordShingles(line, k = SHINGLE_K) {
43
+ const words = (line.toLowerCase().match(/[a-z0-9_]+/g) ?? []);
44
+ const set = new Set();
45
+ if (words.length < k) {
46
+ for (const w of words)
47
+ set.add(w);
48
+ return set;
49
+ }
50
+ for (let i = 0; i + k <= words.length; i++)
51
+ set.add(words.slice(i, i + k).join(' '));
52
+ return set;
53
+ }
54
+ export function jaccard(a, b) {
55
+ if (a.size === 0 && b.size === 0)
56
+ return 1;
57
+ if (a.size === 0 || b.size === 0)
58
+ return 0;
59
+ let inter = 0;
60
+ for (const x of a)
61
+ if (b.has(x))
62
+ inter++;
63
+ return inter / (a.size + b.size - inter);
64
+ }
65
+ import { bm25Scores } from './relevance.js';
66
+ export function crushText(text, opts = {}) {
67
+ const o = { ...DEFAULTS, ...opts };
68
+ const lines = text.split('\n');
69
+ const n = lines.length;
70
+ // Nothing to gain below the budget.
71
+ if (n <= o.maxLines)
72
+ return { text, kept: n, dropped: 0 };
73
+ const headEnd = Math.min(o.headKeep, n);
74
+ const tailStart = Math.max(headEnd, n - o.tailKeep);
75
+ // Pass 1 — decide which indices to keep.
76
+ const keep = new Array(n).fill(false);
77
+ const seenKeys = new Set();
78
+ let budget = o.maxLines;
79
+ const tryKeep = (i, key) => {
80
+ if (keep[i])
81
+ return;
82
+ keep[i] = true;
83
+ seenKeys.add(key);
84
+ budget--;
85
+ };
86
+ // Anchors + signal lines first (highest priority), registering their normalized keys
87
+ // so later near-duplicates of them are suppressed.
88
+ for (let i = 0; i < n; i++) {
89
+ const key = normalizeLineKey(lines[i]);
90
+ const anchor = i < headEnd || i >= tailStart;
91
+ if (anchor || isSignal(lines[i]))
92
+ tryKeep(i, key);
93
+ }
94
+ // Candidate lines for the remaining budget: first occurrence of each not-yet-seen
95
+ // normalized key (so near-duplicates collapse to their first representative).
96
+ const candidates = [];
97
+ const candSeen = new Set(seenKeys);
98
+ for (let i = 0; i < n; i++) {
99
+ if (keep[i])
100
+ continue;
101
+ const key = normalizeLineKey(lines[i]);
102
+ if (key === '' || candSeen.has(key))
103
+ continue;
104
+ candSeen.add(key);
105
+ candidates.push(i);
106
+ }
107
+ // Order candidates: by BM25 relevance to the task when a query is given (keep the
108
+ // lines that matter for what the user is doing), else by original order. Relevance
109
+ // is only ever driven from OUTSIDE the cached prefix (see compressor), so this stays
110
+ // cache-safe. Kept lines are still emitted in original order below.
111
+ let ordered = candidates;
112
+ if (o.query.trim() !== '' && candidates.length > 0) {
113
+ const scores = bm25Scores(o.query, candidates.map(i => lines[i]));
114
+ ordered = candidates
115
+ .map((idx, k) => ({ idx, score: scores[k], k }))
116
+ .sort((a, b) => (b.score - a.score) || (a.k - b.k))
117
+ .map(x => x.idx);
118
+ }
119
+ // Keep candidates, but skip any that is a REWORDED near-duplicate (shingle Jaccard
120
+ // ≥ threshold) of a line already kept in this fill pass — catches dups that survived
121
+ // the exact normalized-key check because their wording differs.
122
+ // Seed with the shingles of lines already kept (anchors + signal) so a candidate that
123
+ // merely rewords an anchor/signal line is also dropped.
124
+ const keptShingles = [];
125
+ for (let i = 0; i < n; i++)
126
+ if (keep[i])
127
+ keptShingles.push(wordShingles(lines[i]));
128
+ for (const i of ordered) {
129
+ if (budget <= 0)
130
+ break;
131
+ const sh = wordShingles(lines[i]);
132
+ if (keptShingles.some(s => jaccard(sh, s) >= NEAR_DUP_JACCARD))
133
+ continue;
134
+ tryKeep(i, normalizeLineKey(lines[i]));
135
+ keptShingles.push(sh);
136
+ }
137
+ // Reconstruct in original order, summarizing runs of dropped lines with one marker.
138
+ const out = [];
139
+ let droppedRun = 0;
140
+ let dropped = 0;
141
+ const flush = () => {
142
+ if (droppedRun > 0) {
143
+ out.push(`... [${droppedRun} lines omitted]`);
144
+ dropped += droppedRun;
145
+ droppedRun = 0;
146
+ }
147
+ };
148
+ for (let i = 0; i < n; i++) {
149
+ if (keep[i]) {
150
+ flush();
151
+ out.push(lines[i]);
152
+ }
153
+ else
154
+ droppedRun++;
155
+ }
156
+ flush();
157
+ if (dropped === 0)
158
+ return { text, kept: n, dropped: 0 };
159
+ return { text: out.join('\n'), kept: n - dropped, dropped };
160
+ }
package/package.json CHANGED
@@ -1,69 +1,69 @@
1
- {
2
- "name": "squeezr-ai",
3
- "version": "1.81.2",
4
- "description": "AI proxy that compresses Claude Code, Claude Desktop, Codex, Codex Desktop, Aider, Gemini CLI and Ollama context windows to save thousands of tokens per session",
5
- "keywords": [
6
- "claude",
7
- "claude-code",
8
- "codex",
9
- "ollama",
10
- "aider",
11
- "gemini",
12
- "token",
13
- "compression",
14
- "proxy",
15
- "llm",
16
- "ai"
17
- ],
18
- "license": "MIT",
19
- "repository": {
20
- "type": "git",
21
- "url": "git+https://github.com/sergioramosv/Squeezr.git"
22
- },
23
- "homepage": "https://github.com/sergioramosv/Squeezr#readme",
24
- "type": "module",
25
- "bin": {
26
- "squeezr": "bin/squeezr.js",
27
- "squeezr-mcp": "dist/mcp.js"
28
- },
29
- "scripts": {
30
- "build": "tsc",
31
- "prepack": "node -e \"['dist/cursorMitm.js','dist/cursorMitm.d.ts','dist/__tests__/cursorMitm.test.js','dist/__tests__/cursorMitm.test.d.ts'].forEach(f=>{try{require('fs').unlinkSync(f)}catch{}})\"",
32
- "dev": "tsx src/index.ts",
33
- "start": "node dist/index.js",
34
- "gain": "node dist/gain.js",
35
- "discover": "node dist/discover.js",
36
- "test": "vitest run",
37
- "test:watch": "vitest",
38
- "test:quality": "vitest run src/__tests__/qualityHarness.test.ts"
39
- },
40
- "files": [
41
- "bin/",
42
- "dist/",
43
- "squeezr.toml"
44
- ],
45
- "dependencies": {
46
- "@anthropic-ai/sdk": "^0.39.0",
47
- "@bufbuild/protobuf": "^2.11.0",
48
- "@hono/node-server": "^1.13.7",
49
- "@modelcontextprotocol/sdk": "^1.29.0",
50
- "@types/diff": "^7.0.2",
51
- "diff": "^9.0.0",
52
- "hono": "^4.7.5",
53
- "node-forge": "^1.4.0",
54
- "openai": "^4.93.0",
55
- "smol-toml": "^1.3.1",
56
- "zod": "^3.24.0"
57
- },
58
- "devDependencies": {
59
- "@types/node": "^22.14.0",
60
- "@types/node-forge": "^1.3.14",
61
- "@vitest/coverage-v8": "^4.1.2",
62
- "tsx": "^4.19.3",
63
- "typescript": "^5.8.3",
64
- "vitest": "^4.1.2"
65
- },
66
- "engines": {
67
- "node": ">=18"
68
- }
69
- }
1
+ {
2
+ "name": "squeezr-ai",
3
+ "version": "1.99.2",
4
+ "description": "AI proxy that compresses Claude Code, Claude Desktop, Codex, Codex Desktop, Aider, Gemini CLI and Ollama context windows to save thousands of tokens per session",
5
+ "keywords": [
6
+ "claude",
7
+ "claude-code",
8
+ "codex",
9
+ "ollama",
10
+ "aider",
11
+ "gemini",
12
+ "token",
13
+ "compression",
14
+ "proxy",
15
+ "llm",
16
+ "ai"
17
+ ],
18
+ "license": "MIT",
19
+ "repository": {
20
+ "type": "git",
21
+ "url": "git+https://github.com/sergioramosv/Squeezr.git"
22
+ },
23
+ "homepage": "https://github.com/sergioramosv/Squeezr#readme",
24
+ "type": "module",
25
+ "bin": {
26
+ "squeezr": "bin/squeezr.js",
27
+ "squeezr-mcp": "dist/mcp.js"
28
+ },
29
+ "scripts": {
30
+ "build": "tsc",
31
+ "prepack": "node -e \"['dist/cursorMitm.js','dist/cursorMitm.d.ts','dist/__tests__/cursorMitm.test.js','dist/__tests__/cursorMitm.test.d.ts'].forEach(f=>{try{require('fs').unlinkSync(f)}catch{}})\"",
32
+ "dev": "tsx src/index.ts",
33
+ "start": "node dist/index.js",
34
+ "gain": "node dist/gain.js",
35
+ "discover": "node dist/discover.js",
36
+ "test": "vitest run",
37
+ "test:watch": "vitest",
38
+ "test:quality": "vitest run src/__tests__/qualityHarness.test.ts"
39
+ },
40
+ "files": [
41
+ "bin/",
42
+ "dist/",
43
+ "squeezr.toml"
44
+ ],
45
+ "dependencies": {
46
+ "@anthropic-ai/sdk": "^0.39.0",
47
+ "@bufbuild/protobuf": "^2.11.0",
48
+ "@hono/node-server": "^1.13.7",
49
+ "@modelcontextprotocol/sdk": "^1.29.0",
50
+ "@types/diff": "^7.0.2",
51
+ "diff": "^9.0.0",
52
+ "hono": "^4.7.5",
53
+ "node-forge": "^1.4.0",
54
+ "openai": "^4.93.0",
55
+ "smol-toml": "^1.3.1",
56
+ "zod": "^3.24.0"
57
+ },
58
+ "devDependencies": {
59
+ "@types/node": "^22.14.0",
60
+ "@types/node-forge": "^1.3.14",
61
+ "@vitest/coverage-v8": "^4.1.2",
62
+ "tsx": "^4.19.3",
63
+ "typescript": "^5.8.3",
64
+ "vitest": "^4.1.2"
65
+ },
66
+ "engines": {
67
+ "node": ">=18"
68
+ }
69
+ }