squeezr-ai 1.90.0 → 1.99.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/__tests__/compressor.test.js +5 -1
- package/dist/__tests__/contentRouter.test.d.ts +1 -0
- package/dist/__tests__/contentRouter.test.js +76 -0
- package/dist/__tests__/controlEndpointGuard.test.d.ts +1 -0
- package/dist/__tests__/controlEndpointGuard.test.js +61 -0
- package/dist/__tests__/doctor.test.js +17 -1
- package/dist/__tests__/jsonCrush.test.js +59 -1
- package/dist/__tests__/outputSavings.test.d.ts +1 -0
- package/dist/__tests__/outputSavings.test.js +49 -0
- package/dist/__tests__/relevance.test.d.ts +1 -0
- package/dist/__tests__/relevance.test.js +66 -0
- package/dist/__tests__/secureKeyFile.test.d.ts +1 -0
- package/dist/__tests__/secureKeyFile.test.js +27 -0
- package/dist/__tests__/serverBinding.test.d.ts +1 -0
- package/dist/__tests__/serverBinding.test.js +18 -0
- package/dist/__tests__/statsOutput.test.d.ts +1 -0
- package/dist/__tests__/statsOutput.test.js +46 -0
- package/dist/__tests__/textCrusher.test.js +63 -1
- package/dist/codexMitm.d.ts +1 -0
- package/dist/codexMitm.js +26 -2
- package/dist/compressor.js +31 -3
- package/dist/contentRouter.d.ts +30 -0
- package/dist/contentRouter.js +112 -0
- package/dist/dashboard.d.ts +1 -1
- package/dist/dashboard.js +26 -7
- package/dist/deterministic.d.ts +1 -1
- package/dist/deterministic.js +14 -13
- package/dist/doctor.d.ts +5 -0
- package/dist/doctor.js +11 -0
- package/dist/index.js +4 -1
- package/dist/jsonCrush.d.ts +4 -0
- package/dist/jsonCrush.js +83 -18
- package/dist/outputSavings.d.ts +20 -0
- package/dist/outputSavings.js +52 -0
- package/dist/relevance.d.ts +27 -0
- package/dist/relevance.js +78 -0
- package/dist/server.js +84 -11
- package/dist/stats.d.ts +17 -0
- package/dist/stats.js +55 -0
- package/dist/systemPrompt.js +3 -2
- package/dist/textCrusher.d.ts +4 -0
- package/dist/textCrusher.js +69 -7
- package/package.json +69 -69
|
@@ -296,7 +296,11 @@ describe('compressGeminiContents', () => {
|
|
|
296
296
|
const cts = makeContents(['g'.repeat(200), 'h'.repeat(200)]);
|
|
297
297
|
await compressGeminiContents(cts, 'my-google-key', baseConfig);
|
|
298
298
|
expect(mockFetch).toHaveBeenCalledWith(expect.stringContaining('generativelanguage.googleapis.com'), expect.any(Object));
|
|
299
|
-
|
|
299
|
+
// Security: the key travels in the x-goog-api-key header, never in the URL.
|
|
300
|
+
const [calledUrl, calledOpts] = mockFetch.mock.calls.at(-1);
|
|
301
|
+
expect(calledUrl).not.toContain('my-google-key');
|
|
302
|
+
expect(calledUrl).not.toContain('key=');
|
|
303
|
+
expect(calledOpts.headers['x-goog-api-key']).toBe('my-google-key');
|
|
300
304
|
});
|
|
301
305
|
it('returns dry-run without modifications', async () => {
|
|
302
306
|
const oldText = 'g'.repeat(200);
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
import { describe, it, expect } from 'vitest';
|
|
2
|
+
import { findObjectArraySpans, crushEmbeddedJson, TABLE_MARKER_RE } from '../contentRouter.js';
|
|
3
|
+
import { retrieveOriginal } from '../expand.js';
|
|
4
|
+
function bigArray(prefix = '', suffix = '') {
|
|
5
|
+
const rows = Array.from({ length: 20 }, (_, i) => ({ id: i, name: `svc-${i}`, status: 'ok', region: 'us-east-1' }));
|
|
6
|
+
return `${prefix}${JSON.stringify(rows)}${suffix}`;
|
|
7
|
+
}
|
|
8
|
+
describe('findObjectArraySpans', () => {
|
|
9
|
+
it('finds a whole-text array of objects', () => {
|
|
10
|
+
const t = JSON.stringify([{ a: 1 }, { a: 2 }]);
|
|
11
|
+
const spans = findObjectArraySpans(t);
|
|
12
|
+
expect(spans.length).toBe(1);
|
|
13
|
+
expect(t.slice(spans[0].start, spans[0].end)).toBe(t);
|
|
14
|
+
});
|
|
15
|
+
it('finds an array embedded between prose', () => {
|
|
16
|
+
const t = bigArray('Here are the pods:\n', '\nDone.');
|
|
17
|
+
const spans = findObjectArraySpans(t);
|
|
18
|
+
expect(spans.length).toBe(1);
|
|
19
|
+
const sub = t.slice(spans[0].start, spans[0].end);
|
|
20
|
+
expect(sub.startsWith('[{')).toBe(true);
|
|
21
|
+
expect(sub.endsWith('}]')).toBe(true);
|
|
22
|
+
});
|
|
23
|
+
it('handles brackets inside string values without breaking', () => {
|
|
24
|
+
const t = 'x ' + JSON.stringify([{ note: 'has ] and } inside' }, { note: 'more [ [ {' }]) + ' y';
|
|
25
|
+
const spans = findObjectArraySpans(t);
|
|
26
|
+
expect(spans.length).toBe(1);
|
|
27
|
+
expect(() => JSON.parse(t.slice(spans[0].start, spans[0].end))).not.toThrow();
|
|
28
|
+
});
|
|
29
|
+
it('finds two separate embedded arrays', () => {
|
|
30
|
+
const t = `first ${JSON.stringify([{ a: 1 }, { a: 2 }])} middle ${JSON.stringify([{ b: 3 }, { b: 4 }])} end`;
|
|
31
|
+
expect(findObjectArraySpans(t).length).toBe(2);
|
|
32
|
+
});
|
|
33
|
+
it('ignores arrays of scalars (not object arrays)', () => {
|
|
34
|
+
expect(findObjectArraySpans('[1, 2, 3, 4, 5]').length).toBe(0);
|
|
35
|
+
});
|
|
36
|
+
});
|
|
37
|
+
describe('crushEmbeddedJson', () => {
|
|
38
|
+
it('crushes an array embedded in prose, preserving the surrounding text', () => {
|
|
39
|
+
const t = bigArray('POD STATUS REPORT\n', '\n(end of report)');
|
|
40
|
+
const res = crushEmbeddedJson(t);
|
|
41
|
+
expect(res.savedChars).toBeGreaterThan(0);
|
|
42
|
+
expect(res.text.startsWith('POD STATUS REPORT')).toBe(true);
|
|
43
|
+
expect(res.text.includes('(end of report)')).toBe(true);
|
|
44
|
+
expect(TABLE_MARKER_RE.test(res.text)).toBe(true);
|
|
45
|
+
});
|
|
46
|
+
it('keeps the embedded original recoverable via expand', () => {
|
|
47
|
+
const t = bigArray('label: ', '');
|
|
48
|
+
const res = crushEmbeddedJson(t);
|
|
49
|
+
const m = res.text.match(TABLE_MARKER_RE);
|
|
50
|
+
expect(m).not.toBeNull();
|
|
51
|
+
const original = retrieveOriginal(m[1]);
|
|
52
|
+
expect(original).not.toBeUndefined();
|
|
53
|
+
expect(() => JSON.parse(original)).not.toThrow();
|
|
54
|
+
});
|
|
55
|
+
it('crushes the whole-text case too (parity with jsonCrush)', () => {
|
|
56
|
+
const res = crushEmbeddedJson(bigArray());
|
|
57
|
+
expect(res.savedChars).toBeGreaterThan(0);
|
|
58
|
+
expect(TABLE_MARKER_RE.test(res.text)).toBe(true);
|
|
59
|
+
});
|
|
60
|
+
it('leaves text with no crushable arrays unchanged', () => {
|
|
61
|
+
const t = 'just a normal log line\nwith no json at all';
|
|
62
|
+
const res = crushEmbeddedJson(t);
|
|
63
|
+
expect(res.savedChars).toBe(0);
|
|
64
|
+
expect(res.text).toBe(t);
|
|
65
|
+
});
|
|
66
|
+
it('leaves a small (non-crushable) embedded array untouched, surroundings intact', () => {
|
|
67
|
+
const t = 'before [{"a":1},{"a":2}] after';
|
|
68
|
+
const res = crushEmbeddedJson(t);
|
|
69
|
+
expect(res.text).toBe(t);
|
|
70
|
+
expect(res.savedChars).toBe(0);
|
|
71
|
+
});
|
|
72
|
+
it('is deterministic', () => {
|
|
73
|
+
const t = bigArray('p ', ' s');
|
|
74
|
+
expect(crushEmbeddedJson(t).text).toBe(crushEmbeddedJson(t).text);
|
|
75
|
+
});
|
|
76
|
+
});
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
import { describe, it, expect } from 'vitest';
|
|
2
|
+
import { app } from '../server.js';
|
|
3
|
+
// Security regression: control endpoints (/squeezr/*) must not be usable from a
|
|
4
|
+
// cross-origin browser context. A malicious page open in the user's browser must
|
|
5
|
+
// not be able to change state or read control responses on http://localhost:<port>.
|
|
6
|
+
// We hit *non-existent* control paths so the guard runs but no real handler fires
|
|
7
|
+
// (403 = guard blocked; 404 = guard allowed and routing missed) — no side effects.
|
|
8
|
+
describe('control-endpoint CSRF/CORS guard', () => {
|
|
9
|
+
it('rejects a cross-origin mutation to a control endpoint with 403', async () => {
|
|
10
|
+
const res = await app.request('/squeezr/__guard_probe__', {
|
|
11
|
+
method: 'POST',
|
|
12
|
+
headers: { origin: 'http://evil.example' },
|
|
13
|
+
});
|
|
14
|
+
expect(res.status).toBe(403);
|
|
15
|
+
});
|
|
16
|
+
it('rejects a file:// (null) origin mutation to a control endpoint', async () => {
|
|
17
|
+
const res = await app.request('/squeezr/__guard_probe__', {
|
|
18
|
+
method: 'POST',
|
|
19
|
+
headers: { origin: 'null' },
|
|
20
|
+
});
|
|
21
|
+
expect(res.status).toBe(403);
|
|
22
|
+
});
|
|
23
|
+
it('allows a mutation with no Origin (curl / MCP / native over loopback)', async () => {
|
|
24
|
+
const res = await app.request('/squeezr/__guard_probe__', { method: 'POST' });
|
|
25
|
+
expect(res.status).toBe(404); // guard passed; route simply does not exist
|
|
26
|
+
});
|
|
27
|
+
it('allows a mutation from a loopback browser origin', async () => {
|
|
28
|
+
const res = await app.request('/squeezr/__guard_probe__', {
|
|
29
|
+
method: 'POST',
|
|
30
|
+
headers: { origin: 'http://localhost:8899' },
|
|
31
|
+
});
|
|
32
|
+
expect(res.status).toBe(404); // guard passed; route simply does not exist
|
|
33
|
+
});
|
|
34
|
+
it('does not guard the proxy endpoints (Cursor needs cross-origin)', async () => {
|
|
35
|
+
const res = await app.request('/v1/__proxy_probe__', {
|
|
36
|
+
method: 'POST',
|
|
37
|
+
headers: { origin: 'http://evil.example' },
|
|
38
|
+
});
|
|
39
|
+
expect(res.status).not.toBe(403);
|
|
40
|
+
});
|
|
41
|
+
it('never reflects a wildcard ACAO on control endpoints', async () => {
|
|
42
|
+
const res = await app.request('/squeezr/health', {
|
|
43
|
+
headers: { origin: 'http://evil.example' },
|
|
44
|
+
});
|
|
45
|
+
expect(res.headers.get('access-control-allow-origin')).not.toBe('*');
|
|
46
|
+
});
|
|
47
|
+
it('reflects the loopback origin (not wildcard) on control endpoints', async () => {
|
|
48
|
+
const res = await app.request('/squeezr/health', {
|
|
49
|
+
headers: { origin: 'http://127.0.0.1:1234' },
|
|
50
|
+
});
|
|
51
|
+
expect(res.headers.get('access-control-allow-origin')).toBe('http://127.0.0.1:1234');
|
|
52
|
+
expect(res.headers.get('vary')).toBe('Origin');
|
|
53
|
+
});
|
|
54
|
+
it('keeps wildcard ACAO on proxy endpoints for browser tooling', async () => {
|
|
55
|
+
const res = await app.request('/v1/__proxy_probe__', {
|
|
56
|
+
method: 'OPTIONS',
|
|
57
|
+
headers: { origin: 'http://evil.example' },
|
|
58
|
+
});
|
|
59
|
+
expect(res.headers.get('access-control-allow-origin')).toBe('*');
|
|
60
|
+
});
|
|
61
|
+
});
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { describe, it, expect } from 'vitest';
|
|
2
|
-
import { checkProxy, checkVersion, checkEnv, checkBypass, computeExitCode, formatDoctor, } from '../doctor.js';
|
|
2
|
+
import { checkProxy, checkVersion, checkEnv, checkBypass, checkSavings, computeExitCode, formatDoctor, } from '../doctor.js';
|
|
3
3
|
describe('checkProxy', () => {
|
|
4
4
|
it('passes when health is present', () => {
|
|
5
5
|
expect(checkProxy({ identity: 'squeezr', version: '1.89.0' }).status).toBe('pass');
|
|
@@ -46,6 +46,22 @@ describe('checkBypass', () => {
|
|
|
46
46
|
expect(checkBypass(undefined).status).toBe('skip');
|
|
47
47
|
});
|
|
48
48
|
});
|
|
49
|
+
describe('checkSavings', () => {
|
|
50
|
+
it('skips when the proxy is down (requests unknown)', () => {
|
|
51
|
+
expect(checkSavings(undefined, undefined).status).toBe('skip');
|
|
52
|
+
});
|
|
53
|
+
it('skips when there is no traffic yet', () => {
|
|
54
|
+
expect(checkSavings(0, 0).status).toBe('skip');
|
|
55
|
+
});
|
|
56
|
+
it('warns when traffic is flowing but nothing is being saved', () => {
|
|
57
|
+
expect(checkSavings(120, 0).status).toBe('warn');
|
|
58
|
+
});
|
|
59
|
+
it('passes when savings are flowing', () => {
|
|
60
|
+
const r = checkSavings(120, 34);
|
|
61
|
+
expect(r.status).toBe('pass');
|
|
62
|
+
expect(r.detail.includes('34')).toBe(true);
|
|
63
|
+
});
|
|
64
|
+
});
|
|
49
65
|
describe('computeExitCode', () => {
|
|
50
66
|
it('0 when all pass/skip', () => {
|
|
51
67
|
expect(computeExitCode([{ name: 'a', status: 'pass', detail: '' }, { name: 'b', status: 'skip', detail: '' }])).toBe(0);
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { describe, it, expect } from 'vitest';
|
|
2
|
-
import { crushJsonArrays, TABLE_MARKER_RE } from '../jsonCrush.js';
|
|
2
|
+
import { crushJsonArrays, TABLE_MARKER_RE, simhash32, hammingDistance } from '../jsonCrush.js';
|
|
3
3
|
import { retrieveOriginal } from '../expand.js';
|
|
4
4
|
function makeRows(n) {
|
|
5
5
|
const rows = [];
|
|
@@ -101,6 +101,49 @@ describe('crushJsonArrays', () => {
|
|
|
101
101
|
const b = crushJsonArrays(input).text;
|
|
102
102
|
expect(a).toBe(b);
|
|
103
103
|
});
|
|
104
|
+
// ── Lossy row-drop (SimHash near-dup collapse) for large arrays ──────────────
|
|
105
|
+
function manyRows(n) {
|
|
106
|
+
return Array.from({ length: n }, (_, i) => ({
|
|
107
|
+
id: i,
|
|
108
|
+
name: `svc-${i}`,
|
|
109
|
+
status: i % 2 === 0 ? 'running' : 'stopped',
|
|
110
|
+
region: 'us-east-1',
|
|
111
|
+
}));
|
|
112
|
+
}
|
|
113
|
+
it('drops near-duplicate rows in a large array but keeps the original recoverable', () => {
|
|
114
|
+
const rows = manyRows(120);
|
|
115
|
+
const input = JSON.stringify(rows);
|
|
116
|
+
const { text, savedChars } = crushJsonArrays(input);
|
|
117
|
+
expect(savedChars).toBeGreaterThan(0);
|
|
118
|
+
// the inline table shows far fewer than 120 rows
|
|
119
|
+
const bodyLines = text.split('\n').filter(l => l.includes('\t'));
|
|
120
|
+
expect(bodyLines.length).toBeLessThan(120);
|
|
121
|
+
// an omitted-rows note is present
|
|
122
|
+
expect(/of 120 rows|rows omitted|near-duplicate/i.test(text)).toBe(true);
|
|
123
|
+
// ALL 120 rows still recoverable via expand
|
|
124
|
+
const m = text.match(TABLE_MARKER_RE);
|
|
125
|
+
expect(JSON.parse(retrieveOriginal(m[1])).length).toBe(120);
|
|
126
|
+
});
|
|
127
|
+
it('ALWAYS keeps an anomalous/error row inline even amid near-duplicates', () => {
|
|
128
|
+
const rows = manyRows(120);
|
|
129
|
+
rows[63] = { id: 63, name: 'svc-63', status: 'CrashLoopBackOff', region: 'us-east-1', error: 'OOMKilled' };
|
|
130
|
+
const input = JSON.stringify(rows);
|
|
131
|
+
const { text } = crushJsonArrays(input);
|
|
132
|
+
expect(text.includes('CrashLoopBackOff')).toBe(true);
|
|
133
|
+
expect(text.includes('OOMKilled')).toBe(true);
|
|
134
|
+
});
|
|
135
|
+
it('keeps ALL rows for arrays below the lossy threshold (no row-drop)', () => {
|
|
136
|
+
const input = JSON.stringify(manyRows(30));
|
|
137
|
+
const { text } = crushJsonArrays(input);
|
|
138
|
+
expect(/rows omitted|near-duplicate/i.test(text)).toBe(false);
|
|
139
|
+
// lines after the marker (0) and header (1) are the data rows
|
|
140
|
+
const dataRows = text.split('\n').slice(2);
|
|
141
|
+
expect(dataRows.length).toBe(30);
|
|
142
|
+
});
|
|
143
|
+
it('row-drop is deterministic', () => {
|
|
144
|
+
const input = JSON.stringify(manyRows(150));
|
|
145
|
+
expect(crushJsonArrays(input).text).toBe(crushJsonArrays(input).text);
|
|
146
|
+
});
|
|
104
147
|
it('crushes a pretty-printed (indented) JSON array too', () => {
|
|
105
148
|
const input = JSON.stringify(Array.from({ length: 8 }, (_, i) => ({ id: i, name: `x${i}`, status: 'ok', region: 'us' })), null, 2);
|
|
106
149
|
const { text, savedChars } = crushJsonArrays(input);
|
|
@@ -108,3 +151,18 @@ describe('crushJsonArrays', () => {
|
|
|
108
151
|
expect(TABLE_MARKER_RE.test(text)).toBe(true);
|
|
109
152
|
});
|
|
110
153
|
});
|
|
154
|
+
describe('simhash32 / hammingDistance', () => {
|
|
155
|
+
it('identical text → identical fingerprint (distance 0)', () => {
|
|
156
|
+
expect(hammingDistance(simhash32('the quick brown fox'), simhash32('the quick brown fox'))).toBe(0);
|
|
157
|
+
});
|
|
158
|
+
it('near-identical rows are closer than unrelated ones (the property that matters)', () => {
|
|
159
|
+
// Realistic row strings (many shared tokens) — SimHash's signal is strongest here.
|
|
160
|
+
const base = simhash32('id 1 name svc-1 status running region us-east-1 node ip-10-0-1-5');
|
|
161
|
+
const near = simhash32('id 2 name svc-2 status running region us-east-1 node ip-10-0-1-5');
|
|
162
|
+
const far = simhash32('completely unrelated payload about quarterly revenue and marketing spend');
|
|
163
|
+
expect(hammingDistance(base, near)).toBeLessThan(hammingDistance(base, far));
|
|
164
|
+
});
|
|
165
|
+
it('is deterministic', () => {
|
|
166
|
+
expect(simhash32('same input here')).toBe(simhash32('same input here'));
|
|
167
|
+
});
|
|
168
|
+
});
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import { describe, it, expect } from 'vitest';
|
|
2
|
+
import { wordNgrams, echoRatio, extractAssistantTextFromSse } from '../outputSavings.js';
|
|
3
|
+
describe('wordNgrams', () => {
|
|
4
|
+
it('builds word n-grams', () => {
|
|
5
|
+
expect(wordNgrams('a b c d', 3).has('a b c')).toBe(true);
|
|
6
|
+
expect(wordNgrams('a b c d', 3).has('b c d')).toBe(true);
|
|
7
|
+
});
|
|
8
|
+
it('empty for text shorter than n', () => {
|
|
9
|
+
expect(wordNgrams('a b', 3).size).toBe(0);
|
|
10
|
+
});
|
|
11
|
+
});
|
|
12
|
+
describe('echoRatio', () => {
|
|
13
|
+
it('~1 when the output only restates context', () => {
|
|
14
|
+
const context = 'the authentication middleware refreshes the token on every request cycle';
|
|
15
|
+
const output = 'the authentication middleware refreshes the token on every request cycle';
|
|
16
|
+
expect(echoRatio(output, context)).toBeGreaterThan(0.9);
|
|
17
|
+
});
|
|
18
|
+
it('~0 when the output is entirely novel', () => {
|
|
19
|
+
const context = 'the authentication middleware refreshes the token';
|
|
20
|
+
const output = 'completely different sentence about unrelated banana harvest logistics today';
|
|
21
|
+
expect(echoRatio(output, context)).toBeLessThan(0.1);
|
|
22
|
+
});
|
|
23
|
+
it('is 0 for empty output', () => {
|
|
24
|
+
expect(echoRatio('', 'some context here at all')).toBe(0);
|
|
25
|
+
});
|
|
26
|
+
it('is between 0 and 1 for partial restating', () => {
|
|
27
|
+
const context = 'shared preamble tokens appear here in the context block';
|
|
28
|
+
const output = 'shared preamble tokens appear here plus a brand new original tail clause';
|
|
29
|
+
const r = echoRatio(output, context);
|
|
30
|
+
expect(r).toBeGreaterThan(0);
|
|
31
|
+
expect(r).toBeLessThan(1);
|
|
32
|
+
});
|
|
33
|
+
it('is deterministic', () => {
|
|
34
|
+
expect(echoRatio('a b c d e', 'a b c d e f')).toBe(echoRatio('a b c d e', 'a b c d e f'));
|
|
35
|
+
});
|
|
36
|
+
});
|
|
37
|
+
describe('extractAssistantTextFromSse', () => {
|
|
38
|
+
it('concatenates text_delta events, unescaping', () => {
|
|
39
|
+
const sse = [
|
|
40
|
+
'event: content_block_delta',
|
|
41
|
+
'data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"Hello "}}',
|
|
42
|
+
'data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"world\\n"}}',
|
|
43
|
+
].join('\n');
|
|
44
|
+
expect(extractAssistantTextFromSse(sse)).toBe('Hello world\n');
|
|
45
|
+
});
|
|
46
|
+
it('returns empty string when there are no text deltas', () => {
|
|
47
|
+
expect(extractAssistantTextFromSse('data: {"type":"message_start"}')).toBe('');
|
|
48
|
+
});
|
|
49
|
+
});
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
import { describe, it, expect } from 'vitest';
|
|
2
|
+
import { tokenize, bm25Scores, rankByRelevance } from '../relevance.js';
|
|
3
|
+
describe('tokenize', () => {
|
|
4
|
+
it('lowercases and splits on non-alphanumerics', () => {
|
|
5
|
+
expect(tokenize('Foo.Bar(baz)')).toEqual(['foo', 'bar', 'baz']);
|
|
6
|
+
});
|
|
7
|
+
it('drops 1-char noise tokens but keeps numbers/identifiers', () => {
|
|
8
|
+
expect(tokenize('a in the db_pool 42')).toEqual(['in', 'the', 'db_pool', '42']);
|
|
9
|
+
});
|
|
10
|
+
it('returns [] for empty/symbol-only input', () => {
|
|
11
|
+
expect(tokenize(' %%$$ ')).toEqual([]);
|
|
12
|
+
});
|
|
13
|
+
});
|
|
14
|
+
describe('bm25Scores', () => {
|
|
15
|
+
it('scores a matching doc above a non-matching one', () => {
|
|
16
|
+
const docs = ['the database connection failed', 'the cat sat on the mat'];
|
|
17
|
+
const s = bm25Scores('database connection', docs);
|
|
18
|
+
expect(s[0]).toBeGreaterThan(s[1]);
|
|
19
|
+
});
|
|
20
|
+
it('weights a RARE query term more than a common one (idf)', () => {
|
|
21
|
+
// "apple" appears in 1 doc (rare), "banana" in 3 (common). A doc matching the
|
|
22
|
+
// rare term should outscore a doc matching only the common one.
|
|
23
|
+
const docs = ['apple', 'banana', 'banana', 'banana'];
|
|
24
|
+
const s = bm25Scores('apple banana', docs);
|
|
25
|
+
expect(s[0]).toBeGreaterThan(s[1]);
|
|
26
|
+
});
|
|
27
|
+
it('returns all-zero scores when the query has no usable terms', () => {
|
|
28
|
+
const docs = ['anything here', 'and here'];
|
|
29
|
+
expect(bm25Scores('', docs)).toEqual([0, 0]);
|
|
30
|
+
expect(bm25Scores('%%%', docs)).toEqual([0, 0]);
|
|
31
|
+
});
|
|
32
|
+
it('gives a non-matching doc a score of 0', () => {
|
|
33
|
+
const docs = ['relevant match here', 'totally unrelated'];
|
|
34
|
+
const s = bm25Scores('relevant', docs);
|
|
35
|
+
expect(s[1]).toBe(0);
|
|
36
|
+
expect(s[0]).toBeGreaterThan(0);
|
|
37
|
+
});
|
|
38
|
+
it('does not let a long repetitive doc dominate unboundedly (length normalization)', () => {
|
|
39
|
+
const short = 'error';
|
|
40
|
+
const long = ('error ' + 'filler '.repeat(200)).trim();
|
|
41
|
+
const s = bm25Scores('error', [short, long]);
|
|
42
|
+
// the concise doc that is mostly the query term should score at least as high
|
|
43
|
+
expect(s[0]).toBeGreaterThanOrEqual(s[1]);
|
|
44
|
+
});
|
|
45
|
+
it('is deterministic', () => {
|
|
46
|
+
const docs = ['alpha beta', 'beta gamma', 'gamma delta'];
|
|
47
|
+
expect(bm25Scores('beta gamma', docs)).toEqual(bm25Scores('beta gamma', docs));
|
|
48
|
+
});
|
|
49
|
+
it('produces one score per doc', () => {
|
|
50
|
+
const docs = ['a b c', 'd e f', 'g h i'];
|
|
51
|
+
expect(bm25Scores('a', docs).length).toBe(3);
|
|
52
|
+
});
|
|
53
|
+
});
|
|
54
|
+
describe('rankByRelevance', () => {
|
|
55
|
+
it('returns document indices sorted by descending relevance', () => {
|
|
56
|
+
const docs = ['no match', 'strong error error match', 'weak error'];
|
|
57
|
+
const order = rankByRelevance('error', docs);
|
|
58
|
+
expect(order[0]).toBe(1); // most "error" hits
|
|
59
|
+
expect(order[order.length - 1]).toBe(0); // no match last
|
|
60
|
+
});
|
|
61
|
+
it('is a stable permutation of all indices', () => {
|
|
62
|
+
const docs = ['a', 'b', 'c', 'd'];
|
|
63
|
+
const order = rankByRelevance('a b', docs);
|
|
64
|
+
expect([...order].sort((x, y) => x - y)).toEqual([0, 1, 2, 3]);
|
|
65
|
+
});
|
|
66
|
+
});
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import { describe, it, expect } from 'vitest';
|
|
2
|
+
import { mkdtempSync, writeFileSync, readFileSync, rmSync, statSync } from 'node:fs';
|
|
3
|
+
import { tmpdir } from 'node:os';
|
|
4
|
+
import { join } from 'node:path';
|
|
5
|
+
import { secureKeyFile } from '../codexMitm.js';
|
|
6
|
+
// Security regression: the CA private key must be locked to the current user.
|
|
7
|
+
// 0o600 is a no-op on Windows, so secureKeyFile applies a per-OS restriction
|
|
8
|
+
// (icacls on Windows, chmod on POSIX). It must never throw — a key we cannot
|
|
9
|
+
// secure is warned about, not fatal — and must leave the file readable by us.
|
|
10
|
+
describe('secureKeyFile', () => {
|
|
11
|
+
it('secures a key file without throwing and keeps it owner-readable', () => {
|
|
12
|
+
const dir = mkdtempSync(join(tmpdir(), 'squeezr-key-'));
|
|
13
|
+
const keyPath = join(dir, 'ca.key');
|
|
14
|
+
writeFileSync(keyPath, 'PRIVATE-KEY-MATERIAL', { mode: 0o600 });
|
|
15
|
+
try {
|
|
16
|
+
expect(() => secureKeyFile(keyPath)).not.toThrow();
|
|
17
|
+
expect(readFileSync(keyPath, 'utf-8')).toBe('PRIVATE-KEY-MATERIAL');
|
|
18
|
+
if (process.platform !== 'win32') {
|
|
19
|
+
// On POSIX the mode must be owner-only (no group/other bits).
|
|
20
|
+
expect(statSync(keyPath).mode & 0o077).toBe(0);
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
finally {
|
|
24
|
+
rmSync(dir, { recursive: true, force: true });
|
|
25
|
+
}
|
|
26
|
+
});
|
|
27
|
+
});
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import { describe, it, expect } from 'vitest';
|
|
2
|
+
import { readFileSync } from 'node:fs';
|
|
3
|
+
import { fileURLToPath } from 'node:url';
|
|
4
|
+
// Security regression: the main proxy holds the user's API keys and exposes
|
|
5
|
+
// state-changing control endpoints, so it MUST bind to loopback only and never
|
|
6
|
+
// be reachable from the LAN. This guards against silently reverting that fix.
|
|
7
|
+
describe('main proxy loopback binding', () => {
|
|
8
|
+
const src = readFileSync(fileURLToPath(new URL('../index.ts', import.meta.url)), 'utf-8');
|
|
9
|
+
it('binds httpServer.listen to 127.0.0.1', () => {
|
|
10
|
+
expect(src).toMatch(/httpServer\.listen\(\s*PORT\s*,\s*['"]127\.0\.0\.1['"]/);
|
|
11
|
+
});
|
|
12
|
+
it('never binds the main proxy to all interfaces', () => {
|
|
13
|
+
expect(src).not.toMatch(/httpServer\.listen\(\s*PORT\s*,\s*['"]0\.0\.0\.0['"]/);
|
|
14
|
+
expect(src).not.toMatch(/httpServer\.listen\(\s*PORT\s*,\s*['"]::['"]/);
|
|
15
|
+
// A two-arg listen(PORT, cb) would bind all interfaces by default — forbid it.
|
|
16
|
+
expect(src).not.toMatch(/httpServer\.listen\(\s*PORT\s*,\s*\(/);
|
|
17
|
+
});
|
|
18
|
+
});
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import { describe, it, expect } from 'vitest';
|
|
2
|
+
import { Stats } from '../stats.js';
|
|
3
|
+
// recordEcho / recordShaping only mutate in-memory counters (persistence happens on the
|
|
4
|
+
// main record() path), so these tests never touch ~/.squeezr. We assert relative deltas
|
|
5
|
+
// so a pre-existing stats.json baseline doesn't matter.
|
|
6
|
+
describe('Stats output metrics', () => {
|
|
7
|
+
it('recordEcho updates sample count and average echo', () => {
|
|
8
|
+
const s = new Stats();
|
|
9
|
+
const before = s.summary().output;
|
|
10
|
+
s.recordEcho(0.2);
|
|
11
|
+
s.recordEcho(0.4);
|
|
12
|
+
const after = s.summary().output;
|
|
13
|
+
expect(after.echo_samples).toBe(before.echo_samples + 2);
|
|
14
|
+
expect(after.avg_echo_pct).toBeGreaterThanOrEqual(0);
|
|
15
|
+
expect(after.avg_echo_pct).toBeLessThanOrEqual(100);
|
|
16
|
+
});
|
|
17
|
+
it('recordShaping counts only the true flags', () => {
|
|
18
|
+
const s = new Stats();
|
|
19
|
+
const before = s.summary().output;
|
|
20
|
+
s.recordShaping(true, false); // steered only
|
|
21
|
+
s.recordShaping(true, true); // steered + effort
|
|
22
|
+
s.recordShaping(false, false); // neither
|
|
23
|
+
const after = s.summary().output;
|
|
24
|
+
expect(after.steered).toBe(before.steered + 2);
|
|
25
|
+
expect(after.effort_lowered).toBe(before.effort_lowered + 1);
|
|
26
|
+
});
|
|
27
|
+
it('avg echo reflects the mean of the samples added', () => {
|
|
28
|
+
const s = new Stats();
|
|
29
|
+
// add a large batch so any baseline is diluted toward our known mean
|
|
30
|
+
for (let i = 0; i < 200; i++)
|
|
31
|
+
s.recordEcho(0.3);
|
|
32
|
+
expect(Math.abs(s.summary().output.avg_echo_pct - 30)).toBeLessThan(10);
|
|
33
|
+
});
|
|
34
|
+
it('recordBypassed counts separately and never touches processed/saved totals', () => {
|
|
35
|
+
const s = new Stats();
|
|
36
|
+
const before = s.summary();
|
|
37
|
+
s.recordBypassed();
|
|
38
|
+
s.recordBypassed();
|
|
39
|
+
const after = s.summary();
|
|
40
|
+
expect(after.bypassed_requests).toBe(before.bypassed_requests + 2);
|
|
41
|
+
// bypassed traffic must NOT inflate processed or saved
|
|
42
|
+
expect(after.total_original_chars).toBe(before.total_original_chars);
|
|
43
|
+
expect(after.total_saved_chars).toBe(before.total_saved_chars);
|
|
44
|
+
expect(after.requests).toBe(before.requests);
|
|
45
|
+
});
|
|
46
|
+
});
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { describe, it, expect } from 'vitest';
|
|
2
|
-
import { crushText, normalizeLineKey } from '../textCrusher.js';
|
|
2
|
+
import { crushText, normalizeLineKey, wordShingles, jaccard } from '../textCrusher.js';
|
|
3
3
|
describe('normalizeLineKey', () => {
|
|
4
4
|
it('collapses numbers, hex and timestamps so near-duplicates share a key', () => {
|
|
5
5
|
const a = normalizeLineKey('2026-07-21T10:00:01Z downloaded chunk 12 (0x1a2b)');
|
|
@@ -68,4 +68,66 @@ describe('crushText', () => {
|
|
|
68
68
|
const res = crushText(repetitiveLog(300));
|
|
69
69
|
expect(/\[\d+ lines omitted/.test(res.text)).toBe(true);
|
|
70
70
|
});
|
|
71
|
+
it('with a query, keeps task-relevant lines that a blind pass would drop', () => {
|
|
72
|
+
// 120 noise lines + one line mentioning the task keyword, positioned in the middle
|
|
73
|
+
// (not an anchor) so only relevance can save it.
|
|
74
|
+
const lines = [];
|
|
75
|
+
for (let i = 0; i < 60; i++)
|
|
76
|
+
lines.push(`background chore ${i} finished`);
|
|
77
|
+
lines.push('updated the authentication middleware token refresh logic');
|
|
78
|
+
for (let i = 0; i < 60; i++)
|
|
79
|
+
lines.push(`background chore ${i + 60} finished`);
|
|
80
|
+
const input = lines.join('\n');
|
|
81
|
+
const withQuery = crushText(input, { maxLines: 20, headKeep: 3, tailKeep: 3, query: 'authentication middleware token' });
|
|
82
|
+
expect(withQuery.text.includes('authentication middleware token refresh')).toBe(true);
|
|
83
|
+
});
|
|
84
|
+
it('stays deterministic for a fixed query', () => {
|
|
85
|
+
const input = repetitiveLog(300);
|
|
86
|
+
const a = crushText(input, { query: 'record ms' }).text;
|
|
87
|
+
const b = crushText(input, { query: 'record ms' }).text;
|
|
88
|
+
expect(a).toBe(b);
|
|
89
|
+
});
|
|
90
|
+
it('collapses REWORDED near-duplicate lines (not just number-varied)', () => {
|
|
91
|
+
// Two wordings of the same idea, no numbers, no signal keywords → exact-key dedup
|
|
92
|
+
// would keep both; shingle near-dup should collapse them.
|
|
93
|
+
const a = 'the worker finished processing the batch quickly and moved on';
|
|
94
|
+
const b = 'the worker completed processing the batch quickly and moved on';
|
|
95
|
+
// Unique anchor lines at head/tail (no "worker") so the variants live in the MIDDLE,
|
|
96
|
+
// where the budget-fill + shingle collapse applies.
|
|
97
|
+
const lines = ['ALPHA HEADER ONE', 'ALPHA HEADER TWO', 'ALPHA HEADER THREE'];
|
|
98
|
+
for (let i = 0; i < 100; i++)
|
|
99
|
+
lines.push(i % 2 === 0 ? a : b);
|
|
100
|
+
lines.push('OMEGA FOOTER ONE', 'OMEGA FOOTER TWO', 'OMEGA FOOTER THREE');
|
|
101
|
+
const res = crushText(lines.join('\n'), { maxLines: 30, headKeep: 3, tailKeep: 3 });
|
|
102
|
+
const distinct = new Set(res.text.split('\n').filter(l => l.includes('worker')));
|
|
103
|
+
// both wordings collapse to (at most) one representative
|
|
104
|
+
expect(distinct.size).toBeLessThanOrEqual(1);
|
|
105
|
+
});
|
|
106
|
+
it('does NOT collapse genuinely different lines', () => {
|
|
107
|
+
const lines = ['UNIQUE HEADER'];
|
|
108
|
+
for (let i = 0; i < 30; i++) {
|
|
109
|
+
lines.push('alpha subsystem initialised the primary cache layer');
|
|
110
|
+
lines.push('gamma module rejected the inbound websocket handshake');
|
|
111
|
+
}
|
|
112
|
+
lines.push('UNIQUE FOOTER');
|
|
113
|
+
const res = crushText(lines.join('\n'), { maxLines: 40, headKeep: 2, tailKeep: 2 });
|
|
114
|
+
expect(res.text.includes('alpha subsystem')).toBe(true);
|
|
115
|
+
expect(res.text.includes('gamma module')).toBe(true);
|
|
116
|
+
});
|
|
117
|
+
});
|
|
118
|
+
describe('wordShingles / jaccard', () => {
|
|
119
|
+
it('reworded lines share most shingles (high jaccard)', () => {
|
|
120
|
+
const a = wordShingles('the worker finished processing the batch quickly');
|
|
121
|
+
const b = wordShingles('the worker completed processing the batch quickly');
|
|
122
|
+
expect(jaccard(a, b)).toBeGreaterThan(0.4);
|
|
123
|
+
});
|
|
124
|
+
it('unrelated lines share few shingles (low jaccard)', () => {
|
|
125
|
+
const a = wordShingles('database connection pool exhausted');
|
|
126
|
+
const b = wordShingles('the cat sat on the warm mat');
|
|
127
|
+
expect(jaccard(a, b)).toBeLessThan(0.1);
|
|
128
|
+
});
|
|
129
|
+
it('identical lines have jaccard 1', () => {
|
|
130
|
+
const a = wordShingles('exactly the same words here now');
|
|
131
|
+
expect(jaccard(a, a)).toBe(1);
|
|
132
|
+
});
|
|
71
133
|
});
|
package/dist/codexMitm.d.ts
CHANGED
package/dist/codexMitm.js
CHANGED
|
@@ -4,7 +4,8 @@ import http from 'node:http';
|
|
|
4
4
|
import https from 'node:https';
|
|
5
5
|
import fs from 'node:fs';
|
|
6
6
|
import crypto from 'node:crypto';
|
|
7
|
-
import {
|
|
7
|
+
import { execFileSync } from 'node:child_process';
|
|
8
|
+
import { homedir, userInfo } from 'node:os';
|
|
8
9
|
import { join } from 'node:path';
|
|
9
10
|
import forge from 'node-forge';
|
|
10
11
|
import { config } from './config.js';
|
|
@@ -15,6 +16,26 @@ const CA_CERT_PATH = join(CA_DIR, 'ca.crt');
|
|
|
15
16
|
export const BUNDLE_PATH = join(CA_DIR, 'bundle.crt');
|
|
16
17
|
export const MITM_PORT = config.mitmPort;
|
|
17
18
|
// ── CA generation ─────────────────────────────────────────────────────────────
|
|
19
|
+
// POSIX permission bits (0o600) are ignored on Windows, so the CA private key
|
|
20
|
+
// would be left readable by other accounts. Lock it down per-OS: an ACL that
|
|
21
|
+
// grants only the current user on Windows, chmod 0o600 on POSIX. A private key
|
|
22
|
+
// we cannot secure is a real risk, so failure is surfaced loudly, never silent.
|
|
23
|
+
export function secureKeyFile(path) {
|
|
24
|
+
if (process.platform === 'win32') {
|
|
25
|
+
const user = process.env.USERNAME || userInfo().username;
|
|
26
|
+
try {
|
|
27
|
+
// /inheritance:r drops inherited ACEs; /grant:r sets the current user as
|
|
28
|
+
// the only principal with Full control.
|
|
29
|
+
execFileSync('icacls', [path, '/inheritance:r', '/grant:r', `${user}:F`], { stdio: 'ignore' });
|
|
30
|
+
}
|
|
31
|
+
catch (err) {
|
|
32
|
+
console.warn(`[squeezr/mitm] WARNING: could not restrict permissions on ${path} — ` +
|
|
33
|
+
`${err.message}. Secure this private key manually.`);
|
|
34
|
+
}
|
|
35
|
+
return;
|
|
36
|
+
}
|
|
37
|
+
fs.chmodSync(path, 0o600);
|
|
38
|
+
}
|
|
18
39
|
function ensureCA() {
|
|
19
40
|
const certsExist = fs.existsSync(CA_KEY_PATH) && fs.existsSync(CA_CERT_PATH);
|
|
20
41
|
if (!certsExist) {
|
|
@@ -35,6 +56,7 @@ function ensureCA() {
|
|
|
35
56
|
]);
|
|
36
57
|
cert.sign(keys.privateKey, forge.md.sha256.create());
|
|
37
58
|
fs.writeFileSync(CA_KEY_PATH, forge.pki.privateKeyToPem(keys.privateKey), { mode: 0o600 });
|
|
59
|
+
secureKeyFile(CA_KEY_PATH);
|
|
38
60
|
fs.writeFileSync(CA_CERT_PATH, forge.pki.certificateToPem(cert), { mode: 0o644 });
|
|
39
61
|
console.log(`[squeezr/mitm] CA generated → ${CA_CERT_PATH}`);
|
|
40
62
|
}
|
|
@@ -438,7 +460,9 @@ export function startMitmProxy() {
|
|
|
438
460
|
if (err.code !== 'EADDRINUSE')
|
|
439
461
|
console.error('[squeezr/mitm] error:', err.message);
|
|
440
462
|
});
|
|
441
|
-
|
|
463
|
+
// Loopback only: the MITM proxy terminates TLS for the user's traffic and must
|
|
464
|
+
// never be reachable from the LAN (matches the other listeners).
|
|
465
|
+
mitmServer.listen(MITM_PORT, '127.0.0.1', () => {
|
|
442
466
|
console.log(`[squeezr/mitm] HTTPS proxy on http://localhost:${MITM_PORT}`);
|
|
443
467
|
});
|
|
444
468
|
}
|