pennyrouter 0.3.43 → 0.3.44
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/launch.js +20 -0
- package/src/launch.test.js +16 -1
- package/standalone/govcloud-rates.json +106 -0
- package/standalone/pricing.py +9 -31
- package/standalone/web-demo/README.md +3 -5
- package/standalone/web-demo/claude-meter.mjs +201 -53
- package/standalone/web-demo/compare.html +1428 -0
- package/standalone/web-demo/gallery-submit.mjs +70 -0
- package/standalone/web-demo/server.mjs +161 -81
- package/standalone/web-demo/telemetry.mjs +71 -0
- package/standalone/web-demo/terminal-preview.html +0 -384
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
import { readFile } from 'node:fs/promises';
|
|
2
|
+
import { homedir } from 'node:os';
|
|
3
|
+
import { join } from 'node:path';
|
|
4
|
+
|
|
5
|
+
// Publishes a finished local comparison to the PennyRouter gallery for review. The CLI's saved
|
|
6
|
+
// Penny key authenticates the upload server to server, so it never reaches the browser.
|
|
7
|
+
|
|
8
|
+
export const appUrl = (process.env.PENNYROUTER_APP_URL || 'https://pennyrouter.com').replace(/\/+$/, '');
|
|
9
|
+
export const maxVideoBytes = 100 * 1024 * 1024;
|
|
10
|
+
|
|
11
|
+
// Mirrors pennyConfigPath() and activePennyProfile() in packages/pennyrouter-cli/src/launch.js.
|
|
12
|
+
function configPath() {
|
|
13
|
+
if (process.platform === 'win32') {
|
|
14
|
+
return join(process.env.APPDATA || process.env.LOCALAPPDATA || homedir(), 'PennyRouter', 'config.json');
|
|
15
|
+
}
|
|
16
|
+
return join(process.env.XDG_CONFIG_HOME || join(homedir(), '.config'), 'pennyrouter', 'config.json');
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
export async function savedPennyKey() {
|
|
20
|
+
try {
|
|
21
|
+
const config = JSON.parse(await readFile(configPath(), 'utf8'));
|
|
22
|
+
return config.profiles?.[config.active_profile || 'cloud']?.api_key || null;
|
|
23
|
+
} catch {
|
|
24
|
+
return null;
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/** Sends the report, the chosen pages, and an optional video; returns the new entry's id. A
|
|
29
|
+
* report signed by its demo sandbox (the subscription launcher) carries that signature. */
|
|
30
|
+
export async function publishToGallery(report, form, { signature = '', reportNonce = '' } = {}) {
|
|
31
|
+
const key = await savedPennyKey();
|
|
32
|
+
if (!key) throw new Error('Connect PennyRouter first: run `npx pennyrouter install`, then try again.');
|
|
33
|
+
const outgoing = new FormData();
|
|
34
|
+
outgoing.set('report', report);
|
|
35
|
+
outgoing.set('signature', signature);
|
|
36
|
+
outgoing.set('report_nonce', reportNonce);
|
|
37
|
+
outgoing.set('meta', String(form.get('meta') ?? ''));
|
|
38
|
+
for (const side of ['native', 'penny']) {
|
|
39
|
+
const page = form.get(side);
|
|
40
|
+
if (!(page instanceof Blob)) throw new Error('Choose a native page and a Penny page.');
|
|
41
|
+
outgoing.set(side, page, `${side}.html`);
|
|
42
|
+
}
|
|
43
|
+
const cover = form.get('cover');
|
|
44
|
+
if (cover instanceof Blob && cover.size) outgoing.set('cover', cover, 'cover');
|
|
45
|
+
const response = await fetch(`${appUrl}/api/gallery/submissions`, {
|
|
46
|
+
method: 'POST', headers: { authorization: `Bearer ${key}` }, body: outgoing,
|
|
47
|
+
signal: AbortSignal.timeout(60_000),
|
|
48
|
+
});
|
|
49
|
+
const body = await response.json().catch(() => ({}));
|
|
50
|
+
if (!response.ok) throw new Error(body.error || `The gallery did not accept the submission (${response.status}).`);
|
|
51
|
+
|
|
52
|
+
const video = form.get('video');
|
|
53
|
+
if (!body.videoUpload || !(video instanceof Blob) || !video.size) return { id: body.id };
|
|
54
|
+
try {
|
|
55
|
+
if (video.size > maxVideoBytes) throw new Error('too large');
|
|
56
|
+
const upload = await fetch(body.videoUpload.signedUrl, {
|
|
57
|
+
method: 'PUT', headers: { 'content-type': body.videoType }, body: video,
|
|
58
|
+
signal: AbortSignal.timeout(10 * 60_000),
|
|
59
|
+
});
|
|
60
|
+
if (!upload.ok) throw new Error(`upload ${upload.status}`);
|
|
61
|
+
const confirm = await fetch(`${appUrl}/api/gallery/submissions/${body.id}/video`, {
|
|
62
|
+
method: 'POST', headers: { authorization: `Bearer ${key}`, 'content-type': 'application/json' },
|
|
63
|
+
body: JSON.stringify({ video_type: body.videoType }), signal: AbortSignal.timeout(30_000),
|
|
64
|
+
});
|
|
65
|
+
if (!confirm.ok) throw new Error(`confirm ${confirm.status}`);
|
|
66
|
+
return { id: body.id };
|
|
67
|
+
} catch {
|
|
68
|
+
return { id: body.id, videoError: 'Your submission was saved, but its video could not be uploaded.' };
|
|
69
|
+
}
|
|
70
|
+
}
|
|
@@ -1,15 +1,18 @@
|
|
|
1
1
|
import { createServer } from 'node:http';
|
|
2
2
|
import { randomUUID } from 'node:crypto';
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
3
|
+
import { mkdir, readFile, stat } from 'node:fs/promises';
|
|
4
|
+
import { execFile } from 'node:child_process';
|
|
5
5
|
import { join, dirname, resolve } from 'node:path';
|
|
6
|
-
import { homedir
|
|
6
|
+
import { homedir } from 'node:os';
|
|
7
7
|
import { fileURLToPath } from 'node:url';
|
|
8
8
|
import { promisify } from 'node:util';
|
|
9
9
|
import { WebSocketServer } from 'ws';
|
|
10
10
|
import pty from 'node-pty';
|
|
11
|
-
import { ClaudeMeter, CodexMeter, saveReport } from './claude-meter.mjs';
|
|
11
|
+
import { ClaudeMeter, CodexMeter, galleryReport, pricingProfiles, saveReport } from './claude-meter.mjs';
|
|
12
|
+
import { publishToGallery } from './gallery-submit.mjs';
|
|
12
13
|
import { listArtifacts, readArtifact } from './artifacts.mjs';
|
|
14
|
+
import { readTelemetry, telemetryEnv, telemetryRoute } from './telemetry.mjs';
|
|
15
|
+
import { Readable } from 'node:stream';
|
|
13
16
|
|
|
14
17
|
const here = dirname(fileURLToPath(import.meta.url));
|
|
15
18
|
const compareDir = dirname(here);
|
|
@@ -18,16 +21,14 @@ const args = process.argv.slice(2);
|
|
|
18
21
|
const claude = args.includes('--claude');
|
|
19
22
|
const harnessArg = args.find((arg) => /^--harness=(claude|codex)$/.test(arg));
|
|
20
23
|
const startupHarness = harnessArg?.slice('--harness='.length) || 'claude';
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
const compare = args.includes('--compare') || nodeMeter;
|
|
24
|
-
const pythonCompare = compare && !nodeMeter && process.platform !== 'win32';
|
|
24
|
+
// --compare-node is the earlier name for --compare, kept so existing commands still work.
|
|
25
|
+
const compare = args.includes('--compare') || args.includes('--compare-node');
|
|
25
26
|
const portArg = args.find((arg) => /^--port=\d+$/.test(arg));
|
|
26
27
|
const port = Number(portArg?.slice(7) ?? 8789);
|
|
27
28
|
if (args.some((arg) => !['--claude', '--compare', '--compare-node', '--harness=claude', '--harness=codex', portArg].includes(arg)) ||
|
|
28
29
|
(claude && compare) || (harnessArg && !compare) ||
|
|
29
30
|
(args.includes('--harness=claude') && args.includes('--harness=codex')) || port < 1 || port > 65535) {
|
|
30
|
-
console.error('Usage: npm start -- [--claude | --compare [--harness=claude|codex]
|
|
31
|
+
console.error('Usage: npm start -- [--claude | --compare [--harness=claude|codex]] [--port=8789]');
|
|
31
32
|
process.exit(1);
|
|
32
33
|
}
|
|
33
34
|
|
|
@@ -35,15 +36,46 @@ const files = new Map([
|
|
|
35
36
|
['/demo-artifact-preview', ['artifact-preview.html', 'text/html; charset=utf-8']],
|
|
36
37
|
['/terminal-unicode.js', ['terminal-unicode.js', 'text/javascript; charset=utf-8']],
|
|
37
38
|
['/addon-unicode11.js', ['node_modules/@xterm/addon-unicode11/lib/addon-unicode11.js', 'text/javascript; charset=utf-8']],
|
|
38
|
-
['/', ['
|
|
39
|
+
['/', ['compare.html', 'text/html; charset=utf-8']],
|
|
39
40
|
['/xterm.css', ['node_modules/@xterm/xterm/css/xterm.css', 'text/css; charset=utf-8']],
|
|
40
41
|
['/xterm.js', ['node_modules/@xterm/xterm/lib/xterm.js', 'text/javascript; charset=utf-8']],
|
|
41
42
|
['/addon-fit.js', ['node_modules/@xterm/addon-fit/lib/addon-fit.js', 'text/javascript; charset=utf-8']],
|
|
42
43
|
]);
|
|
43
44
|
const artifactSessions = new Map();
|
|
45
|
+
const telemetryMeters = new Map();
|
|
46
|
+
// Finished comparisons that may still be published, by one-time publish token.
|
|
47
|
+
const publishable = new Map();
|
|
48
|
+
const publishWindowMs = 60 * 60_000;
|
|
49
|
+
// `npm start` runs from this package's folder; INIT_CWD is where it was invoked.
|
|
50
|
+
const launchDirectory = process.env.INIT_CWD || process.cwd();
|
|
51
|
+
|
|
52
|
+
// Shown in the page header, as the hosted demo shows its checkpoint's versions.
|
|
53
|
+
async function harnessVersion(command) {
|
|
54
|
+
try {
|
|
55
|
+
const { stdout } = await execFileAsync(command, ['--version'],
|
|
56
|
+
{ timeout: 5000, shell: process.platform === 'win32', windowsHide: true });
|
|
57
|
+
return stdout.match(/\d+\.\d+\.\d+/)?.[0] || null;
|
|
58
|
+
} catch { return null; }
|
|
59
|
+
}
|
|
60
|
+
const harnessVersions = Promise.all([harnessVersion('claude'), harnessVersion('codex')])
|
|
61
|
+
.then(([claudeVersion, codexVersion]) => ({ claude: claudeVersion, codex: codexVersion }));
|
|
44
62
|
|
|
45
63
|
const server = createServer(async (request, response) => {
|
|
46
64
|
const url = new URL(request.url, 'http://localhost');
|
|
65
|
+
if (url.pathname === '/gallery/submit') {
|
|
66
|
+
await submitToGallery(request, response);
|
|
67
|
+
return;
|
|
68
|
+
}
|
|
69
|
+
const telemetryMeter = telemetryMeters.get(url.pathname);
|
|
70
|
+
if (telemetryMeter && request.method === 'POST') {
|
|
71
|
+
try {
|
|
72
|
+
telemetryMeter.ingestTelemetry(await readTelemetry(request));
|
|
73
|
+
response.writeHead(200, { 'Content-Type': 'application/json' }).end('{}');
|
|
74
|
+
} catch {
|
|
75
|
+
response.writeHead(400).end();
|
|
76
|
+
}
|
|
77
|
+
return;
|
|
78
|
+
}
|
|
47
79
|
if (['/artifacts', '/artifact'].includes(url.pathname)) {
|
|
48
80
|
const roots = artifactSessions.get(request.headers.authorization?.replace(/^Bearer /, ''));
|
|
49
81
|
const root = roots?.get(url.searchParams.get('side'));
|
|
@@ -64,10 +96,11 @@ const server = createServer(async (request, response) => {
|
|
|
64
96
|
try {
|
|
65
97
|
let body = await readFile(join(here, file[0]));
|
|
66
98
|
if (request.url === '/') {
|
|
99
|
+
const config = { mode: 'local', compare, defaultHarness: startupHarness,
|
|
100
|
+
folders: { left: join(launchDirectory, 'native'), right: join(launchDirectory, 'penny') },
|
|
101
|
+
versions: await harnessVersions };
|
|
67
102
|
body = body.toString('utf8')
|
|
68
|
-
.replace('
|
|
69
|
-
.replace('__PENNY_COMPARE__', JSON.stringify(compare))
|
|
70
|
-
.replace('__PENNY_DEFAULT_HARNESS__', JSON.stringify(startupHarness));
|
|
103
|
+
.replace('__COMPARE_CONFIG__', JSON.stringify(config).replaceAll('<', '\\u003c'));
|
|
71
104
|
}
|
|
72
105
|
response.writeHead(200, { 'Content-Type': file[1], 'Cache-Control': 'no-store' }).end(body);
|
|
73
106
|
} catch (error) {
|
|
@@ -76,6 +109,32 @@ const server = createServer(async (request, response) => {
|
|
|
76
109
|
}
|
|
77
110
|
});
|
|
78
111
|
|
|
112
|
+
async function submitToGallery(request, response) {
|
|
113
|
+
const reply = (status, body) => response.writeHead(status, { 'Content-Type': 'application/json',
|
|
114
|
+
'Cache-Control': 'no-store' }).end(JSON.stringify(body));
|
|
115
|
+
const allowedOrigins = new Set([`http://127.0.0.1:${port}`, `http://localhost:${port}`]);
|
|
116
|
+
const token = request.headers.authorization?.replace(/^Bearer /, '');
|
|
117
|
+
const report = publishable.get(token);
|
|
118
|
+
if (request.method !== 'POST' || !allowedOrigins.has(request.headers.origin) || !report) {
|
|
119
|
+
reply(403, { error: 'This comparison can no longer be published. Finish a new one to publish it.' });
|
|
120
|
+
return;
|
|
121
|
+
}
|
|
122
|
+
if (Number(request.headers['content-length'] || 0) > 120 * 1024 * 1024) {
|
|
123
|
+
reply(413, { error: 'The submission is too large.' });
|
|
124
|
+
return;
|
|
125
|
+
}
|
|
126
|
+
try {
|
|
127
|
+
const form = await new Request('http://localhost/gallery/submit', { method: 'POST',
|
|
128
|
+
headers: { 'content-type': request.headers['content-type'] || '' },
|
|
129
|
+
body: Readable.toWeb(request), duplex: 'half' }).formData();
|
|
130
|
+
const result = await publishToGallery(report, form);
|
|
131
|
+
publishable.delete(token);
|
|
132
|
+
reply(201, result);
|
|
133
|
+
} catch (error) {
|
|
134
|
+
reply(400, { error: error.message || 'Could not publish to the gallery.' });
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
|
|
79
138
|
const wss = new WebSocketServer({ noServer: true, maxPayload: 64 * 1024 });
|
|
80
139
|
server.on('upgrade', (request, socket, head) => {
|
|
81
140
|
const allowedOrigins = new Set([`http://127.0.0.1:${port}`, `http://localhost:${port}`]);
|
|
@@ -87,7 +146,26 @@ server.on('upgrade', (request, socket, head) => {
|
|
|
87
146
|
wss.handleUpgrade(request, socket, head, (ws) => wss.emit('connection', ws));
|
|
88
147
|
});
|
|
89
148
|
|
|
149
|
+
// Launched from inside a Claude Code session (an integrated terminal or a `!` command), this
|
|
150
|
+
// process inherits that session's identity. A pane that keeps it attaches to the parent
|
|
151
|
+
// session and writes no transcript of its own, which leaves its meter empty.
|
|
152
|
+
const inheritedSessionKeys = /^(CLAUDECODE|CLAUDE_PID|CLAUDE_EFFORT|CLAUDE_CODE_(SESSION_ID|CHILD_SESSION|SESSION_ATTENDED|BRIDGE_SESSION_ID|MESSAGING_SOCKET|MESSAGING_TOKEN|ENTRYPOINT|EXECPATH))$/;
|
|
153
|
+
function paneEnvironment() {
|
|
154
|
+
return Object.fromEntries(Object.entries(process.env).filter(([key]) => !inheritedSessionKeys.test(key)));
|
|
155
|
+
}
|
|
156
|
+
|
|
90
157
|
const variants = new Set(['claude', 'penny-claude', 'codex', 'penny-codex']);
|
|
158
|
+
const providers = ['native', 'bedrock', 'foundry', 'vertex'];
|
|
159
|
+
const proxies = ['none', 'litellm', 'bifrost', 'kong', 'portkey', 'other'];
|
|
160
|
+
|
|
161
|
+
// Pricing changes how both panes are costed. Provider and proxy are recorded with the report.
|
|
162
|
+
function validateEnvironment(environment = {}) {
|
|
163
|
+
const { pricing = 'standard', provider = 'native', proxy = 'none' } = environment;
|
|
164
|
+
if (!pricingProfiles.includes(pricing) || !providers.includes(provider) || !proxies.includes(proxy)) {
|
|
165
|
+
throw new Error('Invalid environment. Choose a listed pricing schedule, provider, and proxy.');
|
|
166
|
+
}
|
|
167
|
+
return { pricing, provider, proxy };
|
|
168
|
+
}
|
|
91
169
|
const safeModel = /^[A-Za-z0-9._/-]{1,160}$/;
|
|
92
170
|
|
|
93
171
|
async function validateConfig(config) {
|
|
@@ -101,22 +179,32 @@ async function validateConfig(config) {
|
|
|
101
179
|
!claudeModels.includes(config.model) && !config.model.startsWith('claude-'))) {
|
|
102
180
|
throw new Error('Invalid compare settings. Choose Claude or Codex, a directory, and a valid model name.');
|
|
103
181
|
}
|
|
104
|
-
const expanded = config.cwd
|
|
182
|
+
const expanded = /^~[\\/]/.test(config.cwd) ? join(homedir(), config.cwd.slice(2)) : config.cwd;
|
|
105
183
|
const cwd = resolve(expanded);
|
|
106
|
-
|
|
107
|
-
|
|
184
|
+
const existing = await stat(cwd).catch(() => null);
|
|
185
|
+
if (existing && !existing.isDirectory()) throw new Error(`Not a folder: ${cwd}`);
|
|
186
|
+
if (!existing && !(await stat(dirname(cwd)).catch(() => null))?.isDirectory()) {
|
|
187
|
+
throw new Error(`Folder does not exist: ${dirname(cwd)}`);
|
|
188
|
+
}
|
|
189
|
+
return { ...config, cwd, create: !existing };
|
|
108
190
|
}
|
|
109
191
|
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
if (config.label) args.push('--label', config.label);
|
|
116
|
-
if (side === 'right') args.push('--delta-file', join(state, 'delta.txt'));
|
|
117
|
-
return ['python3', args];
|
|
192
|
+
// Each pane needs its own folder: agents running side by side in one folder would edit the
|
|
193
|
+
// same files and read each other's work. A missing last folder is created; its parent must exist.
|
|
194
|
+
async function preparePaneFolders(left, right) {
|
|
195
|
+
if (left.cwd === right.cwd) {
|
|
196
|
+
throw new Error('Choose a different folder for each pane, so the two agents do not edit the same files.');
|
|
118
197
|
}
|
|
119
|
-
if (
|
|
198
|
+
for (const config of [left, right]) if (config.create) await mkdir(config.cwd);
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
// A harness pane runs one program, never an interactive shell, so echo can be off from the
|
|
202
|
+
// start. Otherwise terminal replies (focus reports, device attributes) that arrive before the
|
|
203
|
+
// harness enters raw mode are printed as text such as ^[[I.
|
|
204
|
+
const quiet = 'stty -echo 2>/dev/null; ';
|
|
205
|
+
|
|
206
|
+
function launchCommand(sessionId, config) {
|
|
207
|
+
if (compare) {
|
|
120
208
|
const codex = config.variant.includes('codex');
|
|
121
209
|
const command = config.variant.startsWith('penny-')
|
|
122
210
|
? `penny ${codex ? 'codex' : 'claude'}` : (codex ? 'codex' : 'claude');
|
|
@@ -126,14 +214,14 @@ function launchCommand(side, state, sessionId, config) {
|
|
|
126
214
|
return [shell, ['/d', '/s', '/c', `${command}${codex ? '' : ` --session-id ${sessionId}`}${modelArg}`]];
|
|
127
215
|
}
|
|
128
216
|
const shell = process.env.SHELL || '/bin/sh';
|
|
129
|
-
return [shell, ['-lc',
|
|
217
|
+
return [shell, ['-lc', `${quiet}exec ${command}${codex ? '' : ` --session-id ${sessionId}`}${modelArg}`]];
|
|
130
218
|
}
|
|
131
219
|
if (process.platform === 'win32') {
|
|
132
220
|
const shell = process.env.ComSpec || 'cmd.exe';
|
|
133
221
|
return claude ? [shell, ['/d', '/s', '/c', 'claude']] : [shell, []];
|
|
134
222
|
}
|
|
135
223
|
const shell = process.env.SHELL || '/bin/sh';
|
|
136
|
-
return claude ? [shell, ['-lc',
|
|
224
|
+
return claude ? [shell, ['-lc', `${quiet}exec claude`]] : [shell, []];
|
|
137
225
|
}
|
|
138
226
|
|
|
139
227
|
wss.on('connection', (ws) => {
|
|
@@ -142,19 +230,14 @@ wss.on('connection', (ws) => {
|
|
|
142
230
|
artifactSessions.set(artifactToken, artifactRoots);
|
|
143
231
|
const sessions = new Map();
|
|
144
232
|
const compareConfig = new Map();
|
|
233
|
+
const telemetryRoutesBySide = new Map();
|
|
234
|
+
let environment = validateEnvironment();
|
|
145
235
|
const meters = new Map();
|
|
146
236
|
const snapshots = new Map();
|
|
147
|
-
let state;
|
|
148
|
-
let delta;
|
|
149
237
|
let meterTimer;
|
|
150
238
|
let polling = false;
|
|
151
239
|
let closed = false;
|
|
152
240
|
let exiting = false;
|
|
153
|
-
const ready = pythonCompare ? mkdtemp(join(tmpdir(), 'penny-compare-web-')).then(async (dir) => {
|
|
154
|
-
state = dir;
|
|
155
|
-
await writeFile(join(dir, 'started'), '');
|
|
156
|
-
await writeFile(join(dir, 'delta.txt'), '');
|
|
157
|
-
}) : Promise.resolve();
|
|
158
241
|
const send = (message) => {
|
|
159
242
|
if (ws.readyState === ws.OPEN) ws.send(JSON.stringify(message));
|
|
160
243
|
};
|
|
@@ -167,14 +250,6 @@ wss.on('connection', (ws) => {
|
|
|
167
250
|
snapshots.set(meter.side, snapshot);
|
|
168
251
|
send({ type: 'meter', data: snapshot });
|
|
169
252
|
}
|
|
170
|
-
const left = snapshots.get('left');
|
|
171
|
-
const right = snapshots.get('right');
|
|
172
|
-
if (left && right) {
|
|
173
|
-
const base = left.totals;
|
|
174
|
-
const current = right.totals;
|
|
175
|
-
send({ type: 'delta', cost: base.cost ? (current.cost - base.cost) / base.cost * 100 : 0,
|
|
176
|
-
tokens: base.total ? (current.total - base.total) / base.total * 100 : 0 });
|
|
177
|
-
}
|
|
178
253
|
} catch (error) {
|
|
179
254
|
send({ type: 'error', side: 'left', error: `Meter: ${error.message}` });
|
|
180
255
|
} finally { polling = false; }
|
|
@@ -183,13 +258,12 @@ wss.on('connection', (ws) => {
|
|
|
183
258
|
if (closed) return;
|
|
184
259
|
closed = true;
|
|
185
260
|
artifactSessions.delete(artifactToken);
|
|
261
|
+
for (const route of telemetryRoutesBySide.values()) telemetryMeters.delete(route);
|
|
186
262
|
for (const session of sessions.values()) {
|
|
187
263
|
try { session.kill(); } catch { /* already exited */ }
|
|
188
264
|
}
|
|
189
265
|
sessions.clear();
|
|
190
266
|
if (meterTimer) clearInterval(meterTimer);
|
|
191
|
-
if (delta) delta.kill();
|
|
192
|
-
if (state) void rm(state, { recursive: true, force: true });
|
|
193
267
|
};
|
|
194
268
|
const handle = async (raw) => {
|
|
195
269
|
let message;
|
|
@@ -199,9 +273,11 @@ wss.on('connection', (ws) => {
|
|
|
199
273
|
if (!compare) throw new Error('Compare setup is unavailable in shell demo mode.');
|
|
200
274
|
const left = await validateConfig(message.config?.left);
|
|
201
275
|
const right = await validateConfig(message.config?.right);
|
|
276
|
+
environment = validateEnvironment(message.environment);
|
|
202
277
|
if (left.model && left.variant.includes('codex') !== right.variant.includes('codex')) {
|
|
203
278
|
throw new Error('Choose the same model family for both panes, or leave Model set to default.');
|
|
204
279
|
}
|
|
280
|
+
await preparePaneFolders(left, right);
|
|
205
281
|
compareConfig.set('left', left);
|
|
206
282
|
compareConfig.set('right', right);
|
|
207
283
|
artifactRoots.set('left', left.cwd);
|
|
@@ -211,6 +287,7 @@ wss.on('connection', (ws) => {
|
|
|
211
287
|
} catch (error) {
|
|
212
288
|
compareConfig.clear();
|
|
213
289
|
artifactRoots.clear();
|
|
290
|
+
environment = validateEnvironment();
|
|
214
291
|
send({ type: 'configuration-error', error: error.message });
|
|
215
292
|
}
|
|
216
293
|
return;
|
|
@@ -218,21 +295,23 @@ wss.on('connection', (ws) => {
|
|
|
218
295
|
if (message.type === 'exit') {
|
|
219
296
|
if (exiting) return;
|
|
220
297
|
exiting = true;
|
|
221
|
-
if (
|
|
222
|
-
try {
|
|
223
|
-
const { stdout } = await execFileAsync('python3', [join(compareDir, 'report.py'),
|
|
224
|
-
state, join(compareDir, 'reports'), '--json']);
|
|
225
|
-
send({ type: 'report', ...JSON.parse(stdout) });
|
|
226
|
-
} catch (error) {
|
|
227
|
-
send({ type: 'report-error', error: error.message });
|
|
228
|
-
}
|
|
229
|
-
} else if (nodeMeter) {
|
|
298
|
+
if (compare) {
|
|
230
299
|
try {
|
|
300
|
+
// Telemetry is exported every second; let the last batch land before closing the books.
|
|
301
|
+
if (telemetryRoutesBySide.size) await new Promise((done) => setTimeout(done, 1500));
|
|
231
302
|
await pollMeters();
|
|
303
|
+
for (const meter of meters.values()) meter.chargeSideCalls({ final: true });
|
|
232
304
|
const left = meters.get('left');
|
|
233
305
|
const right = meters.get('right');
|
|
234
306
|
if (!left || !right) throw new Error('Both compare panes must start before a report can be saved.');
|
|
235
|
-
|
|
307
|
+
const report = galleryReport(left, right, { environment, harnessVersions: await harnessVersions });
|
|
308
|
+
const publishToken = report ? randomUUID() : null;
|
|
309
|
+
if (publishToken) {
|
|
310
|
+
publishable.set(publishToken, report);
|
|
311
|
+
setTimeout(() => publishable.delete(publishToken), publishWindowMs).unref();
|
|
312
|
+
}
|
|
313
|
+
send({ type: 'report', ...await saveReport(left, right, join(compareDir, 'reports'), environment),
|
|
314
|
+
gallery: publishToken ? { publishToken } : null });
|
|
236
315
|
} catch (error) {
|
|
237
316
|
send({ type: 'report-error', error: error.message });
|
|
238
317
|
}
|
|
@@ -248,19 +327,26 @@ wss.on('connection', (ws) => {
|
|
|
248
327
|
try {
|
|
249
328
|
const config = compare
|
|
250
329
|
? compareConfig.get(message.side)
|
|
251
|
-
: { variant: 'claude', cwd:
|
|
330
|
+
: { variant: 'claude', cwd: launchDirectory, label: '', model: '' };
|
|
252
331
|
if (!config) throw new Error('Choose and validate both compare panes before starting.');
|
|
253
|
-
await ready;
|
|
254
332
|
if (closed) return;
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
333
|
+
const sessionId = compare && !config.variant.includes('codex') ? randomUUID() : undefined;
|
|
334
|
+
const [command, commandArgs] = launchCommand(sessionId, config);
|
|
335
|
+
let meter;
|
|
336
|
+
if (compare) {
|
|
337
|
+
const Meter = config.variant.includes('codex') ? CodexMeter : ClaudeMeter;
|
|
338
|
+
const role = config.variant.startsWith('penny-') ? 'PENNY' : 'NATIVE';
|
|
339
|
+
meter = new Meter({ sessionId, cwd: config.cwd, side: message.side, role,
|
|
340
|
+
label: config.label, variant: config.variant, pricingProfile: environment.pricing });
|
|
259
341
|
}
|
|
260
|
-
|
|
261
|
-
const
|
|
262
|
-
|
|
263
|
-
|
|
342
|
+
// Claude panes report background model calls to this server; see telemetry.mjs.
|
|
343
|
+
const route = meter instanceof CodexMeter || !meter ? null : telemetryRoute();
|
|
344
|
+
if (route) {
|
|
345
|
+
telemetryMeters.set(route, meter);
|
|
346
|
+
telemetryRoutesBySide.set(message.side, route);
|
|
347
|
+
}
|
|
348
|
+
const env = { ...paneEnvironment(), TERM: 'xterm-256color',
|
|
349
|
+
...(route ? telemetryEnv(`http://127.0.0.1:${port}`, route) : {}) };
|
|
264
350
|
if (compare && message.side === 'left') {
|
|
265
351
|
for (const key of ['ANTHROPIC_BASE_URL', 'ANTHROPIC_AUTH_TOKEN', 'ANTHROPIC_API_KEY']) {
|
|
266
352
|
delete env[key];
|
|
@@ -273,12 +359,8 @@ wss.on('connection', (ws) => {
|
|
|
273
359
|
sessions.set(message.side, session);
|
|
274
360
|
artifactRoots.set(message.side, config.cwd);
|
|
275
361
|
if (!compare) send({ type: 'artifact-session', token: artifactToken });
|
|
276
|
-
if (
|
|
277
|
-
|
|
278
|
-
const role = config.variant.startsWith('penny-') ? 'PENNY' : 'NATIVE';
|
|
279
|
-
meters.set(message.side, new Meter({ sessionId, cwd: config.cwd,
|
|
280
|
-
side: message.side, role, label: config.label, variant: config.variant }));
|
|
281
|
-
send({ type: 'meter-mode' });
|
|
362
|
+
if (meter) {
|
|
363
|
+
meters.set(message.side, meter);
|
|
282
364
|
if (!meterTimer) meterTimer = setInterval(() => { void pollMeters(); }, 1000);
|
|
283
365
|
}
|
|
284
366
|
session.onData((data) => send({ type: 'output', side: message.side, data }));
|
|
@@ -305,16 +387,14 @@ server.listen(port, '127.0.0.1', () => {
|
|
|
305
387
|
const url = `http://127.0.0.1:${port}`;
|
|
306
388
|
console.log(`Penny Compare live terminal demo: ${url}`);
|
|
307
389
|
console.log(compare
|
|
308
|
-
?
|
|
390
|
+
? 'Compare setup ready. Choose both panes in the browser.'
|
|
309
391
|
: claude ? 'Each pane starts Claude Code.' : 'Each pane starts a shell. Type claude to run Claude Code.');
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
}
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
});
|
|
319
|
-
}
|
|
392
|
+
// Windows hands the URL to the default browser without cmd, whose quoting rules
|
|
393
|
+
// mangle `start "" "<url>"` once Node quotes it as a single argument.
|
|
394
|
+
const [opener, openerArgs] = process.platform === 'win32'
|
|
395
|
+
? ['rundll32.exe', ['url.dll,FileProtocolHandler', url]]
|
|
396
|
+
: [process.platform === 'darwin' ? 'open' : 'xdg-open', [url]];
|
|
397
|
+
execFile(opener, openerArgs, { windowsHide: true }, (error) => {
|
|
398
|
+
if (error) console.error(`Could not open the browser automatically: ${error.message}`);
|
|
399
|
+
});
|
|
320
400
|
});
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
import { randomBytes } from 'node:crypto';
|
|
2
|
+
|
|
3
|
+
// Claude Code reports every model request it makes as an OpenTelemetry `api_request` log
|
|
4
|
+
// event, including background calls (such as the auto-mode reviewer) that never reach the
|
|
5
|
+
// session transcript. Each pane exports those events only to its own path on the Compare
|
|
6
|
+
// server; nothing is sent anywhere else, and prompts stay redacted (Claude Code's default).
|
|
7
|
+
|
|
8
|
+
export const maxTelemetryBytes = 1024 * 1024;
|
|
9
|
+
|
|
10
|
+
// One unguessable path per pane, so a request identifies its pane and outsiders cannot post.
|
|
11
|
+
export function telemetryRoute() {
|
|
12
|
+
return `/otlp/${randomBytes(24).toString('base64url')}/v1/logs`;
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
export function telemetryEnv(origin, route) {
|
|
16
|
+
return {
|
|
17
|
+
CLAUDE_CODE_ENABLE_TELEMETRY: '1',
|
|
18
|
+
OTEL_LOGS_EXPORTER: 'otlp',
|
|
19
|
+
OTEL_METRICS_EXPORTER: 'none',
|
|
20
|
+
OTEL_EXPORTER_OTLP_LOGS_PROTOCOL: 'http/json',
|
|
21
|
+
OTEL_EXPORTER_OTLP_LOGS_ENDPOINT: `${origin}${route}`,
|
|
22
|
+
OTEL_LOGS_EXPORT_INTERVAL: '1000',
|
|
23
|
+
};
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
const value = (attribute) => {
|
|
27
|
+
const raw = attribute?.value || {};
|
|
28
|
+
return raw.stringValue ?? raw.intValue ?? raw.doubleValue ?? raw.boolValue;
|
|
29
|
+
};
|
|
30
|
+
const count = (raw) => Math.max(0, Math.floor(Number(raw) || 0));
|
|
31
|
+
|
|
32
|
+
// The api_request events in one OTLP/HTTP JSON logs export.
|
|
33
|
+
export function apiRequests(body) {
|
|
34
|
+
const requests = [];
|
|
35
|
+
for (const resource of body?.resourceLogs || []) {
|
|
36
|
+
for (const scope of resource?.scopeLogs || []) {
|
|
37
|
+
for (const record of scope?.logRecords || []) {
|
|
38
|
+
const attributes = Object.fromEntries((record?.attributes || []).map((item) => [item?.key, value(item)]));
|
|
39
|
+
if (attributes['event.name'] !== 'api_request') continue;
|
|
40
|
+
// Responses relayed by a gateway can lack Anthropic's request id; the meter then
|
|
41
|
+
// matches the event to the transcript by its usage instead (see claude-meter.mjs).
|
|
42
|
+
const requestId = attributes.request_id ? String(attributes.request_id) : null;
|
|
43
|
+
const key = requestId || `sequence:${attributes['event.sequence'] ?? ''}:${attributes['event.timestamp'] ?? ''}`;
|
|
44
|
+
requests.push({
|
|
45
|
+
key,
|
|
46
|
+
requestId,
|
|
47
|
+
model: String(attributes.model || ''),
|
|
48
|
+
querySource: String(attributes.query_source || ''),
|
|
49
|
+
timestamp: attributes['event.timestamp'] || null,
|
|
50
|
+
fresh_input: count(attributes.input_tokens),
|
|
51
|
+
output: count(attributes.output_tokens),
|
|
52
|
+
cache_read: count(attributes.cache_read_tokens),
|
|
53
|
+
cache_write: count(attributes.cache_creation_tokens),
|
|
54
|
+
});
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
return requests;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
// Reads an OTLP/HTTP JSON export from a request, bounded by maxTelemetryBytes.
|
|
62
|
+
export async function readTelemetry(request) {
|
|
63
|
+
const chunks = [];
|
|
64
|
+
let size = 0;
|
|
65
|
+
for await (const chunk of request) {
|
|
66
|
+
size += chunk.length;
|
|
67
|
+
if (size > maxTelemetryBytes) throw new Error('Telemetry export too large');
|
|
68
|
+
chunks.push(chunk);
|
|
69
|
+
}
|
|
70
|
+
return apiRequests(JSON.parse(Buffer.concat(chunks).toString('utf8')));
|
|
71
|
+
}
|