vellum 0.2.2 → 0.2.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bun.lock +68 -100
- package/package.json +3 -3
- package/src/__tests__/asset-materialize-tool.test.ts +2 -2
- package/src/__tests__/checker.test.ts +104 -0
- package/src/__tests__/config-schema.test.ts +6 -0
- package/src/__tests__/gateway-only-enforcement.test.ts +458 -0
- package/src/__tests__/handlers-twilio-config.test.ts +221 -0
- package/src/__tests__/ipc-snapshot.test.ts +20 -0
- package/src/__tests__/memory-regressions.test.ts +100 -2
- package/src/__tests__/oauth-callback-registry.test.ts +85 -0
- package/src/__tests__/oauth2-gateway-transport.test.ts +298 -0
- package/src/__tests__/provider-commit-message-generator.test.ts +342 -0
- package/src/__tests__/public-ingress-urls.test.ts +206 -0
- package/src/__tests__/session-conflict-gate.test.ts +28 -25
- package/src/__tests__/tool-executor.test.ts +88 -0
- package/src/__tests__/turn-commit.test.ts +64 -0
- package/src/calls/__tests__/twilio-webhook-urls.test.ts +162 -0
- package/src/calls/call-domain.ts +3 -3
- package/src/calls/twilio-config.ts +25 -9
- package/src/calls/twilio-provider.ts +4 -4
- package/src/calls/twilio-routes.ts +10 -2
- package/src/calls/twilio-webhook-urls.ts +47 -0
- package/src/cli/map.ts +30 -6
- package/src/config/defaults.ts +5 -0
- package/src/config/schema.ts +34 -2
- package/src/config/system-prompt.ts +1 -1
- package/src/config/types.ts +1 -0
- package/src/config/vellum-skills/telegram-setup/SKILL.md +1 -5
- package/src/daemon/computer-use-session.ts +2 -1
- package/src/daemon/handlers/config.ts +95 -4
- package/src/daemon/handlers/sessions.ts +2 -2
- package/src/daemon/handlers/work-items.ts +1 -1
- package/src/daemon/ipc-contract-inventory.json +8 -0
- package/src/daemon/ipc-contract.ts +39 -1
- package/src/daemon/ride-shotgun-handler.ts +2 -1
- package/src/daemon/session-agent-loop.ts +37 -2
- package/src/daemon/session-conflict-gate.ts +18 -109
- package/src/daemon/session-tool-setup.ts +7 -0
- package/src/inbound/public-ingress-urls.ts +106 -0
- package/src/memory/attachments-store.ts +0 -1
- package/src/memory/channel-delivery-store.ts +0 -1
- package/src/memory/conflict-intent.ts +114 -0
- package/src/memory/conversation-key-store.ts +0 -1
- package/src/memory/db.ts +346 -149
- package/src/memory/job-handlers/conflict.ts +23 -1
- package/src/memory/runs-store.ts +0 -3
- package/src/memory/schema.ts +0 -4
- package/src/runtime/gateway-client.ts +36 -0
- package/src/runtime/http-server.ts +140 -2
- package/src/runtime/routes/channel-routes.ts +121 -79
- package/src/security/oauth-callback-registry.ts +56 -0
- package/src/security/oauth2.ts +174 -58
- package/src/swarm/backend-claude-code.ts +1 -1
- package/src/tools/assets/search.ts +1 -36
- package/src/tools/browser/api-map.ts +123 -50
- package/src/tools/claude-code/claude-code.ts +131 -1
- package/src/tools/tasks/work-item-list.ts +16 -2
- package/src/workspace/commit-message-enrichment-service.ts +3 -3
- package/src/workspace/provider-commit-message-generator.ts +57 -14
- package/src/workspace/turn-commit.ts +6 -2
package/src/security/oauth2.ts
CHANGED
|
@@ -1,6 +1,10 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* General-purpose OAuth2 Authorization Code flow with PKCE.
|
|
3
3
|
*
|
|
4
|
+
* Supports two callback transports:
|
|
5
|
+
* - loopback: spins up a local HTTP server on 127.0.0.1 (default when no public URL configured)
|
|
6
|
+
* - gateway: uses the gateway's OAuth callback route + in-memory registry (when ingress.publicBaseUrl is set)
|
|
7
|
+
*
|
|
4
8
|
* Moved from integrations/oauth2.ts. Types that were in integrations/types.ts
|
|
5
9
|
* are now inlined here since the integration framework is removed.
|
|
6
10
|
*/
|
|
@@ -39,6 +43,11 @@ export interface OAuth2FlowCallbacks {
|
|
|
39
43
|
openUrl: (url: string) => void;
|
|
40
44
|
}
|
|
41
45
|
|
|
46
|
+
export interface OAuth2FlowOptions {
|
|
47
|
+
/** Which callback transport to use. When omitted, auto-detected from config. */
|
|
48
|
+
callbackTransport?: 'loopback' | 'gateway';
|
|
49
|
+
}
|
|
50
|
+
|
|
42
51
|
export interface OAuth2FlowResult {
|
|
43
52
|
tokens: OAuth2TokenResult;
|
|
44
53
|
grantedScopes: string[];
|
|
@@ -62,20 +71,107 @@ function generateState(): string {
|
|
|
62
71
|
}
|
|
63
72
|
|
|
64
73
|
// ---------------------------------------------------------------------------
|
|
65
|
-
//
|
|
74
|
+
// Token exchange (shared between transports)
|
|
75
|
+
// ---------------------------------------------------------------------------
|
|
76
|
+
|
|
77
|
+
async function exchangeCodeForTokens(
|
|
78
|
+
config: OAuth2Config,
|
|
79
|
+
code: string,
|
|
80
|
+
redirectUri: string,
|
|
81
|
+
codeVerifier: string,
|
|
82
|
+
): Promise<OAuth2FlowResult> {
|
|
83
|
+
const usePKCE = !config.clientSecret;
|
|
84
|
+
|
|
85
|
+
const tokenBody: Record<string, string> = {
|
|
86
|
+
grant_type: 'authorization_code',
|
|
87
|
+
code,
|
|
88
|
+
redirect_uri: redirectUri,
|
|
89
|
+
client_id: config.clientId,
|
|
90
|
+
};
|
|
91
|
+
if (usePKCE) {
|
|
92
|
+
tokenBody.code_verifier = codeVerifier;
|
|
93
|
+
}
|
|
94
|
+
if (config.clientSecret) {
|
|
95
|
+
tokenBody.client_secret = config.clientSecret;
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
const tokenResp = await fetch(config.tokenUrl, {
|
|
99
|
+
method: 'POST',
|
|
100
|
+
headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
|
|
101
|
+
body: new URLSearchParams(tokenBody),
|
|
102
|
+
});
|
|
103
|
+
|
|
104
|
+
if (!tokenResp.ok) {
|
|
105
|
+
const rawBody = await tokenResp.text().catch(() => '');
|
|
106
|
+
let safeDetail: Record<string, unknown> = {};
|
|
107
|
+
let errorCode = '';
|
|
108
|
+
try {
|
|
109
|
+
const parsed = JSON.parse(rawBody) as Record<string, unknown>;
|
|
110
|
+
if (parsed.error) { safeDetail.error = String(parsed.error); errorCode = String(parsed.error); }
|
|
111
|
+
if (parsed.error_description) safeDetail.error_description = String(parsed.error_description);
|
|
112
|
+
} catch {
|
|
113
|
+
safeDetail.error = '[non-JSON response]';
|
|
114
|
+
}
|
|
115
|
+
log.error({ status: tokenResp.status, ...safeDetail }, 'OAuth2 token exchange failed');
|
|
116
|
+
const detail = errorCode ? `HTTP ${tokenResp.status}: ${errorCode}` : `HTTP ${tokenResp.status}`;
|
|
117
|
+
throw new Error(`OAuth2 token exchange failed (${detail})`);
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
const tokenData = await tokenResp.json() as Record<string, unknown>;
|
|
121
|
+
|
|
122
|
+
// Slack V2 OAuth returns user tokens nested under `authed_user`
|
|
123
|
+
const authedUser = tokenData.authed_user as Record<string, unknown> | undefined;
|
|
124
|
+
const tokenSource = authedUser?.access_token ? authedUser : tokenData;
|
|
125
|
+
|
|
126
|
+
const tokens: OAuth2TokenResult = {
|
|
127
|
+
accessToken: (tokenSource.access_token as string) ?? (tokenData.access_token as string),
|
|
128
|
+
refreshToken: (tokenSource.refresh_token as string | undefined) ?? (tokenData.refresh_token as string | undefined),
|
|
129
|
+
expiresIn: (tokenSource.expires_in as number | undefined) ?? (tokenData.expires_in as number | undefined),
|
|
130
|
+
scope: (tokenSource.scope as string | undefined) ?? (tokenData.scope as string | undefined),
|
|
131
|
+
tokenType: (tokenSource.token_type as string | undefined) ?? (tokenData.token_type as string | undefined),
|
|
132
|
+
};
|
|
133
|
+
|
|
134
|
+
const grantedScopes = typeof tokens.scope === 'string'
|
|
135
|
+
? tokens.scope.split(/[ ,]/).filter(Boolean)
|
|
136
|
+
: [...config.scopes];
|
|
137
|
+
|
|
138
|
+
return { tokens, grantedScopes, rawTokenResponse: tokenData };
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
// ---------------------------------------------------------------------------
|
|
142
|
+
// Transport auto-detection
|
|
66
143
|
// ---------------------------------------------------------------------------
|
|
67
144
|
|
|
68
145
|
/**
|
|
69
|
-
*
|
|
146
|
+
* Determine which callback transport to use when not explicitly specified.
|
|
147
|
+
* Uses gateway if ingress.publicBaseUrl is configured, otherwise loopback.
|
|
70
148
|
*/
|
|
71
|
-
|
|
149
|
+
function detectTransport(): 'loopback' | 'gateway' {
|
|
150
|
+
try {
|
|
151
|
+
// Dynamic import avoided — loadConfig is synchronous and already used elsewhere.
|
|
152
|
+
const { loadConfig } = require('../config/loader.js') as typeof import('../config/loader.js');
|
|
153
|
+
const appConfig = loadConfig();
|
|
154
|
+
if (appConfig.ingress?.publicBaseUrl) {
|
|
155
|
+
return 'gateway';
|
|
156
|
+
}
|
|
157
|
+
} catch {
|
|
158
|
+
// Config loading failed — fall back to loopback
|
|
159
|
+
log.debug('Config not available for transport auto-detection, defaulting to loopback');
|
|
160
|
+
}
|
|
161
|
+
return 'loopback';
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
// ---------------------------------------------------------------------------
|
|
165
|
+
// Loopback transport
|
|
166
|
+
// ---------------------------------------------------------------------------
|
|
167
|
+
|
|
168
|
+
async function runLoopbackFlow(
|
|
72
169
|
config: OAuth2Config,
|
|
73
170
|
callbacks: OAuth2FlowCallbacks,
|
|
171
|
+
codeVerifier: string,
|
|
172
|
+
codeChallenge: string,
|
|
173
|
+
state: string,
|
|
74
174
|
): Promise<OAuth2FlowResult> {
|
|
75
|
-
const codeVerifier = generateCodeVerifier();
|
|
76
|
-
const codeChallenge = generateCodeChallenge(codeVerifier);
|
|
77
|
-
const state = generateState();
|
|
78
|
-
|
|
79
175
|
let resolveCode: (value: { code: string; returnedState: string }) => void;
|
|
80
176
|
let rejectCode: (reason: Error) => void;
|
|
81
177
|
|
|
@@ -84,7 +180,6 @@ export async function startOAuth2Flow(
|
|
|
84
180
|
rejectCode = reject;
|
|
85
181
|
});
|
|
86
182
|
|
|
87
|
-
/** How long to wait for the user to complete the OAuth consent flow. */
|
|
88
183
|
const FLOW_TIMEOUT_MS = 120_000;
|
|
89
184
|
|
|
90
185
|
const timeout = setTimeout(() => {
|
|
@@ -153,64 +248,85 @@ export async function startOAuth2Flow(
|
|
|
153
248
|
throw new Error('OAuth2 state mismatch — possible CSRF attack');
|
|
154
249
|
}
|
|
155
250
|
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
if (usePKCE) {
|
|
163
|
-
tokenBody.code_verifier = codeVerifier;
|
|
164
|
-
}
|
|
165
|
-
if (config.clientSecret) {
|
|
166
|
-
tokenBody.client_secret = config.clientSecret;
|
|
167
|
-
}
|
|
251
|
+
return await exchangeCodeForTokens(config, code, redirectUri, codeVerifier);
|
|
252
|
+
} finally {
|
|
253
|
+
clearTimeout(timeout);
|
|
254
|
+
server.stop(true);
|
|
255
|
+
}
|
|
256
|
+
}
|
|
168
257
|
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
body: new URLSearchParams(tokenBody),
|
|
173
|
-
});
|
|
258
|
+
// ---------------------------------------------------------------------------
|
|
259
|
+
// Gateway transport
|
|
260
|
+
// ---------------------------------------------------------------------------
|
|
174
261
|
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
}
|
|
186
|
-
log.error({ status: tokenResp.status, ...safeDetail }, 'OAuth2 token exchange failed');
|
|
187
|
-
const detail = errorCode ? `HTTP ${tokenResp.status}: ${errorCode}` : `HTTP ${tokenResp.status}`;
|
|
188
|
-
throw new Error(`OAuth2 token exchange failed (${detail})`);
|
|
189
|
-
}
|
|
262
|
+
async function runGatewayFlow(
|
|
263
|
+
config: OAuth2Config,
|
|
264
|
+
callbacks: OAuth2FlowCallbacks,
|
|
265
|
+
codeVerifier: string,
|
|
266
|
+
codeChallenge: string,
|
|
267
|
+
state: string,
|
|
268
|
+
): Promise<OAuth2FlowResult> {
|
|
269
|
+
const { loadConfig } = require('../config/loader.js') as typeof import('../config/loader.js');
|
|
270
|
+
const { getOAuthCallbackUrl } = require('../inbound/public-ingress-urls.js') as typeof import('../inbound/public-ingress-urls.js');
|
|
271
|
+
const { registerPendingCallback } = require('./oauth-callback-registry.js') as typeof import('./oauth-callback-registry.js');
|
|
190
272
|
|
|
191
|
-
|
|
273
|
+
const appConfig = loadConfig();
|
|
274
|
+
const redirectUri = getOAuthCallbackUrl(appConfig);
|
|
192
275
|
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
276
|
+
const codePromise = new Promise<string>((resolve, reject) => {
|
|
277
|
+
registerPendingCallback(state, resolve, reject);
|
|
278
|
+
});
|
|
196
279
|
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
280
|
+
const usePKCE = !config.clientSecret;
|
|
281
|
+
const authParams = new URLSearchParams({
|
|
282
|
+
...config.extraParams,
|
|
283
|
+
client_id: config.clientId,
|
|
284
|
+
redirect_uri: redirectUri,
|
|
285
|
+
response_type: 'code',
|
|
286
|
+
scope: config.scopes.join(' '),
|
|
287
|
+
state,
|
|
288
|
+
...(usePKCE ? { code_challenge: codeChallenge, code_challenge_method: 'S256' } : {}),
|
|
289
|
+
});
|
|
204
290
|
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
: [...config.scopes];
|
|
291
|
+
const authUrl = `${config.authUrl}?${authParams}`;
|
|
292
|
+
callbacks.openUrl(authUrl);
|
|
208
293
|
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
294
|
+
const code = await codePromise;
|
|
295
|
+
|
|
296
|
+
return await exchangeCodeForTokens(config, code, redirectUri, codeVerifier);
|
|
297
|
+
}
|
|
298
|
+
|
|
299
|
+
// ---------------------------------------------------------------------------
|
|
300
|
+
// Public API
|
|
301
|
+
// ---------------------------------------------------------------------------
|
|
302
|
+
|
|
303
|
+
/**
|
|
304
|
+
* Run a full OAuth2 authorization code flow with PKCE support.
|
|
305
|
+
*
|
|
306
|
+
* Supports two callback transports:
|
|
307
|
+
* - loopback (default): local HTTP server on 127.0.0.1
|
|
308
|
+
* - gateway: callback via the gateway's OAuth route + in-memory registry
|
|
309
|
+
*
|
|
310
|
+
* Transport is auto-detected based on ingress.publicBaseUrl config unless
|
|
311
|
+
* explicitly specified via options.callbackTransport.
|
|
312
|
+
*/
|
|
313
|
+
export async function startOAuth2Flow(
|
|
314
|
+
config: OAuth2Config,
|
|
315
|
+
callbacks: OAuth2FlowCallbacks,
|
|
316
|
+
options?: OAuth2FlowOptions,
|
|
317
|
+
): Promise<OAuth2FlowResult> {
|
|
318
|
+
const codeVerifier = generateCodeVerifier();
|
|
319
|
+
const codeChallenge = generateCodeChallenge(codeVerifier);
|
|
320
|
+
const state = generateState();
|
|
321
|
+
|
|
322
|
+
const transport = options?.callbackTransport ?? detectTransport();
|
|
323
|
+
log.debug({ transport }, 'OAuth2 flow starting');
|
|
324
|
+
|
|
325
|
+
if (transport === 'gateway') {
|
|
326
|
+
return runGatewayFlow(config, callbacks, codeVerifier, codeChallenge, state);
|
|
213
327
|
}
|
|
328
|
+
|
|
329
|
+
return runLoopbackFlow(config, callbacks, codeVerifier, codeChallenge, state);
|
|
214
330
|
}
|
|
215
331
|
|
|
216
332
|
/**
|
|
@@ -117,7 +117,7 @@ export function createClaudeCodeBackend(): SwarmWorkerBackend {
|
|
|
117
117
|
|
|
118
118
|
const parts: string[] = [`[${message.subtype}] (${message.num_turns} turns, ${(message.duration_ms / 1000).toFixed(1)}s)`];
|
|
119
119
|
if (errors.length > 0) parts.push(`Errors: ${errors.join('; ')}`);
|
|
120
|
-
if (denials.length > 0) parts.push(`Permission denied: ${denials.map(d => d.tool_name).join(', ')}`);
|
|
120
|
+
if (denials.length > 0) parts.push(`Permission denied: ${denials.map((d: { tool_name: string }) => d.tool_name).join(', ')}`);
|
|
121
121
|
resultText += `\n${parts.join('\n')}`;
|
|
122
122
|
}
|
|
123
123
|
}
|
|
@@ -13,7 +13,7 @@ import type { Tool, ToolContext, ToolExecutionResult } from '../types.js';
|
|
|
13
13
|
import type { ToolDefinition } from '../../providers/types.js';
|
|
14
14
|
import { registerTool } from '../registry.js';
|
|
15
15
|
import { getDb } from '../../memory/db.js';
|
|
16
|
-
import { attachments, messageAttachments, messages, conversations
|
|
16
|
+
import { attachments, messageAttachments, messages, conversations } from '../../memory/schema.js';
|
|
17
17
|
import type { StoredAttachment } from '../../memory/attachments-store.js';
|
|
18
18
|
import { isAttachmentVisible, type AttachmentContext } from '../../daemon/media-visibility-policy.js';
|
|
19
19
|
import { getConversationThreadType } from '../../memory/conversation-store.js';
|
|
@@ -103,25 +103,6 @@ function isAttachmentVisibleFromContext(attachmentId: string, currentContext: At
|
|
|
103
103
|
);
|
|
104
104
|
}
|
|
105
105
|
|
|
106
|
-
// ---------------------------------------------------------------------------
|
|
107
|
-
// Assistant ID lookup
|
|
108
|
-
// ---------------------------------------------------------------------------
|
|
109
|
-
|
|
110
|
-
/**
|
|
111
|
-
* Derive the assistant ID that owns a conversation via the conversation_keys
|
|
112
|
-
* table. Returns null when no mapping exists (e.g. native macOS app sessions
|
|
113
|
-
* that use the implicit 'local-assistant' identity).
|
|
114
|
-
*/
|
|
115
|
-
function getAssistantIdForConversation(conversationId: string): string | null {
|
|
116
|
-
const db = getDb();
|
|
117
|
-
const row = db
|
|
118
|
-
.select({ assistantId: conversationKeys.assistantId })
|
|
119
|
-
.from(conversationKeys)
|
|
120
|
-
.where(eq(conversationKeys.conversationId, conversationId))
|
|
121
|
-
.get();
|
|
122
|
-
return row?.assistantId ?? null;
|
|
123
|
-
}
|
|
124
|
-
|
|
125
106
|
// ---------------------------------------------------------------------------
|
|
126
107
|
// Search logic
|
|
127
108
|
// ---------------------------------------------------------------------------
|
|
@@ -131,8 +112,6 @@ export interface AssetSearchParams {
|
|
|
131
112
|
filename?: string;
|
|
132
113
|
recency?: string;
|
|
133
114
|
conversation_id?: string;
|
|
134
|
-
/** Tenant boundary — when set, only attachments belonging to this assistant are returned. */
|
|
135
|
-
assistant_id?: string;
|
|
136
115
|
limit?: number;
|
|
137
116
|
}
|
|
138
117
|
|
|
@@ -163,11 +142,6 @@ export function searchAttachments(params: AssetSearchParams): StoredAttachment[]
|
|
|
163
142
|
}
|
|
164
143
|
}
|
|
165
144
|
|
|
166
|
-
// Tenant boundary — restrict to the active assistant's attachments
|
|
167
|
-
if (params.assistant_id) {
|
|
168
|
-
conditions.push(eq(attachments.assistantId, params.assistant_id));
|
|
169
|
-
}
|
|
170
|
-
|
|
171
145
|
// Conversation scope — join through message_attachments + messages
|
|
172
146
|
if (params.conversation_id) {
|
|
173
147
|
const linkedIds = db
|
|
@@ -207,11 +181,6 @@ export function searchAttachments(params: AssetSearchParams): StoredAttachment[]
|
|
|
207
181
|
bindValues.push(Date.now() - offsetMs);
|
|
208
182
|
}
|
|
209
183
|
}
|
|
210
|
-
if (params.assistant_id) {
|
|
211
|
-
whereParts.push(`a.assistant_id = ?`);
|
|
212
|
-
bindValues.push(params.assistant_id);
|
|
213
|
-
}
|
|
214
|
-
|
|
215
184
|
const limit = Math.min(params.limit ?? DEFAULT_LIMIT, MAX_RESULTS);
|
|
216
185
|
const stmt = raw.prepare(
|
|
217
186
|
`SELECT a.id, a.original_filename, a.mime_type, a.size_bytes, a.kind, a.thumbnail_base64, a.created_at
|
|
@@ -347,9 +316,6 @@ class AssetSearchTool implements Tool {
|
|
|
347
316
|
}
|
|
348
317
|
|
|
349
318
|
try {
|
|
350
|
-
// Scope results to the active assistant to prevent cross-tenant leaks
|
|
351
|
-
const assistantId = getAssistantIdForConversation(context.conversationId) ?? undefined;
|
|
352
|
-
|
|
353
319
|
// Over-fetch with MAX_RESULTS so visibility filtering doesn't
|
|
354
320
|
// under-fill the caller's requested limit.
|
|
355
321
|
const results = searchAttachments({
|
|
@@ -357,7 +323,6 @@ class AssetSearchTool implements Tool {
|
|
|
357
323
|
filename,
|
|
358
324
|
recency,
|
|
359
325
|
conversation_id: conversationId,
|
|
360
|
-
assistant_id: assistantId,
|
|
361
326
|
limit: MAX_RESULTS,
|
|
362
327
|
});
|
|
363
328
|
|
|
@@ -38,12 +38,31 @@ export interface ApiMapResult {
|
|
|
38
38
|
const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
|
|
39
39
|
const NUMERIC_RE = /^\d+$/;
|
|
40
40
|
const HEX_HASH_RE = /^[0-9a-f]{8,}$/i;
|
|
41
|
+
const DATE_RE = /^\d{4}-\d{2}-\d{2}$/;
|
|
42
|
+
|
|
43
|
+
/** URL path patterns that indicate non-API noise. */
|
|
44
|
+
const NOISE_PATH_PATTERNS = [
|
|
45
|
+
/\/web-translations\//,
|
|
46
|
+
/\/cdn-cgi\//,
|
|
47
|
+
/\.properties$/,
|
|
48
|
+
/\.js$/,
|
|
49
|
+
/\.css$/,
|
|
50
|
+
/\.woff2?$/,
|
|
51
|
+
/\.png$/,
|
|
52
|
+
/\.jpg$/,
|
|
53
|
+
/\.svg$/,
|
|
54
|
+
/\.ico$/,
|
|
55
|
+
/\.map$/,
|
|
56
|
+
/\/preference\//,
|
|
57
|
+
/\/userpreference-service\//,
|
|
58
|
+
];
|
|
41
59
|
|
|
42
60
|
/** Returns true when a path segment looks like a dynamic ID. */
|
|
43
61
|
function isIdSegment(segment: string): boolean {
|
|
44
62
|
if (NUMERIC_RE.test(segment)) return true;
|
|
45
63
|
if (UUID_RE.test(segment)) return true;
|
|
46
64
|
if (HEX_HASH_RE.test(segment)) return true;
|
|
65
|
+
if (DATE_RE.test(segment)) return true;
|
|
47
66
|
return false;
|
|
48
67
|
}
|
|
49
68
|
|
|
@@ -69,27 +88,43 @@ function tryParseJson(text: string | undefined): Record<string, unknown> | undef
|
|
|
69
88
|
return undefined;
|
|
70
89
|
}
|
|
71
90
|
|
|
91
|
+
/** Extract GraphQL operation name from request body. */
|
|
92
|
+
function extractGraphQLOperationName(postData: string | undefined): string | null {
|
|
93
|
+
if (!postData) return null;
|
|
94
|
+
const body = tryParseJson(postData);
|
|
95
|
+
if (!body) return null;
|
|
96
|
+
if (typeof body.operationName === 'string' && body.operationName) return body.operationName;
|
|
97
|
+
// Try extracting from query string: "query FooBar { ..." or "mutation FooBar { ..."
|
|
98
|
+
if (typeof body.query === 'string') {
|
|
99
|
+
const named = body.query.match(/(?:query|mutation|subscription)\s+(\w+)/);
|
|
100
|
+
if (named) return named[1];
|
|
101
|
+
// Unnamed query — extract the first field name: "query{fooBar(" or "query { fooBar {"
|
|
102
|
+
const firstField = body.query.match(/(?:query|mutation|subscription)\s*\{?\s*(\w+)/);
|
|
103
|
+
if (firstField) return firstField[1];
|
|
104
|
+
}
|
|
105
|
+
return null;
|
|
106
|
+
}
|
|
107
|
+
|
|
72
108
|
// ---------------------------------------------------------------------------
|
|
73
109
|
// Core analysis
|
|
74
110
|
// ---------------------------------------------------------------------------
|
|
75
111
|
|
|
112
|
+
interface GroupData {
|
|
113
|
+
method: string;
|
|
114
|
+
urlPattern: string;
|
|
115
|
+
exampleUrl: string;
|
|
116
|
+
queryParams: Set<string>;
|
|
117
|
+
requestBodyKeys: Set<string>;
|
|
118
|
+
responseStatus: Set<number>;
|
|
119
|
+
responseBodyKeys: Set<string>;
|
|
120
|
+
count: number;
|
|
121
|
+
}
|
|
122
|
+
|
|
76
123
|
export function analyzeApiMap(
|
|
77
124
|
entries: NetworkRecordedEntry[],
|
|
78
125
|
domain: string,
|
|
79
126
|
): ApiMapResult {
|
|
80
|
-
const groups = new Map<
|
|
81
|
-
string,
|
|
82
|
-
{
|
|
83
|
-
method: string;
|
|
84
|
-
urlPattern: string;
|
|
85
|
-
exampleUrl: string;
|
|
86
|
-
queryParams: Set<string>;
|
|
87
|
-
requestBodyKeys: Set<string>;
|
|
88
|
-
responseStatus: Set<number>;
|
|
89
|
-
responseBodyKeys: Set<string>;
|
|
90
|
-
count: number;
|
|
91
|
-
}
|
|
92
|
-
>();
|
|
127
|
+
const groups = new Map<string, GroupData>();
|
|
93
128
|
|
|
94
129
|
for (const entry of entries) {
|
|
95
130
|
const { request, response } = entry;
|
|
@@ -97,11 +132,30 @@ export function analyzeApiMap(
|
|
|
97
132
|
try {
|
|
98
133
|
parsed = new URL(request.url);
|
|
99
134
|
} catch {
|
|
100
|
-
continue;
|
|
135
|
+
continue;
|
|
101
136
|
}
|
|
102
137
|
|
|
138
|
+
// Skip non-API noise
|
|
139
|
+
if (NOISE_PATH_PATTERNS.some(p => p.test(parsed.pathname))) continue;
|
|
140
|
+
|
|
141
|
+
// Skip non-JSON responses
|
|
142
|
+
const mimeType = response?.mimeType ?? '';
|
|
143
|
+
if (response && !mimeType.includes('json') && !mimeType.includes('graphql')) continue;
|
|
144
|
+
|
|
103
145
|
const method = request.method.toUpperCase();
|
|
104
|
-
const
|
|
146
|
+
const normalizedPath = normalizePathSegments(parsed.pathname);
|
|
147
|
+
const basePattern = `${parsed.hostname}${normalizedPath}`;
|
|
148
|
+
|
|
149
|
+
// For GraphQL endpoints, split by operation name
|
|
150
|
+
let urlPattern = basePattern;
|
|
151
|
+
const isGraphQL = normalizedPath.includes('graphql');
|
|
152
|
+
if (isGraphQL && method === 'POST') {
|
|
153
|
+
const opName = extractGraphQLOperationName(request.postData);
|
|
154
|
+
if (opName) {
|
|
155
|
+
urlPattern = `${basePattern} → ${opName}`;
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
|
|
105
159
|
const key = `${method} ${urlPattern}`;
|
|
106
160
|
|
|
107
161
|
let group = groups.get(key);
|
|
@@ -121,26 +175,23 @@ export function analyzeApiMap(
|
|
|
121
175
|
|
|
122
176
|
group.count++;
|
|
123
177
|
|
|
124
|
-
// Collect query param keys
|
|
125
178
|
for (const paramKey of parsed.searchParams.keys()) {
|
|
126
179
|
group.queryParams.add(paramKey);
|
|
127
180
|
}
|
|
128
181
|
|
|
129
|
-
// Request body keys (POST/PUT/PATCH)
|
|
130
182
|
if (['POST', 'PUT', 'PATCH'].includes(method)) {
|
|
131
183
|
const body = tryParseJson(request.postData);
|
|
132
184
|
if (body) {
|
|
133
185
|
for (const k of Object.keys(body)) {
|
|
134
|
-
|
|
186
|
+
if (k !== 'query' && k !== 'operationName' && k !== 'extensions') {
|
|
187
|
+
group.requestBodyKeys.add(k);
|
|
188
|
+
}
|
|
135
189
|
}
|
|
136
190
|
}
|
|
137
191
|
}
|
|
138
192
|
|
|
139
|
-
// Response status
|
|
140
193
|
if (response) {
|
|
141
194
|
group.responseStatus.add(response.status);
|
|
142
|
-
|
|
143
|
-
// Response body keys
|
|
144
195
|
const resBody = tryParseJson(response.body);
|
|
145
196
|
if (resBody) {
|
|
146
197
|
for (const k of Object.keys(resBody)) {
|
|
@@ -161,13 +212,21 @@ export function analyzeApiMap(
|
|
|
161
212
|
count: g.count,
|
|
162
213
|
}));
|
|
163
214
|
|
|
164
|
-
// Sort
|
|
165
|
-
|
|
215
|
+
// Sort: data endpoints first (low count = unique pages), then boilerplate
|
|
216
|
+
// Within each tier, sort alphabetically by pattern for readability
|
|
217
|
+
endpoints.sort((a, b) => {
|
|
218
|
+
const aIsBoilerplate = a.count > 15;
|
|
219
|
+
const bIsBoilerplate = b.count > 15;
|
|
220
|
+
if (aIsBoilerplate !== bIsBoilerplate) return aIsBoilerplate ? 1 : -1;
|
|
221
|
+
return a.urlPattern.localeCompare(b.urlPattern);
|
|
222
|
+
});
|
|
223
|
+
|
|
224
|
+
const totalApiRequests = endpoints.reduce((sum, ep) => sum + ep.count, 0);
|
|
166
225
|
|
|
167
226
|
return {
|
|
168
227
|
domain,
|
|
169
228
|
analyzedAt: Date.now(),
|
|
170
|
-
totalRequests:
|
|
229
|
+
totalRequests: totalApiRequests,
|
|
171
230
|
endpoints,
|
|
172
231
|
};
|
|
173
232
|
}
|
|
@@ -191,30 +250,44 @@ export function saveApiMap(domain: string, result: ApiMapResult): string {
|
|
|
191
250
|
// ---------------------------------------------------------------------------
|
|
192
251
|
|
|
193
252
|
export function printApiMapTable(result: ApiMapResult): void {
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
253
|
+
const dataEndpoints = result.endpoints.filter(ep => ep.count <= 15);
|
|
254
|
+
const boilerplate = result.endpoints.filter(ep => ep.count > 15);
|
|
255
|
+
|
|
256
|
+
console.log(`\nAPI Map for ${result.domain} — ${result.endpoints.length} endpoints discovered\n`);
|
|
257
|
+
|
|
258
|
+
const stripDomain = (pattern: string) => {
|
|
259
|
+
const idx = pattern.indexOf('/');
|
|
260
|
+
return idx >= 0 ? pattern.slice(idx) : pattern;
|
|
261
|
+
};
|
|
262
|
+
|
|
263
|
+
const printSection = (title: string, eps: ApiEndpoint[]) => {
|
|
264
|
+
if (eps.length === 0) return;
|
|
265
|
+
console.log(` ${title} (${eps.length})\n`);
|
|
266
|
+
|
|
267
|
+
const header = ['Method', 'Endpoint', 'Hits', 'Response Keys'];
|
|
268
|
+
const rows = eps.map((ep) => [
|
|
269
|
+
ep.method,
|
|
270
|
+
stripDomain(ep.urlPattern),
|
|
271
|
+
String(ep.count),
|
|
272
|
+
ep.responseBodyKeys.slice(0, 5).join(', ') || '-',
|
|
273
|
+
]);
|
|
274
|
+
|
|
275
|
+
const widths = header.map((h, i) =>
|
|
276
|
+
Math.min(i === 1 ? 72 : i === 3 ? 50 : 200, Math.max(h.length, ...rows.map((r) => r[i].length))),
|
|
277
|
+
);
|
|
278
|
+
|
|
279
|
+
const sep = widths.map((w) => '-'.repeat(w)).join(' | ');
|
|
280
|
+
const fmt = (row: string[]) =>
|
|
281
|
+
row.map((cell, i) => cell.slice(0, widths[i]).padEnd(widths[i])).join(' | ');
|
|
282
|
+
|
|
283
|
+
console.log(` ${fmt(header)}`);
|
|
284
|
+
console.log(` ${sep}`);
|
|
285
|
+
for (const row of rows) {
|
|
286
|
+
console.log(` ${fmt(row)}`);
|
|
287
|
+
}
|
|
288
|
+
console.log();
|
|
289
|
+
};
|
|
290
|
+
|
|
291
|
+
printSection('DATA ENDPOINTS', dataEndpoints);
|
|
292
|
+
printSection('PAGE-LOAD BOILERPLATE', boilerplate);
|
|
220
293
|
}
|