vellum 0.2.2 → 0.2.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. package/bun.lock +68 -100
  2. package/package.json +3 -3
  3. package/src/__tests__/asset-materialize-tool.test.ts +2 -2
  4. package/src/__tests__/checker.test.ts +104 -0
  5. package/src/__tests__/config-schema.test.ts +6 -0
  6. package/src/__tests__/gateway-only-enforcement.test.ts +458 -0
  7. package/src/__tests__/handlers-twilio-config.test.ts +221 -0
  8. package/src/__tests__/ipc-snapshot.test.ts +20 -0
  9. package/src/__tests__/memory-regressions.test.ts +100 -2
  10. package/src/__tests__/oauth-callback-registry.test.ts +85 -0
  11. package/src/__tests__/oauth2-gateway-transport.test.ts +298 -0
  12. package/src/__tests__/provider-commit-message-generator.test.ts +342 -0
  13. package/src/__tests__/public-ingress-urls.test.ts +206 -0
  14. package/src/__tests__/session-conflict-gate.test.ts +28 -25
  15. package/src/__tests__/tool-executor.test.ts +88 -0
  16. package/src/__tests__/turn-commit.test.ts +64 -0
  17. package/src/calls/__tests__/twilio-webhook-urls.test.ts +162 -0
  18. package/src/calls/call-domain.ts +3 -3
  19. package/src/calls/twilio-config.ts +25 -9
  20. package/src/calls/twilio-provider.ts +4 -4
  21. package/src/calls/twilio-routes.ts +10 -2
  22. package/src/calls/twilio-webhook-urls.ts +47 -0
  23. package/src/cli/map.ts +30 -6
  24. package/src/config/defaults.ts +5 -0
  25. package/src/config/schema.ts +34 -2
  26. package/src/config/system-prompt.ts +1 -1
  27. package/src/config/types.ts +1 -0
  28. package/src/config/vellum-skills/telegram-setup/SKILL.md +1 -5
  29. package/src/daemon/computer-use-session.ts +2 -1
  30. package/src/daemon/handlers/config.ts +95 -4
  31. package/src/daemon/handlers/sessions.ts +2 -2
  32. package/src/daemon/handlers/work-items.ts +1 -1
  33. package/src/daemon/ipc-contract-inventory.json +8 -0
  34. package/src/daemon/ipc-contract.ts +39 -1
  35. package/src/daemon/ride-shotgun-handler.ts +2 -1
  36. package/src/daemon/session-agent-loop.ts +37 -2
  37. package/src/daemon/session-conflict-gate.ts +18 -109
  38. package/src/daemon/session-tool-setup.ts +7 -0
  39. package/src/inbound/public-ingress-urls.ts +106 -0
  40. package/src/memory/attachments-store.ts +0 -1
  41. package/src/memory/channel-delivery-store.ts +0 -1
  42. package/src/memory/conflict-intent.ts +114 -0
  43. package/src/memory/conversation-key-store.ts +0 -1
  44. package/src/memory/db.ts +346 -149
  45. package/src/memory/job-handlers/conflict.ts +23 -1
  46. package/src/memory/runs-store.ts +0 -3
  47. package/src/memory/schema.ts +0 -4
  48. package/src/runtime/gateway-client.ts +36 -0
  49. package/src/runtime/http-server.ts +140 -2
  50. package/src/runtime/routes/channel-routes.ts +121 -79
  51. package/src/security/oauth-callback-registry.ts +56 -0
  52. package/src/security/oauth2.ts +174 -58
  53. package/src/swarm/backend-claude-code.ts +1 -1
  54. package/src/tools/assets/search.ts +1 -36
  55. package/src/tools/browser/api-map.ts +123 -50
  56. package/src/tools/claude-code/claude-code.ts +131 -1
  57. package/src/tools/tasks/work-item-list.ts +16 -2
  58. package/src/workspace/commit-message-enrichment-service.ts +3 -3
  59. package/src/workspace/provider-commit-message-generator.ts +57 -14
  60. package/src/workspace/turn-commit.ts +6 -2
@@ -1,6 +1,10 @@
1
1
  /**
2
2
  * General-purpose OAuth2 Authorization Code flow with PKCE.
3
3
  *
4
+ * Supports two callback transports:
5
+ * - loopback: spins up a local HTTP server on 127.0.0.1 (default when no public URL configured)
6
+ * - gateway: uses the gateway's OAuth callback route + in-memory registry (when ingress.publicBaseUrl is set)
7
+ *
4
8
  * Moved from integrations/oauth2.ts. Types that were in integrations/types.ts
5
9
  * are now inlined here since the integration framework is removed.
6
10
  */
@@ -39,6 +43,11 @@ export interface OAuth2FlowCallbacks {
39
43
  openUrl: (url: string) => void;
40
44
  }
41
45
 
46
+ export interface OAuth2FlowOptions {
47
+ /** Which callback transport to use. When omitted, auto-detected from config. */
48
+ callbackTransport?: 'loopback' | 'gateway';
49
+ }
50
+
42
51
  export interface OAuth2FlowResult {
43
52
  tokens: OAuth2TokenResult;
44
53
  grantedScopes: string[];
@@ -62,20 +71,107 @@ function generateState(): string {
62
71
  }
63
72
 
64
73
  // ---------------------------------------------------------------------------
65
- // Public API
74
+ // Token exchange (shared between transports)
75
+ // ---------------------------------------------------------------------------
76
+
77
+ async function exchangeCodeForTokens(
78
+ config: OAuth2Config,
79
+ code: string,
80
+ redirectUri: string,
81
+ codeVerifier: string,
82
+ ): Promise<OAuth2FlowResult> {
83
+ const usePKCE = !config.clientSecret;
84
+
85
+ const tokenBody: Record<string, string> = {
86
+ grant_type: 'authorization_code',
87
+ code,
88
+ redirect_uri: redirectUri,
89
+ client_id: config.clientId,
90
+ };
91
+ if (usePKCE) {
92
+ tokenBody.code_verifier = codeVerifier;
93
+ }
94
+ if (config.clientSecret) {
95
+ tokenBody.client_secret = config.clientSecret;
96
+ }
97
+
98
+ const tokenResp = await fetch(config.tokenUrl, {
99
+ method: 'POST',
100
+ headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
101
+ body: new URLSearchParams(tokenBody),
102
+ });
103
+
104
+ if (!tokenResp.ok) {
105
+ const rawBody = await tokenResp.text().catch(() => '');
106
+ let safeDetail: Record<string, unknown> = {};
107
+ let errorCode = '';
108
+ try {
109
+ const parsed = JSON.parse(rawBody) as Record<string, unknown>;
110
+ if (parsed.error) { safeDetail.error = String(parsed.error); errorCode = String(parsed.error); }
111
+ if (parsed.error_description) safeDetail.error_description = String(parsed.error_description);
112
+ } catch {
113
+ safeDetail.error = '[non-JSON response]';
114
+ }
115
+ log.error({ status: tokenResp.status, ...safeDetail }, 'OAuth2 token exchange failed');
116
+ const detail = errorCode ? `HTTP ${tokenResp.status}: ${errorCode}` : `HTTP ${tokenResp.status}`;
117
+ throw new Error(`OAuth2 token exchange failed (${detail})`);
118
+ }
119
+
120
+ const tokenData = await tokenResp.json() as Record<string, unknown>;
121
+
122
+ // Slack V2 OAuth returns user tokens nested under `authed_user`
123
+ const authedUser = tokenData.authed_user as Record<string, unknown> | undefined;
124
+ const tokenSource = authedUser?.access_token ? authedUser : tokenData;
125
+
126
+ const tokens: OAuth2TokenResult = {
127
+ accessToken: (tokenSource.access_token as string) ?? (tokenData.access_token as string),
128
+ refreshToken: (tokenSource.refresh_token as string | undefined) ?? (tokenData.refresh_token as string | undefined),
129
+ expiresIn: (tokenSource.expires_in as number | undefined) ?? (tokenData.expires_in as number | undefined),
130
+ scope: (tokenSource.scope as string | undefined) ?? (tokenData.scope as string | undefined),
131
+ tokenType: (tokenSource.token_type as string | undefined) ?? (tokenData.token_type as string | undefined),
132
+ };
133
+
134
+ const grantedScopes = typeof tokens.scope === 'string'
135
+ ? tokens.scope.split(/[ ,]/).filter(Boolean)
136
+ : [...config.scopes];
137
+
138
+ return { tokens, grantedScopes, rawTokenResponse: tokenData };
139
+ }
140
+
141
+ // ---------------------------------------------------------------------------
142
+ // Transport auto-detection
66
143
  // ---------------------------------------------------------------------------
67
144
 
68
145
  /**
69
- * Run a full OAuth2 PKCE authorization code flow using a loopback redirect.
146
+ * Determine which callback transport to use when not explicitly specified.
147
+ * Uses gateway if ingress.publicBaseUrl is configured, otherwise loopback.
70
148
  */
71
- export async function startOAuth2Flow(
149
+ function detectTransport(): 'loopback' | 'gateway' {
150
+ try {
151
+ // Dynamic import avoided — loadConfig is synchronous and already used elsewhere.
152
+ const { loadConfig } = require('../config/loader.js') as typeof import('../config/loader.js');
153
+ const appConfig = loadConfig();
154
+ if (appConfig.ingress?.publicBaseUrl) {
155
+ return 'gateway';
156
+ }
157
+ } catch {
158
+ // Config loading failed — fall back to loopback
159
+ log.debug('Config not available for transport auto-detection, defaulting to loopback');
160
+ }
161
+ return 'loopback';
162
+ }
163
+
164
+ // ---------------------------------------------------------------------------
165
+ // Loopback transport
166
+ // ---------------------------------------------------------------------------
167
+
168
+ async function runLoopbackFlow(
72
169
  config: OAuth2Config,
73
170
  callbacks: OAuth2FlowCallbacks,
171
+ codeVerifier: string,
172
+ codeChallenge: string,
173
+ state: string,
74
174
  ): Promise<OAuth2FlowResult> {
75
- const codeVerifier = generateCodeVerifier();
76
- const codeChallenge = generateCodeChallenge(codeVerifier);
77
- const state = generateState();
78
-
79
175
  let resolveCode: (value: { code: string; returnedState: string }) => void;
80
176
  let rejectCode: (reason: Error) => void;
81
177
 
@@ -84,7 +180,6 @@ export async function startOAuth2Flow(
84
180
  rejectCode = reject;
85
181
  });
86
182
 
87
- /** How long to wait for the user to complete the OAuth consent flow. */
88
183
  const FLOW_TIMEOUT_MS = 120_000;
89
184
 
90
185
  const timeout = setTimeout(() => {
@@ -153,64 +248,85 @@ export async function startOAuth2Flow(
153
248
  throw new Error('OAuth2 state mismatch — possible CSRF attack');
154
249
  }
155
250
 
156
- const tokenBody: Record<string, string> = {
157
- grant_type: 'authorization_code',
158
- code,
159
- redirect_uri: redirectUri,
160
- client_id: config.clientId,
161
- };
162
- if (usePKCE) {
163
- tokenBody.code_verifier = codeVerifier;
164
- }
165
- if (config.clientSecret) {
166
- tokenBody.client_secret = config.clientSecret;
167
- }
251
+ return await exchangeCodeForTokens(config, code, redirectUri, codeVerifier);
252
+ } finally {
253
+ clearTimeout(timeout);
254
+ server.stop(true);
255
+ }
256
+ }
168
257
 
169
- const tokenResp = await fetch(config.tokenUrl, {
170
- method: 'POST',
171
- headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
172
- body: new URLSearchParams(tokenBody),
173
- });
258
+ // ---------------------------------------------------------------------------
259
+ // Gateway transport
260
+ // ---------------------------------------------------------------------------
174
261
 
175
- if (!tokenResp.ok) {
176
- const rawBody = await tokenResp.text().catch(() => '');
177
- let safeDetail: Record<string, unknown> = {};
178
- let errorCode = '';
179
- try {
180
- const parsed = JSON.parse(rawBody) as Record<string, unknown>;
181
- if (parsed.error) { safeDetail.error = String(parsed.error); errorCode = String(parsed.error); }
182
- if (parsed.error_description) safeDetail.error_description = String(parsed.error_description);
183
- } catch {
184
- safeDetail.error = '[non-JSON response]';
185
- }
186
- log.error({ status: tokenResp.status, ...safeDetail }, 'OAuth2 token exchange failed');
187
- const detail = errorCode ? `HTTP ${tokenResp.status}: ${errorCode}` : `HTTP ${tokenResp.status}`;
188
- throw new Error(`OAuth2 token exchange failed (${detail})`);
189
- }
262
+ async function runGatewayFlow(
263
+ config: OAuth2Config,
264
+ callbacks: OAuth2FlowCallbacks,
265
+ codeVerifier: string,
266
+ codeChallenge: string,
267
+ state: string,
268
+ ): Promise<OAuth2FlowResult> {
269
+ const { loadConfig } = require('../config/loader.js') as typeof import('../config/loader.js');
270
+ const { getOAuthCallbackUrl } = require('../inbound/public-ingress-urls.js') as typeof import('../inbound/public-ingress-urls.js');
271
+ const { registerPendingCallback } = require('./oauth-callback-registry.js') as typeof import('./oauth-callback-registry.js');
190
272
 
191
- const tokenData = await tokenResp.json() as Record<string, unknown>;
273
+ const appConfig = loadConfig();
274
+ const redirectUri = getOAuthCallbackUrl(appConfig);
192
275
 
193
- // Slack V2 OAuth returns user tokens nested under `authed_user`
194
- const authedUser = tokenData.authed_user as Record<string, unknown> | undefined;
195
- const tokenSource = authedUser?.access_token ? authedUser : tokenData;
276
+ const codePromise = new Promise<string>((resolve, reject) => {
277
+ registerPendingCallback(state, resolve, reject);
278
+ });
196
279
 
197
- const tokens: OAuth2TokenResult = {
198
- accessToken: (tokenSource.access_token as string) ?? (tokenData.access_token as string),
199
- refreshToken: (tokenSource.refresh_token as string | undefined) ?? (tokenData.refresh_token as string | undefined),
200
- expiresIn: (tokenSource.expires_in as number | undefined) ?? (tokenData.expires_in as number | undefined),
201
- scope: (tokenSource.scope as string | undefined) ?? (tokenData.scope as string | undefined),
202
- tokenType: (tokenSource.token_type as string | undefined) ?? (tokenData.token_type as string | undefined),
203
- };
280
+ const usePKCE = !config.clientSecret;
281
+ const authParams = new URLSearchParams({
282
+ ...config.extraParams,
283
+ client_id: config.clientId,
284
+ redirect_uri: redirectUri,
285
+ response_type: 'code',
286
+ scope: config.scopes.join(' '),
287
+ state,
288
+ ...(usePKCE ? { code_challenge: codeChallenge, code_challenge_method: 'S256' } : {}),
289
+ });
204
290
 
205
- const grantedScopes = typeof tokens.scope === 'string'
206
- ? tokens.scope.split(/[ ,]/).filter(Boolean)
207
- : [...config.scopes];
291
+ const authUrl = `${config.authUrl}?${authParams}`;
292
+ callbacks.openUrl(authUrl);
208
293
 
209
- return { tokens, grantedScopes, rawTokenResponse: tokenData };
210
- } finally {
211
- clearTimeout(timeout);
212
- server.stop(true);
294
+ const code = await codePromise;
295
+
296
+ return await exchangeCodeForTokens(config, code, redirectUri, codeVerifier);
297
+ }
298
+
299
+ // ---------------------------------------------------------------------------
300
+ // Public API
301
+ // ---------------------------------------------------------------------------
302
+
303
+ /**
304
+ * Run a full OAuth2 authorization code flow with PKCE support.
305
+ *
306
+ * Supports two callback transports:
307
+ * - loopback (default): local HTTP server on 127.0.0.1
308
+ * - gateway: callback via the gateway's OAuth route + in-memory registry
309
+ *
310
+ * Transport is auto-detected based on ingress.publicBaseUrl config unless
311
+ * explicitly specified via options.callbackTransport.
312
+ */
313
+ export async function startOAuth2Flow(
314
+ config: OAuth2Config,
315
+ callbacks: OAuth2FlowCallbacks,
316
+ options?: OAuth2FlowOptions,
317
+ ): Promise<OAuth2FlowResult> {
318
+ const codeVerifier = generateCodeVerifier();
319
+ const codeChallenge = generateCodeChallenge(codeVerifier);
320
+ const state = generateState();
321
+
322
+ const transport = options?.callbackTransport ?? detectTransport();
323
+ log.debug({ transport }, 'OAuth2 flow starting');
324
+
325
+ if (transport === 'gateway') {
326
+ return runGatewayFlow(config, callbacks, codeVerifier, codeChallenge, state);
213
327
  }
328
+
329
+ return runLoopbackFlow(config, callbacks, codeVerifier, codeChallenge, state);
214
330
  }
215
331
 
216
332
  /**
@@ -117,7 +117,7 @@ export function createClaudeCodeBackend(): SwarmWorkerBackend {
117
117
 
118
118
  const parts: string[] = [`[${message.subtype}] (${message.num_turns} turns, ${(message.duration_ms / 1000).toFixed(1)}s)`];
119
119
  if (errors.length > 0) parts.push(`Errors: ${errors.join('; ')}`);
120
- if (denials.length > 0) parts.push(`Permission denied: ${denials.map(d => d.tool_name).join(', ')}`);
120
+ if (denials.length > 0) parts.push(`Permission denied: ${denials.map((d: { tool_name: string }) => d.tool_name).join(', ')}`);
121
121
  resultText += `\n${parts.join('\n')}`;
122
122
  }
123
123
  }
@@ -13,7 +13,7 @@ import type { Tool, ToolContext, ToolExecutionResult } from '../types.js';
13
13
  import type { ToolDefinition } from '../../providers/types.js';
14
14
  import { registerTool } from '../registry.js';
15
15
  import { getDb } from '../../memory/db.js';
16
- import { attachments, messageAttachments, messages, conversations, conversationKeys } from '../../memory/schema.js';
16
+ import { attachments, messageAttachments, messages, conversations } from '../../memory/schema.js';
17
17
  import type { StoredAttachment } from '../../memory/attachments-store.js';
18
18
  import { isAttachmentVisible, type AttachmentContext } from '../../daemon/media-visibility-policy.js';
19
19
  import { getConversationThreadType } from '../../memory/conversation-store.js';
@@ -103,25 +103,6 @@ function isAttachmentVisibleFromContext(attachmentId: string, currentContext: At
103
103
  );
104
104
  }
105
105
 
106
- // ---------------------------------------------------------------------------
107
- // Assistant ID lookup
108
- // ---------------------------------------------------------------------------
109
-
110
- /**
111
- * Derive the assistant ID that owns a conversation via the conversation_keys
112
- * table. Returns null when no mapping exists (e.g. native macOS app sessions
113
- * that use the implicit 'local-assistant' identity).
114
- */
115
- function getAssistantIdForConversation(conversationId: string): string | null {
116
- const db = getDb();
117
- const row = db
118
- .select({ assistantId: conversationKeys.assistantId })
119
- .from(conversationKeys)
120
- .where(eq(conversationKeys.conversationId, conversationId))
121
- .get();
122
- return row?.assistantId ?? null;
123
- }
124
-
125
106
  // ---------------------------------------------------------------------------
126
107
  // Search logic
127
108
  // ---------------------------------------------------------------------------
@@ -131,8 +112,6 @@ export interface AssetSearchParams {
131
112
  filename?: string;
132
113
  recency?: string;
133
114
  conversation_id?: string;
134
- /** Tenant boundary — when set, only attachments belonging to this assistant are returned. */
135
- assistant_id?: string;
136
115
  limit?: number;
137
116
  }
138
117
 
@@ -163,11 +142,6 @@ export function searchAttachments(params: AssetSearchParams): StoredAttachment[]
163
142
  }
164
143
  }
165
144
 
166
- // Tenant boundary — restrict to the active assistant's attachments
167
- if (params.assistant_id) {
168
- conditions.push(eq(attachments.assistantId, params.assistant_id));
169
- }
170
-
171
145
  // Conversation scope — join through message_attachments + messages
172
146
  if (params.conversation_id) {
173
147
  const linkedIds = db
@@ -207,11 +181,6 @@ export function searchAttachments(params: AssetSearchParams): StoredAttachment[]
207
181
  bindValues.push(Date.now() - offsetMs);
208
182
  }
209
183
  }
210
- if (params.assistant_id) {
211
- whereParts.push(`a.assistant_id = ?`);
212
- bindValues.push(params.assistant_id);
213
- }
214
-
215
184
  const limit = Math.min(params.limit ?? DEFAULT_LIMIT, MAX_RESULTS);
216
185
  const stmt = raw.prepare(
217
186
  `SELECT a.id, a.original_filename, a.mime_type, a.size_bytes, a.kind, a.thumbnail_base64, a.created_at
@@ -347,9 +316,6 @@ class AssetSearchTool implements Tool {
347
316
  }
348
317
 
349
318
  try {
350
- // Scope results to the active assistant to prevent cross-tenant leaks
351
- const assistantId = getAssistantIdForConversation(context.conversationId) ?? undefined;
352
-
353
319
  // Over-fetch with MAX_RESULTS so visibility filtering doesn't
354
320
  // under-fill the caller's requested limit.
355
321
  const results = searchAttachments({
@@ -357,7 +323,6 @@ class AssetSearchTool implements Tool {
357
323
  filename,
358
324
  recency,
359
325
  conversation_id: conversationId,
360
- assistant_id: assistantId,
361
326
  limit: MAX_RESULTS,
362
327
  });
363
328
 
@@ -38,12 +38,31 @@ export interface ApiMapResult {
38
38
  const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
39
39
  const NUMERIC_RE = /^\d+$/;
40
40
  const HEX_HASH_RE = /^[0-9a-f]{8,}$/i;
41
+ const DATE_RE = /^\d{4}-\d{2}-\d{2}$/;
42
+
43
+ /** URL path patterns that indicate non-API noise. */
44
+ const NOISE_PATH_PATTERNS = [
45
+ /\/web-translations\//,
46
+ /\/cdn-cgi\//,
47
+ /\.properties$/,
48
+ /\.js$/,
49
+ /\.css$/,
50
+ /\.woff2?$/,
51
+ /\.png$/,
52
+ /\.jpg$/,
53
+ /\.svg$/,
54
+ /\.ico$/,
55
+ /\.map$/,
56
+ /\/preference\//,
57
+ /\/userpreference-service\//,
58
+ ];
41
59
 
42
60
  /** Returns true when a path segment looks like a dynamic ID. */
43
61
  function isIdSegment(segment: string): boolean {
44
62
  if (NUMERIC_RE.test(segment)) return true;
45
63
  if (UUID_RE.test(segment)) return true;
46
64
  if (HEX_HASH_RE.test(segment)) return true;
65
+ if (DATE_RE.test(segment)) return true;
47
66
  return false;
48
67
  }
49
68
 
@@ -69,27 +88,43 @@ function tryParseJson(text: string | undefined): Record<string, unknown> | undef
69
88
  return undefined;
70
89
  }
71
90
 
91
+ /** Extract GraphQL operation name from request body. */
92
+ function extractGraphQLOperationName(postData: string | undefined): string | null {
93
+ if (!postData) return null;
94
+ const body = tryParseJson(postData);
95
+ if (!body) return null;
96
+ if (typeof body.operationName === 'string' && body.operationName) return body.operationName;
97
+ // Try extracting from query string: "query FooBar { ..." or "mutation FooBar { ..."
98
+ if (typeof body.query === 'string') {
99
+ const named = body.query.match(/(?:query|mutation|subscription)\s+(\w+)/);
100
+ if (named) return named[1];
101
+ // Unnamed query — extract the first field name: "query{fooBar(" or "query { fooBar {"
102
+ const firstField = body.query.match(/(?:query|mutation|subscription)\s*\{?\s*(\w+)/);
103
+ if (firstField) return firstField[1];
104
+ }
105
+ return null;
106
+ }
107
+
72
108
  // ---------------------------------------------------------------------------
73
109
  // Core analysis
74
110
  // ---------------------------------------------------------------------------
75
111
 
112
+ interface GroupData {
113
+ method: string;
114
+ urlPattern: string;
115
+ exampleUrl: string;
116
+ queryParams: Set<string>;
117
+ requestBodyKeys: Set<string>;
118
+ responseStatus: Set<number>;
119
+ responseBodyKeys: Set<string>;
120
+ count: number;
121
+ }
122
+
76
123
  export function analyzeApiMap(
77
124
  entries: NetworkRecordedEntry[],
78
125
  domain: string,
79
126
  ): ApiMapResult {
80
- const groups = new Map<
81
- string,
82
- {
83
- method: string;
84
- urlPattern: string;
85
- exampleUrl: string;
86
- queryParams: Set<string>;
87
- requestBodyKeys: Set<string>;
88
- responseStatus: Set<number>;
89
- responseBodyKeys: Set<string>;
90
- count: number;
91
- }
92
- >();
127
+ const groups = new Map<string, GroupData>();
93
128
 
94
129
  for (const entry of entries) {
95
130
  const { request, response } = entry;
@@ -97,11 +132,30 @@ export function analyzeApiMap(
97
132
  try {
98
133
  parsed = new URL(request.url);
99
134
  } catch {
100
- continue; // skip malformed URLs
135
+ continue;
101
136
  }
102
137
 
138
+ // Skip non-API noise
139
+ if (NOISE_PATH_PATTERNS.some(p => p.test(parsed.pathname))) continue;
140
+
141
+ // Skip non-JSON responses
142
+ const mimeType = response?.mimeType ?? '';
143
+ if (response && !mimeType.includes('json') && !mimeType.includes('graphql')) continue;
144
+
103
145
  const method = request.method.toUpperCase();
104
- const urlPattern = `${parsed.hostname}${normalizePathSegments(parsed.pathname)}`;
146
+ const normalizedPath = normalizePathSegments(parsed.pathname);
147
+ const basePattern = `${parsed.hostname}${normalizedPath}`;
148
+
149
+ // For GraphQL endpoints, split by operation name
150
+ let urlPattern = basePattern;
151
+ const isGraphQL = normalizedPath.includes('graphql');
152
+ if (isGraphQL && method === 'POST') {
153
+ const opName = extractGraphQLOperationName(request.postData);
154
+ if (opName) {
155
+ urlPattern = `${basePattern} → ${opName}`;
156
+ }
157
+ }
158
+
105
159
  const key = `${method} ${urlPattern}`;
106
160
 
107
161
  let group = groups.get(key);
@@ -121,26 +175,23 @@ export function analyzeApiMap(
121
175
 
122
176
  group.count++;
123
177
 
124
- // Collect query param keys
125
178
  for (const paramKey of parsed.searchParams.keys()) {
126
179
  group.queryParams.add(paramKey);
127
180
  }
128
181
 
129
- // Request body keys (POST/PUT/PATCH)
130
182
  if (['POST', 'PUT', 'PATCH'].includes(method)) {
131
183
  const body = tryParseJson(request.postData);
132
184
  if (body) {
133
185
  for (const k of Object.keys(body)) {
134
- group.requestBodyKeys.add(k);
186
+ if (k !== 'query' && k !== 'operationName' && k !== 'extensions') {
187
+ group.requestBodyKeys.add(k);
188
+ }
135
189
  }
136
190
  }
137
191
  }
138
192
 
139
- // Response status
140
193
  if (response) {
141
194
  group.responseStatus.add(response.status);
142
-
143
- // Response body keys
144
195
  const resBody = tryParseJson(response.body);
145
196
  if (resBody) {
146
197
  for (const k of Object.keys(resBody)) {
@@ -161,13 +212,21 @@ export function analyzeApiMap(
161
212
  count: g.count,
162
213
  }));
163
214
 
164
- // Sort by count descending, then by urlPattern for stability
165
- endpoints.sort((a, b) => b.count - a.count || a.urlPattern.localeCompare(b.urlPattern));
215
+ // Sort: data endpoints first (low count = unique pages), then boilerplate
216
+ // Within each tier, sort alphabetically by pattern for readability
217
+ endpoints.sort((a, b) => {
218
+ const aIsBoilerplate = a.count > 15;
219
+ const bIsBoilerplate = b.count > 15;
220
+ if (aIsBoilerplate !== bIsBoilerplate) return aIsBoilerplate ? 1 : -1;
221
+ return a.urlPattern.localeCompare(b.urlPattern);
222
+ });
223
+
224
+ const totalApiRequests = endpoints.reduce((sum, ep) => sum + ep.count, 0);
166
225
 
167
226
  return {
168
227
  domain,
169
228
  analyzedAt: Date.now(),
170
- totalRequests: entries.length,
229
+ totalRequests: totalApiRequests,
171
230
  endpoints,
172
231
  };
173
232
  }
@@ -191,30 +250,44 @@ export function saveApiMap(domain: string, result: ApiMapResult): string {
191
250
  // ---------------------------------------------------------------------------
192
251
 
193
252
  export function printApiMapTable(result: ApiMapResult): void {
194
- console.log(`\nAPI Map for ${result.domain} — ${result.totalRequests} total requests, ${result.endpoints.length} unique endpoints\n`);
195
-
196
- const header = ['Method', 'URL Pattern', 'Count', 'Status', 'Query Params'];
197
- const rows = result.endpoints.map((ep) => [
198
- ep.method,
199
- ep.urlPattern,
200
- String(ep.count),
201
- ep.responseStatus.join(',') || '-',
202
- ep.queryParams.join(',') || '-',
203
- ]);
204
-
205
- // Calculate column widths
206
- const widths = header.map((h, i) =>
207
- Math.max(h.length, ...rows.map((r) => r[i].length)),
208
- );
209
-
210
- const sep = widths.map((w) => '-'.repeat(w)).join(' | ');
211
- const fmt = (row: string[]) =>
212
- row.map((cell, i) => cell.padEnd(widths[i])).join(' | ');
213
-
214
- console.log(fmt(header));
215
- console.log(sep);
216
- for (const row of rows) {
217
- console.log(fmt(row));
218
- }
219
- console.log();
253
+ const dataEndpoints = result.endpoints.filter(ep => ep.count <= 15);
254
+ const boilerplate = result.endpoints.filter(ep => ep.count > 15);
255
+
256
+ console.log(`\nAPI Map for ${result.domain} — ${result.endpoints.length} endpoints discovered\n`);
257
+
258
+ const stripDomain = (pattern: string) => {
259
+ const idx = pattern.indexOf('/');
260
+ return idx >= 0 ? pattern.slice(idx) : pattern;
261
+ };
262
+
263
+ const printSection = (title: string, eps: ApiEndpoint[]) => {
264
+ if (eps.length === 0) return;
265
+ console.log(` ${title} (${eps.length})\n`);
266
+
267
+ const header = ['Method', 'Endpoint', 'Hits', 'Response Keys'];
268
+ const rows = eps.map((ep) => [
269
+ ep.method,
270
+ stripDomain(ep.urlPattern),
271
+ String(ep.count),
272
+ ep.responseBodyKeys.slice(0, 5).join(', ') || '-',
273
+ ]);
274
+
275
+ const widths = header.map((h, i) =>
276
+ Math.min(i === 1 ? 72 : i === 3 ? 50 : 200, Math.max(h.length, ...rows.map((r) => r[i].length))),
277
+ );
278
+
279
+ const sep = widths.map((w) => '-'.repeat(w)).join(' | ');
280
+ const fmt = (row: string[]) =>
281
+ row.map((cell, i) => cell.slice(0, widths[i]).padEnd(widths[i])).join(' | ');
282
+
283
+ console.log(` ${fmt(header)}`);
284
+ console.log(` ${sep}`);
285
+ for (const row of rows) {
286
+ console.log(` ${fmt(row)}`);
287
+ }
288
+ console.log();
289
+ };
290
+
291
+ printSection('DATA ENDPOINTS', dataEndpoints);
292
+ printSection('PAGE-LOAD BOILERPLATE', boilerplate);
220
293
  }