vellum 0.2.1 → 0.2.7

package/src/tools/browser/api-map.ts

@@ -0,0 +1,293 @@
+/**
+ * Recording analyzer that processes NetworkRecordedEntry[] into a deduplicated
+ * API map. Collapses ID-like path segments into {id} placeholders so repeated
+ * calls to the same endpoint are grouped together.
+ */
+
+import { mkdirSync, writeFileSync } from 'fs';
+import { join } from 'path';
+import { getDataDir } from '../../util/platform.js';
+import type { NetworkRecordedEntry } from './network-recording-types.js';
+
+// ---------------------------------------------------------------------------
+// Types
+// ---------------------------------------------------------------------------
+
+export interface ApiEndpoint {
+  method: string;
+  urlPattern: string;
+  exampleUrl: string;
+  queryParams: string[];
+  requestBodyKeys: string[];
+  responseStatus: number[];
+  responseBodyKeys: string[];
+  count: number;
+}
+
+export interface ApiMapResult {
+  domain: string;
+  analyzedAt: number;
+  totalRequests: number;
+  endpoints: ApiEndpoint[];
+}
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
+const NUMERIC_RE = /^\d+$/;
+const HEX_HASH_RE = /^[0-9a-f]{8,}$/i;
+const DATE_RE = /^\d{4}-\d{2}-\d{2}$/;
+
+/** URL path patterns that indicate non-API noise. */
+const NOISE_PATH_PATTERNS = [
+  /\/web-translations\//,
+  /\/cdn-cgi\//,
+  /\.properties$/,
+  /\.js$/,
+  /\.css$/,
+  /\.woff2?$/,
+  /\.png$/,
+  /\.jpg$/,
+  /\.svg$/,
+  /\.ico$/,
+  /\.map$/,
+  /\/preference\//,
+  /\/userpreference-service\//,
+];
+
+/** Returns true when a path segment looks like a dynamic ID. */
+function isIdSegment(segment: string): boolean {
+  if (NUMERIC_RE.test(segment)) return true;
+  if (UUID_RE.test(segment)) return true;
+  if (HEX_HASH_RE.test(segment)) return true;
+  if (DATE_RE.test(segment)) return true;
+  return false;
+}
+
+/** Replace ID-like path segments with `{id}`. */
+function normalizePathSegments(pathname: string): string {
+  return pathname
+    .split('/')
+    .map((seg) => (isIdSegment(seg) ? '{id}' : seg))
+    .join('/');
+}
+
+/** Safely parse JSON, returning undefined on failure. */
+function tryParseJson(text: string | undefined): Record<string, unknown> | undefined {
+  if (!text) return undefined;
+  try {
+    const parsed = JSON.parse(text);
+    if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) {
+      return parsed as Record<string, unknown>;
+    }
+  } catch {
+    // not JSON
+  }
+  return undefined;
+}
+
+/** Extract GraphQL operation name from request body. */
+function extractGraphQLOperationName(postData: string | undefined): string | null {
+  if (!postData) return null;
+  const body = tryParseJson(postData);
+  if (!body) return null;
+  if (typeof body.operationName === 'string' && body.operationName) return body.operationName;
+  // Try extracting from query string: "query FooBar { ..." or "mutation FooBar { ..."
+  if (typeof body.query === 'string') {
+    const named = body.query.match(/(?:query|mutation|subscription)\s+(\w+)/);
+    if (named) return named[1];
+    // Unnamed query — extract the first field name: "query{fooBar(" or "query { fooBar {"
+    const firstField = body.query.match(/(?:query|mutation|subscription)\s*\{?\s*(\w+)/);
+    if (firstField) return firstField[1];
+  }
+  return null;
+}
+
+// ---------------------------------------------------------------------------
+// Core analysis
+// ---------------------------------------------------------------------------
+
+interface GroupData {
+  method: string;
+  urlPattern: string;
+  exampleUrl: string;
+  queryParams: Set<string>;
+  requestBodyKeys: Set<string>;
+  responseStatus: Set<number>;
+  responseBodyKeys: Set<string>;
+  count: number;
+}
+
+export function analyzeApiMap(
+  entries: NetworkRecordedEntry[],
+  domain: string,
+): ApiMapResult {
+  const groups = new Map<string, GroupData>();
+
+  for (const entry of entries) {
+    const { request, response } = entry;
+    let parsed: URL;
+    try {
+      parsed = new URL(request.url);
+    } catch {
+      continue;
+    }
+
+    // Skip non-API noise
+    if (NOISE_PATH_PATTERNS.some(p => p.test(parsed.pathname))) continue;
+
+    // Skip non-JSON responses
+    const mimeType = response?.mimeType ?? '';
+    if (response && !mimeType.includes('json') && !mimeType.includes('graphql')) continue;
+
+    const method = request.method.toUpperCase();
+    const normalizedPath = normalizePathSegments(parsed.pathname);
+    const basePattern = `${parsed.hostname}${normalizedPath}`;
+
+    // For GraphQL endpoints, split by operation name
+    let urlPattern = basePattern;
+    const isGraphQL = normalizedPath.includes('graphql');
+    if (isGraphQL && method === 'POST') {
+      const opName = extractGraphQLOperationName(request.postData);
+      if (opName) {
+        urlPattern = `${basePattern} → ${opName}`;
+      }
+    }
+
+    const key = `${method} ${urlPattern}`;
+
+    let group = groups.get(key);
+    if (!group) {
+      group = {
+        method,
+        urlPattern,
+        exampleUrl: request.url,
+        queryParams: new Set(),
+        requestBodyKeys: new Set(),
+        responseStatus: new Set(),
+        responseBodyKeys: new Set(),
+        count: 0,
+      };
+      groups.set(key, group);
+    }
+
+    group.count++;
+
+    for (const paramKey of parsed.searchParams.keys()) {
+      group.queryParams.add(paramKey);
+    }
+
+    if (['POST', 'PUT', 'PATCH'].includes(method)) {
+      const body = tryParseJson(request.postData);
+      if (body) {
+        for (const k of Object.keys(body)) {
+          if (k !== 'query' && k !== 'operationName' && k !== 'extensions') {
+            group.requestBodyKeys.add(k);
+          }
+        }
+      }
+    }
+
+    if (response) {
+      group.responseStatus.add(response.status);
+      const resBody = tryParseJson(response.body);
+      if (resBody) {
+        for (const k of Object.keys(resBody)) {
+          group.responseBodyKeys.add(k);
+        }
+      }
+    }
+  }
+
+  const endpoints: ApiEndpoint[] = Array.from(groups.values()).map((g) => ({
+    method: g.method,
+    urlPattern: g.urlPattern,
+    exampleUrl: g.exampleUrl,
+    queryParams: Array.from(g.queryParams).sort(),
+    requestBodyKeys: Array.from(g.requestBodyKeys).sort(),
+    responseStatus: Array.from(g.responseStatus).sort((a, b) => a - b),
+    responseBodyKeys: Array.from(g.responseBodyKeys).sort(),
+    count: g.count,
+  }));
+
+  // Sort: data endpoints first (low count = unique pages), then boilerplate
+  // Within each tier, sort alphabetically by pattern for readability
+  endpoints.sort((a, b) => {
+    const aIsBoilerplate = a.count > 15;
+    const bIsBoilerplate = b.count > 15;
+    if (aIsBoilerplate !== bIsBoilerplate) return aIsBoilerplate ? 1 : -1;
+    return a.urlPattern.localeCompare(b.urlPattern);
+  });
+
+  const totalApiRequests = endpoints.reduce((sum, ep) => sum + ep.count, 0);
+
+  return {
+    domain,
+    analyzedAt: Date.now(),
+    totalRequests: totalApiRequests,
+    endpoints,
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Persistence
+// ---------------------------------------------------------------------------
+
+export function saveApiMap(domain: string, result: ApiMapResult): string {
+  const dir = join(getDataDir(), 'api-maps');
+  mkdirSync(dir, { recursive: true });
+
+  const timestamp = Date.now();
+  const filePath = join(dir, `${domain}-${timestamp}.json`);
+  writeFileSync(filePath, JSON.stringify(result, null, 2));
+  return filePath;
+}
+
+// ---------------------------------------------------------------------------
+// Pretty-print
+// ---------------------------------------------------------------------------
+
+export function printApiMapTable(result: ApiMapResult): void {
+  const dataEndpoints = result.endpoints.filter(ep => ep.count <= 15);
+  const boilerplate = result.endpoints.filter(ep => ep.count > 15);
+
+  console.log(`\nAPI Map for ${result.domain} — ${result.endpoints.length} endpoints discovered\n`);
+
+  const stripDomain = (pattern: string) => {
+    const idx = pattern.indexOf('/');
+    return idx >= 0 ? pattern.slice(idx) : pattern;
+  };
+
+  const printSection = (title: string, eps: ApiEndpoint[]) => {
+    if (eps.length === 0) return;
+    console.log(`  ${title} (${eps.length})\n`);
+
+    const header = ['Method', 'Endpoint', 'Hits', 'Response Keys'];
+    const rows = eps.map((ep) => [
+      ep.method,
+      stripDomain(ep.urlPattern),
+      String(ep.count),
+      ep.responseBodyKeys.slice(0, 5).join(', ') || '-',
+    ]);
+
+    const widths = header.map((h, i) =>
+      Math.min(i === 1 ? 72 : i === 3 ? 50 : 200, Math.max(h.length, ...rows.map((r) => r[i].length))),
+    );
+
+    const sep = widths.map((w) => '-'.repeat(w)).join(' | ');
+    const fmt = (row: string[]) =>
+      row.map((cell, i) => cell.slice(0, widths[i]).padEnd(widths[i])).join(' | ');
+
+    console.log(`  ${fmt(header)}`);
+    console.log(`  ${sep}`);
+    for (const row of rows) {
+      console.log(`  ${fmt(row)}`);
+    }
+    console.log();
+  };
+
+  printSection('DATA ENDPOINTS', dataEndpoints);
+  printSection('PAGE-LOAD BOILERPLATE', boilerplate);
+}

package/src/tools/browser/auto-navigate.ts

@@ -0,0 +1,270 @@
+/**
+ * CDP-based auto-navigation for any domain.
+ *
+ * Drives Chrome through a domain's pages by discovering internal links,
+ * so the NetworkRecorder captures the API surface without manual browsing.
+ */
+
+import { getLogger } from '../../util/logger.js';
+
+const log = getLogger('auto-navigate');
+
+const CDP_BASE = 'http://localhost:9222';
+const MAX_PAGES = 15;
+const PAGE_WAIT_MS = 3500;
+const SCROLL_WAIT_MS = 2000;
+
+/** Minimal CDP client — connects to one page tab. */
+class MiniCDP {
+  private ws: WebSocket | null = null;
+  private nextId = 1;
+  private callbacks = new Map<number, { resolve: (v: unknown) => void; reject: (e: Error) => void }>();
+
+  async connect(wsUrl: string): Promise<void> {
+    return new Promise((resolve, reject) => {
+      const ws = new WebSocket(wsUrl);
+      ws.onopen = () => { this.ws = ws; resolve(); };
+      ws.onerror = (e) => reject(new Error(`CDP error: ${e}`));
+      ws.onclose = () => {
+        this.ws = null;
+        for (const [, cb] of this.callbacks) {
+          cb.reject(new Error('WebSocket closed'));
+        }
+        this.callbacks.clear();
+      };
+      ws.onmessage = (event) => {
+        const msg = JSON.parse(String(event.data));
+        if (msg.id != null) {
+          const cb = this.callbacks.get(msg.id);
+          if (cb) {
+            this.callbacks.delete(msg.id);
+            msg.error ? cb.reject(new Error(msg.error.message)) : cb.resolve(msg.result);
+          }
+        }
+      };
+    });
+  }
+
+  async send(method: string, params?: Record<string, unknown>): Promise<unknown> {
+    if (!this.ws) throw new Error('Not connected');
+    const id = this.nextId++;
+    return new Promise((resolve, reject) => {
+      this.callbacks.set(id, { resolve, reject });
+      this.ws!.send(JSON.stringify({ id, method, params }));
+    });
+  }
+
+  close() { this.ws?.close(); }
+}
+
+/**
+ * Navigate Chrome through a domain's pages to trigger API calls.
+ * Discovers internal links from the DOM and visits up to ~15 unique paths.
+ *
+ * @param domain The domain to crawl (e.g. "example.com").
+ * @param abortSignal Optional signal to stop navigation early.
+ * @returns List of visited page URLs.
+ */
+export async function autoNavigate(domain: string, abortSignal?: { aborted: boolean }): Promise<string[]> {
+  let wsUrl: string | null = null;
+  try {
+    const res = await fetch(`${CDP_BASE}/json/list`);
+    if (!res.ok) {
+      log.warn('CDP not available for auto-navigation');
+      return [];
+    }
+    const targets = (await res.json()) as Array<{ type: string; url: string; webSocketDebuggerUrl: string }>;
+    const domainTab = targets.find(t => {
+      if (t.type !== 'page') return false;
+      try {
+        const hostname = new URL(t.url).hostname;
+        return hostname === domain || hostname.endsWith('.' + domain);
+      } catch { return false; }
+    });
+    wsUrl = domainTab?.webSocketDebuggerUrl ?? targets.find(t => t.type === 'page')?.webSocketDebuggerUrl ?? null;
+  } catch (err) {
+    log.warn({ err }, 'Failed to discover Chrome tabs');
+    return [];
+  }
+
+  if (!wsUrl) {
+    log.warn('No Chrome tab found for auto-navigation');
+    return [];
+  }
+
+  const cdp = new MiniCDP();
+  try {
+    await cdp.connect(wsUrl);
+  } catch (err) {
+    log.warn({ err }, 'Failed to connect CDP for auto-navigation');
+    return [];
+  }
+
+  await cdp.send('Page.enable').catch(() => {});
+
+  const rootUrl = `https://${domain}/`;
+  const visited = new Set<string>();
+  const visitedUrls: string[] = [];
+
+  // Navigate to the domain root first
+  try {
+    await cdp.send('Page.navigate', { url: rootUrl });
+    await sleep(PAGE_WAIT_MS);
+    visited.add('/');
+    visitedUrls.push(rootUrl);
+    log.info({ url: rootUrl }, 'Visited root page');
+  } catch (err) {
+    log.warn({ err }, 'Failed to navigate to domain root');
+    cdp.close();
+    return [];
+  }
+
+  if (abortSignal?.aborted) { cdp.close(); return visitedUrls; }
+
+  // Scroll the root page to trigger lazy content
+  await scrollPage(cdp);
+  await sleep(SCROLL_WAIT_MS);
+
+  // Click common interactive elements on the root page
+  await clickInteractiveElements(cdp);
+  await sleep(SCROLL_WAIT_MS);
+
+  // Discover internal links from the current page
+  let discoveredLinks = await discoverInternalLinks(cdp, domain);
+  log.info({ count: discoveredLinks.length }, 'Discovered internal links from root');
+
+  // Visit discovered pages
+  for (const link of discoveredLinks) {
+    if (abortSignal?.aborted) break;
+    if (visited.size >= MAX_PAGES) break;
+    if (visited.has(link.key)) continue;
+
+    const url = link.url;
+    log.info({ url }, 'Auto-navigate visiting page');
+
+    try {
+      await cdp.send('Page.navigate', { url });
+      await sleep(PAGE_WAIT_MS);
+      visited.add(link.key);
+      visitedUrls.push(url);
+
+      // Scroll to trigger lazy-loaded content
+      await scrollPage(cdp);
+      await sleep(SCROLL_WAIT_MS);
+
+      // Click interactive elements to trigger more API calls
+      await clickInteractiveElements(cdp);
+      await sleep(1500);
+
+      // Discover more links from this page
+      const newLinks = await discoverInternalLinks(cdp, domain);
+      for (const nl of newLinks) {
+        if (!visited.has(nl.key) && !discoveredLinks.some(l => l.key === nl.key)) {
+          discoveredLinks.push(nl);
+        }
+      }
+
+      log.info({ url }, 'Auto-navigate page completed');
+    } catch (err) {
+      log.warn({ err, url }, 'Auto-navigate page failed');
+    }
+  }
+
+  cdp.close();
+  log.info({ visited: visitedUrls.length, total: discoveredLinks.length + 1 }, 'Auto-navigation finished');
+  return visitedUrls;
+}
+
+interface DiscoveredLink {
+  /** Full URL to navigate to (preserves subdomain). */
+  url: string;
+  /** Deduplication key: origin + pathname. */
+  key: string;
+}
+
+/** Extract internal links from the current page DOM, preserving subdomains. */
+async function discoverInternalLinks(cdp: MiniCDP, domain: string): Promise<DiscoveredLink[]> {
+  try {
+    const result = await cdp.send('Runtime.evaluate', {
+      expression: `
+        (function() {
+          const domain = ${JSON.stringify(domain)};
+          const seen = new Set();
+          const links = [];
+          for (const a of document.querySelectorAll('a[href]')) {
+            const href = a.getAttribute('href');
+            if (!href) continue;
+            try {
+              const url = new URL(href, location.origin);
+              if (url.hostname !== domain && !url.hostname.endsWith('.' + domain)) continue;
+              const path = url.pathname;
+              // Skip anchors, query-only links, file downloads, and trivial paths
+              if (path === '/' || path === '') continue;
+              if (path.match(/\\.(png|jpg|jpeg|gif|svg|css|js|woff|pdf|zip)$/i)) continue;
+              const key = url.origin + url.pathname;
+              if (!seen.has(key)) {
+                seen.add(key);
+                links.push({ url: url.origin + url.pathname, key });
+              }
+            } catch { /* skip malformed URLs */ }
+          }
+          return links;
+        })()
+      `,
+      awaitPromise: false,
+      returnByValue: true,
+    }) as { result?: { value?: DiscoveredLink[] } };
+    return result?.result?.value ?? [];
+  } catch {
+    return [];
+  }
+}
+
+/** Scroll the page to trigger lazy-loaded content. */
+async function scrollPage(cdp: MiniCDP): Promise<void> {
+  await cdp.send('Runtime.evaluate', {
+    expression: 'window.scrollBy(0, 800)',
+    awaitPromise: false,
+  }).catch(() => {});
+}
+
+/** Click common interactive elements (tabs, nav buttons) to trigger API calls. */
+async function clickInteractiveElements(cdp: MiniCDP): Promise<void> {
+  const selectors = [
+    'nav a:not([href="/"])',
+    '[role="tab"]',
+    '[role="tablist"] button',
+    'button[data-tab]',
+    '.tab, .nav-tab, .nav-link',
+  ];
+
+  for (const selector of selectors) {
+    await clickInPage(cdp, selector);
+    await sleep(800);
+  }
+}
+
+async function clickInPage(cdp: MiniCDP, selector: string): Promise<boolean> {
+  try {
+    const result = await cdp.send('Runtime.evaluate', {
+      expression: `
+        (function() {
+          const el = document.querySelector(${JSON.stringify(selector)});
+          if (!el) return false;
+          el.scrollIntoView({ block: 'center' });
+          el.click();
+          return true;
+        })()
+      `,
+      awaitPromise: false,
+      returnByValue: true,
+    }) as { result?: { value?: boolean } };
+    return result?.result?.value === true;
+  } catch {
+    return false;
+  }
+}
+
+function sleep(ms: number): Promise<void> {
+  return new Promise(r => setTimeout(r, ms));
+}

package/src/tools/browser/browser-execution.ts

@@ -1,6 +1,7 @@
 import type { ToolContext, ToolExecutionResult } from '../types.js';
 import type { ImageContent } from '../../providers/types.js';
 import { getLogger } from '../../util/logger.js';
+import { truncate } from '../../util/truncate.js';
 import {
   parseUrl,
   isPrivateOrLocalHost,
@@ -826,7 +827,7 @@ export async function executeBrowserWaitFor(
         `document.body?.innerText?.includes(${escaped})`,
         { timeout },
       );
-      return { content: `Text "${text.slice(0, 80)}" appeared on page.`, isError: false };
+      return { content: `Text "${truncate(text, 80)}" appeared on page.`, isError: false };
     }
 
     // duration mode (milliseconds)

package/src/tools/browser/browser-manager.ts

@@ -433,7 +433,7 @@ class BrowserManager {
     if (this.browserCdpSession) {
       try {
         await this.browserCdpSession.detach();
-      } catch {}
+      } catch (e) { log.debug({ err: e }, 'CDP session detach failed during shutdown'); }
       this.browserCdpSession = null;
       this.browserWindowId = null;
     }
@@ -489,7 +489,7 @@ class BrowserManager {
       try {
         await cdp.send('Page.stopScreencast');
         await cdp.detach();
-      } catch {}
+      } catch (e) { log.debug({ err: e }, 'Screencast stop / CDP detach failed during cleanup'); }
       this.cdpSessions.delete(sessionId);
       this.screencastCallbacks.delete(sessionId);
     }

package/src/tools/browser/network-recorder.ts

@@ -6,6 +6,7 @@
  */
 
 import { getLogger } from '../../util/logger.js';
+import { truncate } from '../../util/truncate.js';
 import type { NetworkRecordedEntry, NetworkRecordedRequest, ExtractedCredential } from './network-recording-types.js';
 
 const log = getLogger('network-recorder');
@@ -54,7 +55,7 @@ class DirectCDPClient {
               for (const h of handlers) h(msg.params ?? {});
             }
           }
-        } catch {}
+        } catch (e) { log.debug({ err: e }, 'Failed to parse CDP WebSocket message'); }
       };
     });
   }
@@ -195,7 +196,7 @@ export class NetworkRecorder {
     for (const client of this.pageClients) {
       try {
         await client.send('Network.disable');
-      } catch {}
+      } catch (e) { log.debug({ err: e }, 'Network.disable failed during cleanup'); }
       client.close();
     }
     this.pageClients = [];
@@ -273,7 +274,7 @@ export class NetworkRecorder {
     const method = (request.method as string) ?? 'GET';
     const postData = request.postData as string | undefined;
 
-    log.debug({ url: url.slice(0, 120), method, requestId }, 'Request captured');
+    log.debug({ url: truncate(url, 120, ''), method, requestId }, 'Request captured');
 
     const recordedRequest: NetworkRecordedRequest = { method, url, headers, postData };
     const entry: NetworkRecordedEntry = {
@@ -307,7 +308,7 @@ export class NetworkRecorder {
       this.loginSignals.some(sig => entry.request.url.includes(sig))
     ) {
       this.loginDetectedFired = true;
-      log.info({ url: entry.request.url.slice(0, 120) }, 'Login detected — will auto-stop in 5s');
+      log.info({ url: truncate(entry.request.url, 120, '') }, 'Login detected — will auto-stop in 5s');
       // Delay to let remaining network requests (cookies, session data) settle
       setTimeout(() => this.onLoginDetected?.(), 5000);
     }