vellum 0.0.16 → 0.2.0

package/src/tools/ui-surface/definitions.ts

@@ -0,0 +1,232 @@
+/**
+ * UI surface tool definitions.
+ *
+ * These tools allow the model to show, update, and dismiss just-in-time UI
+ * surfaces (cards, forms, lists, confirmations) on a connected macOS client.
+ * They are proxy tools -- execution is forwarded to the client and never
+ * handled locally by the daemon.
+ */
+
+import { RiskLevel } from '../../permissions/types.js';
+import type { Tool, ToolExecutionResult } from '../types.js';
+import type { ToolDefinition } from '../../providers/types.js';
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+function proxyExecute(): Promise<ToolExecutionResult> {
+  throw new Error('Proxy tool: execution must be forwarded to the connected client');
+}
+
+// ---------------------------------------------------------------------------
+// ui_show
+// ---------------------------------------------------------------------------
+
+export const uiShowTool: Tool = {
+  name: 'ui_show',
+  description:
+    'Show a UI surface to the user. Use display: "inline" (default) to embed in chat, or "panel" for a floating window.\n\n' +
+    'Supported surface types:\n' +
+    '- card: Informational card with title, subtitle, body text, and optional metadata key-value pairs. ' +
+    'Cards support an optional template field for specialized native rendering. ' +
+    'data shape: { title: string, subtitle?: string, body: string, metadata?: Array<{ label: string, value: string }>, template?: string, templateData?: object }\n' +
+    '  Template "weather_forecast": renders an Apple Weather-style forecast widget. ' +
+    'templateData shape: { location: string, currentTemp: number, feelsLike: number, unit: "F"|"C", condition: string, humidity: number, windSpeed: number, windDirection: string, ' +
+    'hourly: Array<{ time: string, icon: string (SF Symbol name), temp: number }>, ' +
+    'forecast: Array<{ day: string, icon: string (SF Symbol name), low: number, high: number, precip: number|null, condition: string }> }\n' +
+    '- table: Data table with columns, selectable rows, and action buttons. ' +
+    'data shape: { columns: Array<{ id: string, label: string }>, rows: Array<{ id: string, cells: Record<string, string>, selectable?: boolean, selected?: boolean }>, selectionMode?: "none"|"single"|"multiple", caption?: string }\n' +
+    '- form: Input form with typed fields. ' +
+    'data shape: { description?: string, fields: Array<{ id: string, type: "text"|"textarea"|"select"|"toggle"|"number"|"password", label: string, placeholder?: string, required?: boolean, defaultValue?: string|number|boolean, options?: Array<{ label: string, value: string }> }>, submitLabel?: string }. ' +
+    'For multi-page forms, use pages array instead of top-level fields: { pages: [{ id: string, title: string, description?: string, fields: [...] }], pageLabels?: { next?: string, back?: string, submit?: string }, submitLabel?: string }\n' +
+    '- list: Selectable list of items. ' +
+    'data shape: { items: Array<{ id: string, title: string, subtitle?: string, icon?: string, selected?: boolean }>, selectionMode: "single"|"multiple"|"none" }\n' +
+    '- confirmation: Yes/no confirmation dialog. ' +
+    'data shape: { message: string, detail?: string, confirmLabel?: string, cancelLabel?: string, destructive?: boolean }\n' +
+    '- dynamic_page: Custom HTML page rendered in a sandboxed container. ' +
+    'data shape: { html: string, width?: number, height?: number, preview?: { title: string, subtitle?: string, description?: string, icon?: string (emoji), metrics?: Array<{ label: string, value: string }> } }. ' +
+    'When preview is provided, a compact preview card is shown inline in chat with the title, subtitle, description, metric pills, and a "View Output" button that opens the full page.\n' +
+    '- file_upload: File upload dialog where the user can drag-and-drop or browse for files. ' +
+    'data shape: { prompt: string, acceptedTypes?: string[], maxFiles?: number }\n\n' +
+    'Action payload conventions:\n' +
+    '- Multi-select tables: use `window.vellum.sendAction(actionId, { selectedIds: [...] })` to send selected row IDs\n' +
+    '- Bulk actions: include `selectedRows` array with full row data for context',
+  category: 'ui-surface',
+  defaultRiskLevel: RiskLevel.Low,
+  executionMode: 'proxy',
+
+  getDefinition(): ToolDefinition {
+    return {
+      name: this.name,
+      description: this.description,
+      input_schema: {
+        type: 'object',
+        properties: {
+          surface_type: {
+            type: 'string',
+            enum: ['card', 'form', 'list', 'table', 'confirmation', 'dynamic_page', 'file_upload'],
+            description: 'The type of surface to display',
+          },
+          title: {
+            type: 'string',
+            description: 'Optional title for the surface window',
+          },
+          data: {
+            type: 'object',
+            description: 'Surface data; structure depends on surface_type (see tool description)',
+          },
+          actions: {
+            type: 'array',
+            items: {
+              type: 'object',
+              properties: {
+                id: { type: 'string', description: 'Unique action identifier' },
+                label: { type: 'string', description: 'Button label text' },
+                style: {
+                  type: 'string',
+                  enum: ['primary', 'secondary', 'destructive'],
+                  description: 'Visual style of the button',
+                },
+              },
+              required: ['id', 'label'],
+            },
+            description: 'Optional action buttons to display on the surface',
+          },
+          display: {
+            type: 'string',
+            enum: ['inline', 'panel'],
+            description: 'Where to render the surface. "inline" embeds it in the chat message. "panel" shows a floating window. Defaults to "inline".',
+          },
+          await_action: {
+            type: 'boolean',
+            description: 'Whether to block until the user interacts with an action. Defaults to true when actions are provided.',
+          },
+        },
+        required: ['surface_type', 'data'],
+      },
+    };
+  },
+
+  execute: proxyExecute,
+};
+
+// ---------------------------------------------------------------------------
+// ui_update
+// ---------------------------------------------------------------------------
+
+export const uiUpdateTool: Tool = {
+  name: 'ui_update',
+  description: "Update an existing surface's data. The provided data object is merged into the surface's current data.",
+  category: 'ui-surface',
+  defaultRiskLevel: RiskLevel.Low,
+  executionMode: 'proxy',
+
+  getDefinition(): ToolDefinition {
+    return {
+      name: this.name,
+      description: this.description,
+      input_schema: {
+        type: 'object',
+        properties: {
+          surface_id: {
+            type: 'string',
+            description: 'The ID of the surface to update',
+          },
+          data: {
+            type: 'object',
+            description: 'Partial data to merge into the existing surface data',
+          },
+        },
+        required: ['surface_id', 'data'],
+      },
+    };
+  },
+
+  execute: proxyExecute,
+};
+
+// ---------------------------------------------------------------------------
+// ui_dismiss
+// ---------------------------------------------------------------------------
+
+export const uiDismissTool: Tool = {
+  name: 'ui_dismiss',
+  description: 'Dismiss a currently displayed surface.',
+  category: 'ui-surface',
+  defaultRiskLevel: RiskLevel.Low,
+  executionMode: 'proxy',
+
+  getDefinition(): ToolDefinition {
+    return {
+      name: this.name,
+      description: this.description,
+      input_schema: {
+        type: 'object',
+        properties: {
+          surface_id: {
+            type: 'string',
+            description: 'The ID of the surface to dismiss',
+          },
+        },
+        required: ['surface_id'],
+      },
+    };
+  },
+
+  execute: proxyExecute,
+};
+
+// ---------------------------------------------------------------------------
+// request_file
+// ---------------------------------------------------------------------------
+
+export const requestFileTool: Tool = {
+  name: 'request_file',
+  description:
+    'Request a file or image from the user. Shows a file upload dialog where the user can drag-and-drop or browse for files. ' +
+    'Use this when you need the user to share a file (image, document, PDF, etc.) to continue the conversation. ' +
+    'The result contains the uploaded file data including base64 content and MIME type.',
+  category: 'ui-surface',
+  defaultRiskLevel: RiskLevel.Low,
+  executionMode: 'proxy',
+
+  getDefinition(): ToolDefinition {
+    return {
+      name: this.name,
+      description: this.description,
+      input_schema: {
+        type: 'object',
+        properties: {
+          prompt: {
+            type: 'string',
+            description: 'What to ask the user for, e.g. "Please share the design file you\'d like me to review"',
+          },
+          accepted_types: {
+            type: 'array',
+            items: { type: 'string' },
+            description: 'MIME type filters, e.g. ["image/*", "application/pdf"]. If omitted, all supported file types are accepted.',
+          },
+          max_files: {
+            type: 'number',
+            description: 'Maximum number of files to accept. Defaults to 1.',
+          },
+        },
+        required: ['prompt'],
+      },
+    };
+  },
+
+  execute: proxyExecute,
+};
+
+// ---------------------------------------------------------------------------
+// All tools exported as array for convenience
+// ---------------------------------------------------------------------------
+
+export const allUiSurfaceTools: Tool[] = [
+  uiShowTool,
+  uiUpdateTool,
+  uiDismissTool,
+  requestFileTool,
+];

package/src/tools/ui-surface/registry.ts

@@ -0,0 +1,14 @@
+/**
+ * Registers all UI surface proxy tools with the daemon's tool registry.
+ *
+ * Called once at daemon startup via initializeTools().
+ */
+
+import { registerTool } from '../registry.js';
+import { allUiSurfaceTools } from './definitions.js';
+
+export function registerUiSurfaceTools(): void {
+  for (const tool of allUiSurfaceTools) {
+    registerTool(tool);
+  }
+}

package/src/tools/watch/screen-watch.ts

@@ -0,0 +1,128 @@
+import crypto from 'node:crypto';
+import { RiskLevel } from '../../permissions/types.js';
+import type { Tool, ToolContext, ToolExecutionResult } from '../types.js';
+import type { ToolDefinition } from '../../providers/types.js';
+import { getLogger } from '../../util/logger.js';
+import {
+  watchSessions,
+  getActiveWatchSession,
+  fireWatchStartNotifier,
+  fireWatchCompletionNotifier,
+} from './watch-state.js';
+import type { WatchSession } from './watch-state.js';
+
+const log = getLogger('screen-watch');
+
+class ScreenWatchTool implements Tool {
+  name = 'start_screen_watch';
+  description = 'Start observing the screen at regular intervals for a specified duration. Captures OCR text from the active window and provides periodic commentary.';
+  category = 'observation';
+  defaultRiskLevel = RiskLevel.Low;
+
+  getDefinition(): ToolDefinition {
+    return {
+      name: this.name,
+      description: this.description,
+      input_schema: {
+        type: 'object',
+        properties: {
+          duration_minutes: {
+            type: 'number',
+            description: 'How long to watch in minutes (1-15, default 5)',
+          },
+          interval_seconds: {
+            type: 'number',
+            description: 'Seconds between screen captures (5-30, default 10)',
+          },
+          focus_area: {
+            type: 'string',
+            description: 'What to focus on observing',
+          },
+        },
+        required: ['focus_area'],
+      },
+    };
+  }
+
+  async execute(input: Record<string, unknown>, context: ToolContext): Promise<ToolExecutionResult> {
+    const { sessionId } = context;
+
+    // Validate focus_area
+    const focusArea = typeof input.focus_area === 'string' && input.focus_area.length > 0
+      ? input.focus_area
+      : undefined;
+
+    if (!focusArea) {
+      return {
+        content: 'Error: focus_area is required and must be a non-empty string',
+        isError: true,
+      };
+    }
+
+    // Clamp duration to 1-15 minutes
+    let durationMinutes = typeof input.duration_minutes === 'number' ? input.duration_minutes : 5;
+    durationMinutes = Math.max(1, Math.min(15, durationMinutes));
+
+    // Clamp interval to 5-30 seconds
+    let intervalSeconds = typeof input.interval_seconds === 'number' ? input.interval_seconds : 10;
+    intervalSeconds = Math.max(5, Math.min(30, intervalSeconds));
+
+    // Check for existing active session
+    const existing = getActiveWatchSession(sessionId);
+    if (existing) {
+      return {
+        content: `Error: An active watch session already exists for this conversation (watchId: ${existing.watchId}, focus: "${existing.focusArea}"). Cancel or wait for it to complete before starting a new one.`,
+        isError: true,
+      };
+    }
+
+    // Generate watchId
+    const watchId = crypto.randomUUID().slice(0, 8);
+    const now = Date.now();
+    const durationSeconds = durationMinutes * 60;
+
+    // Create session
+    const session: WatchSession = {
+      watchId,
+      sessionId,
+      focusArea,
+      durationSeconds,
+      intervalSeconds,
+      observations: [],
+      commentaryCount: 0,
+      status: 'active',
+      startedAt: now,
+    };
+
+    // Store in sessions map
+    watchSessions.set(watchId, session);
+
+    // Fire start notifier
+    fireWatchStartNotifier(sessionId, session);
+
+    // Set timeout for duration expiry
+    session.timeoutHandle = setTimeout(() => {
+      session.status = 'completing';
+      session.timeoutHandle = undefined;
+      log.info({ watchId, focusArea }, 'Watch session duration expired, marking as completing');
+      fireWatchCompletionNotifier(sessionId, session);
+    }, durationSeconds * 1000);
+
+    log.info({ watchId, sessionId, focusArea, durationMinutes, intervalSeconds }, 'Screen watch session started');
+
+    const expectedCaptures = Math.floor(durationSeconds / intervalSeconds);
+    const content = [
+      `Screen watch started (watchId: ${watchId})`,
+      `Focus: ${focusArea}`,
+      `Duration: ${durationMinutes} minute${durationMinutes !== 1 ? 's' : ''}`,
+      `Interval: every ${intervalSeconds} seconds`,
+      `Expected captures: ~${expectedCaptures}`,
+      `Started at: ${new Date(now).toISOString()}`,
+      `Expected end: ${new Date(now + durationSeconds * 1000).toISOString()}`,
+    ].join('\n');
+
+    return { content, isError: false };
+  }
+}
+
+export const screenWatchTool = new ScreenWatchTool();

package/src/tools/watch/watch-state.ts

@@ -0,0 +1,119 @@
+import { getLogger } from '../../util/logger.js';
+
+const log = getLogger('watch');
+
+export interface WatchObservationEntry {
+  ocrText: string;
+  appName?: string;
+  windowTitle?: string;
+  bundleIdentifier?: string;
+  timestamp: number;
+  captureIndex: number;
+}
+
+export interface WatchSession {
+  watchId: string;
+  sessionId: string;
+  focusArea: string;
+  durationSeconds: number;
+  intervalSeconds: number;
+  observations: WatchObservationEntry[];
+  commentaryCount: number;
+  status: 'active' | 'completing' | 'completed' | 'cancelled';
+  startedAt: number;
+  timeoutHandle?: ReturnType<typeof setTimeout>;
+  /** Guards against concurrent generateSummary calls */
+  summaryInFlight?: boolean;
+  /** Whether this session was started via ride shotgun (no live commentary) */
+  isRideShotgun?: boolean;
+  /** Learn mode records network traffic alongside screen observations */
+  isLearnMode?: boolean;
+  /** Domain filter for network recording in learn mode */
+  targetDomain?: string;
+  /** Recording ID for learn mode sessions */
+  recordingId?: string;
+  /** Path where the learn recording was successfully saved (undefined if save failed) */
+  savedRecordingPath?: string;
+}
+
+/** Module-level map of watch sessions keyed by watchId. */
+export const watchSessions = new Map<string, WatchSession>();
+
+// ── Start notifiers ─────────────────────────────────────────────────
+const startNotifiers = new Map<string, (session: WatchSession) => void>();
+
+export function registerWatchStartNotifier(sessionId: string, callback: (session: WatchSession) => void): void {
+  startNotifiers.set(sessionId, callback);
+}
+
+export function unregisterWatchStartNotifier(sessionId: string): void {
+  startNotifiers.delete(sessionId);
+}
+
+export function fireWatchStartNotifier(sessionId: string, session: WatchSession): void {
+  startNotifiers.get(sessionId)?.(session);
+}
+
+// ── Commentary notifiers ────────────────────────────────────────────
+const commentaryNotifiers = new Map<string, (session: WatchSession) => void>();
+
+export function registerWatchCommentaryNotifier(sessionId: string, callback: (session: WatchSession) => void): void {
+  commentaryNotifiers.set(sessionId, callback);
+}
+
+export function unregisterWatchCommentaryNotifier(sessionId: string): void {
+  commentaryNotifiers.delete(sessionId);
+}
+
+export function fireWatchCommentaryNotifier(sessionId: string, session: WatchSession): void {
+  commentaryNotifiers.get(sessionId)?.(session);
+}
+
+// ── Completion notifiers ────────────────────────────────────────────
+const completionNotifiers = new Map<string, (session: WatchSession) => void>();
+
+export function registerWatchCompletionNotifier(sessionId: string, callback: (session: WatchSession) => void): void {
+  completionNotifiers.set(sessionId, callback);
+}
+
+export function unregisterWatchCompletionNotifier(sessionId: string): void {
+  completionNotifiers.delete(sessionId);
+}
+
+export function fireWatchCompletionNotifier(sessionId: string, session: WatchSession): void {
+  completionNotifiers.get(sessionId)?.(session);
+}
+
+// ── Session helpers ─────────────────────────────────────────────────
+
+/** Find the first active watch session for a given sessionId. */
+export function getActiveWatchSession(sessionId: string): WatchSession | undefined {
+  for (const session of watchSessions.values()) {
+    if (session.sessionId === sessionId && session.status === 'active') {
+      return session;
+    }
+  }
+  return undefined;
+}
+
+/** Add an observation to a watch session. */
+export function addObservation(watchId: string, observation: WatchObservationEntry): void {
+  const session = watchSessions.get(watchId);
+  if (!session) {
+    log.warn({ watchId }, 'Cannot add observation: session not found');
+    return;
+  }
+  session.observations.push(observation);
+  log.info({ watchId, captureIndex: observation.captureIndex }, 'Observation added');
+}
+
+/** Remove completed/cancelled sessions for a given sessionId. */
+export function pruneWatchSessions(sessionId: string): void {
+  for (const [watchId, session] of watchSessions) {
+    if (session.sessionId !== sessionId) continue;
+    if (session.status === 'completed' || session.status === 'cancelled') {
+      if (session.timeoutHandle) clearTimeout(session.timeoutHandle);
+      watchSessions.delete(watchId);
+    }
+  }
+}

package/src/tools/watcher/create.ts

@@ -0,0 +1,110 @@
+import { RiskLevel } from '../../permissions/types.js';
+import type { Tool, ToolContext, ToolExecutionResult } from '../types.js';
+import type { ToolDefinition } from '../../providers/types.js';
+import { registerTool } from '../registry.js';
+import { createWatcher } from '../../watcher/watcher-store.js';
+import { getWatcherProvider, listWatcherProviders } from '../../watcher/provider-registry.js';
+
+class WatcherCreateTool implements Tool {
+  name = 'watcher_create';
+  description = 'Create a new watcher that polls an external service for events and processes them with an action prompt';
+  category = 'watcher';
+  defaultRiskLevel = RiskLevel.Medium;
+
+  getDefinition(): ToolDefinition {
+    return {
+      name: this.name,
+      description: this.description,
+      input_schema: {
+        type: 'object',
+        properties: {
+          name: {
+            type: 'string',
+            description: 'A human-readable name for this watcher (e.g. "My Gmail")',
+          },
+          provider: {
+            type: 'string',
+            description: 'The provider to poll (e.g. "gmail")',
+          },
+          action_prompt: {
+            type: 'string',
+            description: 'Instructions for the LLM on how to handle detected events. This prompt is sent along with event data to a background conversation.',
+          },
+          poll_interval_ms: {
+            type: 'number',
+            description: 'How often to poll in milliseconds. Defaults to 60000 (1 minute). Minimum 15000.',
+          },
+          credential_service: {
+            type: 'string',
+            description: 'Override the credential service to use. Defaults to the provider\'s required service.',
+          },
+          config: {
+            type: 'object',
+            description: 'Provider-specific configuration (e.g. filter criteria)',
+          },
+        },
+        required: ['name', 'provider', 'action_prompt'],
+      },
+    };
+  }
+
+  async execute(input: Record<string, unknown>, _context: ToolContext): Promise<ToolExecutionResult> {
+    const name = input.name as string;
+    const providerId = input.provider as string;
+    const actionPrompt = input.action_prompt as string;
+    const pollIntervalMs = (input.poll_interval_ms as number) ?? undefined;
+    const config = input.config as Record<string, unknown> | undefined;
+
+    if (!name || typeof name !== 'string') {
+      return { content: 'Error: name is required and must be a string', isError: true };
+    }
+    if (!providerId || typeof providerId !== 'string') {
+      return { content: 'Error: provider is required and must be a string', isError: true };
+    }
+    if (!actionPrompt || typeof actionPrompt !== 'string') {
+      return { content: 'Error: action_prompt is required and must be a string', isError: true };
+    }
+
+    const provider = getWatcherProvider(providerId);
+    if (!provider) {
+      const available = listWatcherProviders().map((p) => p.id).join(', ') || 'none';
+      return { content: `Error: Unknown provider "${providerId}". Available: ${available}`, isError: true };
+    }
+
+    if (pollIntervalMs !== undefined && pollIntervalMs < 15000) {
+      return { content: 'Error: poll_interval_ms must be at least 15000 (15 seconds)', isError: true };
+    }
+
+    const credentialService = (input.credential_service as string) ?? provider.requiredCredentialService;
+
+    try {
+      const watcher = createWatcher({
+        name,
+        providerId,
+        actionPrompt,
+        credentialService,
+        pollIntervalMs,
+        configJson: config ? JSON.stringify(config) : null,
+      });
+
+      const intervalSec = Math.round(watcher.pollIntervalMs / 1000);
+      return {
+        content: [
+          'Watcher created successfully.',
+          `  Name: ${watcher.name}`,
+          `  Provider: ${provider.displayName}`,
+          `  Poll interval: ${intervalSec}s`,
+          `  Credential: ${watcher.credentialService}`,
+          `  Status: ${watcher.status}`,
+          `  ID: ${watcher.id}`,
+        ].join('\n'),
+        isError: false,
+      };
+    } catch (err) {
+      const msg = err instanceof Error ? err.message : String(err);
+      return { content: `Error creating watcher: ${msg}`, isError: true };
+    }
+  }
+}
+
+registerTool(new WatcherCreateTool());

package/src/tools/watcher/delete.ts

@@ -0,0 +1,53 @@
+import { RiskLevel } from '../../permissions/types.js';
+import type { Tool, ToolContext, ToolExecutionResult } from '../types.js';
+import type { ToolDefinition } from '../../providers/types.js';
+import { registerTool } from '../registry.js';
+import { getWatcher, deleteWatcher } from '../../watcher/watcher-store.js';
+
+class WatcherDeleteTool implements Tool {
+  name = 'watcher_delete';
+  description = 'Delete a watcher and all its event history';
+  category = 'watcher';
+  defaultRiskLevel = RiskLevel.Medium;
+
+  getDefinition(): ToolDefinition {
+    return {
+      name: this.name,
+      description: this.description,
+      input_schema: {
+        type: 'object',
+        properties: {
+          watcher_id: {
+            type: 'string',
+            description: 'The ID of the watcher to delete',
+          },
+        },
+        required: ['watcher_id'],
+      },
+    };
+  }
+
+  async execute(input: Record<string, unknown>, _context: ToolContext): Promise<ToolExecutionResult> {
+    const watcherId = input.watcher_id as string;
+    if (!watcherId || typeof watcherId !== 'string') {
+      return { content: 'Error: watcher_id is required', isError: true };
+    }
+
+    const watcher = getWatcher(watcherId);
+    if (!watcher) {
+      return { content: `Error: Watcher not found: ${watcherId}`, isError: true };
+    }
+
+    const deleted = deleteWatcher(watcherId);
+    if (!deleted) {
+      return { content: `Error: Failed to delete watcher: ${watcherId}`, isError: true };
+    }
+
+    return {
+      content: `Watcher deleted: "${watcher.name}"`,
+      isError: false,
+    };
+  }
+}
+
+registerTool(new WatcherDeleteTool());