vellum 0.2.12 → 0.2.14

package/src/memory/jobs-worker.ts

@@ -85,8 +85,9 @@ export async function runMemoryJobsOnce(
   // Periodic stale item sweep (throttled to at most once per hour)
   sweepStaleItems(config);
 
+  const batchSize = Math.max(1, config.memory.jobs.batchSize);
   const concurrency = Math.max(1, config.memory.jobs.workerConcurrency);
-  const jobs = claimMemoryJobs(concurrency);
+  const jobs = claimMemoryJobs(batchSize);
   if (jobs.length === 0) {
     if (enableScheduledCleanup) {
       maybeEnqueueScheduledCleanupJobs(config);
@@ -94,36 +95,47 @@ export async function runMemoryJobsOnce(
     return 0;
   }
 
-  let processed = 0;
+  // Group jobs by type so same-type jobs run sequentially (preventing
+  // checkpoint races for backfill, etc.), while different types run concurrently.
+  const jobsByType = new Map<string, MemoryJob[]>();
   for (const job of jobs) {
-    try {
-      await processJob(job, config);
-      completeMemoryJob(job.id);
-      processed += 1;
-    } catch (err) {
-      if (err instanceof BackendUnavailableError) {
-        // Backend not configured yet -- put the job back with exponential backoff.
-        const result = deferMemoryJob(job.id);
-        if (result === 'failed') {
-          log.warn({ jobId: job.id, type: job.type }, 'Embedding backend unavailable, job exceeded max deferrals');
-        } else {
-          log.debug({ jobId: job.id, type: job.type }, 'Embedding backend unavailable, deferring job');
-        }
-      } else {
-        const message = err instanceof Error ? err.message : String(err);
-        const category = classifyError(err);
-        if (category === 'retryable') {
-          const delay = retryDelayForAttempt(job.attempts + 1);
-          failMemoryJob(job.id, message, {
-            retryDelayMs: delay,
-            maxAttempts: RETRY_MAX_ATTEMPTS,
-          });
-          log.warn({ err, jobId: job.id, type: job.type, delay, category }, 'Memory job failed (retryable)');
-        } else {
-          failMemoryJob(job.id, message, { maxAttempts: 1 });
-          log.warn({ err, jobId: job.id, type: job.type, category }, 'Memory job failed (fatal)');
+    let group = jobsByType.get(job.type);
+    if (!group) {
+      group = [];
+      jobsByType.set(job.type, group);
+    }
+    group.push(job);
+  }
+
+  let processed = 0;
+  const typeGroups = [...jobsByType.values()];
+
+  // Run type groups concurrently (up to workerConcurrency at a time).
+  // Within each group, jobs are processed sequentially.
+  for (let i = 0; i < typeGroups.length; i += concurrency) {
+    const groupChunk = typeGroups.slice(i, i + concurrency);
+    const groupResults = await Promise.allSettled(
+      groupChunk.map(async (group) => {
+        let groupProcessed = 0;
+        for (const job of group) {
+          try {
+            await processJob(job, config);
+            completeMemoryJob(job.id);
+            groupProcessed += 1;
+          } catch (err) {
+            handleJobError(job, err);
+          }
         }
+        return groupProcessed;
+      }),
+    );
+    for (const result of groupResults) {
+      if (result.status === 'fulfilled') {
+        processed += result.value;
       }
+      // Errors within groups are already handled per-job above;
+      // a rejected group promise would only come from an unexpected
+      // error in the loop itself, which is unlikely.
     }
   }
   if (enableScheduledCleanup) {
@@ -132,6 +144,33 @@ export async function runMemoryJobsOnce(
   return processed;
 }
 
+// ── Job error handling ─────────────────────────────────────────────
+
+function handleJobError(job: MemoryJob, err: unknown): void {
+  if (err instanceof BackendUnavailableError) {
+    const result = deferMemoryJob(job.id);
+    if (result === 'failed') {
+      log.warn({ jobId: job.id, type: job.type }, 'Embedding backend unavailable, job exceeded max deferrals');
+    } else {
+      log.debug({ jobId: job.id, type: job.type }, 'Embedding backend unavailable, deferring job');
+    }
+  } else {
+    const message = err instanceof Error ? err.message : String(err);
+    const category = classifyError(err);
+    if (category === 'retryable') {
+      const delay = retryDelayForAttempt(job.attempts + 1);
+      failMemoryJob(job.id, message, {
+        retryDelayMs: delay,
+        maxAttempts: RETRY_MAX_ATTEMPTS,
+      });
+      log.warn({ err, jobId: job.id, type: job.type, delay, category }, 'Memory job failed (retryable)');
+    } else {
+      failMemoryJob(job.id, message, { maxAttempts: 1 });
+      log.warn({ err, jobId: job.id, type: job.type, category }, 'Memory job failed (fatal)');
+    }
+  }
+}
+
 // ── Job dispatch ───────────────────────────────────────────────────
 
 async function processJob(job: MemoryJob, config: AssistantConfig): Promise<void> {

package/src/memory/runs-store.ts

@@ -6,7 +6,7 @@
  *   running → needs_secret       → running → completed | failed
  */
 
-import { eq, inArray } from 'drizzle-orm';
+import { and, eq, inArray } from 'drizzle-orm';
 import { v4 as uuid } from 'uuid';
 import { getDb } from './db.js';
 import { messageRuns } from './schema.js';
@@ -25,12 +25,6 @@ export interface PendingConfirmation {
   executionTarget?: 'sandbox' | 'host';
   allowlistOptions?: Array<{ label: string; pattern: string }>;
   scopeOptions?: Array<{ label: string; scope: string }>;
-  /** Principal kind that initiated this tool use (e.g. 'core' or 'skill'). */
-  principalKind?: string;
-  /** Skill ID when principalKind is 'skill'. */
-  principalId?: string;
-  /** Content-hash of the skill source for version tracking. */
-  principalVersion?: string;
   /** When false, the client should hide "always allow" / trust-rule persistence affordances. */
   persistentDecisionsAllowed?: boolean;
 }
@@ -222,6 +216,59 @@ export function failRun(runId: string, error: string): void {
     .run();
 }
 
+// ---------------------------------------------------------------------------
+// Pending-confirmation lookups
+// ---------------------------------------------------------------------------
+
+/** Summary of a run awaiting confirmation, used by channel approval flows. */
+export interface PendingRunInfo {
+  runId: string;
+  /** The prompter-level request ID stored inside PendingConfirmation.toolUseId. */
+  requestId: string;
+  toolName: string;
+  input: Record<string, unknown>;
+  riskLevel: string;
+}
+
+/**
+ * Find all runs in `needs_confirmation` state for a given conversation.
+ *
+ * Returns structured info for each pending run so channel adapters can
+ * render approval prompts and route decisions without reaching into
+ * raw DB rows.
+ */
+export function getPendingConfirmationsByConversation(conversationId: string): PendingRunInfo[] {
+  const db = getDb();
+  const rows = db
+    .select()
+    .from(messageRuns)
+    .where(
+      and(
+        eq(messageRuns.conversationId, conversationId),
+        eq(messageRuns.status, 'needs_confirmation'),
+      ),
+    )
+    .all();
+
+  const results: PendingRunInfo[] = [];
+  for (const row of rows) {
+    const run = rowToRun(row);
+    if (!run.pendingConfirmation) continue;
+    results.push({
+      runId: run.id,
+      requestId: run.pendingConfirmation.toolUseId,
+      toolName: run.pendingConfirmation.toolName,
+      input: run.pendingConfirmation.input,
+      riskLevel: run.pendingConfirmation.riskLevel,
+    });
+  }
+  return results;
+}
+
+// ---------------------------------------------------------------------------
+// Orphan recovery
+// ---------------------------------------------------------------------------
+
 /**
  * Mark all non-terminal runs as failed.
  * Called on startup to recover from daemon restarts that left runs

package/src/memory/schema.ts

@@ -548,6 +548,8 @@ export const callSessions = sqliteTable('call_sessions', {
   toNumber: text('to_number').notNull(),
   task: text('task'),
   status: text('status').notNull().default('initiated'),
+  callerIdentityMode: text('caller_identity_mode'),
+  callerIdentitySource: text('caller_identity_source'),
   startedAt: integer('started_at'),
   endedAt: integer('ended_at'),
   lastError: text('last_error'),
@@ -586,3 +588,21 @@ export const processedCallbacks = sqliteTable('processed_callbacks', {
   claimId: text('claim_id'),
   createdAt: integer('created_at').notNull(),
 });
+
+// ── External Conversation Bindings ───────────────────────────────────
+// UNIQUE (source_channel, external_chat_id) enforced via idx_ext_conv_bindings_channel_chat_unique in db.ts
+
+export const externalConversationBindings = sqliteTable('external_conversation_bindings', {
+  conversationId: text('conversation_id')
+    .primaryKey()
+    .references(() => conversations.id, { onDelete: 'cascade' }),
+  sourceChannel: text('source_channel').notNull(),
+  externalChatId: text('external_chat_id').notNull(),
+  externalUserId: text('external_user_id'),
+  displayName: text('display_name'),
+  username: text('username'),
+  createdAt: integer('created_at').notNull(),
+  updatedAt: integer('updated_at').notNull(),
+  lastInboundAt: integer('last_inbound_at'),
+  lastOutboundAt: integer('last_outbound_at'),
+});

package/src/messaging/provider.ts

@@ -38,6 +38,15 @@ export interface MessagingProvider {
   getThreadReplies?(token: string, conversationId: string, threadId: string, options?: HistoryOptions): Promise<Message[]>;
   markRead?(token: string, conversationId: string, messageId?: string): Promise<void>;
 
+  /**
+   * Override the default credential check used by getConnectedProviders().
+   * When present, the registry calls this instead of looking for
+   * credential:${credentialService}:access_token. Useful for providers
+   * that don't use OAuth (e.g. Telegram bot tokens stored under a
+   * non-standard key).
+   */
+  isConnected?(): boolean;
+
   /** Platform-specific capabilities for tool routing (e.g. 'reactions', 'threads', 'labels'). */
   capabilities: Set<string>;
 }

package/src/messaging/providers/telegram-bot/adapter.ts

@@ -0,0 +1,162 @@
+/**
+ * Telegram Bot messaging provider adapter.
+ *
+ * Enables proactive outbound messaging to Telegram chats via the gateway's
+ * /deliver/telegram endpoint. Unlike Slack/Gmail which use direct API calls
+ * with OAuth tokens, Telegram delivery is proxied through the gateway which
+ * owns the bot token and handles Telegram API retries.
+ *
+ * The `token` parameter in MessagingProvider methods is unused for Telegram
+ * because delivery is authenticated via the gateway's bearer token, not
+ * a per-user OAuth token.
+ */
+
+import type { MessagingProvider } from '../../provider.js';
+import type {
+  Conversation,
+  Message,
+  SearchResult,
+  SendResult,
+  ConnectionInfo,
+  ListOptions,
+  HistoryOptions,
+  SearchOptions,
+  SendOptions,
+} from '../../provider-types.js';
+import { getSecureKey } from '../../../security/secure-keys.js';
+import { readHttpToken } from '../../../util/platform.js';
+import { getOrCreateConversation } from '../../../memory/conversation-key-store.js';
+import * as externalConversationStore from '../../../memory/external-conversation-store.js';
+import * as telegram from './client.js';
+
+/** Resolve the gateway base URL, preferring GATEWAY_INTERNAL_BASE_URL if set. */
+function getGatewayUrl(): string {
+  if (process.env.GATEWAY_INTERNAL_BASE_URL) {
+    return process.env.GATEWAY_INTERNAL_BASE_URL.replace(/\/+$/, "");
+  }
+  const port = Number(process.env.GATEWAY_PORT) || 7830;
+  return `http://127.0.0.1:${port}`;
+}
+
+/** Read the runtime HTTP bearer token used to authenticate with the gateway. */
+function getBearerToken(): string {
+  const token = readHttpToken();
+  if (!token) {
+    throw new Error('No runtime HTTP bearer token available — is the daemon running?');
+  }
+  return token;
+}
+
+/** Read the Telegram bot token from the credential vault. */
+function getBotToken(): string | undefined {
+  return getSecureKey('credential:telegram:bot_token');
+}
+
+export const telegramBotMessagingProvider: MessagingProvider = {
+  id: 'telegram',
+  displayName: 'Telegram',
+  credentialService: 'telegram',
+  capabilities: new Set(['send']),
+
+  /**
+   * Custom connectivity check. The standard registry check looks for
+   * credential:telegram:access_token, but the Telegram bot token is
+   * stored as credential:telegram:bot_token. This method lets the
+   * registry detect that Telegram credentials exist.
+   *
+   * Both bot_token and webhook_secret are required — the gateway's
+   * /deliver/telegram endpoint rejects requests without the webhook
+   * secret, so partial credentials would cause every send to fail.
+   */
+  isConnected(): boolean {
+    return getBotToken() !== undefined && !!getSecureKey('credential:telegram:webhook_secret');
+  },
+
+  async testConnection(_token: string): Promise<ConnectionInfo> {
+    const botToken = getBotToken();
+    if (!botToken) {
+      return {
+        connected: false,
+        user: 'unknown',
+        platform: 'telegram',
+        metadata: { error: 'No bot token found. Run the telegram-setup skill.' },
+      };
+    }
+
+    try {
+      const resp = await telegram.getMe(botToken);
+      if (!resp.ok || !resp.result) {
+        return {
+          connected: false,
+          user: 'unknown',
+          platform: 'telegram',
+          metadata: { error: resp.description ?? 'getMe failed' },
+        };
+      }
+
+      return {
+        connected: true,
+        user: resp.result.username ?? resp.result.first_name,
+        platform: 'telegram',
+        metadata: {
+          botId: resp.result.id,
+          botUsername: resp.result.username,
+          botName: resp.result.first_name,
+        },
+      };
+    } catch (e) {
+      return {
+        connected: false,
+        user: 'unknown',
+        platform: 'telegram',
+        metadata: { error: e instanceof Error ? e.message : 'getMe failed' },
+      };
+    }
+  },
+
+  async sendMessage(_token: string, conversationId: string, text: string, _options?: SendOptions): Promise<SendResult> {
+    const gatewayUrl = getGatewayUrl();
+    const bearerToken = getBearerToken();
+
+    await telegram.sendMessage(gatewayUrl, bearerToken, conversationId, text);
+
+    // Upsert external conversation binding so deleted/reset syncs are
+    // resurrected when an outbound message is sent. This ensures the
+    // conversation key mapping and binding exist for the next inbound.
+    try {
+      const sourceChannel = 'telegram';
+      const conversationKey = `${sourceChannel}:${conversationId}`;
+      const { conversationId: internalId } = getOrCreateConversation(conversationKey);
+      externalConversationStore.upsertOutboundBinding({
+        conversationId: internalId,
+        sourceChannel,
+        externalChatId: conversationId,
+      });
+    } catch {
+      // Best-effort — don't fail the send if binding upsert fails
+    }
+
+    return {
+      id: `tg-${Date.now()}`,
+      timestamp: Date.now(),
+      conversationId,
+    };
+  },
+
+  // Telegram Bot API does not support listing conversations. Bots only
+  // interact with chats where users have initiated contact or the bot
+  // has been added to a group.
+  async listConversations(_token: string, _options?: ListOptions): Promise<Conversation[]> {
+    return [];
+  },
+
+  // Telegram Bot API does not provide message history retrieval.
+  async getHistory(_token: string, _conversationId: string, _options?: HistoryOptions): Promise<Message[]> {
+    return [];
+  },
+
+  // Telegram Bot API does not support message search.
+  async search(_token: string, _query: string, _options?: SearchOptions): Promise<SearchResult> {
+    return { total: 0, messages: [], hasMore: false };
+  },
+};

package/src/messaging/providers/telegram-bot/client.ts

@@ -0,0 +1,104 @@
+/**
+ * Low-level Telegram operations.
+ *
+ * Outbound message delivery routes through the gateway's /deliver/telegram
+ * endpoint, which handles bot token management and Telegram API retries.
+ * Connection verification calls the Telegram Bot API directly with the
+ * stored bot token.
+ */
+
+import type { TelegramGetMeResponse } from './types.js';
+
+const TELEGRAM_API_BASE = 'https://api.telegram.org';
+const DELIVERY_TIMEOUT_MS = 30_000;
+
+export class TelegramApiError extends Error {
+  constructor(
+    public readonly status: number,
+    message: string,
+  ) {
+    super(message);
+    this.name = 'TelegramApiError';
+  }
+}
+
+/**
+ * Verify a bot token by calling Telegram's getMe API directly.
+ * Used for testConnection() — the only operation that bypasses the gateway.
+ */
+export async function getMe(botToken: string): Promise<TelegramGetMeResponse> {
+  const resp = await fetch(`${TELEGRAM_API_BASE}/bot${botToken}/getMe`, {
+    method: 'POST',
+    signal: AbortSignal.timeout(DELIVERY_TIMEOUT_MS),
+  });
+
+  if (!resp.ok) {
+    throw new TelegramApiError(
+      resp.status,
+      `Telegram getMe failed with status ${resp.status}`,
+    );
+  }
+
+  return resp.json() as Promise<TelegramGetMeResponse>;
+}
+
+/**
+ * Send a text message to a Telegram chat via the gateway's deliver endpoint.
+ */
+export async function sendMessage(
+  gatewayUrl: string,
+  bearerToken: string,
+  chatId: string,
+  text: string,
+): Promise<void> {
+  await deliverToGateway(gatewayUrl, bearerToken, { chatId, text });
+}
+
+/**
+ * Send a message with attachments to a Telegram chat via the gateway.
+ */
+export async function sendMessageWithAttachments(
+  gatewayUrl: string,
+  bearerToken: string,
+  chatId: string,
+  text: string | undefined,
+  attachmentIds: string[],
+): Promise<void> {
+  await deliverToGateway(gatewayUrl, bearerToken, {
+    chatId,
+    text,
+    attachments: attachmentIds.map((id) => ({ id })),
+  });
+}
+
+/** Payload accepted by the gateway's /deliver/telegram endpoint. */
+interface DeliverPayload {
+  chatId: string;
+  text?: string;
+  attachments?: Array<{ id: string }>;
+}
+
+async function deliverToGateway(
+  gatewayUrl: string,
+  bearerToken: string,
+  payload: DeliverPayload,
+): Promise<void> {
+  const url = `${gatewayUrl}/deliver/telegram`;
+  const resp = await fetch(url, {
+    method: 'POST',
+    headers: {
+      'Content-Type': 'application/json',
+      Authorization: `Bearer ${bearerToken}`,
+    },
+    body: JSON.stringify(payload),
+    signal: AbortSignal.timeout(DELIVERY_TIMEOUT_MS),
+  });
+
+  if (!resp.ok) {
+    const body = await resp.text().catch(() => '<unreadable>');
+    throw new TelegramApiError(
+      resp.status,
+      `Gateway /deliver/telegram failed (${resp.status}): ${body}`,
+    );
+  }
+}

package/src/messaging/providers/telegram-bot/types.ts

@@ -0,0 +1,15 @@
+/** Telegram Bot API types used by the messaging provider. */
+
+export interface TelegramUser {
+  id: number;
+  is_bot: boolean;
+  first_name: string;
+  last_name?: string;
+  username?: string;
+}
+
+export interface TelegramGetMeResponse {
+  ok: boolean;
+  result?: TelegramUser;
+  description?: string;
+}

package/src/messaging/registry.ts

@@ -23,6 +23,7 @@ export function getMessagingProvider(id: string): MessagingProvider {
 /** Return all registered providers that have stored credentials. */
 export function getConnectedProviders(): MessagingProvider[] {
   return Array.from(providers.values()).filter((p) => {
+    if (p.isConnected) return p.isConnected();
     const token = getSecureKey(`credential:${p.credentialService}:access_token`);
     return token !== undefined;
   });