takos-control 1.0.0

package/src/application/services/agent/runner-events.ts

@@ -0,0 +1,169 @@
+/**
+ * Agent Runner Event Emission
+ *
+ * Event emission helpers for the AgentRunner, including sequencing,
+ * DB persistence, and Durable Object relay.
+ */
+
+import type { RunStatus, Env } from '../../../shared/types';
+import type { AgentEvent } from './agent-models';
+import type { EventEmissionError } from './runner-types';
+import { getDb, runEvents } from '../../../infra/db';
+import type { RunTerminalPayload } from '../run-notifier';
+import {
+  buildTerminalPayload,
+  buildRunNotifierEmitRequest,
+  getRunNotifierStub,
+  buildRunNotifierEmitPayload,
+} from '../run-notifier';
+import { logError, logWarn } from '../../../shared/utils/logger';
+import {
+  MAX_EVENT_EMISSION_ERRORS as MAX_EMISSION_ERRORS,
+} from '../../../shared/config/limits';
+import type { SqlDatabaseBinding } from '../../../shared/types/bindings.ts';
+
+// ── Event emission helpers ──────────────────────────────────────────
+
+const MAX_EVENT_EMISSION_ERRORS = MAX_EMISSION_ERRORS;
+const EVENT_EMISSION_MAX_BACKOFF_MS = 4_000;
+
+export interface EventEmitterState {
+  eventSequence: number;
+  pendingEventEmissions: number;
+  eventEmissionErrors: EventEmissionError[];
+}
+
+export function buildTerminalEventPayloadImpl(
+  runId: string,
+  status: 'completed' | 'failed' | 'cancelled',
+  details: Record<string, unknown>,
+  sessionId: string | null,
+): RunTerminalPayload {
+  return buildTerminalPayload(runId, status, details, sessionId);
+}
+
+/**
+ * Emit a sequenced event for the run (to DB and WebSocket).
+ */
+export async function emitEventImpl(
+  state: EventEmitterState,
+  env: Env,
+  db: SqlDatabaseBinding,
+  runId: string,
+  spaceId: string,
+  getCurrentSessionId: () => Promise<string | null>,
+  type: AgentEvent['type'],
+  data: Record<string, unknown>,
+  options?: { skipDb?: boolean },
+  remoteEmit?: (input: {
+    runId: string;
+    type: AgentEvent['type'];
+    data: Record<string, unknown>;
+    sequence: number;
+    skipDb?: boolean;
+  }) => Promise<void>,
+): Promise<void> {
+  const now = new Date().toISOString();
+  const sequence = ++state.eventSequence;
+  state.pendingEventEmissions++;
+
+  // For terminal events, ensure we have the latest session_id from DB
+  let eventData = data;
+  if ((type === 'completed' || type === 'error' || type === 'cancelled') && data.run) {
+    const sessionId = await getCurrentSessionId();
+    eventData = {
+      ...data,
+      run: {
+        ...(data.run as Record<string, unknown>),
+        session_id: sessionId,
+      },
+    };
+  }
+
+  const skipDb = options?.skipDb ?? false;
+  const offloadEnabled = Boolean(env.TAKOS_OFFLOAD);
+  let legacyEventId: number | null = null;
+  const isTerminal = type === 'completed' || type === 'error' || type === 'cancelled';
+
+  try {
+    if (remoteEmit) {
+      await remoteEmit({
+        runId,
+        type,
+        data: eventData,
+        sequence,
+        skipDb,
+      });
+      return;
+    }
+
+    // Skip D1 write when R2 offload is enabled — the RunNotifierDO writes
+    // events to R2 segments, making D1 redundant.
+    if (!skipDb && !offloadEnabled) {
+      const drizzleDb = getDb(db);
+      const persisted = await drizzleDb.insert(runEvents).values({
+        runId,
+        type,
+        data: JSON.stringify({ ...eventData, _sequence: sequence }),
+        createdAt: now,
+      }).returning({ id: runEvents.id }).get();
+      legacyEventId = persisted?.id ?? null;
+    }
+
+    const stub = getRunNotifierStub(env, runId);
+    const payload = buildRunNotifierEmitPayload(runId, type, eventData, legacyEventId);
+
+    let emitOk = false;
+    const doEmit = async () => {
+      const emitRes = await stub.fetch(buildRunNotifierEmitRequest(payload));
+      if (!emitRes.ok) {
+        const body = await emitRes.text().catch(() => '');
+        throw new Error(`DO emit non-OK ${emitRes.status}: ${body}`);
+      }
+      emitOk = true;
+    };
+
+    try {
+      await doEmit();
+    } catch (firstErr) {
+      if (isTerminal) {
+        const TERMINAL_MAX_RETRIES = 3;
+        for (let attempt = 1; attempt <= TERMINAL_MAX_RETRIES; attempt++) {
+          const backoffMs = Math.min(1000 * Math.pow(2, attempt - 1), EVENT_EMISSION_MAX_BACKOFF_MS);
+          await new Promise(resolve => setTimeout(resolve, backoffMs));
+          try {
+            await doEmit();
+            break;
+          } catch (retryErr) {
+            const retryMsg = retryErr instanceof Error ? retryErr.message : String(retryErr);
+            if (attempt === TERMINAL_MAX_RETRIES) {
+              logError(`CRITICAL: Terminal event '${type}' emit failed after ${TERMINAL_MAX_RETRIES} retries (run=${runId})`, retryMsg, { module: 'emitevent' });
+            } else {
+              logWarn(`Terminal event '${type}' retry ${attempt}/${TERMINAL_MAX_RETRIES} failed (run=${runId})`, { module: 'emitevent', detail: retryMsg });
+            }
+          }
+        }
+      }
+      if (!emitOk) {
+        const msg = firstErr instanceof Error ? firstErr.message : String(firstErr);
+        logError(`DO emit failed for ${type}`, msg, { module: 'emitevent' });
+      }
+    }
+  } catch (err) {
+    const errorMsg = err instanceof Error ? err.message : String(err);
+    logError(`Event emission error for ${type} (run=${runId})`, errorMsg, { module: 'emitevent' });
+    if (isTerminal) {
+      logError(`CRITICAL: Terminal event '${type}' lost for run=${runId}`, undefined, { module: 'emitevent' });
+    }
+
+    if (state.eventEmissionErrors.length < MAX_EVENT_EMISSION_ERRORS) {
+      state.eventEmissionErrors.push({
+        type,
+        error: errorMsg,
+        timestamp: now,
+      });
+    }
+  } finally {
+    state.pendingEventEmissions--;
+  }
+}

package/src/application/services/agent/runner-history.ts

@@ -0,0 +1,368 @@
+/**
+ * Agent Runner Messages & Conversation History
+ *
+ * Run status persistence, conversation history building, and
+ * message-related helpers extracted from runner.ts.
+ */
+
+import type { RunStatus, Env } from '../../../shared/types';
+import type { AgentMessage, ToolCall } from './agent-models';
+import { getDb, runs, threads, messages } from '../../../infra/db';
+import { and, eq, sql, desc } from 'drizzle-orm';
+import { resolveHistoryTokenBudget } from './model-catalog';
+import { estimateTokens } from './prompt-budget';
+import { readMessageFromR2 } from '../offload/messages';
+import { buildThreadContextSystemMessage, queryRelevantThreadMessages } from './thread-context';
+import { logInfo, logWarn } from '../../../shared/utils/logger';
+import {
+  THREAD_RETRIEVAL_TOP_K,
+  THREAD_RETRIEVAL_MIN_SCORE,
+  THREAD_CONTEXT_MAX_CHARS,
+} from '../../../shared/config/limits';
+import { safeJsonParseOrDefault } from '../../../shared/utils';
+import {
+  buildDelegationSystemMessage,
+  buildDelegationUserMessage,
+  getDelegationPacketFromRunInput,
+} from './delegation';
+import type { SqlDatabaseBinding } from '../../../shared/types/bindings.ts';
+
+// ── Run status persistence ──────────────────────────────────────────
+
+/**
+ * Update run status in the database.
+ */
+export async function updateRunStatusImpl(
+  db: SqlDatabaseBinding,
+  runId: string,
+  totalUsage: { inputTokens: number; outputTokens: number },
+  status: RunStatus,
+  output?: string,
+  error?: string,
+): Promise<void> {
+  const drizzleDb = getDb(db);
+  const now = new Date().toISOString();
+
+  const updateData: {
+    status: string;
+    startedAt?: string;
+    completedAt?: string;
+    output?: string;
+    error?: string;
+    usage: string;
+  } = {
+    status,
+    usage: JSON.stringify(totalUsage),
+  };
+
+  if (status === 'running') {
+    updateData.startedAt = now;
+  }
+
+  if (status === 'completed' || status === 'failed' || status === 'cancelled') {
+    updateData.completedAt = now;
+  }
+
+  if (output !== undefined) {
+    updateData.output = output;
+  }
+
+  if (error !== undefined) {
+    updateData.error = error;
+  }
+
+  const condition = status === 'cancelled'
+    ? eq(runs.id, runId)
+    : and(eq(runs.id, runId), sql`${runs.status} != 'cancelled'`);
+
+  await drizzleDb.update(runs).set(updateData).where(condition);
+}
+
+// ── Conversation history helpers ────────────────────────────────────
+
+/** Type guard to validate tool_calls array structure */
+export function isValidToolCallsArray(value: unknown): value is ToolCall[] {
+  if (!Array.isArray(value)) return false;
+  return value.every(item => {
+    if (typeof item !== 'object' || item === null) return false;
+    const obj = item as Record<string, unknown>;
+    return (
+      typeof obj.id === 'string' &&
+      typeof obj.name === 'string' &&
+      typeof obj.arguments === 'object' &&
+      obj.arguments !== null
+    );
+  });
+}
+
+export interface ConversationHistoryDeps {
+  db: SqlDatabaseBinding;
+  env: Env;
+  threadId: string;
+  runId: string;
+  spaceId: string;
+  aiModel: string;
+}
+
+export function normalizeRunStatus(value: string | null | undefined): RunStatus | null {
+  return value === 'pending'
+    || value === 'queued'
+    || value === 'running'
+    || value === 'completed'
+    || value === 'failed'
+    || value === 'cancelled'
+    ? value
+    : null;
+}
+
+type MessageAttachmentRef = {
+  file_id: string;
+  path?: string;
+  name: string;
+  mime_type?: string | null;
+  size?: number;
+};
+
+function parseMessageAttachmentRefs(metadata: string | null | undefined): MessageAttachmentRef[] {
+  if (!metadata) return [];
+  try {
+    const parsed = JSON.parse(metadata) as unknown;
+    if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) return [];
+    const attachments = (parsed as Record<string, unknown>).attachments;
+    if (!Array.isArray(attachments)) return [];
+    const parsedAttachments: MessageAttachmentRef[] = [];
+    for (const entry of attachments) {
+      if (!entry || typeof entry !== 'object' || Array.isArray(entry)) continue;
+      const value = entry as Record<string, unknown>;
+      if (typeof value.file_id !== 'string' || typeof value.name !== 'string') continue;
+      parsedAttachments.push({
+        file_id: value.file_id,
+        path: typeof value.path === 'string' ? value.path : undefined,
+        name: value.name,
+        mime_type: typeof value.mime_type === 'string' ? value.mime_type : null,
+        size: typeof value.size === 'number' ? value.size : undefined,
+      });
+    }
+    return parsedAttachments;
+  } catch {
+    return [];
+  }
+}
+
+function appendAttachmentContext(content: string, attachments: MessageAttachmentRef[]): string {
+  if (attachments.length === 0) return content;
+
+  const lines = [
+    'Attached workspace storage files are available for this message.',
+    'Use workspace_files_read with file_id or path if you need to inspect them.',
+    ...attachments.map((attachment) => {
+      const parts = [
+        attachment.path || attachment.name,
+        `file_id: ${attachment.file_id}`,
+      ];
+      if (attachment.mime_type) parts.push(`mime_type: ${attachment.mime_type}`);
+      if (typeof attachment.size === 'number') parts.push(`size: ${attachment.size}`);
+      return `- ${parts.join(', ')}`;
+    }),
+  ];
+
+  const attachmentContext = lines.join('\n');
+  return content.trim()
+    ? `${content}\n\n${attachmentContext}`
+    : attachmentContext;
+}
+
+export async function buildConversationHistory(deps: ConversationHistoryDeps): Promise<AgentMessage[]> {
+  const { db: dbBinding, env, threadId, runId, spaceId, aiModel } = deps;
+  const db = getDb(dbBinding);
+  const startedAt = Date.now();
+
+  let threadSummary: string | null = null;
+  let threadKeyPointsJson = '[]';
+
+  const thread = await db.select({
+    summary: threads.summary,
+    keyPoints: threads.keyPoints,
+  }).from(threads).where(eq(threads.id, threadId)).get();
+
+  if (thread) {
+    threadSummary = thread.summary ?? null;
+    threadKeyPointsJson = thread.keyPoints || '[]';
+  }
+
+  const tokenBudget = resolveHistoryTokenBudget(aiModel, env.MODEL_CONTEXT_WINDOWS);
+
+  // Fetch recent messages (generous upper bound; trimmed by token budget below)
+  const MAX_FETCH = 500;
+  const rows = await db.select({
+    id: messages.id,
+    role: messages.role,
+    content: messages.content,
+    r2Key: messages.r2Key,
+    toolCalls: messages.toolCalls,
+    toolCallId: messages.toolCallId,
+    metadata: messages.metadata,
+    sequence: messages.sequence,
+  }).from(messages).where(eq(messages.threadId, threadId))
+    .orderBy(desc(messages.sequence))
+    .limit(MAX_FETCH)
+    .all();
+
+  rows.reverse(); // chronological
+
+  // Hydrate offloaded message payloads from R2 (best-effort).
+  if (env.TAKOS_OFFLOAD) {
+    const bucket = env.TAKOS_OFFLOAD;
+    const candidates = rows
+      .map((m, idx) => ({ idx, key: m.r2Key }))
+      .filter((x) => typeof x.key === 'string' && x.key.length > 0) as Array<{ idx: number; key: string }>;
+
+    const concurrency = 20;
+    for (let i = 0; i < candidates.length; i += concurrency) {
+      const batch = candidates.slice(i, i + concurrency);
+      await Promise.all(batch.map(async ({ idx, key }) => {
+        const persisted = await readMessageFromR2(bucket, key);
+        if (!persisted) return;
+        if (persisted.id !== rows[idx].id) return;
+        if (persisted.thread_id !== threadId) return;
+        rows[idx].content = persisted.content;
+        rows[idx].toolCalls = persisted.tool_calls;
+        rows[idx].toolCallId = persisted.tool_call_id;
+        rows[idx].metadata = persisted.metadata;
+      }));
+    }
+  }
+
+  const excludeSequences = new Set<number>();
+  let lastUserQuery = '';
+
+  // Build all candidate messages (newest first in rows, but rows is already reversed to chronological)
+  interface CandidateMessage { msg: AgentMessage; sequence: number; tokens: number }
+  const candidates: CandidateMessage[] = [];
+
+  for (const msg of rows) {
+    excludeSequences.add(msg.sequence);
+    if (msg.role === 'user') {
+      lastUserQuery = appendAttachmentContext(msg.content, parseMessageAttachmentRefs(msg.metadata));
+    }
+
+    const attachments = msg.role === 'user'
+      ? parseMessageAttachmentRefs(msg.metadata)
+      : [];
+    const agentMsg: AgentMessage = {
+      role: msg.role as AgentMessage['role'],
+      content: appendAttachmentContext(msg.content, attachments),
+    };
+
+    if (msg.toolCalls) {
+      try {
+        const parsed = JSON.parse(msg.toolCalls);
+        if (isValidToolCallsArray(parsed)) {
+          agentMsg.tool_calls = parsed;
+        } else {
+          logWarn('Invalid tool_calls structure, skipping', { module: 'services/agent/conversation-history' });
+        }
+      } catch (parseError) {
+        logWarn('Failed to parse tool_calls from message', { module: 'services/agent/conversation-history', error: parseError instanceof Error ? parseError.message : String(parseError) });
+      }
+    }
+
+    if (msg.toolCallId) {
+      agentMsg.tool_call_id = msg.toolCallId;
+    }
+
+    const tokens = estimateTokens(agentMsg.content || '')
+      + (agentMsg.tool_calls ? estimateTokens(JSON.stringify(agentMsg.tool_calls)) : 0);
+    candidates.push({ msg: agentMsg, sequence: msg.sequence, tokens });
+  }
+
+  // Trim from the front (oldest) to fit within token budget, keeping most recent messages
+  let totalTokens = 0;
+  for (const c of candidates) totalTokens += c.tokens;
+
+  let trimIndex = 0;
+  while (trimIndex < candidates.length - 1 && totalTokens > tokenBudget) {
+    totalTokens -= candidates[trimIndex].tokens;
+    trimIndex++;
+  }
+
+  const trimmed = candidates.slice(trimIndex);
+  const agentMessages = trimmed.map(c => c.msg);
+  const oldestRecentSequence = trimmed.length > 0 ? trimmed[0].sequence : undefined;
+
+  let retrieved: Awaited<ReturnType<typeof queryRelevantThreadMessages>> = [];
+  try {
+    retrieved = await queryRelevantThreadMessages({
+      env,
+      spaceId,
+      threadId,
+      query: lastUserQuery,
+      topK: THREAD_RETRIEVAL_TOP_K,
+      minScore: THREAD_RETRIEVAL_MIN_SCORE,
+      beforeSequence: oldestRecentSequence,
+      excludeSequences,
+    });
+  } catch (err) {
+    logWarn(`Vector search failed for thread ${threadId}`, { module: 'thread_context', detail: err });
+  }
+
+  const contextMsg = buildThreadContextSystemMessage({
+    summary: threadSummary,
+    keyPointsJson: threadKeyPointsJson,
+    retrieved,
+    maxChars: THREAD_CONTEXT_MAX_CHARS,
+  });
+  if (contextMsg) {
+    agentMessages.unshift(contextMsg);
+  }
+
+  // For sub-agent runs: prefer the structured delegation packet over broad parent history inheritance.
+  try {
+    const runRow = await db.select({
+      parentRunId: runs.parentRunId,
+      input: runs.input,
+    }).from(runs).where(eq(runs.id, runId)).get();
+    if (runRow?.parentRunId) {
+      const delegationPacket = getDelegationPacketFromRunInput(runRow.input);
+      if (delegationPacket) {
+        agentMessages.unshift(buildDelegationSystemMessage(delegationPacket));
+        agentMessages.push(buildDelegationUserMessage(delegationPacket));
+      } else {
+        const parsed = safeJsonParseOrDefault<Record<string, unknown> | unknown>(runRow.input || '{}', {});
+        const task = parsed && typeof parsed === 'object' && !Array.isArray(parsed)
+          ? (parsed as Record<string, unknown>).task
+          : null;
+        if (typeof task === 'string' && task.trim()) {
+          agentMessages.push({
+            role: 'user',
+            content:
+              `[Delegated sub-task from parent agent (run: ${runRow.parentRunId})]\n\n` +
+              task.trim(),
+          });
+        }
+      }
+    }
+  } catch (err) {
+    // Non-fatal: if we can't inject the task, the sub-agent still has the thread context
+    logWarn(`Failed to inject task for run ${runId}`, { module: 'sub_agent', detail: err });
+  }
+
+  // Lightweight benchmark log (helps validate context optimization in production logs).
+  try {
+    let chars = 0;
+    for (const msg of agentMessages) {
+      chars += (msg.content || '').length;
+      if (msg.tool_calls) {
+        chars += JSON.stringify(msg.tool_calls).length;
+      }
+    }
+    const estTokens = Math.ceil(chars / 4);
+    const elapsedMs = Date.now() - startedAt;
+    logInfo(`built thread=${threadId} model=${aiModel} budget=${tokenBudget} ` +
+      `fetched=${rows.length} used=${trimmed.length} retrieved=${retrieved.length} estTokens=${estTokens} ms=${elapsedMs}`, { module: 'thread_context' });
+  } catch {
+    // ignore
+  }
+
+  return agentMessages;
+}

package/src/application/services/agent/runner-types.ts

@@ -0,0 +1,73 @@
+/**
+ * Internal types and constants for the Agent Runner module.
+ */
+
+import { MAX_TOTAL_TOOL_CALLS_PER_RUN, MAX_TOOL_EXECUTIONS_HISTORY } from '../../../shared/config/limits';
+
+export const MAX_TOTAL_TOOL_CALLS = MAX_TOTAL_TOOL_CALLS_PER_RUN;
+
+export interface ToolExecution {
+  name: string;
+  arguments: Record<string, unknown>;
+  result?: string;
+  error?: string;
+  startedAt: number;
+  duration_ms?: number;
+}
+
+export interface EventEmissionError {
+  type: string;
+  error: string;
+  timestamp: string;
+}
+
+/**
+ * Combine multiple AbortSignals into a single one that aborts
+ * when any of the input signals abort.
+ */
+export function anySignal(signals: AbortSignal[]): AbortSignal {
+  const controller = new AbortController();
+  for (const signal of signals) {
+    if (signal.aborted) {
+      controller.abort(signal.reason);
+      return controller.signal;
+    }
+    signal.addEventListener('abort', () => controller.abort(signal.reason), { once: true });
+  }
+  return controller.signal;
+}
+
+/** Truncate error message to prevent excessive output */
+export function sanitizeErrorMessage(error: string): string {
+  return error.length > 10000 ? error.slice(0, 10000) + '...' : error;
+}
+
+/** Truncate very large argument values for practical output size */
+export function redactSensitiveArgs(args: Record<string, unknown>): Record<string, unknown> {
+  const processed: Record<string, unknown> = {};
+
+  for (const [key, value] of Object.entries(args)) {
+    if (typeof value === 'string' && value.length > 10000) {
+      processed[key] = value.slice(0, 1000) + `... [truncated:${value.length} chars]`;
+    } else {
+      processed[key] = value;
+    }
+  }
+
+  return processed;
+}
+
+/** Max tool execution history entries */
+export const MAX_TOOL_EXECUTIONS = MAX_TOOL_EXECUTIONS_HISTORY;
+
+/** Add a tool execution, evicting oldest 50% when at capacity */
+export function addToolExecution(
+  toolExecutions: ToolExecution[],
+  execution: ToolExecution,
+): void {
+  if (toolExecutions.length >= MAX_TOOL_EXECUTIONS) {
+    const removeCount = Math.max(1, Math.floor(MAX_TOOL_EXECUTIONS * 0.5));
+    toolExecutions.splice(0, removeCount);
+  }
+  toolExecutions.push(execution);
+}