work-ally 0.2.0-alpha.1

package/bridge/src/runtime-client-transport.ts

@@ -0,0 +1,309 @@
+import { spawn } from 'node:child_process';
+import { createInterface } from 'node:readline';
+import { logError, logInfo, logWarn } from './logger.ts';
+import type { ConnectionLifecycleEvent } from './types.ts';
+
+interface RuntimeClientTransportOptions {
+  codexHome: string | null;
+  codexBin: string;
+  spawnImpl: typeof spawn;
+  connectTimeoutMs: number;
+  disconnectTimeoutMs: number;
+  reconnectBaseDelayMs: number;
+  reconnectMaxDelayMs: number;
+  initialize(): Promise<void>;
+  onMessage(line: string): void;
+  rejectAllPending(error: Error): void;
+  rejectAllTurns(error: Error): void;
+  activeTurnCount(): number;
+  emitConnectionEvent(event: ConnectionLifecycleEvent): void;
+}
+
+function reconnectDelayForAttempt(attempt: number, baseMs: number, maxMs: number): number {
+  const base = Math.max(250, baseMs);
+  const max = Math.max(base, maxMs);
+  const delay = base * Math.pow(2, Math.max(0, attempt - 1));
+  return Math.min(max, delay);
+}
+
+export class RuntimeClientTransport {
+  private readonly options: RuntimeClientTransportOptions;
+  private child: ReturnType<typeof spawn> | null = null;
+  private stdoutInterface: ReturnType<typeof createInterface> | null = null;
+  private connectPromise: Promise<void> | null = null;
+  private reconnectTimer: NodeJS.Timeout | null = null;
+  private reconnectAttempt = 0;
+  private reconnectNotified = false;
+  private manualDisconnect = false;
+
+  constructor(options: RuntimeClientTransportOptions) {
+    this.options = options;
+  }
+
+  isReady(): boolean {
+    return Boolean(
+      this.child
+      && this.child.exitCode === null
+      && !this.child.killed
+      && this.child.stdin
+      && !this.child.stdin.destroyed
+      && !this.child.stdin.writableEnded,
+    );
+  }
+
+  sendRaw(payload: unknown): void {
+    if (!this.isReady() || !this.child?.stdin) {
+      throw new Error('runtime stdio is not connected');
+    }
+    this.child.stdin.write(`${JSON.stringify(payload)}\n`);
+  }
+
+  scheduleReconnect(reason: string, reconnect: () => Promise<void>): void {
+    if (this.manualDisconnect || this.reconnectTimer) {
+      return;
+    }
+    const attempt = this.reconnectAttempt + 1;
+    const delayMs = reconnectDelayForAttempt(
+      attempt,
+      this.options.reconnectBaseDelayMs,
+      this.options.reconnectMaxDelayMs,
+    );
+    this.reconnectAttempt = attempt;
+    this.reconnectNotified = true;
+    logWarn('runtime', 'reconnect_scheduled', { transport: 'stdio', attempt, delayMs, reason });
+    if (attempt === 1) {
+      this.options.emitConnectionEvent({
+        source: 'runtime',
+        state: 'disconnected',
+        reason,
+        attempt,
+        nextRetryDelayMs: delayMs,
+      });
+    }
+    this.options.emitConnectionEvent({
+      source: 'runtime',
+      state: 'reconnecting',
+      reason,
+      attempt,
+      nextRetryDelayMs: delayMs,
+    });
+    this.reconnectTimer = setTimeout(() => {
+      this.reconnectTimer = null;
+      reconnect().catch((error) => {
+        const message = error instanceof Error ? error.message : String(error);
+        this.scheduleReconnect(message, reconnect);
+      });
+    }, delayMs);
+    this.reconnectTimer.unref?.();
+  }
+
+  handleTransportClosed(reason: string, scheduleReconnect: (reason: string) => void): void {
+    logWarn('runtime', 'transport_closed', { transport: 'stdio', reason });
+    this.clearTransportHandles();
+    this.connectPromise = null;
+    const closedError = new Error(reason);
+    this.options.rejectAllPending(closedError);
+    if (this.manualDisconnect) {
+      this.options.rejectAllTurns(closedError);
+      return;
+    }
+    const activeTurnCount = this.options.activeTurnCount();
+    if (activeTurnCount > 0) {
+      logWarn('runtime', 'turns_waiting_for_pull_reconcile_after_disconnect', {
+        activeTurnCount,
+        reason,
+      });
+    }
+    scheduleReconnect(reason);
+  }
+
+  async connect(): Promise<void> {
+    this.manualDisconnect = false;
+    if (this.isReady()) {
+      return;
+    }
+    if (this.connectPromise) {
+      return this.connectPromise;
+    }
+
+    this.connectPromise = new Promise<void>((resolve, reject) => {
+      logInfo('runtime', 'connect_start', { transport: 'stdio', codexBin: this.options.codexBin });
+      const child = this.options.spawnImpl(this.options.codexBin, ['app-server', '--listen', 'stdio://'], {
+        stdio: ['pipe', 'pipe', 'pipe'],
+        env: {
+          ...process.env,
+          ...(this.options.codexHome ? { CODEX_HOME: this.options.codexHome } : {}),
+        },
+      });
+      this.child = child;
+      this.stdoutInterface = createInterface({
+        input: child.stdout,
+        crlfDelay: Infinity,
+      });
+      this.stdoutInterface.on('line', (line) => {
+        if (!line) {
+          return;
+        }
+        try {
+          this.options.onMessage(line);
+        } catch (error) {
+          logWarn('runtime', 'stdout_line_parse_failed', {
+            error: error instanceof Error ? (error.stack || error.message) : String(error),
+            line,
+          });
+        }
+      });
+      child.stderr.on('data', (chunk) => {
+        const text = String(chunk).trim();
+        if (!text) {
+          return;
+        }
+        logWarn('runtime', 'child_stderr', { text });
+      });
+      child.stdin.on('error', (error) => {
+        logWarn('runtime', 'stdin_error', {
+          error: error instanceof Error ? (error.stack || error.message) : String(error),
+        });
+      });
+
+      let settled = false;
+      const timeout = setTimeout(() => {
+        if (settled) {
+          return;
+        }
+        settled = true;
+        try {
+          child.kill('SIGTERM');
+        } catch {
+          // noop
+        }
+        reject(new Error(`runtime connect timed out: stdio://${this.options.codexBin}`));
+      }, this.options.connectTimeoutMs);
+
+      const fail = (error: unknown) => {
+        if (settled) {
+          return;
+        }
+        settled = true;
+        clearTimeout(timeout);
+        reject(error instanceof Error ? error : new Error(String(error)));
+      };
+
+      child.once('spawn', async () => {
+        logInfo('runtime', 'connect_open', {
+          transport: 'stdio',
+          pid: child.pid || null,
+        });
+        this.clearReconnectTimer();
+        try {
+          await this.options.initialize();
+          if (this.reconnectNotified) {
+            this.options.emitConnectionEvent({
+              source: 'runtime',
+              state: 'reconnected',
+              attempt: this.reconnectAttempt || null,
+              reason: null,
+              nextRetryDelayMs: null,
+            });
+          }
+          this.reconnectAttempt = 0;
+          this.reconnectNotified = false;
+          if (!settled) {
+            settled = true;
+            clearTimeout(timeout);
+            resolve();
+          }
+        } catch (error) {
+          fail(error);
+          try {
+            child.kill('SIGTERM');
+          } catch {
+            // noop
+          }
+        }
+      });
+
+      child.once('error', (error) => {
+        logError('runtime', 'connect_error', {
+          transport: 'stdio',
+          error: error instanceof Error ? (error.stack || error.message) : String(error),
+        });
+        fail(error);
+      });
+
+      child.once('close', (code, signal) => {
+        const reason = `runtime child process exited${code !== null ? ` (code: ${code})` : ''}${signal ? ` (signal: ${signal})` : ''}`;
+        if (!settled) {
+          fail(new Error(reason));
+        }
+        this.handleTransportClosed(reason, (message) => this.scheduleReconnect(message, () => this.connect()));
+      });
+    });
+
+    try {
+      await this.connectPromise;
+    } catch (error) {
+      this.connectPromise = null;
+      this.clearTransportHandles();
+      throw error;
+    }
+  }
+
+  async disconnect(): Promise<void> {
+    this.manualDisconnect = true;
+    this.clearReconnectTimer();
+    const child = this.child;
+    this.connectPromise = null;
+    if (!child || child.exitCode !== null || child.killed) {
+      this.clearTransportHandles();
+      return;
+    }
+
+    await new Promise<void>((resolve) => {
+      let settled = false;
+      const done = () => {
+        if (settled) {
+          return;
+        }
+        settled = true;
+        resolve();
+      };
+      const timer = setTimeout(done, this.options.disconnectTimeoutMs);
+      child.once('close', () => {
+        clearTimeout(timer);
+        done();
+      });
+      try {
+        child.kill('SIGTERM');
+      } catch {
+        clearTimeout(timer);
+        done();
+      }
+    });
+
+    this.clearTransportHandles();
+  }
+
+  private clearReconnectTimer(): void {
+    if (!this.reconnectTimer) {
+      return;
+    }
+    clearTimeout(this.reconnectTimer);
+    this.reconnectTimer = null;
+  }
+
+  private clearTransportHandles(): void {
+    if (this.stdoutInterface) {
+      this.stdoutInterface.removeAllListeners();
+      this.stdoutInterface.close();
+      this.stdoutInterface = null;
+    }
+    if (this.child) {
+      this.child.removeAllListeners();
+      this.child.stdout?.removeAllListeners();
+      this.child.stderr?.removeAllListeners();
+      this.child.stdin?.removeAllListeners();
+      this.child = null;
+    }
+  }
+}

package/bridge/src/runtime-client-turn-poll.ts

@@ -0,0 +1,224 @@
+import { buildReplyFromThreadItems, consumeCompletedAgentMessageProgress, noteAgentMessageItem, type RuntimeTurnState } from './runtime-client-agent-messages.ts';
+import { enrichTerminalReply, mergeTerminalTurnResult, normalizeTurnError, readTurnSnapshot } from './runtime-client-turn-read.ts';
+import { logInfo, logWarn } from './logger.ts';
+import { normalizeRuntimeError } from './runtime-client-protocol.ts';
+import type { RuntimeProgressMeta, RuntimeThreadStatus, RuntimeTurnResult } from './types.ts';
+
+interface TurnFinalPollDeps {
+  turnStates: Map<string, RuntimeTurnState>;
+  request: (method: string, params: unknown) => Promise<any>;
+  cacheThreadStatus: (threadId: string, status: RuntimeThreadStatus | null | undefined) => RuntimeThreadStatus | null;
+  readPersistedFinalReply: (turnId: string) => string | null;
+  resolveTurnResult: (turnId: string, result: RuntimeTurnResult) => boolean;
+  rejectTurnResult: (turnId: string, error: Error) => boolean;
+  isRecoverableRuntimeError: (error: unknown) => boolean;
+  safeEmitTurnProgress: (turnId: string, text: string, meta?: RuntimeProgressMeta) => Promise<void>;
+}
+
+interface TurnFinalPollOptions {
+  pollMs: number;
+  transportTimeoutMs: number;
+  idleReconcileMs: number;
+  idleAwaitingAuthoritativeResultTimeoutMs: number;
+}
+
+function sleep(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+function isWaitingOnHumanStatus(status: RuntimeThreadStatus | null | undefined): boolean {
+  return status?.type === 'active' && status.activeFlags.some((flag) => flag === 'waitingOnApproval' || flag === 'waitingOnUserInput');
+}
+
+async function observePolledTurnItems(
+  turnStates: Map<string, RuntimeTurnState>,
+  turnId: string,
+  items: Array<any>,
+  safeEmitTurnProgress: (turnId: string, text: string, meta?: RuntimeProgressMeta) => Promise<void>,
+): Promise<void> {
+  const turn = turnStates.get(turnId);
+  if (!turn || !turn.onProgress) {
+    return;
+  }
+  for (const item of items) {
+    if (item?.type !== 'agentMessage' || typeof item.id !== 'string') {
+      continue;
+    }
+    noteAgentMessageItem(turnStates, turnId, item);
+    const completed = consumeCompletedAgentMessageProgress(turnStates, turnId, item.id);
+    if (!completed || completed.phase === 'final_answer') {
+      continue;
+    }
+    await safeEmitTurnProgress(turnId, completed.text, {
+      kind: 'agent_message',
+      phase: completed.phase,
+      turnId,
+      itemId: item.id,
+    });
+  }
+}
+
+export async function pollTurnFinalState(
+  deps: TurnFinalPollDeps,
+  threadId: string,
+  turnId: string,
+  options: TurnFinalPollOptions,
+): Promise<void> {
+  let waitingOnHumanLogged = false;
+  let lastWaitingStatusKey: string | null = null;
+  let lastSuccessfulReadAt = Date.now();
+  let idleSinceAt: number | null = null;
+  let idleAwaitingAuthoritativeResultSinceAt: number | null = null;
+  let idleAwaitingAuthoritativeResultLogged = false;
+  let lastError: Error | null = null;
+
+  while (deps.turnStates.has(turnId)) {
+    try {
+      const snapshot = await readTurnSnapshot({
+        request: deps.request,
+        cacheThreadStatus: deps.cacheThreadStatus,
+      }, threadId, turnId);
+      threadId = snapshot.threadId;
+      lastSuccessfulReadAt = Date.now();
+      lastError = null;
+
+      const items = Array.isArray(snapshot.turn?.items) ? snapshot.turn.items : [];
+      await observePolledTurnItems(deps.turnStates, turnId, items, deps.safeEmitTurnProgress);
+
+      if (snapshot.turn && typeof snapshot.turn.status === 'string' && snapshot.turn.status !== 'inProgress') {
+        const resolvedBase = {
+          threadId: snapshot.threadId,
+          turnId,
+          status: snapshot.turn.status,
+          reply: buildReplyFromThreadItems(items),
+          error: normalizeTurnError(snapshot.turn.error),
+        } satisfies RuntimeTurnResult;
+        const completedEventResult = deps.turnStates.get(turnId)?.completedEventResult || null;
+        const resolved = await enrichTerminalReply({
+          readPersistedFinalReply: deps.readPersistedFinalReply,
+        }, mergeTerminalTurnResult(completedEventResult, resolvedBase));
+        logInfo('runtime', 'turn_final_state_polled', {
+          threadId: resolved.threadId,
+          turnId: resolved.turnId,
+          status: resolved.status,
+        });
+        deps.resolveTurnResult(turnId, resolved);
+        return;
+      }
+
+      const status = snapshot.threadStatus;
+      if (isWaitingOnHumanStatus(status)) {
+        idleSinceAt = null;
+        idleAwaitingAuthoritativeResultSinceAt = null;
+        idleAwaitingAuthoritativeResultLogged = false;
+        const waitingStatusKey = status.activeFlags.slice().sort().join(',');
+        if (waitingStatusKey !== lastWaitingStatusKey) {
+          lastWaitingStatusKey = waitingStatusKey;
+          const turn = deps.turnStates.get(turnId);
+          await turn?.onThreadStatus?.(status, {
+            threadId,
+            turnId,
+          });
+        }
+        if (!waitingOnHumanLogged) {
+          waitingOnHumanLogged = true;
+          logInfo('runtime', 'turn_final_state_poll_waiting_on_human', {
+            threadId,
+            turnId,
+            activeFlags: status?.activeFlags || [],
+          });
+        }
+        await sleep(options.pollMs);
+        continue;
+      }
+      waitingOnHumanLogged = false;
+      lastWaitingStatusKey = null;
+
+      if (status?.type === 'active') {
+        idleSinceAt = null;
+        idleAwaitingAuthoritativeResultSinceAt = null;
+        idleAwaitingAuthoritativeResultLogged = false;
+        await sleep(options.pollMs);
+        continue;
+      }
+
+      if (status?.type === 'idle') {
+        if (idleSinceAt == null) {
+          idleSinceAt = Date.now();
+        }
+        if (Date.now() - idleSinceAt >= options.idleReconcileMs) {
+          const completedEventResult = deps.turnStates.get(turnId)?.completedEventResult || null;
+          if (completedEventResult) {
+            logInfo('runtime', 'turn_final_state_reconciled_from_completed_event', {
+              threadId,
+              turnId,
+              status: completedEventResult.status,
+            });
+            deps.resolveTurnResult(turnId, completedEventResult);
+            return;
+          }
+          if (idleAwaitingAuthoritativeResultSinceAt == null) {
+            idleAwaitingAuthoritativeResultSinceAt = Date.now();
+          }
+          if (!idleAwaitingAuthoritativeResultLogged) {
+            idleAwaitingAuthoritativeResultLogged = true;
+            logWarn('runtime', 'turn_reconcile_waiting_for_authoritative_result', {
+              threadId,
+              turnId,
+              idleReconcileMs: options.idleReconcileMs,
+              timeoutMs: options.idleAwaitingAuthoritativeResultTimeoutMs,
+            });
+          }
+          if (Date.now() - idleAwaitingAuthoritativeResultSinceAt >= options.idleAwaitingAuthoritativeResultTimeoutMs) {
+            const error = new Error(`runtime turn reconciliation failed: idle thread without completed turn event (${turnId})`);
+            logWarn('runtime', 'turn_reconcile_failed', {
+              threadId,
+              turnId,
+              idleDurationMs: Date.now() - idleSinceAt,
+              timeoutMs: options.idleAwaitingAuthoritativeResultTimeoutMs,
+            });
+            deps.rejectTurnResult(turnId, error);
+            return;
+          }
+        }
+        await sleep(options.pollMs);
+        continue;
+      }
+
+      idleSinceAt = null;
+      idleAwaitingAuthoritativeResultSinceAt = null;
+      idleAwaitingAuthoritativeResultLogged = false;
+    } catch (error) {
+      const runtimeError = normalizeRuntimeError(error);
+      lastError = runtimeError;
+      if (!deps.isRecoverableRuntimeError(runtimeError)) {
+        logWarn('runtime', 'turn_final_state_poll_failed', {
+          threadId,
+          turnId,
+          error: runtimeError.stack || runtimeError.message,
+        });
+        deps.rejectTurnResult(turnId, runtimeError);
+        return;
+      }
+      logWarn('runtime', 'turn_final_state_poll_failed', {
+        threadId,
+        turnId,
+        error: runtimeError.stack || runtimeError.message,
+      });
+    }
+
+    if (Date.now() - lastSuccessfulReadAt >= options.transportTimeoutMs) {
+      const error = lastError || new Error('runtime stdio is not connected');
+      logWarn('runtime', 'turn_final_state_poll_transport_timeout', {
+        threadId,
+        turnId,
+        timeoutMs: options.transportTimeoutMs,
+        error: error.stack || error.message,
+      });
+      deps.rejectTurnResult(turnId, error);
+      return;
+    }
+
+    await sleep(options.pollMs);
+  }
+}