taskode 0.4.0

package/src/orchestrator.js

@@ -0,0 +1,655 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import { applyWorkspaceChanges, buildRunArtifacts } from './review.js';
+
+export class Orchestrator {
+  constructor({ config, tracker, workspaceManager, agentRunner, store, workflowRuntime = null }) {
+    this.config = config;
+    this.tracker = tracker;
+    this.workspaceManager = workspaceManager;
+    this.agentRunner = agentRunner;
+    this.store = store;
+    this.workflowRuntime = workflowRuntime;
+    this.timer = null;
+    this.timerIntervalMs = null;
+    this.lastWorkflowReloadError = null;
+
+    const persistedRuntime = store.loadRuntime();
+    this.runtime = normalizeRuntime(config, persistedRuntime, store.listRetries());
+    this.recoverPersistedRuns();
+  }
+
+  start() {
+    if (this.timer) return;
+    this.tick();
+    this.syncTimerInterval();
+  }
+
+  stop() {
+    if (this.timer) clearInterval(this.timer);
+    this.timer = null;
+    this.timerIntervalMs = null;
+    this.store.saveRuntime(this.runtime);
+  }
+
+  getState() {
+    return {
+      ...this.runtime,
+      running: Object.values(this.runtime.running),
+      claimed: Object.values(this.runtime.claimed),
+      retries: Object.values(this.runtime.retries),
+      workflow: this.workflowRuntime?.getMetadata?.() || null
+    };
+  }
+
+  async tick() {
+    this.runtime.last_tick_at = new Date().toISOString();
+
+    try {
+      this.refreshWorkflowConfig();
+      await this.dispatchDueRetries();
+
+      const issues = await this.tracker.fetchCandidates();
+      const sorted = issues.sort((a, b) => (a.priority ?? 999) - (b.priority ?? 999));
+
+      for (const issue of sorted) {
+        if (this.runtime.claimed[issue.id]) continue;
+
+        const task = this.store.getTask(issue.identifier);
+        if (issue.blocked_by?.length) {
+          if (task && task.status !== 'blocked') {
+            this.store.updateTask(task.id, { status: 'blocked' });
+          }
+          continue;
+        }
+
+        if (task?.status === 'blocked') {
+          this.store.updateTask(task.id, { status: 'todo' });
+        }
+
+        if (task && ['review', 'done', 'cancelled'].includes(task.status)) continue;
+
+        if (Object.keys(this.runtime.running).length >= this.config.maxConcurrentAgents) continue;
+        if (!stateSlotsAvailable(issue, this.runtime.running, this.config.maxConcurrentAgentsByState, this.config.maxConcurrentAgents)) continue;
+        if (!this.workerSlotsAvailable()) continue;
+
+        await this.dispatch(issue, false);
+      }
+
+      await this.reconcileRunningIssues();
+      this.store.saveRuntime(this.runtime);
+    } catch (error) {
+      this.store.appendSystemLog(`tick error: ${error.message}`);
+    }
+  }
+
+  async approveRun(runId) {
+    const run = this.store.getRun(runId);
+    if (!run) throw new Error(`Run not found: ${runId}`);
+    if (run.status !== 'succeeded') throw new Error('Only succeeded runs can be approved');
+
+    const task = this.store.getTask(run.issueIdentifier);
+    if (!run.workspacePath) throw new Error(`Run ${runId} has no workspace`);
+
+    const changes = applyWorkspaceChanges({
+      projectRoot: this.config.projectRoot,
+      workspacePath: run.workspacePath,
+      changes: run.changedFiles,
+      ignoreNames: this.config.workspaceIgnoreNames
+    });
+
+    this.store.updateRun(run.id, {
+      reviewStatus: 'approved',
+      appliedAt: new Date().toISOString(),
+      changedFiles: changes
+    });
+
+    if (task) {
+      this.store.updateTask(task.id, { status: 'done' });
+    }
+
+    await this.onRunApproved(run, task);
+
+    if (this.config.review.autoCleanupApproved) {
+      await this.workspaceManager.cleanupByPath(run.workspacePath);
+    }
+
+    this.store.appendAudit({
+      action: 'run_approved',
+      actor: 'system',
+      metadata: { runId: run.id, taskId: task?.id || null }
+    });
+
+    return this.store.getRun(run.id);
+  }
+
+  async rejectRun(runId, reason = '') {
+    const run = this.store.getRun(runId);
+    if (!run) throw new Error(`Run not found: ${runId}`);
+
+    const task = this.store.getTask(run.issueIdentifier);
+    this.store.updateRun(run.id, { reviewStatus: 'rejected' });
+
+    if (task) {
+      this.store.updateTask(task.id, { status: 'todo' });
+    }
+
+    await this.onRunRejected(run, task, reason);
+
+    this.store.appendAudit({
+      action: 'run_rejected',
+      actor: 'system',
+      metadata: { runId: run.id, taskId: task?.id || null, reason }
+    });
+
+    return this.store.getRun(run.id);
+  }
+
+  async requeueTask(taskId) {
+    const task = this.store.getTask(taskId);
+    if (!task) throw new Error(`Task not found: ${taskId}`);
+
+    this.store.updateTask(task.id, { status: 'todo' });
+    this.store.removeRetry(task.externalId || task.id);
+
+    this.store.appendAudit({
+      action: 'task_requeued',
+      actor: 'system',
+      metadata: { taskId: task.id }
+    });
+
+    return this.store.getTask(task.id);
+  }
+
+  recoverPersistedRuns() {
+    const staleRuns = Object.values(this.runtime.running);
+    if (!staleRuns.length) return;
+
+    for (const staleRun of staleRuns) {
+      const now = new Date().toISOString();
+      const attempt = staleRun.attempt || 1;
+      const task = this.store.getTask(staleRun.issue_identifier);
+
+      if (staleRun.run_id) {
+        this.store.updateRun(staleRun.run_id, {
+          status: 'interrupted',
+          summary: 'Run interrupted by orchestrator restart',
+          finishedAt: now,
+          error: 'orchestrator restart'
+        });
+      }
+
+      if (task && !['done', 'review', 'cancelled'].includes(task.status)) {
+        this.store.updateTask(task.id, { status: 'todo' });
+      }
+
+      if (attempt < this.config.maxRetryAttempts) {
+        const retryEntry = {
+          issue_id: staleRun.issue_id,
+          identifier: staleRun.issue_identifier,
+          attempt,
+          due_at_ms: Date.now(),
+          error: 'orchestrator restart'
+        };
+        this.runtime.retries[staleRun.issue_id] = retryEntry;
+        this.store.setRetry(staleRun.issue_id, retryEntry);
+      }
+
+      delete this.runtime.claimed[staleRun.issue_id];
+      delete this.runtime.running[staleRun.issue_id];
+    }
+
+    this.store.appendSystemLog(`Recovered ${staleRuns.length} interrupted runs after restart`);
+    this.store.saveRuntime(this.runtime);
+  }
+
+  async dispatchDueRetries() {
+    const dueRetries = Object.values(this.runtime.retries)
+      .filter((entry) => entry.due_at_ms <= Date.now())
+      .sort((a, b) => a.due_at_ms - b.due_at_ms);
+
+    for (const retry of dueRetries) {
+      if (Object.keys(this.runtime.running).length >= this.config.maxConcurrentAgents) return;
+      const issues = await this.tracker.fetchByIds([retry.issue_id]);
+      const issue = issues[0];
+
+      if (!issue) {
+        delete this.runtime.retries[retry.issue_id];
+        this.store.removeRetry(retry.issue_id);
+        continue;
+      }
+
+      if (!stateSlotsAvailable(issue, this.runtime.running, this.config.maxConcurrentAgentsByState, this.config.maxConcurrentAgents)) {
+        continue;
+      }
+
+      await this.dispatch(issue, true);
+    }
+  }
+
+  async dispatch(issue, fromRetry) {
+    const preferredWorkerHost = fromRetry ? this.runtime.retries[issue.id]?.worker_host || null : null;
+    const workerHost = this.selectWorkerHost(preferredWorkerHost);
+    if (workerHost === NO_WORKER_CAPACITY) {
+      return false;
+    }
+
+    const attempt = fromRetry ? (this.runtime.retries[issue.id]?.attempt || 1) + 1 : 1;
+    const run = this.store.createRun({
+      taskId: issue.id,
+      issueIdentifier: issue.identifier,
+      attempt,
+      status: 'running',
+      summary: `Running ${issue.identifier}`
+    });
+
+    this.runtime.claimed[issue.id] = { issue_id: issue.id, identifier: issue.identifier };
+    this.runtime.running[issue.id] = {
+      issue_id: issue.id,
+      issue_identifier: issue.identifier,
+      attempt,
+      run_id: run.id,
+      workspace_key: null,
+      issue_state: issue.state || null,
+      worker_host: workerHost,
+      started_at: new Date().toISOString()
+    };
+    this.store.updateRunSession(run.id, { workerHost });
+
+    const task = this.store.getTask(issue.identifier);
+    if (task) {
+      this.store.updateTask(task.id, { status: 'in_progress' });
+    }
+
+    if (fromRetry) this.runtime.metrics.retried += 1;
+    this.runtime.metrics.dispatched += 1;
+
+    this.store.appendAudit({
+      action: 'dispatch',
+      actor: 'system',
+      metadata: { issueId: issue.id, runId: run.id, attempt }
+    });
+
+    this.executeIssue(issue, run, workerHost).catch((error) => {
+      this.store.appendRunLog(run.id, `runner crash: ${error.message}`);
+    });
+    return true;
+  }
+
+  async executeIssue(issue, run, workerHost) {
+    let workspace = null;
+    let result = null;
+
+    try {
+      workspace = await this.workspaceManager.ensure(issue, workerHost);
+
+      if (this.runtime.running[issue.id]) {
+        this.runtime.running[issue.id].workspace_key = workspace.workspace_key;
+        this.runtime.running[issue.id].worker_host = workerHost;
+      }
+
+      this.store.updateRun(run.id, {
+        workspacePath: workspace.path,
+        workspaceKey: workspace.workspace_key
+      });
+
+      const prompt = renderPrompt(this.config.promptTemplate, issue);
+      this.store.appendRunLog(run.id, `workspace: ${workspace.path}`);
+      await this.workspaceManager.runBeforeRunHook(workspace.path, issue, workerHost);
+
+      result = await this.agentRunner.run({
+        issue,
+        workspacePath: workspace.path,
+        prompt,
+        runId: run.id,
+        tracker: this.tracker,
+        workerHost
+      });
+    } catch (error) {
+      result = {
+        status: 'failed',
+        code: 1,
+        output: '',
+        error: error.message,
+        durationMs: 0,
+        telemetry: {}
+      };
+    } finally {
+      if (workspace?.path) {
+        await this.workspaceManager.runAfterRunHook(workspace.path, issue, workerHost);
+      }
+    }
+
+    await this.completeRun(issue, run, workspace, result);
+  }
+
+  async completeRun(issue, run, workspace, result) {
+    const finishedAt = new Date().toISOString();
+    const task = this.store.getTask(issue.identifier);
+    const logFile = writeLogFile(this.config.logsRoot, run.id, result);
+    const artifacts = buildRunArtifacts({
+      projectRoot: this.config.projectRoot,
+      workspacePath: workspace?.path,
+      logsRoot: this.config.logsRoot,
+      runId: run.id,
+      ignoreNames: this.config.workspaceIgnoreNames
+    });
+
+    this.store.updateRunSession(run.id, result.telemetry || {});
+
+    if (result.status === 'succeeded') {
+      this.runtime.codex_totals = addTokenTotals(this.runtime.codex_totals, result.telemetry);
+      if (result.telemetry?.rateLimits) {
+        this.runtime.codex_rate_limits = result.telemetry.rateLimits;
+      }
+      this.runtime.metrics.succeeded += 1;
+      this.store.updateRun(run.id, {
+        status: 'succeeded',
+        summary: `Issue ${issue.identifier} completed in ${result.durationMs}ms`,
+        finishedAt,
+        workspacePath: workspace?.path || null,
+        workspaceKey: workspace?.workspace_key || null,
+        logFile,
+        diffFile: artifacts.diffFile,
+        changedFiles: artifacts.changedFiles,
+        diffSummary: artifacts.diffSummary,
+        reviewStatus: 'pending',
+        error: null
+      });
+
+      if (task) {
+        this.store.updateTask(task.id, { status: 'review' });
+      }
+
+      delete this.runtime.retries[issue.id];
+      this.store.removeRetry(issue.id);
+      await this.onIssueSuccess(issue, run);
+    } else {
+      this.runtime.codex_totals = addTokenTotals(this.runtime.codex_totals, result.telemetry);
+      if (result.telemetry?.rateLimits) {
+        this.runtime.codex_rate_limits = result.telemetry.rateLimits;
+      }
+      this.runtime.metrics.failed += 1;
+      const attempt = this.runtime.running[issue.id]?.attempt || 1;
+      const backoffMs = Math.min(this.config.maxRetryBackoffMs, 1000 * (2 ** attempt));
+      const workerHost = this.runtime.running[issue.id]?.worker_host || null;
+
+      this.store.updateRun(run.id, {
+        status: 'failed',
+        summary: `Issue ${issue.identifier} failed: ${result.error || 'unknown error'}`,
+        finishedAt,
+        workspacePath: workspace?.path || null,
+        workspaceKey: workspace?.workspace_key || null,
+        logFile,
+        diffFile: artifacts.diffFile,
+        changedFiles: artifacts.changedFiles,
+        diffSummary: artifacts.diffSummary,
+        reviewStatus: 'none',
+        error: result.error || 'unknown error'
+      });
+
+      if (attempt >= this.config.maxRetryAttempts) {
+        delete this.runtime.retries[issue.id];
+        this.store.removeRetry(issue.id);
+        if (task) {
+          this.store.updateTask(task.id, { status: 'failed' });
+        }
+        await this.onIssueFailure(issue, run, `max retry reached (${attempt})`);
+      } else {
+        const retryEntry = {
+          issue_id: issue.id,
+          identifier: issue.identifier,
+          attempt,
+          due_at_ms: Date.now() + backoffMs,
+          error: result.error || 'unknown error',
+          worker_host: workerHost
+        };
+
+        this.runtime.retries[issue.id] = retryEntry;
+        this.store.setRetry(issue.id, retryEntry);
+
+        if (task) {
+          this.store.updateTask(task.id, { status: 'todo' });
+        }
+
+        await this.onIssueFailure(issue, run, `retry in ${backoffMs}ms`);
+      }
+    }
+
+    delete this.runtime.running[issue.id];
+    delete this.runtime.claimed[issue.id];
+
+    this.store.appendAudit({
+      action: 'run_finished',
+      actor: 'system',
+      metadata: { issueId: issue.id, runId: run.id, status: result.status }
+    });
+  }
+
+  async onIssueSuccess(issue, run) {
+    if (this.config.tracker.kind === 'linear') {
+      await this.tracker.writeComment(issue.id, `Taskode run ${run.id} succeeded and is ready for review.`);
+      await this.tracker.transitionState(issue.id, 'Human Review');
+      return;
+    }
+
+    if (this.config.tracker.kind === 'github') {
+      await this.tracker.writeComment(issue.id, `Taskode run ${run.id} succeeded and is ready for review.`);
+    }
+  }
+
+  async onIssueFailure(issue, run, note) {
+    if (this.config.tracker.kind === 'linear') {
+      await this.tracker.writeComment(issue.id, `Taskode run ${run.id} failed: ${note}`);
+      await this.tracker.transitionState(issue.id, 'Rework');
+      return;
+    }
+
+    if (this.config.tracker.kind === 'github') {
+      await this.tracker.writeComment(issue.id, `Taskode run ${run.id} failed: ${note}`);
+    }
+  }
+
+  async onRunApproved(run, task) {
+    if (this.config.tracker.kind === 'linear') {
+      await this.tracker.writeComment(run.taskId, `Taskode run ${run.id} was approved and applied.`);
+      await this.tracker.transitionState(run.taskId, 'Done');
+      return;
+    }
+
+    if (this.config.tracker.kind === 'github') {
+      await this.tracker.writeComment(run.taskId, `Taskode run ${run.id} was approved and applied.`);
+      await this.tracker.transitionState(run.taskId, 'Done');
+      return;
+    }
+
+    if (task) {
+      this.store.appendAudit({
+        action: 'local_task_done',
+        actor: 'system',
+        metadata: { taskId: task.id, runId: run.id }
+      });
+    }
+  }
+
+  async onRunRejected(run, task, reason) {
+    const note = reason ? `: ${reason}` : '';
+
+    if (this.config.tracker.kind === 'linear') {
+      await this.tracker.writeComment(run.taskId, `Taskode run ${run.id} was rejected${note}`);
+      await this.tracker.transitionState(run.taskId, 'Rework');
+      return;
+    }
+
+    if (this.config.tracker.kind === 'github') {
+      await this.tracker.writeComment(run.taskId, `Taskode run ${run.id} was rejected${note}`);
+    }
+
+    if (task) {
+      this.store.appendAudit({
+        action: 'local_task_rework',
+        actor: 'system',
+        metadata: { taskId: task.id, runId: run.id, reason }
+      });
+    }
+  }
+
+  async reconcileRunningIssues() {
+    const runningIds = Object.keys(this.runtime.running);
+    if (!runningIds.length) return;
+
+    const trackerIssues = await this.tracker.fetchByIds(runningIds);
+    const trackerStateById = new Map(trackerIssues.map((issue) => [issue.id, issue]));
+
+    for (const issueId of runningIds) {
+      const issue = trackerStateById.get(issueId);
+      if (!issue) continue;
+
+      if (this.config.tracker.terminalStates.includes(issue.state)) {
+        const running = this.runtime.running[issueId];
+        this.runtime.metrics.stopped += 1;
+        delete this.runtime.running[issueId];
+        delete this.runtime.claimed[issueId];
+        delete this.runtime.retries[issueId];
+        this.store.removeRetry(issueId);
+
+        const task = this.store.getTask(issue.identifier);
+        if (task) {
+          this.store.updateTask(task.id, { status: 'done' });
+        }
+
+        if (running?.workspace_key) {
+          await this.workspaceManager.cleanup({
+            workspaceKey: running.workspace_key,
+            identifier: issue.identifier,
+            workerHost: running.worker_host || null
+          });
+        }
+      }
+    }
+  }
+
+  refreshWorkflowConfig() {
+    if (!this.workflowRuntime) return;
+
+    const { changed, error } = this.workflowRuntime.refresh();
+
+    if (error && error.message !== this.lastWorkflowReloadError) {
+      this.lastWorkflowReloadError = error.message;
+      this.store.appendSystemLog(`workflow reload failed: ${error.message}`);
+      return;
+    }
+
+    if (!error && this.lastWorkflowReloadError) {
+      this.store.appendSystemLog('workflow reload recovered');
+      this.lastWorkflowReloadError = null;
+    }
+
+    if (changed) {
+      this.runtime.poll_interval_ms = this.config.pollIntervalMs;
+      this.runtime.max_concurrent_agents = this.config.maxConcurrentAgents;
+      this.store.appendSystemLog(`workflow reloaded: ${this.config.workflowPath}`);
+      this.syncTimerInterval();
+    }
+  }
+
+  syncTimerInterval() {
+    if (!this.timer && this.timerIntervalMs === this.config.pollIntervalMs) return;
+    if (this.timer) clearInterval(this.timer);
+    this.timerIntervalMs = this.config.pollIntervalMs;
+    this.timer = setInterval(() => this.tick(), this.timerIntervalMs);
+  }
+
+  workerSlotsAvailable(preferredWorkerHost = null) {
+    return this.selectWorkerHost(preferredWorkerHost) !== NO_WORKER_CAPACITY;
+  }
+
+  selectWorkerHost(preferredWorkerHost = null) {
+    const hosts = this.config.worker?.sshHosts || [];
+    if (!hosts.length) {
+      return null;
+    }
+
+    const availableHosts = hosts.filter((host) => this.workerHostSlotsAvailable(host));
+    if (!availableHosts.length) {
+      return NO_WORKER_CAPACITY;
+    }
+
+    if (preferredWorkerHost && availableHosts.includes(preferredWorkerHost)) {
+      return preferredWorkerHost;
+    }
+
+    return availableHosts
+      .map((host, index) => ({ host, index, used: this.runningWorkerHostCount(host) }))
+      .sort((left, right) => left.used - right.used || left.index - right.index)[0]
+      .host;
+  }
+
+  workerHostSlotsAvailable(workerHost) {
+    const limit = this.config.worker?.maxConcurrentAgentsPerHost;
+    if (!limit) {
+      return true;
+    }
+    return this.runningWorkerHostCount(workerHost) < limit;
+  }
+
+  runningWorkerHostCount(workerHost) {
+    return Object.values(this.runtime.running).filter((entry) => entry.worker_host === workerHost).length;
+  }
+}
+
+function normalizeRuntime(config, persistedRuntime, retries) {
+  return {
+    poll_interval_ms: config.pollIntervalMs,
+    max_concurrent_agents: config.maxConcurrentAgents,
+    running: { ...(persistedRuntime.running || {}) },
+    claimed: { ...(persistedRuntime.claimed || {}) },
+    retries: retries || {},
+    last_tick_at: persistedRuntime.last_tick_at || null,
+    codex_totals: persistedRuntime.codex_totals || {
+      input_tokens: 0,
+      output_tokens: 0,
+      total_tokens: 0
+    },
+    codex_rate_limits: persistedRuntime.codex_rate_limits || null,
+    metrics: {
+      dispatched: Number(persistedRuntime.metrics?.dispatched || 0),
+      succeeded: Number(persistedRuntime.metrics?.succeeded || 0),
+      failed: Number(persistedRuntime.metrics?.failed || 0),
+      stopped: Number(persistedRuntime.metrics?.stopped || 0),
+      retried: Number(persistedRuntime.metrics?.retried || 0)
+    }
+  };
+}
+
+function renderPrompt(template, issue) {
+  return template
+    .replaceAll('{{ issue.identifier }}', issue.identifier || issue.id)
+    .replaceAll('{{ issue.title }}', issue.title || '')
+    .replaceAll('{{ issue.description }}', issue.description || '');
+}
+
+function writeLogFile(logsRoot, runId, result) {
+  fs.mkdirSync(logsRoot, { recursive: true });
+  const logFile = path.join(logsRoot, `${runId}.log`);
+  const body = [result.output, result.error].filter(Boolean).join('\n').trim();
+  fs.writeFileSync(logFile, body ? `${body}\n` : '', 'utf8');
+  return logFile;
+}
+
+function stateSlotsAvailable(issue, running, limits, fallbackLimit) {
+  const normalizedState = String(issue.state || '').trim().toLowerCase();
+  const limit = Number(limits?.[normalizedState] || fallbackLimit);
+  const used = Object.values(running).filter((entry) => String(entry.issue_state || '').trim().toLowerCase() === normalizedState).length;
+  return limit > used;
+}
+
+function addTokenTotals(currentTotals, telemetry) {
+  return {
+    input_tokens: Number(currentTotals?.input_tokens || 0) + Number(telemetry?.inputTokens || 0),
+    output_tokens: Number(currentTotals?.output_tokens || 0) + Number(telemetry?.outputTokens || 0),
+    total_tokens: Number(currentTotals?.total_tokens || 0) + Number(telemetry?.totalTokens || 0)
+  };
+}
+
+const NO_WORKER_CAPACITY = Symbol('no-worker-capacity');