takos-runtime-service 1.0.0

package/src/runtime/actions/job-manager.ts

@@ -0,0 +1,213 @@
+import * as fs from 'fs/promises';
+import { createLogger } from 'takos-common/logger';
+import { pushLog } from '../logging.js';
+import { type SecretsSanitizer } from './secrets.js';
+import { SANDBOX_LIMITS } from '../../shared/config.js';
+
+const logger = createLogger({ service: 'takos-runtime', defaultFields: { module: 'actions' } });
+
+// ---------------------------------------------------------------------------
+// Types
+// ---------------------------------------------------------------------------
+
+export interface StepDefinition {
+  name?: string;
+  run?: string;
+  uses?: string;
+  with?: Record<string, unknown>;
+  env?: Record<string, string>;
+  if?: string;
+  'continue-on-error'?: boolean;
+  'timeout-minutes'?: number;
+}
+
+export interface ActiveJob {
+  id: string;
+  spaceId: string;
+  repoId: string;
+  ref: string;
+  sha: string;
+  workflowPath: string;
+  jobName: string;
+  workspacePath: string;
+  status: 'running' | 'completed' | 'failed';
+  steps: StepDefinition[];
+  env: Record<string, string>;
+  secrets: Record<string, string>;
+  secretsSanitizer: SecretsSanitizer;
+  logs: string[];
+  currentStep: number;
+  startedAt: number;
+  completedAt?: number;
+  conclusion?: 'success' | 'failure' | 'cancelled';
+  outputs: Record<string, string>;
+}
+
+// ---------------------------------------------------------------------------
+// Constants
+// ---------------------------------------------------------------------------
+
+const JOB_CLEANUP_INTERVAL_MS = 60 * 60 * 1000; // 1 hour (reduced from 24h to catch stale jobs sooner)
+const MAX_JOB_AGE_MS = 7 * 24 * 60 * 60 * 1000;
+const COMPLETED_JOB_RETENTION_MS = 5 * 60 * 1000;
+
+// ---------------------------------------------------------------------------
+// Job Manager
+// ---------------------------------------------------------------------------
+
+export class JobManager {
+  readonly jobs = new Map<string, ActiveJob>();
+  private cleanupInterval: ReturnType<typeof setInterval> | null = null;
+  private cleanupRunning = false;
+
+  countRunningJobsForSpace(spaceId: string): number {
+    return Array.from(this.jobs.values())
+      .filter(job => job.status === 'running' && job.spaceId === spaceId)
+      .length;
+  }
+
+  countRunningJobsGlobal(): number {
+    return Array.from(this.jobs.values())
+      .filter(job => job.status === 'running')
+      .length;
+  }
+
+  /**
+   * Check whether a new job can be started.
+   * Enforces both per-space and global concurrency limits.
+   */
+  canStartJob(spaceId: string): { allowed: boolean; reason?: string } {
+    const globalRunning = this.countRunningJobsGlobal();
+    if (globalRunning >= SANDBOX_LIMITS.maxConcurrentJobs) {
+      return { allowed: false, reason: `Global concurrent job limit reached (${globalRunning}/${SANDBOX_LIMITS.maxConcurrentJobs})` };
+    }
+    const wsRunning = this.countRunningJobsForSpace(spaceId);
+    if (wsRunning >= SANDBOX_LIMITS.maxConcurrentJobs) {
+      return { allowed: false, reason: `Space concurrent job limit reached (${wsRunning}/${SANDBOX_LIMITS.maxConcurrentJobs})` };
+    }
+    return { allowed: true };
+  }
+
+  /** Delete a job from the active map after clearing its sanitizer and cleaning up job directory. */
+  async purgeJob(jobId: string): Promise<void> {
+    const job = this.jobs.get(jobId);
+    if (!job) return;
+    // Delete from map first to prevent concurrent access during cleanup
+    this.jobs.delete(jobId);
+    job.secretsSanitizer.clear();
+    // Ensure job directory is cleaned up (best-effort, may already be removed)
+    await removeJobDirSafe(job.workspacePath, jobId, 'purged job');
+  }
+
+  /** Schedule removal of a completed job after the retention window. */
+  scheduleJobCleanup(jobId: string): void {
+    const tryPurge = (): void => {
+      const job = this.jobs.get(jobId);
+      if (!job) return;
+      if (job.status === 'running') {
+        // Still running -- retry after another retention window
+        setTimeout(tryPurge, COMPLETED_JOB_RETENTION_MS);
+        return;
+      }
+      void this.purgeJob(jobId);
+    };
+    setTimeout(tryPurge, COMPLETED_JOB_RETENTION_MS);
+  }
+
+  /** Mark a job failed, cleanup job directory, and schedule retention cleanup. */
+  async failCloseJob(
+    jobId: string,
+    job: ActiveJob,
+    reason: string,
+  ): Promise<void> {
+    job.status = 'failed';
+    job.conclusion = 'failure';
+    job.completedAt = Date.now();
+    pushLog(job.logs, reason, job.secretsSanitizer);
+    await removeJobDirSafe(job.workspacePath, jobId, 'failed job');
+    this.scheduleJobCleanup(jobId);
+  }
+
+  // -- Lifecycle-managed cleanup of stale jobs --------------------------------
+
+  startCleanup(): void {
+    if (this.cleanupInterval) return;
+
+    this.cleanupInterval = setInterval(() => {
+      if (this.cleanupRunning) return;
+      this.cleanupRunning = true;
+
+      void this.cleanupStaleJobs()
+        .catch((err) => {
+          logger.error('Error in periodic job cleanup', { error: err });
+        })
+        .finally(() => {
+          this.cleanupRunning = false;
+        });
+    }, JOB_CLEANUP_INTERVAL_MS);
+  }
+
+  stopCleanup(): void {
+    if (!this.cleanupInterval) return;
+    clearInterval(this.cleanupInterval);
+    this.cleanupInterval = null;
+  }
+
+  // -- Private helpers -------------------------------------------------------
+
+  private async cleanupStaleJobs(): Promise<void> {
+    const now = Date.now();
+    for (const [jobId, job] of this.jobs.entries()) {
+      const age = now - job.startedAt;
+
+      // Clean up jobs that exceeded maximum age regardless of status
+      if (age > MAX_JOB_AGE_MS) {
+        logger.info('Cleaning up stale job (max age exceeded)', { jobId, status: job.status });
+        job.secretsSanitizer.clear();
+        await removeJobDirSafe(job.workspacePath, jobId, 'stale job');
+        this.jobs.delete(jobId);
+        continue;
+      }
+
+      // Detect and fail running jobs that exceeded the maximum job duration
+      if (job.status === 'running' && age > SANDBOX_LIMITS.maxJobDuration) {
+        logger.warn('Failing job that exceeded max duration', { jobId, durationMs: age });
+        await this.failCloseJob(jobId, job, `Job exceeded maximum duration of ${SANDBOX_LIMITS.maxJobDuration}ms`);
+      }
+    }
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Standalone helpers (not dependent on instance state)
+// ---------------------------------------------------------------------------
+
+/** Remove a job's working directory, logging failures instead of throwing. */
+export async function removeJobDirSafe(
+  workspacePath: string,
+  jobId: string,
+  context: string,
+): Promise<void> {
+  try {
+    await fs.rm(workspacePath, { recursive: true, force: true });
+  } catch (rmErr) {
+    logger.error(`Failed to remove ${context} job directory`, { jobId, error: rmErr });
+  }
+}
+
+
+/** Sanitize a key-value map through the job's secrets sanitizer. */
+export function sanitizeOutputs(
+  outputs: Record<string, string>,
+  sanitizer: SecretsSanitizer,
+): Record<string, string> {
+  return Object.fromEntries(
+    Object.entries(outputs).map(([k, v]) => [k, sanitizer.sanitize(v)])
+  );
+}
+
+// ---------------------------------------------------------------------------
+// Default singleton for backwards compatibility
+// ---------------------------------------------------------------------------
+
+export const jobManager = new JobManager();

package/src/runtime/actions/process-spawner.ts

@@ -0,0 +1,275 @@
+import { spawn, type ChildProcess } from 'child_process';
+import * as fs from 'fs/promises';
+import * as path from 'path';
+import { randomUUID } from 'crypto';
+import { pushLog } from '../logging.js';
+import { SANDBOX_LIMITS } from '../../shared/config.js';
+import { getErrorMessage } from 'takos-common/errors';
+import { gracefulKill } from '../../utils/process-kill.js';
+import type { ExecutorStepResult } from './executor.js';
+
+// ---------------------------------------------------------------------------
+// Types
+// ---------------------------------------------------------------------------
+
+interface CommandFiles {
+  output: string;
+  env: string;
+  path: string;
+  summary: string;
+}
+
+interface PreparedCommandFiles {
+  envVars: Record<string, string>;
+  files: CommandFiles;
+}
+
+interface SpawnOptions {
+  timeout: number;
+  cwd: string;
+  shell?: boolean;
+}
+
+interface SpawnContext {
+  env: Record<string, string>;
+  logs: string[];
+  outputs: Record<string, string>;
+  workspacePath: string;
+  parseWorkflowCommands: (text: string) => void;
+  parseKeyValueFile: (content: string) => Record<string, string>;
+  parsePathFile: (content: string) => string[];
+}
+
+// ---------------------------------------------------------------------------
+// Result builders
+// ---------------------------------------------------------------------------
+
+export function failureResult(
+  stderr: string,
+  outputs: Record<string, string> = {},
+  exitCode: number = 1
+): ExecutorStepResult {
+  return { exitCode, stdout: '', stderr, outputs, conclusion: 'failure' };
+}
+
+export function successResult(stdout: string, outputs: Record<string, string>): ExecutorStepResult {
+  return { exitCode: 0, stdout, stderr: '', outputs, conclusion: 'success' };
+}
+
+// ---------------------------------------------------------------------------
+// Command file management
+// ---------------------------------------------------------------------------
+
+export async function prepareCommandFiles(workspacePath: string): Promise<PreparedCommandFiles> {
+  const commandDir = path.join(workspacePath, '.runner', 'commands');
+  await fs.mkdir(commandDir, { recursive: true });
+
+  const id = randomUUID();
+  const files: CommandFiles = {
+    output: path.join(commandDir, `output-${id}.txt`),
+    env: path.join(commandDir, `env-${id}.txt`),
+    path: path.join(commandDir, `path-${id}.txt`),
+    summary: path.join(commandDir, `summary-${id}.md`),
+  };
+
+  await Promise.all(Object.values(files).map(f => fs.writeFile(f, '')));
+
+  return {
+    envVars: {
+      GITHUB_OUTPUT: files.output,
+      GITHUB_ENV: files.env,
+      GITHUB_PATH: files.path,
+      GITHUB_STEP_SUMMARY: files.summary,
+    },
+    files,
+  };
+}
+
+export async function applyCommandFiles(
+  prepared: PreparedCommandFiles,
+  ctx: SpawnContext
+): Promise<void> {
+  const readFile = (filePath: string): Promise<string> =>
+    fs.readFile(filePath, 'utf-8').catch(() => '');
+
+  const [outputContent, envContent, pathContent] = await Promise.all([
+    readFile(prepared.files.output),
+    readFile(prepared.files.env),
+    readFile(prepared.files.path),
+  ]);
+
+  if (outputContent) Object.assign(ctx.outputs, ctx.parseKeyValueFile(outputContent));
+  if (envContent) Object.assign(ctx.env, ctx.parseKeyValueFile(envContent));
+  if (pathContent) {
+    for (const entry of ctx.parsePathFile(pathContent)) {
+      ctx.env.PATH = entry + path.delimiter + (ctx.env.PATH || '');
+    }
+  }
+}
+
+export async function cleanupCommandFiles(prepared: PreparedCommandFiles): Promise<void> {
+  await Promise.all(
+    Object.values(prepared.files).map(f => fs.rm(f, { force: true }))
+  );
+}
+
+// ---------------------------------------------------------------------------
+// Runtime env construction
+// ---------------------------------------------------------------------------
+
+export function createRuntimeEnv(
+  env: Record<string, string>,
+  workspacePath: string,
+  commandFileEnv: Record<string, string>
+): Record<string, string> {
+  const runnerBase = path.join(workspacePath, '.runner');
+  const runtimeEnv: Record<string, string> = {
+    ...env,
+    ...commandFileEnv,
+    RUNNER_TEMP: path.join(runnerBase, 'temp'),
+    RUNNER_TOOL_CACHE: path.join(runnerBase, 'tool-cache'),
+  };
+
+  for (const key of ['PATH', 'HOME'] as const) {
+    if (!runtimeEnv[key] && process.env[key]) {
+      runtimeEnv[key] = process.env[key]!;
+    }
+  }
+
+  const existingNodeOpts = runtimeEnv.NODE_OPTIONS || '';
+  if (!existingNodeOpts.includes('--max-old-space-size')) {
+    runtimeEnv.NODE_OPTIONS = `${existingNodeOpts} --max-old-space-size=2048`.trim();
+  }
+
+  return runtimeEnv;
+}
+
+// ---------------------------------------------------------------------------
+// Process spawning
+// ---------------------------------------------------------------------------
+
+function handleStdoutData(
+  data: Buffer,
+  state: { stdout: string },
+  child: ChildProcess,
+  ctx: SpawnContext
+): void {
+  const text = data.toString('utf-8');
+  state.stdout += text;
+  if (state.stdout.length > SANDBOX_LIMITS.maxOutputSize) {
+    pushLog(ctx.logs, '[WARNING] Output size limit exceeded, truncating...');
+    state.stdout = state.stdout.slice(0, SANDBOX_LIMITS.maxOutputSize);
+    child.kill('SIGTERM');
+  }
+  ctx.parseWorkflowCommands(text);
+}
+
+function handleStderrData(
+  data: Buffer,
+  state: { stderr: string }
+): void {
+  const text = data.toString('utf-8');
+  state.stderr += text;
+  if (state.stderr.length > SANDBOX_LIMITS.maxOutputSize) {
+    state.stderr = state.stderr.slice(0, SANDBOX_LIMITS.maxOutputSize);
+  }
+}
+
+export async function spawnWithTimeout(
+  command: string,
+  args: string[],
+  options: SpawnOptions,
+  ctx: SpawnContext
+): Promise<ExecutorStepResult> {
+  ctx.outputs = {};
+  ctx.logs = [];
+  const prepared = await prepareCommandFiles(ctx.workspacePath);
+
+  return new Promise((resolve) => {
+    const state = { stdout: '', stderr: '', isTimedOut: false };
+    let child: ChildProcess;
+
+    try {
+      child = spawn(command, args, {
+        cwd: options.cwd,
+        env: createRuntimeEnv(ctx.env, ctx.workspacePath, prepared.envVars),
+        stdio: ['ignore', 'pipe', 'pipe'],
+        shell: options.shell === true ? true : false,
+      });
+    } catch (err) {
+      void cleanupCommandFiles(prepared);
+      resolve(failureResult(`Failed to spawn process: ${getErrorMessage(err)}`, ctx.outputs));
+      return;
+    }
+
+    let forceKillHandle: NodeJS.Timeout | undefined;
+    const timeoutHandle = setTimeout(() => {
+      state.isTimedOut = true;
+      pushLog(ctx.logs, `[TIMEOUT] Command timed out after ${options.timeout}ms`);
+      forceKillHandle = gracefulKill(child);
+    }, options.timeout);
+
+    function clearTimers(): void {
+      clearTimeout(timeoutHandle);
+      if (forceKillHandle) clearTimeout(forceKillHandle);
+    }
+
+    child.stdout?.on('data', (data: Buffer) => {
+      handleStdoutData(data, state, child, ctx);
+    });
+    child.stdout?.on('error', (err) => {
+      state.stderr += `\nstdout stream error: ${err.message}`;
+    });
+
+    child.stderr?.on('data', (data: Buffer) => {
+      handleStderrData(data, state);
+    });
+    child.stderr?.on('error', (err) => {
+      state.stderr += `\nstderr stream error: ${err.message}`;
+    });
+
+    child.on('error', (err) => {
+      clearTimers();
+      void cleanupCommandFiles(prepared);
+      resolve({
+        exitCode: 1,
+        stdout: state.stdout,
+        stderr: state.stderr + `\nSpawn error: ${err.message}`,
+        outputs: ctx.outputs,
+        conclusion: 'failure',
+      });
+    });
+
+    child.on('close', (code, signal) => {
+      clearTimers();
+      void (async () => {
+        try {
+          await applyCommandFiles(prepared, ctx);
+        } catch (err) {
+          pushLog(ctx.logs, `[WARNING] Failed to parse command files: ${getErrorMessage(err)}`);
+        }
+        await cleanupCommandFiles(prepared);
+
+        if (state.isTimedOut) {
+          resolve({
+            exitCode: 124,
+            stdout: state.stdout,
+            stderr: state.stderr + '\nCommand timed out',
+            outputs: ctx.outputs,
+            conclusion: 'failure',
+          });
+          return;
+        }
+
+        const exitCode = code ?? (signal ? 128 : 1);
+        resolve({
+          exitCode,
+          stdout: state.stdout,
+          stderr: state.stderr,
+          outputs: ctx.outputs,
+          conclusion: exitCode === 0 ? 'success' : 'failure',
+        });
+      })();
+    });
+  });
+}

package/src/runtime/actions/secrets.ts

@@ -0,0 +1,162 @@
+const SECRET_MASK = '***';
+
+/**
+ * Maximum length for a single secret value to prevent ReDoS.
+ * Secrets longer than this are handled via string replacement only (no regex).
+ */
+const MAX_SECRET_REGEX_LENGTH = 4096;
+
+export class SecretsSanitizer {
+  private secretValues: Set<string> = new Set();
+  private secretPatterns: RegExp[] = [];
+  /** Secrets too long for safe regex conversion — handled via string replacement only. */
+  private longSecrets: Set<string> = new Set();
+
+  private addValues(values: Iterable<string>): void {
+    for (const value of values) {
+      if (value.length > 0) {
+        this.secretValues.add(value);
+      }
+    }
+    this.buildPatterns();
+  }
+
+  registerSecrets(secrets: Record<string, string>): void {
+    this.addValues(Object.values(secrets));
+  }
+
+  registerSecretValues(values: string[]): void {
+    this.addValues(values);
+  }
+
+  private buildPatterns(): void {
+    this.secretPatterns = [];
+    this.longSecrets = new Set();
+    for (const secret of this.secretValues) {
+      // Skip regex for long secrets to prevent ReDoS
+      if (secret.length > MAX_SECRET_REGEX_LENGTH) {
+        this.longSecrets.add(secret);
+        continue;
+      }
+      const escaped = secret.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+      try {
+        this.secretPatterns.push(new RegExp(escaped, 'g'));
+      } catch (err) {
+        // Regex construction failed — fall back to string replacement
+        this.longSecrets.add(secret);
+      }
+    }
+  }
+
+  sanitize(text: string): string {
+    if (!text || this.secretValues.size === 0) return text;
+
+    let sanitized = text;
+    for (const pattern of this.secretPatterns) {
+      pattern.lastIndex = 0;
+      sanitized = sanitized.replace(pattern, SECRET_MASK);
+    }
+    // String replacement fallback for long secrets and regex-failed secrets
+    for (const secret of this.longSecrets) {
+      if (sanitized.includes(secret)) {
+        sanitized = sanitized.split(secret).join(SECRET_MASK);
+      }
+    }
+    return sanitized;
+  }
+
+  sanitizeLogs(logs: string[]): string[] {
+    return logs.map(log => this.sanitize(log));
+  }
+
+  clear(): void {
+    this.secretValues.clear();
+    this.secretPatterns = [];
+    this.longSecrets.clear();
+  }
+}
+
+/**
+ * Commands that would directly dump environment variables containing secrets.
+ * These are blocked (not just warned) to prevent secret leakage.
+ */
+const SECRET_EXPOSING_COMMANDS: Array<{ pattern: RegExp; description: string }> = [
+  { pattern: /^\s*env\s*$/, description: 'bare "env" dumps all environment variables' },
+  { pattern: /^\s*printenv\s*$/, description: 'bare "printenv" dumps all environment variables' },
+  { pattern: /^\s*export\s+-p\s*$/, description: '"export -p" dumps all exported variables' },
+];
+
+/**
+ * Detect if a shell command might expose secrets via env/printenv/set.
+ * Returns a description of the risk if detected, or null if safe.
+ */
+export function mightExposeSecrets(command: string): string | null {
+  for (const line of command.split('\n')) {
+    const trimmed = line.trim();
+    if (!trimmed || trimmed.startsWith('#')) continue;
+    for (const { pattern, description } of SECRET_EXPOSING_COMMANDS) {
+      if (pattern.test(trimmed)) {
+        return description;
+      }
+    }
+  }
+  return null;
+}
+
+/**
+ * Check if a command should be blocked because it would expose secrets.
+ * Unlike mightExposeSecrets(), this returns true only for commands that
+ * would definitely dump all environment variables.
+ */
+export function shouldBlockForSecretExposure(command: string): boolean {
+  return mightExposeSecrets(command) !== null;
+}
+
+export function createSecretsSanitizer(
+  secrets: Record<string, string>,
+  extraValues: string[] = []
+): SecretsSanitizer {
+  const sanitizer = new SecretsSanitizer();
+  sanitizer.registerSecrets(secrets);
+  if (extraValues.length > 0) {
+    sanitizer.registerSecretValues(extraValues);
+  }
+  return sanitizer;
+}
+
+// ---------------------------------------------------------------------------
+// --- Sensitive environment detection ---
+// ---------------------------------------------------------------------------
+
+const EXTRA_SENSITIVE_ENV_PATTERNS = [
+  /SECRET/i,
+  /PASSWORD/i,
+  /TOKEN/i,
+  /API_KEY/i,
+  /PRIVATE_KEY/i,
+  /ACCESS_KEY/i,
+  /AUTH/i,
+];
+
+const EXTRA_SENSITIVE_ENV_KEYS = new Set([
+  'TAKOS_TOKEN',
+  'TAKOS_SESSION_ID',
+]);
+
+export function collectSensitiveEnvValues(env?: Record<string, string>): string[] {
+  if (!env) return [];
+  const values: string[] = [];
+
+  for (const [key, value] of Object.entries(env)) {
+    if (!value) continue;
+    if (EXTRA_SENSITIVE_ENV_KEYS.has(key)) {
+      values.push(value);
+      continue;
+    }
+    if (EXTRA_SENSITIVE_ENV_PATTERNS.some((pattern) => pattern.test(key))) {
+      values.push(value);
+    }
+  }
+
+  return values;
+}