singleton-pipeline 0.4.0-beta.0

package/packages/cli/src/index.js

@@ -0,0 +1,107 @@
+#!/usr/bin/env node
+import { Command } from 'commander';
+import path from 'node:path';
+import fs from 'node:fs/promises';
+import { style, line } from './theme.js';
+import { scanAgents } from './scanner.js';
+import { runPipeline } from './executor.js';
+import { newAgentCommand } from './commands/new.js';
+import { replCommand } from './commands/repl.js';
+
+const program = new Command();
+
+function groupAgentsByProvider(agents) {
+  return agents.reduce((groups, agent) => {
+    const provider = agent.provider || 'unknown';
+    if (!groups.has(provider)) groups.set(provider, []);
+    groups.get(provider).push(agent);
+    return groups;
+  }, new Map());
+}
+
+program
+  .name('singleton')
+  .description('Singleton Pipeline Builder — scan agents and build pipelines')
+  .version('0.4.0-beta.0');
+
+program
+  .command('scan')
+  .description('Scan a repo for agent .md files')
+  .argument('<path>', 'Path to repo to scan')
+  .option('-o, --output <file>', 'Output JSON file', '.singleton/agents.json')
+  .action(async (repoPath, opts) => {
+    const absPath = path.resolve(repoPath);
+    console.log(style.info(`Scanning ${absPath}...`));
+
+    const agents = await scanAgents(absPath);
+
+    if (agents.length === 0) {
+      console.log(style.warn('No agents found (no .md files with ## Config section).'));
+      return;
+    }
+
+    console.log(style.success(`\nFound ${agents.length} agent(s):\n`));
+    const groups = groupAgentsByProvider(agents);
+    [...groups.entries()].forEach(([provider, providerAgents]) => {
+      console.log(style.muted('  ════════════════════════════════════════'));
+      console.log(style.heading(`  ${provider}`) + style.muted(` (${providerAgents.length})`));
+      console.log(style.muted('  ════════════════════════════════════════'));
+      console.log();
+
+      providerAgents.forEach((a, index) => {
+        console.log(style.id(`    ${a.id}`) + style.muted(` — ${a.description || '(no description)'}`));
+        console.log(style.muted(`      file:    ${path.relative(absPath, a.file)}`));
+        console.log(style.info(`      source:`) + style.muted(`  ${a.source || 'repo'}`));
+        if (a.permission_mode) console.log(style.warn(`      permission:`) + style.muted(` ${a.permission_mode}`));
+        console.log(style.success(`      in:`) + style.muted(`      ${a.inputs.join(', ') || '(none)'}`));
+        console.log(style.id(`      out:`) + style.muted(`     ${a.outputs.join(', ') || '(none)'}`));
+        if (a.tags?.length) console.log(style.muted(`      tags:    ${a.tags.join(', ')}`));
+        if (index < providerAgents.length - 1) console.log(style.muted('    ──────────────────────────────────────'));
+      });
+      console.log();
+    });
+
+    const outPath = path.resolve(absPath, opts.output);
+    await fs.mkdir(path.dirname(outPath), { recursive: true });
+    await fs.writeFile(outPath, JSON.stringify({ scannedAt: new Date().toISOString(), root: absPath, agents }, null, 2));
+    console.log(line.success(`Saved to ${path.relative(process.cwd(), outPath)}`));
+  });
+
+program
+  .command('serve')
+  .description('Start the pipeline builder server + web UI')
+  .option('-p, --port <port>', 'Server port', '4317')
+  .option('-r, --root <path>', 'Project root to scan', process.cwd())
+  .action(async (opts) => {
+    const { startServer } = await import('../../server/src/index.js');
+    await startServer({ port: Number(opts.port), root: path.resolve(opts.root) });
+  });
+
+program
+  .command('new')
+  .description('Create a new agent .md file interactively')
+  .option('-r, --root <path>', 'Project root to scan', process.cwd())
+  .action(async (opts) => {
+    await newAgentCommand(opts);
+  });
+
+program
+  .command('run')
+  .description('Run (dry-run) a pipeline JSON')
+  .requiredOption('--pipeline <file>', 'Pipeline JSON file')
+  .option('--dry-run', 'Skip API calls, just show resolved plan')
+  .option('--verbose', 'Show prompts and outputs in the right panel')
+  .option('--debug', 'Pause before each step for manual review')
+  .action(async (opts) => {
+    await runPipeline(opts.pipeline, { dryRun: opts.dryRun, verbose: opts.verbose, debug: opts.debug });
+  });
+
+program
+  .command('repl', { isDefault: true })
+  .description('Interactive Singleton shell (default)')
+  .option('-r, --root <path>', 'Project root', process.cwd())
+  .action(async (opts) => {
+    await replCommand(opts);
+  });
+
+program.parseAsync(process.argv);

package/packages/cli/src/parser.js

@@ -0,0 +1,78 @@
+const CONFIG_HEADER = /^##\s+Config\s*$/m;
+const PROMPT_HEADER = /^##\s+(Prompt|System|Instructions)\s*$/m;
+const HR = /^---\s*$/m;
+const KV_LINE = /^\s*-\s+\*\*([^*]+)\*\*\s*:\s*(.+?)\s*$/;
+const REQUIRED = ['id', 'description', 'inputs', 'outputs'];
+const LIST_KEYS = new Set(['inputs', 'outputs', 'tags', 'allowed_paths', 'blocked_paths']);
+
+export function parseAgentFileDetailed(content, file) {
+  // Strip YAML frontmatter if present
+  const stripped = content.startsWith('---')
+    ? content.replace(/^---[\s\S]*?---\n?/, '')
+    : content;
+  const configMatch = stripped.match(CONFIG_HEADER);
+  if (!configMatch) {
+    return { agent: null, error: 'missing "## Config" section' };
+  }
+
+  const afterConfig = stripped.slice(configMatch.index + configMatch[0].length);
+
+  // Find end of config block: next ## header, ---, or EOF
+  const nextHeader = afterConfig.match(/^##\s+/m);
+  const hr = afterConfig.match(HR);
+  const ends = [nextHeader?.index, hr?.index].filter((i) => typeof i === 'number');
+  const endIdx = ends.length ? Math.min(...ends) : afterConfig.length;
+
+  const configBlock = afterConfig.slice(0, endIdx);
+  const config = {};
+  for (const line of configBlock.split('\n')) {
+    const m = line.match(KV_LINE);
+    if (!m) continue;
+    const key = m[1].trim();
+    const raw = m[2].trim();
+    config[key] = LIST_KEYS.has(key)
+      ? raw.split(',').map((s) => s.trim()).filter(Boolean)
+      : raw;
+  }
+
+  for (const k of REQUIRED) {
+    if (config[k] === undefined || (LIST_KEYS.has(k) && config[k].length === 0 && k !== 'tags')) {
+      return { agent: null, error: `missing required field: ${k}` };
+    }
+  }
+
+  // Extract prompt body
+  const promptMatch = stripped.match(PROMPT_HEADER);
+  let prompt = '';
+  if (promptMatch) {
+    prompt = stripped.slice(promptMatch.index + promptMatch[0].length).trim();
+  } else {
+    const hrAll = stripped.match(HR);
+    if (hrAll) prompt = stripped.slice(hrAll.index + hrAll[0].length).trim();
+  }
+
+  const agent = {
+    id: config.id,
+    description: config.description || '',
+    inputs: config.inputs || [],
+    outputs: config.outputs || [],
+    tags: config.tags || [],
+    provider: config.provider,
+    model: config.model,
+    runner_agent: config.runner_agent || config.opencode_agent,
+    opencode_agent: config.opencode_agent,
+    permission_mode: config.permission_mode,
+    security_profile: config.security_profile,
+    allowed_paths: config.allowed_paths || [],
+    blocked_paths: config.blocked_paths || [],
+    estimated_tokens: config.estimated_tokens ? Number(config.estimated_tokens) : undefined,
+    file,
+    prompt
+  };
+
+  return { agent, error: null };
+}
+
+export function parseAgentFile(content, file) {
+  return parseAgentFileDetailed(content, file).agent;
+}

package/packages/cli/src/runners/_shared.js

@@ -0,0 +1,83 @@
+// Shared helpers used by multiple runners (opencode, codex, …).
+// Each runner emits JSONL events with slightly different shapes; these helpers
+// normalize the common patterns: parsing lines, walking nested events to find
+// usage/cost, and extracting assistant text from messages with varying schemas.
+
+export function safeJsonParse(line) {
+  try {
+    return JSON.parse(line);
+  } catch {
+    return null;
+  }
+}
+
+export function extractText(value) {
+  if (typeof value === 'string') return value;
+  if (!value || typeof value !== 'object') return '';
+
+  if (typeof value.text === 'string') return value.text;
+  if (typeof value.content === 'string') return value.content;
+  if (typeof value.delta === 'string') return value.delta;
+  if (typeof value.message === 'string') return value.message;
+
+  if (Array.isArray(value.content)) {
+    return value.content
+      .map((item) => extractText(item))
+      .filter(Boolean)
+      .join('\n');
+  }
+
+  if (Array.isArray(value.parts)) {
+    return value.parts
+      .map((item) => extractText(item))
+      .filter(Boolean)
+      .join('\n');
+  }
+
+  if (value.message && typeof value.message === 'object') return extractText(value.message);
+  if (value.part && typeof value.part === 'object') return extractText(value.part);
+  if (value.data && typeof value.data === 'object') return extractText(value.data);
+  if (value.result && typeof value.result === 'object') return extractText(value.result);
+
+  return '';
+}
+
+export function findUsage(value) {
+  if (!value || typeof value !== 'object') return null;
+
+  const input = value.input_tokens
+    ?? value.inputTokens
+    ?? value.tokens?.input
+    ?? value.usage?.input_tokens
+    ?? value.usage?.inputTokens
+    ?? (Object.prototype.hasOwnProperty.call(value, 'input') ? value.input : undefined);
+  const output = value.output_tokens
+    ?? value.outputTokens
+    ?? value.tokens?.output
+    ?? value.usage?.output_tokens
+    ?? value.usage?.outputTokens
+    ?? (Object.prototype.hasOwnProperty.call(value, 'output') ? value.output : undefined);
+  if (input !== undefined || output !== undefined) {
+    return { input: input ?? null, output: output ?? null };
+  }
+
+  for (const nested of Object.values(value)) {
+    const usage = findUsage(nested);
+    if (usage) return usage;
+  }
+
+  return null;
+}
+
+export function findCostUsd(value) {
+  if (!value || typeof value !== 'object') return null;
+  const direct = value.costUsd ?? value.cost_usd ?? value.total_cost_usd ?? value.usage?.costUsd ?? value.usage?.cost_usd;
+  if (direct !== undefined && direct !== null && !Number.isNaN(Number(direct))) return Number(direct);
+
+  for (const nested of Object.values(value)) {
+    const cost = findCostUsd(nested);
+    if (cost !== null) return cost;
+  }
+
+  return null;
+}

package/packages/cli/src/runners/claude.js

@@ -0,0 +1,119 @@
+import { spawn } from 'node:child_process';
+
+const DEFAULT_TIMEOUT_MS = Number(process.env.SINGLETON_RUNNER_TIMEOUT_MS) || 10 * 60 * 1000;
+const ALLOWED_PERMISSION_MODES = new Set(['bypassPermissions']);
+const READ_ONLY_DENY_TOOLS = ['Write', 'Edit', 'Bash', 'NotebookEdit'];
+
+export function buildClaudePermissionArgs(securityPolicy = {}, permissionMode = '') {
+  // Legacy escape hatch: when permission_mode is explicitly set on the agent or
+  // step, honor it as-is and skip the security_policy mapping. This preserves
+  // backward compatibility with pipelines authored before security_policy
+  // support landed.
+  if (permissionMode) {
+    if (!ALLOWED_PERMISSION_MODES.has(permissionMode)) {
+      throw new Error(`unsupported Claude permission_mode: ${permissionMode}`);
+    }
+    return ['--permission-mode', permissionMode];
+  }
+
+  const profile = securityPolicy.profile || 'workspace-write';
+  const args = [];
+
+  if (profile === 'dangerous') {
+    args.push('--permission-mode', 'bypassPermissions');
+    return args;
+  }
+
+  if (profile === 'read-only') {
+    args.push('--disallowedTools', READ_ONLY_DENY_TOOLS.join(','));
+    return args;
+  }
+
+  // restricted-write & workspace-write: Claude Code does not support per-path
+  // tool filtering, so we let it edit and rely on Singleton's post-run snapshot
+  // diff to reject writes outside allowed_paths. acceptEdits avoids interactive
+  // prompts in -p mode.
+  args.push('--permission-mode', 'acceptEdits');
+  return args;
+}
+
+export const claudeRunner = {
+  id: 'claude',
+  command: 'claude',
+
+  async run({
+    cwd,
+    systemPrompt,
+    userPrompt,
+    model,
+    permissionMode = '',
+    securityPolicy,
+    timeoutMs = DEFAULT_TIMEOUT_MS,
+  }) {
+    const args = [
+      '-p',
+      '--output-format',
+      'json',
+      '--system-prompt',
+      systemPrompt,
+      ...buildClaudePermissionArgs(securityPolicy, permissionMode),
+    ];
+
+    if (model) args.push('--model', model);
+
+    const raw = await new Promise((resolve, reject) => {
+      const child = spawn('claude', args, { cwd, stdio: ['pipe', 'pipe', 'pipe'] });
+      let stdout = '';
+      let stderr = '';
+      let timedOut = false;
+
+      const timer = setTimeout(() => {
+        timedOut = true;
+        child.kill('SIGTERM');
+        setTimeout(() => child.kill('SIGKILL'), 5000).unref();
+      }, timeoutMs);
+
+      child.stdout.on('data', (d) => (stdout += d.toString()));
+      child.stderr.on('data', (d) => (stderr += d.toString()));
+      child.on('error', (err) => {
+        clearTimeout(timer);
+        reject(err);
+      });
+      child.on('close', (code) => {
+        clearTimeout(timer);
+        if (timedOut) {
+          reject(new Error(`claude timed out after ${Math.round(timeoutMs / 1000)}s`));
+          return;
+        }
+        if (code !== 0) {
+          reject(new Error(`claude exited ${code}: ${stderr.trim() || stdout.trim()}`));
+          return;
+        }
+
+        try {
+          resolve(JSON.parse(stdout));
+        } catch (err) {
+          reject(new Error(`failed to parse claude output: ${err.message}\n${stdout.slice(0, 500)}`));
+        }
+      });
+
+      child.stdin.write(userPrompt);
+      child.stdin.end();
+    });
+
+    return {
+      text: typeof raw.result === 'string' ? raw.result : JSON.stringify(raw),
+      metadata: {
+        provider: 'claude',
+        model: raw.model ?? model ?? null,
+        turns: Number(raw.num_turns || 0) || null,
+        costUsd: Number(raw.total_cost_usd || 0) || null,
+        tokens: {
+          input: raw.usage?.input_tokens ?? null,
+          output: raw.usage?.output_tokens ?? null,
+        },
+        raw,
+      },
+    };
+  },
+};

package/packages/cli/src/runners/codex-instructions.js

@@ -0,0 +1,75 @@
+import fs from 'node:fs/promises';
+import path from 'node:path';
+
+const PROJECT_DOC_FILENAMES = ['AGENTS.override.md', 'AGENTS.md'];
+const SKIP_DIRS = new Set(['.git', '.singleton', 'node_modules', 'dist', 'build', '.next', '.cache', 'coverage']);
+
+function isSubdir(parent, child) {
+  const rel = path.relative(parent, child);
+  return rel === '' || (!rel.startsWith('..') && !path.isAbsolute(rel));
+}
+
+async function readFirstNonEmptyFile(dir, names) {
+  for (const name of names) {
+    const filePath = path.join(dir, name);
+    try {
+      const content = (await fs.readFile(filePath, 'utf8')).trim();
+      if (!content) continue;
+      return { filePath, content };
+    } catch {
+      // ignore missing files
+    }
+  }
+  return null;
+}
+
+async function collectInstructionFiles(rootDir, rel = '', out = []) {
+  const abs = rel ? path.join(rootDir, rel) : rootDir;
+  const entries = await fs.readdir(abs, { withFileTypes: true });
+
+  const found = await readFirstNonEmptyFile(abs, PROJECT_DOC_FILENAMES);
+  if (found) out.push(found);
+
+  for (const entry of entries) {
+    if (!entry.isDirectory()) continue;
+    if (SKIP_DIRS.has(entry.name)) continue;
+    await collectInstructionFiles(rootDir, rel ? path.join(rel, entry.name) : entry.name, out);
+  }
+
+  return out;
+}
+
+export async function discoverCodexProjectInstructions(projectRoot, currentDir) {
+  const root = projectRoot || currentDir;
+  if (!root) return { text: '', files: [] };
+
+  if (!projectRoot || !currentDir || !isSubdir(projectRoot, currentDir)) {
+    const single = await readFirstNonEmptyFile(currentDir || projectRoot, PROJECT_DOC_FILENAMES);
+    return single
+      ? { text: `<!-- ${path.basename(single.filePath)} -->\n${single.content}`, files: [single.filePath] }
+      : { text: '', files: [] };
+  }
+
+  const discovered = await collectInstructionFiles(root);
+  discovered.sort((a, b) => {
+    const relA = path.relative(root, a.filePath);
+    const relB = path.relative(root, b.filePath);
+    const depthA = relA.split(path.sep).length;
+    const depthB = relB.split(path.sep).length;
+    if (depthA !== depthB) return depthA - depthB;
+    return relA.localeCompare(relB);
+  });
+
+  const chunks = [];
+  const files = [];
+
+  for (const found of discovered) {
+    files.push(found.filePath);
+    chunks.push(`<!-- ${path.relative(root, found.filePath)} -->\n${found.content}`);
+  }
+
+  return {
+    text: chunks.join('\n\n').trim(),
+    files,
+  };
+}

package/packages/cli/src/runners/codex.js

@@ -0,0 +1,162 @@
+import fs from 'node:fs/promises';
+import os from 'node:os';
+import path from 'node:path';
+import { spawn } from 'node:child_process';
+import { discoverCodexProjectInstructions } from './codex-instructions.js';
+import { findUsage, safeJsonParse } from './_shared.js';
+
+const DEFAULT_TIMEOUT_MS = Number(process.env.SINGLETON_RUNNER_TIMEOUT_MS) || 10 * 60 * 1000;
+
+function buildPrompt(systemPrompt, userPrompt, projectInstructions = '') {
+  const parts = ['Follow the system instructions exactly.', ''];
+
+  if (projectInstructions) {
+    parts.push('<codex_project_instructions>');
+    parts.push(projectInstructions);
+    parts.push('</codex_project_instructions>');
+    parts.push('');
+  }
+
+  parts.push('<system>');
+  parts.push(systemPrompt);
+  parts.push('</system>');
+  parts.push('');
+  parts.push('<user>');
+  parts.push(userPrompt);
+  parts.push('</user>');
+  parts.push('');
+
+  return parts.join('\n');
+}
+
+export function buildCodexSandboxArgs(securityPolicy = {}) {
+  const profile = securityPolicy.profile || 'workspace-write';
+
+  // Codex CLI sandbox modes: read-only, workspace-write,
+  // workspace-write-with-network, danger-full-access. Codex has no per-path
+  // filter, so restricted-write maps to workspace-write at the runner level
+  // and Singleton's post-run snapshot diff enforces the allowed_paths.
+  if (profile === 'read-only') return ['--sandbox', 'read-only'];
+  if (profile === 'dangerous') return ['--sandbox', 'danger-full-access'];
+  return ['--sandbox', 'workspace-write'];
+}
+
+export function buildCodexArgs({ prompt, model, outputFile, securityPolicy } = {}) {
+  const args = [
+    'exec',
+    '--json',
+    '--ephemeral',
+    '--skip-git-repo-check',
+    ...buildCodexSandboxArgs(securityPolicy),
+    '--output-last-message',
+    outputFile,
+  ];
+
+  if (model) args.push('--model', model);
+  args.push('-');
+  return args;
+}
+
+export const codexRunner = {
+  id: 'codex',
+  command: 'codex',
+
+  async run({
+    cwd,
+    projectRoot = cwd,
+    currentDir = cwd,
+    systemPrompt,
+    userPrompt,
+    model,
+    securityPolicy,
+    timeoutMs = DEFAULT_TIMEOUT_MS,
+  }) {
+    const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'singleton-codex-'));
+    const outputFile = path.join(tempDir, 'last-message.txt');
+    const projectInstructions = await discoverCodexProjectInstructions(projectRoot, currentDir);
+    const prompt = buildPrompt(systemPrompt, userPrompt, projectInstructions.text);
+
+    const args = buildCodexArgs({ prompt, model, outputFile, securityPolicy });
+
+    const { events, stderr } = await new Promise((resolve, reject) => {
+      const child = spawn('codex', args, {
+        cwd,
+        stdio: ['pipe', 'pipe', 'pipe'],
+        env: {
+          ...process.env,
+          CODEX_HOME: process.env.CODEX_HOME || path.join(os.homedir(), '.codex'),
+        },
+      });
+
+      const stdoutChunks = [];
+      let stderrText = '';
+      let timedOut = false;
+
+      const timer = setTimeout(() => {
+        timedOut = true;
+        child.kill('SIGTERM');
+        setTimeout(() => child.kill('SIGKILL'), 5000).unref();
+      }, timeoutMs);
+
+      child.stdout.on('data', (d) => stdoutChunks.push(d.toString()));
+      child.stderr.on('data', (d) => (stderrText += d.toString()));
+      child.on('error', (err) => {
+        clearTimeout(timer);
+        reject(err);
+      });
+      child.on('close', (code) => {
+        clearTimeout(timer);
+        const stdout = stdoutChunks.join('');
+        const events = stdout
+          .split('\n')
+          .map((line) => line.trim())
+          .filter(Boolean)
+          .map(safeJsonParse)
+          .filter(Boolean);
+
+        if (timedOut) {
+          reject(new Error(`codex timed out after ${Math.round(timeoutMs / 1000)}s`));
+          return;
+        }
+        if (code !== 0) {
+          const eventError = events.findLast?.((event) => event.type === 'error')?.message
+            || [...events].reverse().find((event) => event.type === 'error')?.message;
+          reject(new Error(`codex exited ${code}: ${eventError || stderrText.trim() || 'unknown error'}`));
+          return;
+        }
+
+        resolve({ events, stderr: stderrText });
+      });
+
+      child.stdin.write(prompt);
+      child.stdin.end();
+    });
+
+    let text = '';
+    try {
+      text = (await fs.readFile(outputFile, 'utf8')).trim();
+    } catch {
+      text = '';
+    }
+
+    const turns = events.filter((event) => event.type === 'turn.started').length || null;
+    const usage = [...events]
+      .reverse()
+      .map(findUsage)
+      .find(Boolean) || null;
+
+    try { await fs.rm(tempDir, { recursive: true, force: true }); } catch { /* ignore */ }
+
+    return {
+      text,
+      metadata: {
+        provider: 'codex',
+        model: model || null,
+        turns,
+        costUsd: null,
+        tokens: usage,
+        raw: { events, stderr, projectInstructionFiles: projectInstructions.files },
+      },
+    };
+  },
+};