yaml-flow 3.0.0 → 3.1.1

package/examples/event-graph/ci-cd-pipeline.ts

@@ -0,0 +1,243 @@
+/**
+ * Event Graph Example: CI/CD Pipeline
+ *
+ * Demonstrates:
+ *  - External event injection (human approval gate)
+ *  - Conditional routing via `on`
+ *  - Failure tokens via `on_failure`
+ *  - Retry configuration
+ *  - Stuck detection
+ *
+ * Run with: npx tsx examples/event-graph/ci-cd-pipeline.ts
+ */
+
+import {
+  next,
+  apply,
+  createInitialExecutionState,
+} from '../../src/event-graph/index.js';
+import type { GraphConfig } from '../../src/event-graph/index.js';
+
+// ============================================================================
+// 1. Define the graph
+// ============================================================================
+
+const graph: GraphConfig = {
+  id: 'ci-cd-pipeline',
+  settings: {
+    completion: 'goal-reached',
+    goal: ['deployed'],
+    conflict_strategy: 'alphabetical',
+  },
+  tasks: {
+    build: {
+      provides: ['build-artifact'],
+      description: 'Compile the project',
+    },
+    unit_tests: {
+      requires: ['build-artifact'],
+      provides: ['tests-passed'],
+      retry: { max_attempts: 2 },
+      on_failure: ['tests-failed'],
+      description: 'Run unit tests',
+    },
+    lint: {
+      requires: ['build-artifact'],
+      provides: ['lint-passed'],
+      description: 'Run linter',
+    },
+    security_scan: {
+      requires: ['build-artifact'],
+      provides: ['scan-passed'],
+      on: {
+        clean: ['scan-passed'],
+        vulnerable: ['scan-blocked'],
+      },
+      description: 'Run security scan',
+    },
+    approve: {
+      // Needs all checks AND a human approval token
+      requires: ['tests-passed', 'lint-passed', 'scan-passed', 'human-approval'],
+      provides: ['approved'],
+      description: 'Human approval gate',
+    },
+    deploy: {
+      requires: ['approved'],
+      provides: ['deployed'],
+      retry: { max_attempts: 3 },
+      on_failure: ['deploy-failed'],
+      description: 'Deploy to production',
+    },
+    notify_failure: {
+      // Activates if tests fail, scan is blocked, or deploy fails
+      requires: ['tests-failed'],
+      provides: ['failure-notified'],
+      description: 'Notify team of failure',
+    },
+    notify_blocked: {
+      requires: ['scan-blocked'],
+      provides: ['block-notified'],
+      description: 'Notify team of security block',
+    },
+  },
+};
+
+// ============================================================================
+// 2. Simulated task executor
+// ============================================================================
+
+// Simulate tasks with controlled outcomes
+const taskOutcomes: Record<string, { success: boolean; result?: string }> = {
+  build: { success: true },
+  unit_tests: { success: true },
+  lint: { success: true },
+  security_scan: { success: true, result: 'clean' },
+  approve: { success: true },
+  deploy: { success: true },
+  notify_failure: { success: true },
+  notify_blocked: { success: true },
+};
+
+async function executeTask(taskName: string): Promise<{ success: boolean; result?: string; error?: string }> {
+  await new Promise((r) => setTimeout(r, Math.random() * 100 + 20));
+  const outcome = taskOutcomes[taskName] ?? { success: true };
+
+  if (!outcome.success) {
+    console.log(`  ✗ ${taskName} FAILED`);
+    return { success: false, error: `${taskName} execution error` };
+  }
+
+  console.log(`  ✓ ${taskName} completed${outcome.result ? ` (result: ${outcome.result})` : ''}`);
+  return { success: true, result: outcome.result };
+}
+
+// ============================================================================
+// 3. Driver loop with external event injection
+// ============================================================================
+
+async function main() {
+  let state = createInitialExecutionState(graph, 'pipeline-42');
+  let iteration = 0;
+  let approvalInjected = false;
+
+  console.log('CI/CD Pipeline — Event Graph Demo');
+  console.log('==================================\n');
+
+  while (iteration < 20) {
+    iteration++;
+
+    // Simulate human approval after all automated checks pass
+    if (
+      !approvalInjected &&
+      state.availableOutputs.includes('tests-passed') &&
+      state.availableOutputs.includes('lint-passed') &&
+      state.availableOutputs.includes('scan-passed')
+    ) {
+      console.log('\n  🔔 All checks passed — simulating human approval...');
+      state = apply(
+        state,
+        { type: 'inject-tokens', tokens: ['human-approval'], timestamp: new Date().toISOString() },
+        graph
+      );
+      approvalInjected = true;
+    }
+
+    const schedule = next(graph, state);
+
+    console.log(`\n[iteration ${iteration}] eligible: [${schedule.eligibleTasks.join(', ')}]`);
+
+    if (schedule.isComplete) {
+      console.log('\n✅ Pipeline complete! Deployment successful.');
+      console.log('Final outputs:', state.availableOutputs);
+      break;
+    }
+
+    if (schedule.stuckDetection.is_stuck) {
+      console.error('\n❌ Pipeline stuck:', schedule.stuckDetection.stuck_description);
+      console.log('Blocked tasks:', schedule.stuckDetection.tasks_blocked);
+      break;
+    }
+
+    if (schedule.eligibleTasks.length === 0) {
+      console.log('   (no eligible tasks — waiting for events)');
+      break;
+    }
+
+    // Start + execute eligible tasks
+    const ts = new Date().toISOString();
+    for (const taskName of schedule.eligibleTasks) {
+      state = apply(state, { type: 'task-started', taskName, timestamp: ts }, graph);
+    }
+
+    const results = await Promise.all(schedule.eligibleTasks.map(executeTask));
+
+    for (let i = 0; i < results.length; i++) {
+      const taskName = schedule.eligibleTasks[i];
+      const r = results[i];
+      const ts2 = new Date().toISOString();
+
+      if (r.success) {
+        state = apply(state, { type: 'task-completed', taskName, result: r.result, timestamp: ts2 }, graph);
+      } else {
+        state = apply(state, { type: 'task-failed', taskName, error: r.error!, timestamp: ts2 }, graph);
+      }
+    }
+
+    console.log(`   outputs: [${state.availableOutputs.join(', ')}]`);
+  }
+
+  // ------------------------------------------------------------------
+  // Now demonstrate a failure scenario
+  // ------------------------------------------------------------------
+  console.log('\n\n========== Failure Scenario ==========\n');
+
+  // Override: make security scan find vulnerabilities
+  taskOutcomes.security_scan = { success: true, result: 'vulnerable' };
+
+  state = createInitialExecutionState(graph, 'pipeline-43');
+  iteration = 0;
+  approvalInjected = false;
+
+  while (iteration < 20) {
+    iteration++;
+    const schedule = next(graph, state);
+
+    console.log(`\n[iteration ${iteration}] eligible: [${schedule.eligibleTasks.join(', ')}]`);
+
+    if (schedule.isComplete) {
+      console.log('\n✅ Complete.');
+      break;
+    }
+
+    if (schedule.stuckDetection.is_stuck) {
+      console.log('\n⚠️  Pipeline blocked (as expected with vulnerability).');
+      console.log('Reason:', schedule.stuckDetection.stuck_description);
+      break;
+    }
+
+    if (schedule.eligibleTasks.length === 0) break;
+
+    const ts = new Date().toISOString();
+    for (const taskName of schedule.eligibleTasks) {
+      state = apply(state, { type: 'task-started', taskName, timestamp: ts }, graph);
+    }
+
+    const results = await Promise.all(schedule.eligibleTasks.map(executeTask));
+
+    for (let i = 0; i < results.length; i++) {
+      const taskName = schedule.eligibleTasks[i];
+      const r = results[i];
+      const ts2 = new Date().toISOString();
+
+      if (r.success) {
+        state = apply(state, { type: 'task-completed', taskName, result: r.result, timestamp: ts2 }, graph);
+      } else {
+        state = apply(state, { type: 'task-failed', taskName, error: r.error!, timestamp: ts2 }, graph);
+      }
+    }
+
+    console.log(`   outputs: [${state.availableOutputs.join(', ')}]`);
+  }
+}
+
+main().catch(console.error);

package/examples/event-graph/executor-diamond.ts

@@ -0,0 +1,165 @@
+/**
+ * Event Graph Example: Executor-driven Diamond DAG (Library Mode)
+ *
+ * A diamond-shaped graph with parallel fan-out and fan-in.
+ * Tasks simulate real-world async work with random delays.
+ *
+ *        fetch
+ *       /     \
+ *    parse   validate
+ *       \     /
+ *       combine
+ *          |
+ *        report
+ *
+ * Run with: npx tsx examples/event-graph/executor-diamond.ts
+ */
+
+import {
+  next, apply, createInitialExecutionState,
+} from '../../src/event-graph/index.js';
+import type { GraphConfig, ExecutionState } from '../../src/event-graph/types.js';
+
+// ============================================================================
+// 1. Define the diamond graph
+// ============================================================================
+
+const graph: GraphConfig = {
+  id: 'diamond-dag',
+  settings: {
+    completion: 'all-tasks-done',
+    execution_mode: 'eligibility-mode',
+    conflict_strategy: 'parallel-all',
+  },
+  tasks: {
+    fetch: {
+      provides: ['raw-payload'],
+    },
+    parse: {
+      requires: ['raw-payload'],
+      provides: ['parsed-records'],
+    },
+    validate: {
+      requires: ['raw-payload'],
+      provides: ['validation-report'],
+    },
+    combine: {
+      requires: ['parsed-records', 'validation-report'],
+      provides: ['final-dataset'],
+    },
+    report: {
+      requires: ['final-dataset'],
+      provides: ['report-sent'],
+    },
+  },
+};
+
+// ============================================================================
+// 2. Simulate async executors with random delays
+// ============================================================================
+
+const taskSimulations: Record<string, () => Promise<{ ok: boolean; detail: string }>> = {
+  fetch: async () => {
+    await sleep(randomMs(100, 300));
+    return { ok: true, detail: 'fetched 5MB payload' };
+  },
+  parse: async () => {
+    await sleep(randomMs(150, 400));
+    return { ok: true, detail: 'parsed 12,000 records' };
+  },
+  validate: async () => {
+    await sleep(randomMs(200, 500));
+    const passRate = 95 + Math.floor(Math.random() * 5);
+    return { ok: true, detail: `${passRate}% pass rate` };
+  },
+  combine: async () => {
+    await sleep(randomMs(100, 250));
+    return { ok: true, detail: 'merged parsed + validated' };
+  },
+  report: async () => {
+    await sleep(randomMs(50, 150));
+    return { ok: true, detail: 'emailed to stakeholders' };
+  },
+};
+
+// ============================================================================
+// 3. Execution loop — you drive, engine decides
+// ============================================================================
+
+async function run() {
+  console.log('=== Executor-driven Diamond DAG ===\n');
+  console.log('  Graph shape: fetch → [parse, validate] → combine → report\n');
+
+  let state: ExecutionState = createInitialExecutionState(graph, 'diamond-1');
+  let iteration = 0;
+  const startTime = Date.now();
+
+  while (iteration < 20) {
+    iteration++;
+    const result = next(graph, state);
+
+    if (result.isComplete) {
+      const elapsed = Date.now() - startTime;
+      console.log(`\n✅ All tasks complete in ${elapsed}ms (${iteration} iterations)\n`);
+      break;
+    }
+
+    if (result.eligibleTasks.length === 0) {
+      if (result.stuckDetection.is_stuck) {
+        console.log(`⚠️  Stuck: ${result.stuckDetection.stuck_description}`);
+        break;
+      }
+      continue;
+    }
+
+    console.log(`[iteration ${iteration}] dispatching: [${result.eligibleTasks.join(', ')}]`);
+
+    // Start all eligible
+    const ts = new Date().toISOString();
+    for (const taskName of result.eligibleTasks) {
+      state = apply(state, { type: 'task-started', taskName, timestamp: ts }, graph);
+    }
+
+    // Execute in parallel — this is where your real handlers would go
+    const execResults = await Promise.all(
+      result.eligibleTasks.map(async (taskName) => {
+        const sim = taskSimulations[taskName];
+        const r = await sim();
+        console.log(`  [${taskName}] ${r.ok ? '✓' : '✗'} ${r.detail}`);
+        return { taskName, ...r };
+      }),
+    );
+
+    // Feed results back
+    for (const r of execResults) {
+      const ts2 = new Date().toISOString();
+      if (r.ok) {
+        state = apply(state, { type: 'task-completed', taskName: r.taskName, timestamp: ts2 }, graph);
+      } else {
+        state = apply(state, { type: 'task-failed', taskName: r.taskName, error: 'failed', timestamp: ts2 }, graph);
+      }
+    }
+
+    console.log(`  → outputs: [${state.availableOutputs.join(', ')}]`);
+  }
+
+  // Summary
+  console.log('=== Execution Summary ===');
+  for (const [name, task] of Object.entries(state.tasks)) {
+    console.log(`  ${name}: ${task.status}`);
+  }
+}
+
+run();
+
+// ============================================================================
+// Util
+// ============================================================================
+
+function sleep(ms: number): Promise<void> {
+  return new Promise(resolve => setTimeout(resolve, ms));
+}
+
+function randomMs(min: number, max: number): number {
+  return min + Math.floor(Math.random() * (max - min));
+}

package/examples/event-graph/executor-pipeline.ts

@@ -0,0 +1,161 @@
+/**
+ * Event Graph Example: Executor-driven Pipeline (Library Mode)
+ *
+ * Same ETL pipeline as reactive-pipeline.ts, but YOU drive the loop.
+ * Each task simulates async work with random sleep.
+ *
+ * Demonstrates:
+ *  - Manual executor loop with next/apply
+ *  - validateGraph for static config validation before running
+ *  - validateLiveGraph for runtime state-consistency after running
+ *
+ * Contrast with reactive-pipeline.ts where the graph drives itself.
+ *
+ * Run with: npx tsx examples/event-graph/executor-pipeline.ts
+ */
+
+import {
+  next, apply, createInitialExecutionState, validateGraph,
+} from '../../src/event-graph/index.js';
+import type { GraphConfig, ExecutionState } from '../../src/event-graph/types.js';
+import { validateLiveGraph } from '../../src/continuous-event-graph/index.js';
+
+// ============================================================================
+// 1. Define the graph (same as reactive-pipeline)
+// ============================================================================
+
+const graph: GraphConfig = {
+  id: 'etl-pipeline',
+  settings: {
+    completion: 'all-tasks-done',
+    execution_mode: 'eligibility-mode',
+    conflict_strategy: 'parallel-all',
+  },
+  tasks: {
+    extract: {
+      provides: ['raw-data'],
+    },
+    validate: {
+      requires: ['raw-data'],
+      provides: ['valid-data'],
+    },
+    enrich: {
+      requires: ['valid-data'],
+      provides: ['enriched-data'],
+    },
+    load: {
+      requires: ['enriched-data'],
+      provides: ['loaded'],
+    },
+  },
+};
+
+// ============================================================================
+// 2. Task handlers — simulate async work with random sleep
+// ============================================================================
+
+async function executeTask(taskName: string): Promise<{ ok: boolean; error?: string }> {
+  const delay = 100 + Math.floor(Math.random() * 400); // 100–500ms
+  console.log(`  [${taskName}] executing... (${delay}ms)`);
+  await sleep(delay);
+
+  // Simulate occasional failure (10% chance)
+  if (Math.random() < 0.1) {
+    console.log(`  [${taskName}] FAILED`);
+    return { ok: false, error: `${taskName} timed out` };
+  }
+
+  console.log(`  [${taskName}] done`);
+  return { ok: true };
+}
+
+// ============================================================================
+// 3. YOU drive the execution loop
+// ============================================================================
+
+async function run() {
+  console.log('=== Executor-driven ETL Pipeline (Library Mode) ===\n');
+
+  // Pre-flight: validate static config before running
+  const configValidation = validateGraph(graph);
+  console.log(`Config validation: ${configValidation.valid ? '✅ valid' : '❌ invalid'} (${configValidation.issues.length} issues)`);
+  for (const issue of configValidation.issues) {
+    console.log(`  [${issue.severity}] ${issue.code}: ${issue.message}`);
+  }
+  if (!configValidation.valid) return;
+
+  let state: ExecutionState = createInitialExecutionState(graph, 'exec-1');
+  let iteration = 0;
+
+  while (iteration < 20) {
+    iteration++;
+    const result = next(graph, state);
+
+    console.log(`\n[iteration ${iteration}] eligible: [${result.eligibleTasks.join(', ')}]`);
+
+    if (result.isComplete) {
+      console.log('\n✅ Pipeline complete!');
+      break;
+    }
+
+    if (result.stuckDetection.is_stuck) {
+      console.log(`\n⚠️  Pipeline stuck: ${result.stuckDetection.stuck_description}`);
+      break;
+    }
+
+    if (result.eligibleTasks.length === 0) {
+      console.log('   (waiting for running tasks...)');
+      break;
+    }
+
+    // Mark all eligible tasks as started
+    const ts = new Date().toISOString();
+    for (const taskName of result.eligibleTasks) {
+      state = apply(state, { type: 'task-started', taskName, timestamp: ts }, graph);
+    }
+
+    // Execute in parallel — simulate real async work
+    const results = await Promise.all(
+      result.eligibleTasks.map(async (taskName) => {
+        const r = await executeTask(taskName);
+        return { taskName, ...r };
+      }),
+    );
+
+    // Feed results back into the reducer
+    for (const r of results) {
+      const ts2 = new Date().toISOString();
+      if (r.ok) {
+        state = apply(state, { type: 'task-completed', taskName: r.taskName, timestamp: ts2 }, graph);
+      } else {
+        state = apply(state, { type: 'task-failed', taskName: r.taskName, error: r.error!, timestamp: ts2 }, graph);
+      }
+    }
+
+    console.log(`   outputs: [${state.availableOutputs.join(', ')}]`);
+  }
+
+  // Final state
+  console.log('\n=== Final State ===');
+  for (const [name, task] of Object.entries(state.tasks)) {
+    console.log(`  ${name}: ${task.status} (${task.executionCount}x)`);
+  }
+
+  // Post-run: validate runtime state consistency
+  console.log('\n=== Runtime Validation ===');
+  const runtimeValidation = validateLiveGraph({ config: graph, state });
+  console.log(`  Valid: ${runtimeValidation.valid} (${runtimeValidation.issues.length} issues)`);
+  for (const issue of runtimeValidation.issues) {
+    console.log(`    [${issue.severity}] ${issue.code}: ${issue.message}`);
+  }
+}
+
+run();
+
+// ============================================================================
+// Util
+// ============================================================================
+
+function sleep(ms: number): Promise<void> {
+  return new Promise(resolve => setTimeout(resolve, ms));
+}

package/examples/event-graph/research-pipeline.ts

@@ -0,0 +1,137 @@
+/**
+ * Event Graph Example: Research Pipeline
+ *
+ * Demonstrates:
+ *  - Parallel task execution (fan-out / fan-in)
+ *  - Goal-based completion
+ *  - The standard next() / apply() driver loop
+ *
+ * Run with: npx tsx examples/event-graph/research-pipeline.ts
+ */
+
+import {
+  next,
+  apply,
+  createInitialExecutionState,
+} from '../../src/event-graph/index.js';
+import type { GraphConfig } from '../../src/event-graph/index.js';
+
+// ============================================================================
+// 1. Define the graph
+// ============================================================================
+
+const graph: GraphConfig = {
+  id: 'research-pipeline',
+  settings: {
+    completion: 'goal-reached',
+    goal: ['final-report'],
+    conflict_strategy: 'parallel-all',
+  },
+  tasks: {
+    fetch_sources: {
+      provides: ['raw-sources'],
+      description: 'Fetch source documents from the web',
+    },
+    analyse_sentiment: {
+      requires: ['raw-sources'],
+      provides: ['sentiment-result'],
+      description: 'Run sentiment analysis on sources',
+    },
+    analyse_entities: {
+      requires: ['raw-sources'],
+      provides: ['entity-result'],
+      description: 'Extract named entities from sources',
+    },
+    merge_analysis: {
+      requires: ['sentiment-result', 'entity-result'],
+      provides: ['merged-analysis'],
+      description: 'Merge both analysis results',
+    },
+    generate_report: {
+      requires: ['merged-analysis'],
+      provides: ['final-report'],
+      description: 'Generate a final report',
+    },
+  },
+};
+
+// ============================================================================
+// 2. Simulated task executor
+// ============================================================================
+
+async function executeTask(taskName: string): Promise<string | undefined> {
+  // Simulate varying work durations
+  const delay = Math.random() * 200 + 50;
+  await new Promise((r) => setTimeout(r, delay));
+
+  console.log(`  ✓ ${taskName} completed (${delay.toFixed(0)}ms)`);
+  return undefined; // no special result key → default provides
+}
+
+// ============================================================================
+// 3. Driver loop
+// ============================================================================
+
+async function main() {
+  let state = createInitialExecutionState(graph, 'exec-1');
+  let iteration = 0;
+
+  console.log('Research Pipeline — Event Graph Demo');
+  console.log('====================================\n');
+
+  while (true) {
+    iteration++;
+    const schedule = next(graph, state);
+
+    console.log(`[iteration ${iteration}] eligible: [${schedule.eligibleTasks.join(', ')}]`);
+
+    if (schedule.isComplete) {
+      console.log('\n✅ Pipeline complete!');
+      console.log('Available outputs:', state.availableOutputs);
+      break;
+    }
+
+    if (schedule.stuckDetection.is_stuck) {
+      console.error('\n❌ Pipeline stuck:', schedule.stuckDetection.stuck_description);
+      break;
+    }
+
+    if (schedule.eligibleTasks.length === 0) {
+      console.log('   (waiting for running tasks to complete)');
+      // In a real system you'd await running task results here.
+      break;
+    }
+
+    // Mark all eligible tasks as started
+    const ts = new Date().toISOString();
+    for (const taskName of schedule.eligibleTasks) {
+      state = apply(state, { type: 'task-started', taskName, timestamp: ts }, graph);
+    }
+
+    // Execute in parallel
+    const results = await Promise.all(
+      schedule.eligibleTasks.map(async (taskName) => {
+        try {
+          const result = await executeTask(taskName);
+          return { taskName, ok: true, result };
+        } catch (err: unknown) {
+          return { taskName, ok: false, error: (err as Error).message };
+        }
+      })
+    );
+
+    // Feed results back into the reducer
+    for (const r of results) {
+      const ts = new Date().toISOString();
+      if (r.ok) {
+        state = apply(state, { type: 'task-completed', taskName: r.taskName, result: r.result, timestamp: ts }, graph);
+      } else {
+        state = apply(state, { type: 'task-failed', taskName: r.taskName, error: r.error!, timestamp: ts }, graph);
+      }
+    }
+
+    console.log(`   outputs so far: [${state.availableOutputs.join(', ')}]\n`);
+  }
+}
+
+main().catch(console.error);