yaml-flow 3.0.0 → 3.1.1

package/examples/flows/ai-conversation.yaml

@@ -0,0 +1,116 @@
+# yaml-language-server: $schema=../schema/flow.schema.json
+
+# AI Conversation Flow with Validation Loop
+# Demonstrates retry logic, circuit breakers, and multi-step AI workflows
+
+settings:
+  start_step: generate_response
+  max_total_steps: 50
+  timeout_ms: 60000
+
+steps:
+  generate_response:
+    description: "Generate AI response based on user input"
+    expects_data:
+      - user_message
+      - conversation_history
+    produces_data:
+      - ai_response
+      - confidence_score
+    transitions:
+      success: validate_response
+      failure: error_state
+    retry:
+      max_attempts: 3
+      delay_ms: 1000
+      backoff_multiplier: 2
+
+  validate_response:
+    description: "Validate the AI response meets quality criteria"
+    expects_data:
+      - ai_response
+      - confidence_score
+    produces_data:
+      - validation_result
+      - validation_feedback
+    transitions:
+      valid: check_approval
+      needs_refinement: refine_response
+      failure: error_state
+
+  refine_response:
+    description: "Refine AI response based on validation feedback"
+    expects_data:
+      - ai_response
+      - validation_feedback
+      - user_message
+    produces_data:
+      - ai_response
+      - confidence_score
+    transitions:
+      success: validate_response
+      failure: error_state
+    circuit_breaker:
+      max_iterations: 3
+      on_open: max_refinements
+
+  check_approval:
+    description: "Present response and check for user approval"
+    expects_data:
+      - ai_response
+    produces_data:
+      - user_approved
+      - user_feedback
+    transitions:
+      approved: success_state
+      rejected: incorporate_feedback
+      timeout: timeout_state
+
+  incorporate_feedback:
+    description: "Incorporate user feedback into response"
+    expects_data:
+      - ai_response
+      - user_feedback
+      - user_message
+    produces_data:
+      - ai_response
+      - confidence_score
+    transitions:
+      success: validate_response
+      failure: error_state
+    circuit_breaker:
+      max_iterations: 5
+      on_open: max_feedback_loops
+
+terminal_states:
+  success_state:
+    description: "User approved the AI response"
+    return_intent: "approved"
+    return_artifacts:
+      - ai_response
+      - conversation_history
+
+  error_state:
+    description: "Flow encountered an error"
+    return_intent: "error"
+    return_artifacts: false
+
+  timeout_state:
+    description: "User did not respond in time"
+    return_intent: "timeout"
+    return_artifacts:
+      - ai_response
+
+  max_refinements:
+    description: "Maximum refinement attempts reached"
+    return_intent: "max_refinements"
+    return_artifacts:
+      - ai_response
+      - validation_feedback
+
+  max_feedback_loops:
+    description: "Maximum feedback incorporation attempts reached"
+    return_intent: "max_feedback"
+    return_artifacts:
+      - ai_response
+      - user_feedback

package/examples/flows/order-processing.yaml

@@ -0,0 +1,143 @@
+# yaml-language-server: $schema=../schema/flow.schema.json
+
+# Order Processing Workflow
+# Demonstrates a practical e-commerce order flow with inventory, payment, and shipping
+
+settings:
+  start_step: validate_order
+  max_total_steps: 20
+  timeout_ms: 120000
+
+steps:
+  validate_order:
+    description: "Validate order details"
+    expects_data:
+      - order
+    produces_data:
+      - validated_order
+      - validation_errors
+    transitions:
+      valid: check_inventory
+      invalid: order_invalid
+      failure: error_state
+
+  check_inventory:
+    description: "Check product availability"
+    expects_data:
+      - validated_order
+    produces_data:
+      - inventory_status
+      - reserved_items
+    transitions:
+      available: process_payment
+      partial: partial_availability
+      unavailable: out_of_stock
+      failure: error_state
+    retry:
+      max_attempts: 2
+      delay_ms: 500
+
+  partial_availability:
+    description: "Handle partial availability - ask user to proceed or cancel"
+    expects_data:
+      - validated_order
+      - inventory_status
+    produces_data:
+      - user_decision
+      - adjusted_order
+    transitions:
+      proceed: process_payment
+      cancel: order_cancelled
+      failure: error_state
+
+  process_payment:
+    description: "Process payment for the order"
+    expects_data:
+      - validated_order
+      - reserved_items
+    produces_data:
+      - payment_result
+      - transaction_id
+    transitions:
+      success: create_shipment
+      declined: payment_failed
+      failure: error_state
+    retry:
+      max_attempts: 3
+      delay_ms: 2000
+      backoff_multiplier: 1.5
+
+  create_shipment:
+    description: "Create shipping label and schedule pickup"
+    expects_data:
+      - validated_order
+      - transaction_id
+    produces_data:
+      - shipment_id
+      - tracking_number
+      - estimated_delivery
+    transitions:
+      success: order_complete
+      failure: shipment_error
+
+  shipment_error:
+    description: "Handle shipment creation failure - refund and notify"
+    expects_data:
+      - transaction_id
+      - validated_order
+    produces_data:
+      - refund_status
+    transitions:
+      refunded: order_refunded
+      failure: manual_intervention
+
+terminal_states:
+  order_complete:
+    description: "Order successfully processed"
+    return_intent: "success"
+    return_artifacts:
+      - transaction_id
+      - tracking_number
+      - estimated_delivery
+
+  order_invalid:
+    description: "Order validation failed"
+    return_intent: "invalid_order"
+    return_artifacts:
+      - validation_errors
+
+  out_of_stock:
+    description: "Products out of stock"
+    return_intent: "out_of_stock"
+    return_artifacts:
+      - inventory_status
+
+  order_cancelled:
+    description: "Order cancelled by user"
+    return_intent: "cancelled"
+    return_artifacts: false
+
+  payment_failed:
+    description: "Payment was declined"
+    return_intent: "payment_declined"
+    return_artifacts:
+      - payment_result
+
+  order_refunded:
+    description: "Order refunded due to shipment failure"
+    return_intent: "refunded"
+    return_artifacts:
+      - refund_status
+      - transaction_id
+
+  manual_intervention:
+    description: "Requires manual intervention"
+    return_intent: "manual_required"
+    return_artifacts:
+      - transaction_id
+      - validated_order
+
+  error_state:
+    description: "System error"
+    return_intent: "error"
+    return_artifacts: false

package/examples/flows/simple-greeting.yaml

@@ -0,0 +1,54 @@
+# yaml-language-server: $schema=../schema/flow.schema.json
+
+# Simple Greeting Flow
+# Demonstrates basic flow structure with validation and error handling
+
+settings:
+  start_step: greet
+  max_total_steps: 10
+
+steps:
+  greet:
+    description: "Generate a greeting message"
+    produces_data:
+      - greeting
+      - user_name
+    transitions:
+      success: validate
+      failure: error_state
+
+  validate:
+    description: "Validate the greeting was generated correctly"
+    expects_data:
+      - greeting
+      - user_name
+    produces_data:
+      - is_valid
+    transitions:
+      success: personalize
+      invalid: error_state
+      failure: error_state
+
+  personalize:
+    description: "Create personalized response"
+    expects_data:
+      - greeting
+      - user_name
+    produces_data:
+      - final_message
+    transitions:
+      success: success_state
+      failure: error_state
+
+terminal_states:
+  success_state:
+    description: "Flow completed successfully"
+    return_intent: "success"
+    return_artifacts:
+      - final_message
+      - user_name
+
+  error_state:
+    description: "Flow encountered an error"
+    return_intent: "error"
+    return_artifacts: false

package/examples/graph-of-graphs/multi-stage-etl.ts

@@ -0,0 +1,307 @@
+/**
+ * Graph-of-Graphs Example: Multi-Stage ETL Pipeline
+ *
+ * An outer event-graph orchestrates an ETL pipeline where:
+ *  - "extract" stage: batch × inner EVENT-GRAPH (parallel source extraction)
+ *  - "transform" stage: batch × inner STEP-MACHINE (sequential validation pipeline)
+ *  - "load" + "validate" stages: run in parallel after transform
+ *
+ * Demonstrates:
+ *  - Mixed sub-graph modes (event-graph + step-machine in same pipeline)
+ *  - Config templates shared across both inner configs
+ *  - Variables resolved per-item in the batch
+ *  - Fan-out / fan-in in the outer graph
+ *
+ * Outer graph:
+ *   discover-sources → extract-batch → transform-batch → [load ∥ validate] → finalize
+ *
+ * Inner extract graph (event-graph, per source):
+ *   connect → [fetch-metadata ∥ fetch-schema] → snapshot-data
+ *
+ * Inner transform flow (step-machine, per record):
+ *   parse → validate → normalize → enrich → (accept | reject)
+ *
+ * Run with: npx tsx examples/graph-of-graphs/multi-stage-etl.ts
+ */
+
+import {
+  next, apply, createInitialExecutionState,
+} from '../../src/event-graph/index.js';
+import { createStepMachine } from '../../src/step-machine/index.js';
+import { batch } from '../../src/batch/index.js';
+import { resolveVariables, resolveConfigTemplates } from '../../src/config/index.js';
+import type { GraphConfig } from '../../src/event-graph/types.js';
+import type { StepFlowConfig, StepHandler } from '../../src/step-machine/types.js';
+
+// ============================================================================
+// 1. Inner configs
+// ============================================================================
+
+// --- Extract: event-graph (parallel metadata + schema fetch) ---
+const extractGraphTemplate: Record<string, unknown> = {
+  id: 'source-extractor',
+  'config-templates': {
+    DB_CONN: { driver: 'pg', timeout: 10000, host: '${DB_HOST}' },
+  },
+  settings: {
+    completion: 'all-tasks-complete' as const,
+  },
+  tasks: {
+    connect: {
+      provides: ['connected'],
+      config: { 'config-template': 'DB_CONN', database: '${SOURCE_DB}' },
+    },
+    'fetch-metadata': {
+      requires: ['connected'],
+      provides: ['metadata-ready'],
+      config: { 'config-template': 'DB_CONN', query: 'SELECT * FROM information_schema.tables' },
+    },
+    'fetch-schema': {
+      requires: ['connected'],
+      provides: ['schema-ready'],
+      config: { 'config-template': 'DB_CONN', query: 'SELECT * FROM information_schema.columns' },
+    },
+    'snapshot-data': {
+      requires: ['metadata-ready', 'schema-ready'],
+      provides: ['snapshot-complete'],
+      config: { 'config-template': 'DB_CONN', 'cmd-args': 'pg_dump ${SOURCE_DB}' },
+    },
+  },
+};
+
+// --- Transform: step-machine (sequential validation pipeline) ---
+const transformFlow: StepFlowConfig = {
+  id: 'record-transformer',
+  settings: { start_step: 'parse', max_total_steps: 10 },
+  steps: {
+    parse: {
+      produces_data: ['parsed_record'],
+      transitions: { success: 'validate', error: 'reject' },
+    },
+    validate: {
+      expects_data: ['parsed_record'],
+      produces_data: ['validation_result'],
+      transitions: { valid: 'normalize', invalid: 'reject' },
+    },
+    normalize: {
+      expects_data: ['parsed_record'],
+      produces_data: ['normalized_record'],
+      transitions: { done: 'enrich' },
+    },
+    enrich: {
+      expects_data: ['normalized_record'],
+      produces_data: ['enriched_record'],
+      transitions: { done: 'accept' },
+    },
+  },
+  terminal_states: {
+    accept: { return_intent: 'accepted', return_artifacts: ['enriched_record'] },
+    reject: { return_intent: 'rejected', return_artifacts: ['validation_result'] },
+  },
+};
+
+const transformHandlers: Record<string, StepHandler> = {
+  parse: async (input) => {
+    const raw = input.raw_data as string || '';
+    if (!raw) return { result: 'error' };
+    return { result: 'success', data: { parsed_record: JSON.parse(raw) } };
+  },
+  validate: async (input) => {
+    const rec = input.parsed_record as Record<string, unknown>;
+    if (!rec.id || !rec.name) return { result: 'invalid', data: { validation_result: 'missing required fields' } };
+    return { result: 'valid', data: { validation_result: 'ok' } };
+  },
+  normalize: async (input) => {
+    const rec = input.parsed_record as Record<string, unknown>;
+    return { result: 'done', data: { normalized_record: { ...rec, name: (rec.name as string).trim().toLowerCase() } } };
+  },
+  enrich: async (input) => {
+    const rec = input.normalized_record as Record<string, unknown>;
+    return { result: 'done', data: { enriched_record: { ...rec, enriched_at: new Date().toISOString() } } };
+  },
+};
+
+// ============================================================================
+// 2. Outer graph
+// ============================================================================
+
+const outerGraph: GraphConfig = {
+  id: 'etl-pipeline',
+  settings: { completion: 'all-tasks-complete' },
+  tasks: {
+    'discover-sources': {
+      provides: ['sources-discovered'],
+    },
+    'extract-batch': {
+      requires: ['sources-discovered'],
+      provides: ['extraction-complete'],
+    },
+    'transform-batch': {
+      requires: ['extraction-complete'],
+      provides: ['transform-complete'],
+    },
+    'load-to-warehouse': {
+      requires: ['transform-complete'],
+      provides: ['load-complete'],
+    },
+    'validate-integrity': {
+      requires: ['transform-complete'],
+      provides: ['validation-complete'],
+    },
+    'finalize': {
+      requires: ['load-complete', 'validation-complete'],
+      provides: ['pipeline-done'],
+    },
+  },
+};
+
+// ============================================================================
+// 3. Sub-graph drivers
+// ============================================================================
+
+/** Drive one source through the extract event-graph */
+async function runExtractGraph(source: { id: string; db: string }) {
+  const config = resolveVariables(
+    resolveConfigTemplates(extractGraphTemplate),
+    { SOURCE_DB: source.db, DB_HOST: 'db.internal' },
+  ) as unknown as GraphConfig;
+
+  let state = createInitialExecutionState(config, `extract-${source.id}`);
+  while (true) {
+    const { eligibleTasks, isComplete } = next(config, state);
+    if (isComplete || eligibleTasks.length === 0) break;
+    await Promise.all(
+      eligibleTasks.map(async (taskName) => {
+        state = apply(state, { type: 'task-started', taskName, timestamp: new Date().toISOString() }, config);
+        await new Promise((r) => setTimeout(r, 5 + Math.random() * 15)); // simulate work
+        state = apply(state, { type: 'task-completed', taskName, timestamp: new Date().toISOString() }, config);
+      }),
+    );
+  }
+  return { sourceId: source.id, tokens: state.availableOutputs };
+}
+
+/** Drive one record through the transform step-machine */
+async function runTransformFlow(record: { id: string; raw_data: string }) {
+  const machine = createStepMachine(transformFlow, transformHandlers);
+  return machine.run({ raw_data: record.raw_data });
+}
+
+// ============================================================================
+// 4. Sample data
+// ============================================================================
+
+const sources = [
+  { id: 'src-orders', db: 'orders_db' },
+  { id: 'src-users', db: 'users_db' },
+  { id: 'src-products', db: 'products_db' },
+];
+
+const records = [
+  { id: 'rec-1', raw_data: '{"id": 1, "name": "  Alice "}' },
+  { id: 'rec-2', raw_data: '{"id": 2, "name": " Bob  "}' },
+  { id: 'rec-3', raw_data: '{"name": "no-id"}' },             // will be rejected (no id)
+  { id: 'rec-4', raw_data: '' },                                // will fail to parse
+  { id: 'rec-5', raw_data: '{"id": 5, "name": " Charlie "}' },
+  { id: 'rec-6', raw_data: '{"id": 6, "name": " Diana  "}' },
+];
+
+// ============================================================================
+// 5. Outer graph handlers
+// ============================================================================
+
+const outerHandlers: Record<string, () => Promise<void>> = {
+  'discover-sources': async () => {
+    console.log(`  Found ${sources.length} data sources`);
+  },
+
+  'extract-batch': async () => {
+    console.log(`  Extracting from ${sources.length} sources (concurrency: 2, mode: event-graph)`);
+    const result = await batch(sources, {
+      concurrency: 2,
+      processor: runExtractGraph,
+      onItemComplete: (src, res) => {
+        console.log(`    ✓ ${src.id} (${src.db}): [${res.tokens.join(', ')}]`);
+      },
+    });
+    console.log(`  Extract done: ${result.completed}/${result.total} in ${result.durationMs}ms`);
+  },
+
+  'transform-batch': async () => {
+    console.log(`  Transforming ${records.length} records (concurrency: 4, mode: step-machine)`);
+    const result = await batch(records, {
+      concurrency: 4,
+      processor: runTransformFlow,
+      onItemComplete: (rec, res) => {
+        console.log(`    ✓ ${rec.id}: ${res.intent} — [${res.stepHistory.join(' → ')}]`);
+      },
+      onItemError: (rec, err) => {
+        console.log(`    ✗ ${rec.id}: ${err.message}`);
+      },
+    });
+    console.log(`  Transform done: ${result.completed} ok, ${result.failed} failed in ${result.durationMs}ms`);
+
+    // Show accepted vs rejected
+    const accepted = result.items.filter((i) => i.status === 'completed' && i.result?.intent === 'accepted');
+    const rejected = result.items.filter((i) => i.status === 'completed' && i.result?.intent === 'rejected');
+    console.log(`  Accepted: ${accepted.length}, Rejected: ${rejected.length}`);
+  },
+
+  'load-to-warehouse': async () => {
+    console.log('  Loading accepted records to data warehouse');
+    await new Promise((r) => setTimeout(r, 20));
+  },
+
+  'validate-integrity': async () => {
+    console.log('  Running integrity checks on loaded data');
+    await new Promise((r) => setTimeout(r, 15));
+  },
+
+  'finalize': async () => {
+    console.log('  Generating ETL summary report');
+  },
+};
+
+// ============================================================================
+// 6. Drive outer graph
+// ============================================================================
+
+async function main() {
+  console.log('=== Multi-Stage ETL Pipeline (Graph-of-Graphs) ===');
+  console.log('Outer: event-graph | Extract sub: event-graph | Transform sub: step-machine\n');
+
+  let state = createInitialExecutionState(outerGraph, 'etl-run-1');
+  const now = () => new Date().toISOString();
+
+  while (true) {
+    const { eligibleTasks, isComplete } = next(outerGraph, state);
+    if (isComplete) break;
+    if (eligibleTasks.length === 0) {
+      console.log('\nPipeline stuck!');
+      break;
+    }
+
+    // Note: load-to-warehouse and validate-integrity will run in PARALLEL
+    // because they both only require transform-complete
+    if (eligibleTasks.length > 1) {
+      console.log(`\n▶ [parallel] ${eligibleTasks.join(' + ')}`);
+    }
+    await Promise.all(
+      eligibleTasks.map(async (taskName) => {
+        if (eligibleTasks.length === 1) console.log(`\n▶ ${taskName}`);
+        state = apply(state, { type: 'task-started', taskName, timestamp: now() }, outerGraph);
+        try {
+          await outerHandlers[taskName]();
+          state = apply(state, { type: 'task-completed', taskName, timestamp: now() }, outerGraph);
+        } catch (err: any) {
+          state = apply(state, { type: 'task-failed', taskName, error: err.message, timestamp: now() }, outerGraph);
+        }
+      }),
+    );
+  }
+
+  console.log('\n=== ETL Pipeline Complete ===');
+  console.log('Final tokens:', state.availableOutputs);
+}
+
+main().catch(console.error);