yaml-flow 7.1.0 → 8.0.0

package/examples/npm-libs/flows/ai-conversation.yaml

@@ -1,116 +0,0 @@
-# yaml-language-server: $schema=../schema/flow.schema.json
-
-# AI Conversation Flow with Validation Loop
-# Demonstrates retry logic, circuit breakers, and multi-step AI workflows
-
-settings:
-  start_step: generate_response
-  max_total_steps: 50
-  timeout_ms: 60000
-
-steps:
-  generate_response:
-    description: "Generate AI response based on user input"
-    expects_data:
-      - user_message
-      - conversation_history
-    produces_data:
-      - ai_response
-      - confidence_score
-    transitions:
-      success: validate_response
-      failure: error_state
-    retry:
-      max_attempts: 3
-      delay_ms: 1000
-      backoff_multiplier: 2
-
-  validate_response:
-    description: "Validate the AI response meets quality criteria"
-    expects_data:
-      - ai_response
-      - confidence_score
-    produces_data:
-      - validation_result
-      - validation_feedback
-    transitions:
-      valid: check_approval
-      needs_refinement: refine_response
-      failure: error_state
-
-  refine_response:
-    description: "Refine AI response based on validation feedback"
-    expects_data:
-      - ai_response
-      - validation_feedback
-      - user_message
-    produces_data:
-      - ai_response
-      - confidence_score
-    transitions:
-      success: validate_response
-      failure: error_state
-    circuit_breaker:
-      max_iterations: 3
-      on_open: max_refinements
-
-  check_approval:
-    description: "Present response and check for user approval"
-    expects_data:
-      - ai_response
-    produces_data:
-      - user_approved
-      - user_feedback
-    transitions:
-      approved: success_state
-      rejected: incorporate_feedback
-      timeout: timeout_state
-
-  incorporate_feedback:
-    description: "Incorporate user feedback into response"
-    expects_data:
-      - ai_response
-      - user_feedback
-      - user_message
-    produces_data:
-      - ai_response
-      - confidence_score
-    transitions:
-      success: validate_response
-      failure: error_state
-    circuit_breaker:
-      max_iterations: 5
-      on_open: max_feedback_loops
-
-terminal_states:
-  success_state:
-    description: "User approved the AI response"
-    return_intent: "approved"
-    return_artifacts:
-      - ai_response
-      - conversation_history
-
-  error_state:
-    description: "Flow encountered an error"
-    return_intent: "error"
-    return_artifacts: false
-
-  timeout_state:
-    description: "User did not respond in time"
-    return_intent: "timeout"
-    return_artifacts:
-      - ai_response
-
-  max_refinements:
-    description: "Maximum refinement attempts reached"
-    return_intent: "max_refinements"
-    return_artifacts:
-      - ai_response
-      - validation_feedback
-
-  max_feedback_loops:
-    description: "Maximum feedback incorporation attempts reached"
-    return_intent: "max_feedback"
-    return_artifacts:
-      - ai_response
-      - user_feedback

package/examples/npm-libs/flows/order-processing.yaml

@@ -1,143 +0,0 @@
-# yaml-language-server: $schema=../schema/flow.schema.json
-
-# Order Processing Workflow
-# Demonstrates a practical e-commerce order flow with inventory, payment, and shipping
-
-settings:
-  start_step: validate_order
-  max_total_steps: 20
-  timeout_ms: 120000
-
-steps:
-  validate_order:
-    description: "Validate order details"
-    expects_data:
-      - order
-    produces_data:
-      - validated_order
-      - validation_errors
-    transitions:
-      valid: check_inventory
-      invalid: order_invalid
-      failure: error_state
-
-  check_inventory:
-    description: "Check product availability"
-    expects_data:
-      - validated_order
-    produces_data:
-      - inventory_status
-      - reserved_items
-    transitions:
-      available: process_payment
-      partial: partial_availability
-      unavailable: out_of_stock
-      failure: error_state
-    retry:
-      max_attempts: 2
-      delay_ms: 500
-
-  partial_availability:
-    description: "Handle partial availability - ask user to proceed or cancel"
-    expects_data:
-      - validated_order
-      - inventory_status
-    produces_data:
-      - user_decision
-      - adjusted_order
-    transitions:
-      proceed: process_payment
-      cancel: order_cancelled
-      failure: error_state
-
-  process_payment:
-    description: "Process payment for the order"
-    expects_data:
-      - validated_order
-      - reserved_items
-    produces_data:
-      - payment_result
-      - transaction_id
-    transitions:
-      success: create_shipment
-      declined: payment_failed
-      failure: error_state
-    retry:
-      max_attempts: 3
-      delay_ms: 2000
-      backoff_multiplier: 1.5
-
-  create_shipment:
-    description: "Create shipping label and schedule pickup"
-    expects_data:
-      - validated_order
-      - transaction_id
-    produces_data:
-      - shipment_id
-      - tracking_number
-      - estimated_delivery
-    transitions:
-      success: order_complete
-      failure: shipment_error
-
-  shipment_error:
-    description: "Handle shipment creation failure - refund and notify"
-    expects_data:
-      - transaction_id
-      - validated_order
-    produces_data:
-      - refund_status
-    transitions:
-      refunded: order_refunded
-      failure: manual_intervention
-
-terminal_states:
-  order_complete:
-    description: "Order successfully processed"
-    return_intent: "success"
-    return_artifacts:
-      - transaction_id
-      - tracking_number
-      - estimated_delivery
-
-  order_invalid:
-    description: "Order validation failed"
-    return_intent: "invalid_order"
-    return_artifacts:
-      - validation_errors
-
-  out_of_stock:
-    description: "Products out of stock"
-    return_intent: "out_of_stock"
-    return_artifacts:
-      - inventory_status
-
-  order_cancelled:
-    description: "Order cancelled by user"
-    return_intent: "cancelled"
-    return_artifacts: false
-
-  payment_failed:
-    description: "Payment was declined"
-    return_intent: "payment_declined"
-    return_artifacts:
-      - payment_result
-
-  order_refunded:
-    description: "Order refunded due to shipment failure"
-    return_intent: "refunded"
-    return_artifacts:
-      - refund_status
-      - transaction_id
-
-  manual_intervention:
-    description: "Requires manual intervention"
-    return_intent: "manual_required"
-    return_artifacts:
-      - transaction_id
-      - validated_order
-
-  error_state:
-    description: "System error"
-    return_intent: "error"
-    return_artifacts: false

package/examples/npm-libs/flows/simple-greeting.yaml

@@ -1,54 +0,0 @@
-# yaml-language-server: $schema=../schema/flow.schema.json
-
-# Simple Greeting Flow
-# Demonstrates basic flow structure with validation and error handling
-
-settings:
-  start_step: greet
-  max_total_steps: 10
-
-steps:
-  greet:
-    description: "Generate a greeting message"
-    produces_data:
-      - greeting
-      - user_name
-    transitions:
-      success: validate
-      failure: error_state
-
-  validate:
-    description: "Validate the greeting was generated correctly"
-    expects_data:
-      - greeting
-      - user_name
-    produces_data:
-      - is_valid
-    transitions:
-      success: personalize
-      invalid: error_state
-      failure: error_state
-
-  personalize:
-    description: "Create personalized response"
-    expects_data:
-      - greeting
-      - user_name
-    produces_data:
-      - final_message
-    transitions:
-      success: success_state
-      failure: error_state
-
-terminal_states:
-  success_state:
-    description: "Flow completed successfully"
-    return_intent: "success"
-    return_artifacts:
-      - final_message
-      - user_name
-
-  error_state:
-    description: "Flow encountered an error"
-    return_intent: "error"
-    return_artifacts: false

package/examples/npm-libs/graph-of-graphs/multi-stage-etl.ts

@@ -1,307 +0,0 @@
-/**
- * Graph-of-Graphs Example: Multi-Stage ETL Pipeline
- *
- * An outer event-graph orchestrates an ETL pipeline where:
- *  - "extract" stage: batch × inner EVENT-GRAPH (parallel source extraction)
- *  - "transform" stage: batch × inner STEP-MACHINE (sequential validation pipeline)
- *  - "load" + "validate" stages: run in parallel after transform
- *
- * Demonstrates:
- *  - Mixed sub-graph modes (event-graph + step-machine in same pipeline)
- *  - Config templates shared across both inner configs
- *  - Variables resolved per-item in the batch
- *  - Fan-out / fan-in in the outer graph
- *
- * Outer graph:
- *   discover-bsources → extract-batch → transform-batch → [load ∥ validate] → finalize
- *
- * Inner extract graph (event-graph, per source):
- *   connect → [fetch-metadata ∥ fetch-schema] → snapshot-data
- *
- * Inner transform flow (step-machine, per record):
- *   parse → validate → normalize → enrich → (accept | reject)
- *
- * Run with: npx tsx examples/npm-libs/graph-of-graphs/multi-stage-etl.ts
- */
-
-import {
-  next, apply, createInitialExecutionState,
-} from '../../src/event-graph/index.js';
-import { createStepMachine } from '../../src/step-machine/index.js';
-import { batch } from '../../src/batch/index.js';
-import { resolveVariables, resolveConfigTemplates } from '../../src/config/index.js';
-import type { GraphConfig } from '../../src/event-graph/types.js';
-import type { StepFlowConfig, StepHandler } from '../../src/step-machine/types.js';
-
-// ============================================================================
-// 1. Inner configs
-// ============================================================================
-
-// --- Extract: event-graph (parallel metadata + schema fetch) ---
-const extractGraphTemplate: Record<string, unknown> = {
-  id: 'source-extractor',
-  'config-templates': {
-    DB_CONN: { driver: 'pg', timeout: 10000, host: '${DB_HOST}' },
-  },
-  settings: {
-    completion: 'all-tasks-complete' as const,
-  },
-  tasks: {
-    connect: {
-      provides: ['connected'],
-      config: { 'config-template': 'DB_CONN', database: '${SOURCE_DB}' },
-    },
-    'fetch-metadata': {
-      requires: ['connected'],
-      provides: ['metadata-ready'],
-      config: { 'config-template': 'DB_CONN', query: 'SELECT * FROM information_schema.tables' },
-    },
-    'fetch-schema': {
-      requires: ['connected'],
-      provides: ['schema-ready'],
-      config: { 'config-template': 'DB_CONN', query: 'SELECT * FROM information_schema.columns' },
-    },
-    'snapshot-data': {
-      requires: ['metadata-ready', 'schema-ready'],
-      provides: ['snapshot-complete'],
-      config: { 'config-template': 'DB_CONN', 'cmd-args': 'pg_dump ${SOURCE_DB}' },
-    },
-  },
-};
-
-// --- Transform: step-machine (sequential validation pipeline) ---
-const transformFlow: StepFlowConfig = {
-  id: 'record-transformer',
-  settings: { start_step: 'parse', max_total_steps: 10 },
-  steps: {
-    parse: {
-      produces_data: ['parsed_record'],
-      transitions: { success: 'validate', error: 'reject' },
-    },
-    validate: {
-      expects_data: ['parsed_record'],
-      produces_data: ['validation_result'],
-      transitions: { valid: 'normalize', invalid: 'reject' },
-    },
-    normalize: {
-      expects_data: ['parsed_record'],
-      produces_data: ['normalized_record'],
-      transitions: { done: 'enrich' },
-    },
-    enrich: {
-      expects_data: ['normalized_record'],
-      produces_data: ['enriched_record'],
-      transitions: { done: 'accept' },
-    },
-  },
-  terminal_states: {
-    accept: { return_intent: 'accepted', return_artifacts: ['enriched_record'] },
-    reject: { return_intent: 'rejected', return_artifacts: ['validation_result'] },
-  },
-};
-
-const transformHandlers: Record<string, StepHandler> = {
-  parse: async (input) => {
-    const raw = input.raw_data as string || '';
-    if (!raw) return { result: 'error' };
-    return { result: 'success', data: { parsed_record: JSON.parse(raw) } };
-  },
-  validate: async (input) => {
-    const rec = input.parsed_record as Record<string, unknown>;
-    if (!rec.id || !rec.name) return { result: 'invalid', data: { validation_result: 'missing required fields' } };
-    return { result: 'valid', data: { validation_result: 'ok' } };
-  },
-  normalize: async (input) => {
-    const rec = input.parsed_record as Record<string, unknown>;
-    return { result: 'done', data: { normalized_record: { ...rec, name: (rec.name as string).trim().toLowerCase() } } };
-  },
-  enrich: async (input) => {
-    const rec = input.normalized_record as Record<string, unknown>;
-    return { result: 'done', data: { enriched_record: { ...rec, enriched_at: new Date().toISOString() } } };
-  },
-};
-
-// ============================================================================
-// 2. Outer graph
-// ============================================================================
-
-const outerGraph: GraphConfig = {
-  id: 'etl-pipeline',
-  settings: { completion: 'all-tasks-complete' },
-  tasks: {
-    'discover-bsources': {
-      provides: ['source_defs-discovered'],
-    },
-    'extract-batch': {
-      requires: ['source_defs-discovered'],
-      provides: ['extraction-complete'],
-    },
-    'transform-batch': {
-      requires: ['extraction-complete'],
-      provides: ['transform-complete'],
-    },
-    'load-to-warehouse': {
-      requires: ['transform-complete'],
-      provides: ['load-complete'],
-    },
-    'validate-integrity': {
-      requires: ['transform-complete'],
-      provides: ['validation-complete'],
-    },
-    'finalize': {
-      requires: ['load-complete', 'validation-complete'],
-      provides: ['pipeline-done'],
-    },
-  },
-};
-
-// ============================================================================
-// 3. Sub-graph drivers
-// ============================================================================
-
-/** Drive one source through the extract event-graph */
-async function runExtractGraph(source: { id: string; db: string }) {
-  const config = resolveVariables(
-    resolveConfigTemplates(extractGraphTemplate),
-    { SOURCE_DB: source.db, DB_HOST: 'db.internal' },
-  ) as unknown as GraphConfig;
-
-  let state = createInitialExecutionState(config, `extract-${source.id}`);
-  while (true) {
-    const { eligibleTasks, isComplete } = next(config, state);
-    if (isComplete || eligibleTasks.length === 0) break;
-    await Promise.all(
-      eligibleTasks.map(async (taskName) => {
-        state = apply(state, { type: 'task-started', taskName, timestamp: new Date().toISOString() }, config);
-        await new Promise((r) => setTimeout(r, 5 + Math.random() * 15)); // simulate work
-        state = apply(state, { type: 'task-completed', taskName, timestamp: new Date().toISOString() }, config);
-      }),
-    );
-  }
-  return { sourceId: source.id, tokens: state.availableOutputs };
-}
-
-/** Drive one record through the transform step-machine */
-async function runTransformFlow(record: { id: string; raw_data: string }) {
-  const machine = createStepMachine(transformFlow, transformHandlers);
-  return machine.run({ raw_data: record.raw_data });
-}
-
-// ============================================================================
-// 4. Sample data
-// ============================================================================
-
-const source_defs = [
-  { id: 'src-orders', db: 'orders_db' },
-  { id: 'src-users', db: 'users_db' },
-  { id: 'src-products', db: 'products_db' },
-];
-
-const records = [
-  { id: 'rec-1', raw_data: '{"id": 1, "name": "  Alice "}' },
-  { id: 'rec-2', raw_data: '{"id": 2, "name": " Bob  "}' },
-  { id: 'rec-3', raw_data: '{"name": "no-id"}' },             // will be rejected (no id)
-  { id: 'rec-4', raw_data: '' },                                // will fail to parse
-  { id: 'rec-5', raw_data: '{"id": 5, "name": " Charlie "}' },
-  { id: 'rec-6', raw_data: '{"id": 6, "name": " Diana  "}' },
-];
-
-// ============================================================================
-// 5. Outer graph handlers
-// ============================================================================
-
-const outerHandlers: Record<string, () => Promise<void>> = {
-  'discover-bsources': async () => {
-    console.log(`  Found ${source_defs.length} data source_defs`);
-  },
-
-  'extract-batch': async () => {
-    console.log(`  Extracting from ${source_defs.length} source_defs (concurrency: 2, mode: event-graph)`);
-    const result = await batch(source_defs, {
-      concurrency: 2,
-      processor: runExtractGraph,
-      onItemComplete: (src, res) => {
-        console.log(`    ✓ ${src.id} (${src.db}): [${res.tokens.join(', ')}]`);
-      },
-    });
-    console.log(`  Extract done: ${result.completed}/${result.total} in ${result.durationMs}ms`);
-  },
-
-  'transform-batch': async () => {
-    console.log(`  Transforming ${records.length} records (concurrency: 4, mode: step-machine)`);
-    const result = await batch(records, {
-      concurrency: 4,
-      processor: runTransformFlow,
-      onItemComplete: (rec, res) => {
-        console.log(`    ✓ ${rec.id}: ${res.intent} — [${res.stepHistory.join(' → ')}]`);
-      },
-      onItemError: (rec, err) => {
-        console.log(`    ✗ ${rec.id}: ${err.message}`);
-      },
-    });
-    console.log(`  Transform done: ${result.completed} ok, ${result.failed} failed in ${result.durationMs}ms`);
-
-    // Show accepted vs rejected
-    const accepted = result.items.filter((i) => i.status === 'completed' && i.result?.intent === 'accepted');
-    const rejected = result.items.filter((i) => i.status === 'completed' && i.result?.intent === 'rejected');
-    console.log(`  Accepted: ${accepted.length}, Rejected: ${rejected.length}`);
-  },
-
-  'load-to-warehouse': async () => {
-    console.log('  Loading accepted records to data warehouse');
-    await new Promise((r) => setTimeout(r, 20));
-  },
-
-  'validate-integrity': async () => {
-    console.log('  Running integrity checks on loaded data');
-    await new Promise((r) => setTimeout(r, 15));
-  },
-
-  'finalize': async () => {
-    console.log('  Generating ETL summary report');
-  },
-};
-
-// ============================================================================
-// 6. Drive outer graph
-// ============================================================================
-
-async function main() {
-  console.log('=== Multi-Stage ETL Pipeline (Graph-of-Graphs) ===');
-  console.log('Outer: event-graph | Extract sub: event-graph | Transform sub: step-machine\n');
-
-  let state = createInitialExecutionState(outerGraph, 'etl-run-1');
-  const now = () => new Date().toISOString();
-
-  while (true) {
-    const { eligibleTasks, isComplete } = next(outerGraph, state);
-    if (isComplete) break;
-    if (eligibleTasks.length === 0) {
-      console.log('\nPipeline stuck!');
-      break;
-    }
-
-    // Note: load-to-warehouse and validate-integrity will run in PARALLEL
-    // because they both only require transform-complete
-    if (eligibleTasks.length > 1) {
-      console.log(`\n▶ [parallel] ${eligibleTasks.join(' + ')}`);
-    }
-    await Promise.all(
-      eligibleTasks.map(async (taskName) => {
-        if (eligibleTasks.length === 1) console.log(`\n▶ ${taskName}`);
-        state = apply(state, { type: 'task-started', taskName, timestamp: now() }, outerGraph);
-        try {
-          await outerHandlers[taskName]();
-          state = apply(state, { type: 'task-completed', taskName, timestamp: now() }, outerGraph);
-        } catch (err: any) {
-          state = apply(state, { type: 'task-failed', taskName, error: err.message, timestamp: now() }, outerGraph);
-        }
-      }),
-    );
-  }
-
-  console.log('\n=== ETL Pipeline Complete ===');
-  console.log('Final tokens:', state.availableOutputs);
-}
-
-main().catch(console.error);