reqon-dsl 0.2.0 → 0.3.0

package/dist/interpreter/executor.js

@@ -1,50 +1,87 @@
+/**
+ * ---
+ * purpose: Mission executor - orchestrates pipeline execution
+ * inputs:
+ *   - ReqonProgram - parsed AST
+ *   - ExecutorConfig - auth, stores, callbacks, debug settings
+ * outputs:
+ *   - ExecutionResult - success/errors, stores, duration
+ * related:
+ *   - ./context.ts - execution state (variables, stores, sources)
+ *   - ./evaluator.ts - expression evaluation
+ *   - ./fetch-handler.ts - HTTP requests
+ *   - ./step-handlers/ - individual step type handlers
+ *   - ./source-manager.ts - auth provider management
+ * ---
+ */
 import { isParallelStage } from '../ast/nodes.js';
-import { createContext, setVariable } from './context.js';
+import { createContext, childContext, setVariable } from './context.js';
 import { evaluate } from './evaluator.js';
-import { HttpClient, BearerAuthProvider, OAuth2AuthProvider } from './http.js';
-import { createStore, resolveStoreType } from '../stores/index.js';
-import { loadOAS } from '../oas/index.js';
+import { SourceManager } from './source-manager.js';
+import { StoreManager } from './store-manager.js';
 import { AdaptiveRateLimiter } from '../auth/rate-limiter.js';
 import { CircuitBreaker } from '../auth/circuit-breaker.js';
 import { createExecutionState, findResumePoint, FileExecutionStore, } from '../execution/index.js';
-import { FileSyncStore, } from '../sync/index.js';
+import { FileSyncStore } from '../sync/index.js';
 import { FetchHandler } from './fetch-handler.js';
-import { ForHandler, MapHandler, ValidateHandler, StoreHandler, MatchHandler, WebhookHandler, SkipSignal, RetrySignal, JumpSignal, QueueSignal, } from './step-handlers/index.js';
+import { ForHandler, MapHandler, ValidateHandler, StoreHandler, MatchHandler, ApplyHandler, WebhookHandler, PauseHandler, SkipSignal, AbortError, RetrySignal, JumpSignal, QueueSignal, } from './step-handlers/index.js';
+import { createStructuredLogger } from '../observability/index.js';
+import { PauseSignal } from './signals.js';
+import { FileTraceStore, createTraceRecorder, } from '../trace/index.js';
+import { FilePauseStore, createPauseManager, } from '../pause/index.js';
 export class MissionExecutor {
     config;
     ctx;
     errors = [];
     actionsRun = [];
-    oasSources = new Map();
-    sourceConfigs = new Map();
+    transforms = new Map();
     rateLimiter;
     circuitBreaker;
+    sourceManager;
+    storeManager;
     executionStore;
     executionState;
     syncStore;
     missionName;
+    eventEmitter;
+    logger;
+    stepIndex = 0;
+    debugController;
+    traceRecorder;
+    traceStore;
+    pauseManager;
+    pauseStore;
+    currentStageIndex = 0;
+    currentPauseId;
     constructor(config = {}) {
         this.config = config;
         this.ctx = createContext();
         this.rateLimiter = new AdaptiveRateLimiter();
         this.circuitBreaker = new CircuitBreaker();
+        // Initialize managers (logger set after verbose callbacks configured)
+        this.sourceManager = new SourceManager({ auth: config.auth, missionDir: config.missionDir }, { rateLimiter: this.rateLimiter, circuitBreaker: this.circuitBreaker });
+        this.storeManager = new StoreManager({
+            customStores: config.stores,
+            developmentMode: config.developmentMode,
+            dataDir: config.dataDir,
+        });
         // Set up rate limit callbacks with default logging if verbose
         const callbacks = config.rateLimitCallbacks ?? {};
         if (config.verbose && !callbacks.onRateLimited) {
             callbacks.onRateLimited = (event) => {
-                console.log(`[Reqon] Rate limited on ${event.source}${event.endpoint ? `:${event.endpoint}` : ''} - ` +
+                this.log(`Rate limited on ${event.source}${event.endpoint ? `:${event.endpoint}` : ''} - ` +
                     `waiting ${event.waitSeconds}s (strategy: ${event.strategy})`);
             };
         }
         if (config.verbose && !callbacks.onResumed) {
             callbacks.onResumed = (event) => {
-                console.log(`[Reqon] Rate limit cleared for ${event.source}${event.endpoint ? `:${event.endpoint}` : ''} ` +
+                this.log(`Rate limit cleared for ${event.source}${event.endpoint ? `:${event.endpoint}` : ''} ` +
                     `(waited ${event.waitedSeconds}s)`);
             };
         }
         if (config.verbose && !callbacks.onWaiting) {
             callbacks.onWaiting = (event) => {
-                console.log(`[Reqon] Still waiting for ${event.source}${event.endpoint ? `:${event.endpoint}` : ''} - ` +
+                this.log(`Still waiting for ${event.source}${event.endpoint ? `:${event.endpoint}` : ''} - ` +
                     `${event.waitSeconds}s remaining (elapsed: ${event.elapsedSeconds}s)`);
             };
         }
@@ -53,33 +90,71 @@ export class MissionExecutor {
         const cbCallbacks = config.circuitBreakerCallbacks ?? {};
         if (config.verbose && !cbCallbacks.onOpen) {
             cbCallbacks.onOpen = (event) => {
-                console.log(`[Reqon] Circuit breaker OPEN for ${event.source}${event.endpoint ? `:${event.endpoint}` : ''} - ` +
+                this.log(`Circuit breaker OPEN for ${event.source}${event.endpoint ? `:${event.endpoint}` : ''} - ` +
                     `${event.failures} failures (${event.reason ?? 'threshold exceeded'})`);
             };
         }
         if (config.verbose && !cbCallbacks.onHalfOpen) {
             cbCallbacks.onHalfOpen = (event) => {
-                console.log(`[Reqon] Circuit breaker HALF-OPEN for ${event.source}${event.endpoint ? `:${event.endpoint}` : ''} - ` +
+                this.log(`Circuit breaker HALF-OPEN for ${event.source}${event.endpoint ? `:${event.endpoint}` : ''} - ` +
                     `testing recovery`);
             };
         }
         if (config.verbose && !cbCallbacks.onClose) {
             cbCallbacks.onClose = (event) => {
-                console.log(`[Reqon] Circuit breaker CLOSED for ${event.source}${event.endpoint ? `:${event.endpoint}` : ''} - ` +
+                this.log(`Circuit breaker CLOSED for ${event.source}${event.endpoint ? `:${event.endpoint}` : ''} - ` +
                     `recovery successful`);
             };
         }
         if (config.verbose && !cbCallbacks.onRejected) {
             cbCallbacks.onRejected = (event) => {
-                console.log(`[Reqon] Request REJECTED by circuit breaker for ${event.source}${event.endpoint ? `:${event.endpoint}` : ''} - ` +
+                this.log(`Request REJECTED by circuit breaker for ${event.source}${event.endpoint ? `:${event.endpoint}` : ''} - ` +
                     `retry in ${Math.ceil(event.nextAttemptIn / 1000)}s`);
             };
         }
         this.circuitBreaker.setCallbacks(cbCallbacks);
         // Initialize execution store if persistence enabled
         if (config.persistState) {
-            this.executionStore = config.executionStore ?? new FileExecutionStore(`${config.dataDir ?? '.reqon-data'}/executions`);
+            this.executionStore =
+                config.executionStore ??
+                    new FileExecutionStore(`${config.dataDir ?? '.reqon-data'}/executions`);
+        }
+        // Initialize event emitter if provided
+        this.eventEmitter = config.eventEmitter;
+        // Initialize logger if verbose or provided
+        if (config.logger) {
+            this.logger = config.logger;
+        }
+        else if (config.verbose) {
+            this.logger = createStructuredLogger({
+                prefix: 'Reqon',
+                level: 'debug',
+                context: {},
+            });
         }
+        // Update managers with log function now that logger is configured
+        this.sourceManager = new SourceManager({ auth: config.auth, missionDir: config.missionDir, log: (msg) => this.log(msg) }, { rateLimiter: this.rateLimiter, circuitBreaker: this.circuitBreaker });
+        this.storeManager = new StoreManager({
+            customStores: config.stores,
+            developmentMode: config.developmentMode,
+            dataDir: config.dataDir,
+            log: (msg) => this.log(msg),
+        });
+        // Initialize debug controller if provided
+        this.debugController = config.debugController;
+        // Initialize trace store
+        this.traceStore =
+            config.traceStore ?? new FileTraceStore(`${config.dataDir ?? '.reqon-data'}/traces`);
+        // Initialize pause store and manager
+        this.pauseStore =
+            config.pauseStore ?? new FilePauseStore(`${config.dataDir ?? '.reqon-data'}/pauses`);
+        this.pauseManager =
+            config.pauseManager ??
+                createPauseManager({
+                    store: this.pauseStore,
+                    webhookServer: config.webhookServer,
+                    log: (msg) => this.log(msg),
+                });
     }
     async execute(program) {
         const startTime = Date.now();
@@ -96,6 +171,18 @@ export class MissionExecutor {
         }
         // Initialize or resume execution state
         await this.initializeExecutionState(mission);
+        // Initialize trace recorder if tracing is enabled
+        if (mission.trace && this.traceStore && this.executionState) {
+            this.traceRecorder = createTraceRecorder({
+                executionId: this.executionState.id,
+                mission: mission.name,
+                mode: mission.trace.mode,
+                store: this.traceStore,
+                metadata: this.config.metadata,
+                streaming: true, // Stream snapshots as they happen
+            });
+            this.log(`Tracing enabled (mode: ${mission.trace.mode})`);
+        }
         try {
             await this.executeMission(mission);
             // Mark execution as completed
@@ -107,22 +194,32 @@ export class MissionExecutor {
             }
         }
         catch (error) {
-            this.errors.push({
-                action: 'mission',
-                step: 'execute',
-                message: error.message,
-                details: error,
-            });
-            // Mark execution as failed
-            if (this.executionState) {
-                this.executionState.status = 'failed';
-                this.executionState.completedAt = new Date();
-                this.executionState.duration = Date.now() - startTime;
-                await this.saveExecutionState();
+            // PauseSignal is not an error - execution was intentionally paused
+            if (error instanceof PauseSignal) {
+                this.log('Execution paused');
+                this.currentPauseId = error.pauseId;
+                // State is already set to 'paused' in checkPause() or pause handler
+                // Don't record as error, just let execution end
+            }
+            else {
+                this.errors.push({
+                    action: 'mission',
+                    step: 'execute',
+                    message: error.message,
+                    details: error,
+                });
+                // Mark execution as failed
+                if (this.executionState) {
+                    this.executionState.status = 'failed';
+                    this.executionState.completedAt = new Date();
+                    this.executionState.duration = Date.now() - startTime;
+                    await this.saveExecutionState();
+                }
             }
         }
         const duration = Date.now() - startTime;
-        const success = this.errors.length === 0;
+        const isPaused = this.executionState?.status === 'paused';
+        const success = this.errors.length === 0 && !isPaused;
         // Emit onExecutionComplete callback - count stages in a single pass
         const stageCounts = this.executionState?.stages.reduce((acc, s) => {
             if (s.status === 'completed')
@@ -142,6 +239,34 @@ export class MissionExecutor {
             stagesFailed,
             errors: this.errors,
         });
+        // Emit mission.complete, mission.paused, or mission.failed event
+        if (isPaused) {
+            this.eventEmitter?.emit('mission.paused', {
+                stagesCompleted,
+                executionId: this.executionState?.id,
+            });
+        }
+        else if (success) {
+            this.eventEmitter?.emit('mission.complete', {
+                success: true,
+                stagesCompleted,
+                stagesFailed,
+                stagesSkipped: this.executionState?.stages.filter((s) => s.status === 'skipped').length ?? 0,
+                errorCount: this.errors.length,
+            });
+        }
+        else {
+            const failedStage = this.executionState?.stages.find((s) => s.status === 'failed');
+            this.eventEmitter?.emit('mission.failed', {
+                error: this.errors[0]?.message ?? 'Unknown error',
+                failedStage: failedStage?.action,
+                stagesCompleted,
+            });
+        }
+        // Finalize trace if enabled
+        if (this.traceRecorder) {
+            await this.traceRecorder.finalize(success);
+        }
         return {
             success,
             duration,
@@ -150,6 +275,8 @@ export class MissionExecutor {
             stores: this.ctx.stores,
             executionId: this.executionState?.id,
             state: this.executionState,
+            traceId: this.traceRecorder ? this.executionState?.id : undefined,
+            pauseId: this.currentPauseId,
         };
     }
     async initializeExecutionState(mission) {
@@ -190,6 +317,13 @@ export class MissionExecutor {
             isResume,
             metadata: this.config.metadata,
         });
+        // Emit mission.start event
+        this.eventEmitter?.emit('mission.start', {
+            stageCount: mission.pipeline.stages.length,
+            isResume,
+            resumeFromStage: isResume ? findResumePoint(this.executionState) : undefined,
+            metadata: this.config.metadata,
+        });
     }
     async saveExecutionState() {
         if (this.executionStore && this.executionState) {
@@ -227,20 +361,23 @@ export class MissionExecutor {
         this.log(`Executing mission: ${mission.name}`);
         this.missionName = mission.name;
         // Initialize sync store
-        this.syncStore = this.config.syncStore ?? new FileSyncStore(mission.name, `${this.config.dataDir ?? '.reqon-data'}/sync`);
-        // Initialize sources (HTTP clients)
-        for (const source of mission.sources) {
-            await this.initializeSource(source);
-        }
-        // Initialize stores
-        for (const store of mission.stores) {
-            await this.initializeStore(store);
-        }
+        this.syncStore =
+            this.config.syncStore ??
+                new FileSyncStore(mission.name, `${this.config.dataDir ?? '.reqon-data'}/sync`);
+        // Initialize sources using SourceManager
+        await this.sourceManager.initializeSources(mission.sources, this.ctx);
+        // Initialize stores using StoreManager
+        await this.storeManager.initializeStores(mission.stores, this.ctx);
         // Initialize schemas (for match step schema matching)
         for (const schema of mission.schemas) {
             this.ctx.schemas.set(schema.name, schema);
             this.log(`Registered schema: ${schema.name}`);
         }
+        // Initialize transforms
+        for (const transform of mission.transforms) {
+            this.transforms.set(transform.name, transform);
+            this.log(`Registered transform: ${transform.name}`);
+        }
         // Build action lookup
         const actions = new Map();
         for (const action of mission.actions) {
@@ -256,6 +393,8 @@ export class MissionExecutor {
         // Execute pipeline
         for (let i = 0; i < mission.pipeline.stages.length; i++) {
             const stage = mission.pipeline.stages[i];
+            // Check for pause request at safe point (between stages)
+            await this.checkPause();
             // Skip already completed stages when resuming
             if (i < resumeIndex) {
                 this.log(`Skipping ${this.getStageName(stage)} (already completed)`);
@@ -268,9 +407,12 @@ export class MissionExecutor {
                     this.log(`Skipping ${this.getStageName(stage)} (condition not met)`);
                     this.updateStageState(i, { status: 'skipped' });
                     await this.saveExecutionState();
+                    this.updateControlServerState();
                     continue;
                 }
             }
+            // Track current stage index for pause handler
+            this.currentStageIndex = i;
             // Execute stage (parallel or sequential)
             if (isParallelStage(stage)) {
                 await this.executeParallelStage(i, stage, actions, mission);
@@ -278,6 +420,8 @@ export class MissionExecutor {
             else if (stage.action) {
                 await this.executeSequentialStage(i, stage.action, actions, mission);
             }
+            // Update control server with latest state after each stage
+            this.updateControlServerState();
         }
     }
     getStageName(stage) {
@@ -303,6 +447,13 @@ export class MissionExecutor {
             stageName: actionName,
             totalStages: mission.pipeline.stages.length,
         });
+        // Emit stage.start event
+        this.eventEmitter?.emit('stage.start', {
+            stageIndex,
+            stageName: actionName,
+            totalStages: mission.pipeline.stages.length,
+            isParallel: false,
+        });
         try {
             await this.executeAction(action);
             this.actionsRun.push(action.name);
@@ -319,6 +470,12 @@ export class MissionExecutor {
                 success: true,
                 duration: Date.now() - stageStartTime,
             });
+            // Emit stage.complete event
+            this.eventEmitter?.emit('stage.complete', {
+                stageIndex,
+                stageName: actionName,
+                success: true,
+            });
         }
         catch (error) {
             // Mark stage as failed
@@ -338,6 +495,13 @@ export class MissionExecutor {
                 duration: Date.now() - stageStartTime,
                 error: error.message,
             });
+            // Emit stage.complete event (failure)
+            this.eventEmitter?.emit('stage.complete', {
+                stageIndex,
+                stageName: actionName,
+                success: false,
+                error: error.message,
+            });
             throw error; // Re-throw to stop execution
         }
     }
@@ -365,10 +529,18 @@ export class MissionExecutor {
             stageName,
             totalStages: mission.pipeline.stages.length,
         });
+        // Emit stage.start event (parallel)
+        this.eventEmitter?.emit('stage.start', {
+            stageIndex,
+            stageName,
+            totalStages: mission.pipeline.stages.length,
+            isParallel: true,
+            parallelActions: actionNames,
+        });
         this.log(`Executing parallel stage: ${stageName}`);
         try {
             // Execute all actions in parallel
-            const results = await Promise.allSettled(actionDefs.map(action => this.executeAction(action)));
+            const results = await Promise.allSettled(actionDefs.map((action) => this.executeAction(action)));
             // Check for failures
             const failures = [];
             for (let i = 0; i < results.length; i++) {
@@ -381,7 +553,7 @@ export class MissionExecutor {
                 }
             }
             if (failures.length > 0) {
-                const errorMsg = failures.map(f => `${f.name}: ${f.error.message}`).join('; ');
+                const errorMsg = failures.map((f) => `${f.name}: ${f.error.message}`).join('; ');
                 throw new Error(`Parallel stage failed: ${errorMsg}`);
             }
             // Mark stage as completed
@@ -397,6 +569,12 @@ export class MissionExecutor {
                 success: true,
                 duration: Date.now() - stageStartTime,
             });
+            // Emit stage.complete event (success)
+            this.eventEmitter?.emit('stage.complete', {
+                stageIndex,
+                stageName,
+                success: true,
+            });
         }
         catch (error) {
             // Mark stage as failed
@@ -416,154 +594,129 @@ export class MissionExecutor {
                 duration: Date.now() - stageStartTime,
                 error: error.message,
             });
-            throw error; // Re-throw to stop execution
-        }
-    }
-    async initializeSource(source) {
-        // Store source config for later reference
-        this.sourceConfigs.set(source.name, source);
-        const authConfig = this.config.auth?.[source.name];
-        let authProvider;
-        if (authConfig) {
-            if (authConfig.type === 'bearer' && authConfig.token) {
-                authProvider = new BearerAuthProvider(authConfig.token);
-            }
-            else if (authConfig.type === 'oauth2' && authConfig.accessToken) {
-                authProvider = new OAuth2AuthProvider({
-                    accessToken: authConfig.accessToken,
-                    refreshToken: authConfig.refreshToken,
-                    tokenEndpoint: authConfig.tokenEndpoint,
-                    clientId: authConfig.clientId,
-                    clientSecret: authConfig.clientSecret,
-                });
-            }
-        }
-        // If source has OAS spec, load it
-        let baseUrl = source.config.base;
-        if (source.specPath) {
-            try {
-                const oasSource = await loadOAS(source.specPath);
-                this.oasSources.set(source.name, oasSource);
-                // Use base URL from OAS if not explicitly provided
-                if (!baseUrl) {
-                    baseUrl = oasSource.baseUrl;
-                }
-                this.log(`Loaded OAS spec for ${source.name}: ${oasSource.operations.size} operations`);
-            }
-            catch (error) {
-                throw new Error(`Failed to load OAS spec for ${source.name}: ${error.message}`);
-            }
-        }
-        if (!baseUrl) {
-            throw new Error(`Source ${source.name} has no base URL (provide 'base' or OAS spec with servers)`);
-        }
-        // Configure rate limiter for this source
-        if (source.config.rateLimit) {
-            this.rateLimiter.configure(source.name, {
-                strategy: source.config.rateLimit.strategy,
-                maxWait: source.config.rateLimit.maxWait,
-                fallbackRpm: source.config.rateLimit.fallbackRpm,
-            });
-            this.log(`Rate limit config for ${source.name}: strategy=${source.config.rateLimit.strategy ?? 'pause'}, ` +
-                `maxWait=${source.config.rateLimit.maxWait ?? 300}s`);
-        }
-        // Configure circuit breaker for this source
-        if (source.config.circuitBreaker) {
-            this.circuitBreaker.configure(source.name, {
-                failureThreshold: source.config.circuitBreaker.failureThreshold,
-                // Convert seconds to milliseconds for the circuit breaker
-                resetTimeout: source.config.circuitBreaker.resetTimeout
-                    ? source.config.circuitBreaker.resetTimeout * 1000
-                    : undefined,
-                successThreshold: source.config.circuitBreaker.successThreshold,
-                failureWindow: source.config.circuitBreaker.failureWindow
-                    ? source.config.circuitBreaker.failureWindow * 1000
-                    : undefined,
+            // Emit stage.complete event (failure)
+            this.eventEmitter?.emit('stage.complete', {
+                stageIndex,
+                stageName,
+                success: false,
+                error: error.message,
             });
-            this.log(`Circuit breaker config for ${source.name}: ` +
-                `failureThreshold=${source.config.circuitBreaker.failureThreshold ?? 5}, ` +
-                `resetTimeout=${source.config.circuitBreaker.resetTimeout ?? 30}s`);
-        }
-        const client = new HttpClient({
-            baseUrl,
-            auth: authProvider,
-            rateLimiter: this.rateLimiter,
-            circuitBreaker: this.circuitBreaker,
-            sourceName: source.name,
-        });
-        this.ctx.sources.set(source.name, client);
-        this.log(`Initialized source: ${source.name}`);
-    }
-    async initializeStore(store) {
-        // Check for custom store adapter
-        if (this.config.stores?.[store.name]) {
-            this.ctx.stores.set(store.name, this.config.stores[store.name]);
-            this.log(`Initialized store: ${store.name} (custom adapter)`);
-            return;
+            throw error; // Re-throw to stop execution
         }
-        // Use store factory to create appropriate adapter
-        const developmentMode = this.config.developmentMode ?? true;
-        const storeType = resolveStoreType(store.storeType, developmentMode);
-        const adapter = createStore({
-            type: storeType,
-            name: store.target,
-            baseDir: this.config.dataDir,
-        });
-        this.ctx.stores.set(store.name, adapter);
-        this.log(`Initialized store: ${store.name} (${storeType}${storeType !== store.storeType ? ` <- ${store.storeType}` : ''})`);
     }
     async executeAction(action) {
         this.log(`Executing action: ${action.name}`);
+        // Create a child context for this action with its own response scope
+        // This allows parallel actions to have independent response values
+        const actionCtx = childContext(this.ctx);
         for (const step of action.steps) {
-            await this.executeStep(step, action.name);
+            await this.executeStep(step, action.name, actionCtx);
         }
     }
     async executeStep(step, actionName, ctx) {
         // Use provided context or default to this.ctx
+        // NOTE: ctx is used for action-scoped operations (response, variables)
+        // this.ctx is still used for mission-level resources (stores, sources)
         const execCtx = ctx ?? this.ctx;
-        const originalCtx = this.ctx;
-        // Temporarily use the provided context
-        if (ctx) {
-            this.ctx = ctx;
+        // Track step index for events
+        const currentStepIndex = this.stepIndex++;
+        const stepType = this.getStepType(step.type);
+        // Emit step.start event
+        this.eventEmitter?.emit('step.start', {
+            actionName,
+            stepIndex: currentStepIndex,
+            stepType,
+        });
+        const stepStartTime = Date.now();
+        // Record trace snapshot before step
+        if (this.traceRecorder) {
+            await this.traceRecorder.recordBeforeStep(actionName, currentStepIndex, stepType, execCtx);
+        }
+        // Debug pause point - before executing step
+        if (this.debugController) {
+            const location = {
+                action: actionName,
+                stepIndex: currentStepIndex,
+                stepType,
+            };
+            if (this.debugController.shouldPause(location)) {
+                const snapshot = this.captureDebugSnapshot(actionName, currentStepIndex, stepType, { type: 'step' }, execCtx);
+                const command = await this.debugController.pause(snapshot);
+                this.handleDebugCommand(command);
+            }
         }
         try {
             switch (step.type) {
                 case 'FetchStep':
-                    await this.executeFetch(step);
+                    await this.executeFetch(step, execCtx);
                     break;
                 case 'ForStep':
-                    await this.executeFor(step, actionName);
+                    await this.executeFor(step, actionName, execCtx);
                     break;
                 case 'MapStep':
-                    await this.executeMap(step);
+                    await this.executeMap(step, execCtx);
                     break;
                 case 'ValidateStep':
-                    await this.executeValidate(step);
+                    await this.executeValidate(step, execCtx);
                     break;
                 case 'StoreStep':
-                    await this.executeStore(step);
+                    await this.executeStore(step, execCtx);
                     break;
                 case 'MatchStep':
-                    await this.executeMatch(step, actionName);
+                    await this.executeMatch(step, actionName, execCtx);
                     break;
                 case 'LetStep':
-                    await this.executeLet(step);
+                    await this.executeLet(step, execCtx);
+                    break;
+                case 'ApplyStep':
+                    await this.executeApply(step, execCtx);
                     break;
                 case 'WebhookStep':
-                    await this.executeWebhook(step);
+                    await this.executeWebhook(step, execCtx);
+                    break;
+                case 'PauseStep':
+                    await this.executePause(step, actionName, currentStepIndex, execCtx);
                     break;
                 default:
                     throw new Error(`Unknown step type: ${step.type}`);
             }
+            const stepDuration = Date.now() - stepStartTime;
+            // Record trace snapshot after step
+            if (this.traceRecorder) {
+                await this.traceRecorder.recordAfterStep(actionName, currentStepIndex, stepType, execCtx, stepDuration);
+            }
+            // Emit step.complete event (success)
+            this.eventEmitter?.emit('step.complete', {
+                actionName,
+                stepIndex: currentStepIndex,
+                stepType,
+                success: true,
+            });
         }
         catch (error) {
             // Re-throw flow control signals without recording as errors
             if (error instanceof SkipSignal ||
                 error instanceof RetrySignal ||
                 error instanceof JumpSignal ||
-                error instanceof QueueSignal) {
+                error instanceof QueueSignal ||
+                error instanceof PauseSignal) {
+                // Emit step.complete for flow control (not an error)
+                this.eventEmitter?.emit('step.complete', {
+                    actionName,
+                    stepIndex: currentStepIndex,
+                    stepType,
+                    success: true, // Flow control is not a failure
+                });
                 throw error;
             }
+            // Emit step.complete event (failure)
+            this.eventEmitter?.emit('step.complete', {
+                actionName,
+                stepIndex: currentStepIndex,
+                stepType,
+                success: false,
+                error: error.message,
+            });
             // AbortError is a controlled abort, still record it
             this.errors.push({
                 action: actionName,
@@ -573,91 +726,267 @@ export class MissionExecutor {
             });
             throw error;
         }
-        finally {
-            // Restore original context
-            if (ctx) {
-                this.ctx = originalCtx;
-            }
-        }
     }
-    async executeFetch(step) {
+    async executeFetch(step, ctx) {
         const fetchHandler = new FetchHandler({
-            ctx: this.ctx,
-            oasSources: this.oasSources,
-            sourceConfigs: this.sourceConfigs,
+            ctx,
+            oasSources: this.sourceManager.getAllOASSources(),
+            sourceConfigs: this.sourceManager.getAllSourceConfigs(),
             syncStore: this.syncStore,
             missionName: this.missionName,
             executionId: this.executionState?.id,
             dryRun: this.config.dryRun,
             log: (msg) => this.log(msg),
+            emit: this.eventEmitter
+                ? (type, payload) => this.eventEmitter.emit(type, payload)
+                : undefined,
         });
         const result = await fetchHandler.execute(step);
-        this.ctx.response = result.data;
+        ctx.response = result.data;
         // Update sync checkpoint after successful fetch
         if (result.checkpointKey && this.syncStore) {
             await fetchHandler.recordCheckpoint(result.checkpointKey, step, result.data);
         }
     }
-    async executeFor(step, actionName) {
+    async executeFor(step, actionName, ctx) {
         const handler = new ForHandler({
-            ctx: this.ctx,
+            ctx,
             log: (msg) => this.log(msg),
+            emit: this.eventEmitter
+                ? (type, payload) => this.eventEmitter.emit(type, payload)
+                : undefined,
             executeStep: (s, a, c) => this.executeStep(s, a, c),
             actionName,
+            debugController: this.debugController,
+            captureDebugSnapshot: this.debugController
+                ? (action, stepIndex, stepType, pauseReason, ctx) => this.captureDebugSnapshot(action, stepIndex, stepType, pauseReason, ctx)
+                : undefined,
+            handleDebugCommand: this.debugController
+                ? (cmd) => this.handleDebugCommand(cmd)
+                : undefined,
+            checkPause: this.config.controlServer ? () => this.checkPause() : undefined,
         });
         await handler.execute(step);
     }
-    async executeMap(step) {
+    async executeMap(step, ctx) {
         const handler = new MapHandler({
-            ctx: this.ctx,
+            ctx,
             log: (msg) => this.log(msg),
+            emit: this.eventEmitter
+                ? (type, payload) => this.eventEmitter.emit(type, payload)
+                : undefined,
         });
         await handler.execute(step);
     }
-    async executeValidate(step) {
+    async executeValidate(step, ctx) {
         const handler = new ValidateHandler({
-            ctx: this.ctx,
+            ctx,
             log: (msg) => this.log(msg),
+            emit: this.eventEmitter
+                ? (type, payload) => this.eventEmitter.emit(type, payload)
+                : undefined,
         });
         await handler.execute(step);
     }
-    async executeStore(step) {
+    async executeStore(step, ctx) {
         const handler = new StoreHandler({
-            ctx: this.ctx,
+            ctx,
             log: (msg) => this.log(msg),
+            emit: this.eventEmitter
+                ? (type, payload) => this.eventEmitter.emit(type, payload)
+                : undefined,
         });
         await handler.execute(step);
     }
-    async executeMatch(step, actionName) {
+    async executeMatch(step, actionName, ctx) {
         const handler = new MatchHandler({
-            ctx: this.ctx,
+            ctx,
             log: (msg) => this.log(msg),
+            emit: this.eventEmitter
+                ? (type, payload) => this.eventEmitter.emit(type, payload)
+                : undefined,
             executeStep: (s, a, c) => this.executeStep(s, a, c),
             actionName,
+            debugController: this.debugController,
+            captureDebugSnapshot: this.debugController
+                ? (action, stepIndex, stepType, pauseReason, execCtx) => this.captureDebugSnapshot(action, stepIndex, stepType, pauseReason, execCtx)
+                : undefined,
+            handleDebugCommand: this.debugController
+                ? (cmd) => this.handleDebugCommand(cmd)
+                : undefined,
         });
         await handler.execute(step);
         // Flow control signals (SkipSignal, RetrySignal, etc.) will propagate up
     }
-    async executeLet(step) {
-        const value = evaluate(step.value, this.ctx);
-        setVariable(this.ctx, step.name, value);
+    async executeLet(step, ctx) {
+        const value = evaluate(step.value, ctx);
+        setVariable(ctx, step.name, value);
         this.log(`Set variable '${step.name}' = ${JSON.stringify(value)}`);
     }
-    async executeWebhook(step) {
+    async executeApply(step, ctx) {
+        const transform = this.transforms.get(step.transform);
+        if (!transform) {
+            throw new Error(`Transform '${step.transform}' not found`);
+        }
+        const handler = new ApplyHandler({
+            ctx,
+            log: (msg) => this.log(msg),
+            transform,
+        });
+        await handler.execute(step);
+    }
+    async executeWebhook(step, ctx) {
         if (!this.config.webhookServer) {
             throw new Error('Webhook server not configured. Use --webhook flag or configure webhookServer in executor config.');
         }
         const handler = new WebhookHandler({
-            ctx: this.ctx,
+            ctx,
             webhookServer: this.config.webhookServer,
             executionId: this.executionState?.id ?? 'ephemeral',
             log: (msg) => this.log(msg),
+            emit: this.eventEmitter
+                ? (type, payload) => this.eventEmitter.emit(type, payload)
+                : undefined,
         });
         await handler.execute(step);
     }
+    async executePause(step, actionName, stepIndex, ctx) {
+        if (!this.pauseManager) {
+            throw new Error('Pause manager not configured');
+        }
+        // Mark execution state as paused before creating pause
+        if (this.executionState) {
+            this.executionState.status = 'paused';
+            await this.saveExecutionState();
+        }
+        const handler = new PauseHandler({
+            ctx,
+            log: (msg) => this.log(msg),
+            emit: this.eventEmitter
+                ? (type, payload) => this.eventEmitter.emit(type, payload)
+                : undefined,
+            pauseManager: this.pauseManager,
+            executionId: this.executionState?.id ?? 'ephemeral',
+            mission: this.missionName ?? 'unknown',
+            actionName,
+            stageIndex: this.currentStageIndex,
+            stepIndex,
+        });
+        // This will throw PauseSignal
+        await handler.execute(step);
+    }
     log(message) {
-        if (this.config.verbose) {
+        if (this.logger) {
+            this.logger.info(message);
+        }
+        else if (this.config.verbose) {
             console.log(`[Reqon] ${message}`);
         }
     }
+    /**
+     * Check if pause has been requested and handle it
+     * Should be called at safe pause points (between stages, loop iterations)
+     */
+    async checkPause() {
+        if (!this.config.controlServer?.isPauseRequested()) {
+            return;
+        }
+        this.log('Pause requested - saving state and pausing execution');
+        // Save state as paused
+        if (this.executionState) {
+            this.executionState.status = 'paused';
+            await this.saveExecutionState();
+        }
+        // Clear the pause request (it's been handled)
+        this.config.controlServer.clearPauseRequest();
+        // Throw pause signal to stop execution
+        throw new PauseSignal();
+    }
+    /**
+     * Update control server with current state
+     */
+    updateControlServerState() {
+        if (this.config.controlServer && this.executionState) {
+            this.config.controlServer.updateState(this.executionState);
+        }
+    }
+    getStepType(stepType) {
+        const mapping = {
+            FetchStep: 'fetch',
+            ForStep: 'for',
+            MapStep: 'map',
+            ValidateStep: 'validate',
+            StoreStep: 'store',
+            MatchStep: 'match',
+            LetStep: 'let',
+            WebhookStep: 'webhook',
+            PauseStep: 'pause',
+        };
+        return mapping[stepType] ?? 'fetch';
+    }
+    /** Get the event emitter (for external access) */
+    getEventEmitter() {
+        return this.eventEmitter;
+    }
+    /** Get the structured logger (for external access) */
+    getLogger() {
+        return this.logger;
+    }
+    /** Get the debug controller (for external access) */
+    getDebugController() {
+        return this.debugController;
+    }
+    /** Capture current execution state for debugging */
+    captureDebugSnapshot(action, stepIndex, stepType, pauseReason, ctx) {
+        // Collect variables from context chain
+        const variables = {};
+        let current = ctx;
+        while (current) {
+            for (const [key, value] of current.variables) {
+                if (!(key in variables)) {
+                    variables[key] = value;
+                }
+            }
+            current = current.parent;
+        }
+        // Collect store info
+        const stores = {};
+        for (const [name, _store] of ctx.stores) {
+            stores[name] = {
+                type: ctx.storeTypes.get(name) ?? 'unknown',
+                count: -1, // Would need async call to get count
+            };
+        }
+        return {
+            mission: this.missionName ?? 'unknown',
+            action,
+            stepIndex,
+            stepType,
+            pauseReason,
+            variables,
+            stores,
+            response: ctx.response,
+        };
+    }
+    /** Handle debug command and update state */
+    handleDebugCommand(cmd) {
+        if (!this.debugController)
+            return;
+        switch (cmd.type) {
+            case 'abort':
+                throw new AbortError('Execution aborted by debugger');
+            case 'continue':
+                this.debugController.mode = 'run';
+                break;
+            case 'step':
+                this.debugController.mode = 'step';
+                break;
+            case 'step-into':
+                this.debugController.mode = 'step-into';
+                break;
+            case 'step-over':
+                this.debugController.mode = 'step-over';
+                break;
+        }
+    }
 }