reqon-dsl 0.2.0

package/dist/interpreter/executor.js

@@ -0,0 +1,663 @@
+import { isParallelStage } from '../ast/nodes.js';
+import { createContext, setVariable } from './context.js';
+import { evaluate } from './evaluator.js';
+import { HttpClient, BearerAuthProvider, OAuth2AuthProvider } from './http.js';
+import { createStore, resolveStoreType } from '../stores/index.js';
+import { loadOAS } from '../oas/index.js';
+import { AdaptiveRateLimiter } from '../auth/rate-limiter.js';
+import { CircuitBreaker } from '../auth/circuit-breaker.js';
+import { createExecutionState, findResumePoint, FileExecutionStore, } from '../execution/index.js';
+import { FileSyncStore, } from '../sync/index.js';
+import { FetchHandler } from './fetch-handler.js';
+import { ForHandler, MapHandler, ValidateHandler, StoreHandler, MatchHandler, WebhookHandler, SkipSignal, RetrySignal, JumpSignal, QueueSignal, } from './step-handlers/index.js';
+export class MissionExecutor {
+    config;
+    ctx;
+    errors = [];
+    actionsRun = [];
+    oasSources = new Map();
+    sourceConfigs = new Map();
+    rateLimiter;
+    circuitBreaker;
+    executionStore;
+    executionState;
+    syncStore;
+    missionName;
+    constructor(config = {}) {
+        this.config = config;
+        this.ctx = createContext();
+        this.rateLimiter = new AdaptiveRateLimiter();
+        this.circuitBreaker = new CircuitBreaker();
+        // Set up rate limit callbacks with default logging if verbose
+        const callbacks = config.rateLimitCallbacks ?? {};
+        if (config.verbose && !callbacks.onRateLimited) {
+            callbacks.onRateLimited = (event) => {
+                console.log(`[Reqon] Rate limited on ${event.source}${event.endpoint ? `:${event.endpoint}` : ''} - ` +
+                    `waiting ${event.waitSeconds}s (strategy: ${event.strategy})`);
+            };
+        }
+        if (config.verbose && !callbacks.onResumed) {
+            callbacks.onResumed = (event) => {
+                console.log(`[Reqon] Rate limit cleared for ${event.source}${event.endpoint ? `:${event.endpoint}` : ''} ` +
+                    `(waited ${event.waitedSeconds}s)`);
+            };
+        }
+        if (config.verbose && !callbacks.onWaiting) {
+            callbacks.onWaiting = (event) => {
+                console.log(`[Reqon] Still waiting for ${event.source}${event.endpoint ? `:${event.endpoint}` : ''} - ` +
+                    `${event.waitSeconds}s remaining (elapsed: ${event.elapsedSeconds}s)`);
+            };
+        }
+        this.rateLimiter.setCallbacks(callbacks);
+        // Set up circuit breaker callbacks with default logging if verbose
+        const cbCallbacks = config.circuitBreakerCallbacks ?? {};
+        if (config.verbose && !cbCallbacks.onOpen) {
+            cbCallbacks.onOpen = (event) => {
+                console.log(`[Reqon] Circuit breaker OPEN for ${event.source}${event.endpoint ? `:${event.endpoint}` : ''} - ` +
+                    `${event.failures} failures (${event.reason ?? 'threshold exceeded'})`);
+            };
+        }
+        if (config.verbose && !cbCallbacks.onHalfOpen) {
+            cbCallbacks.onHalfOpen = (event) => {
+                console.log(`[Reqon] Circuit breaker HALF-OPEN for ${event.source}${event.endpoint ? `:${event.endpoint}` : ''} - ` +
+                    `testing recovery`);
+            };
+        }
+        if (config.verbose && !cbCallbacks.onClose) {
+            cbCallbacks.onClose = (event) => {
+                console.log(`[Reqon] Circuit breaker CLOSED for ${event.source}${event.endpoint ? `:${event.endpoint}` : ''} - ` +
+                    `recovery successful`);
+            };
+        }
+        if (config.verbose && !cbCallbacks.onRejected) {
+            cbCallbacks.onRejected = (event) => {
+                console.log(`[Reqon] Request REJECTED by circuit breaker for ${event.source}${event.endpoint ? `:${event.endpoint}` : ''} - ` +
+                    `retry in ${Math.ceil(event.nextAttemptIn / 1000)}s`);
+            };
+        }
+        this.circuitBreaker.setCallbacks(cbCallbacks);
+        // Initialize execution store if persistence enabled
+        if (config.persistState) {
+            this.executionStore = config.executionStore ?? new FileExecutionStore(`${config.dataDir ?? '.reqon-data'}/executions`);
+        }
+    }
+    async execute(program) {
+        const startTime = Date.now();
+        // Find mission definition
+        const mission = program.statements.find((s) => s.type === 'MissionDefinition');
+        if (!mission) {
+            return {
+                success: false,
+                duration: Date.now() - startTime,
+                actionsRun: [],
+                errors: [{ action: '', step: '', message: 'No mission found in program' }],
+                stores: this.ctx.stores,
+            };
+        }
+        // Initialize or resume execution state
+        await this.initializeExecutionState(mission);
+        try {
+            await this.executeMission(mission);
+            // Mark execution as completed
+            if (this.executionState) {
+                this.executionState.status = 'completed';
+                this.executionState.completedAt = new Date();
+                this.executionState.duration = Date.now() - startTime;
+                await this.saveExecutionState();
+            }
+        }
+        catch (error) {
+            this.errors.push({
+                action: 'mission',
+                step: 'execute',
+                message: error.message,
+                details: error,
+            });
+            // Mark execution as failed
+            if (this.executionState) {
+                this.executionState.status = 'failed';
+                this.executionState.completedAt = new Date();
+                this.executionState.duration = Date.now() - startTime;
+                await this.saveExecutionState();
+            }
+        }
+        const duration = Date.now() - startTime;
+        const success = this.errors.length === 0;
+        // Emit onExecutionComplete callback - count stages in a single pass
+        const stageCounts = this.executionState?.stages.reduce((acc, s) => {
+            if (s.status === 'completed')
+                acc.completed++;
+            else if (s.status === 'failed')
+                acc.failed++;
+            return acc;
+        }, { completed: 0, failed: 0 });
+        const stagesCompleted = stageCounts?.completed ?? this.actionsRun.length;
+        const stagesFailed = stageCounts?.failed ?? (success ? 0 : 1);
+        this.config.progress?.onExecutionComplete?.({
+            executionId: this.executionState?.id ?? 'ephemeral',
+            mission: mission.name,
+            success,
+            duration,
+            stagesCompleted,
+            stagesFailed,
+            errors: this.errors,
+        });
+        return {
+            success,
+            duration,
+            actionsRun: this.actionsRun,
+            errors: this.errors,
+            stores: this.ctx.stores,
+            executionId: this.executionState?.id,
+            state: this.executionState,
+        };
+    }
+    async initializeExecutionState(mission) {
+        let isResume = false;
+        if (this.executionStore) {
+            // Resume from previous execution?
+            if (this.config.resumeFrom) {
+                const previous = await this.executionStore.load(this.config.resumeFrom);
+                if (previous) {
+                    this.executionState = previous;
+                    this.executionState.status = 'running';
+                    this.log(`Resuming execution ${previous.id} from previous run`);
+                    await this.saveExecutionState();
+                    isResume = true;
+                }
+                else {
+                    this.log(`Warning: Could not find execution ${this.config.resumeFrom} to resume`);
+                }
+            }
+            if (!this.executionState) {
+                // Create new execution state
+                const stages = mission.pipeline.stages.map((s) => this.getStageName(s));
+                this.executionState = createExecutionState({
+                    mission: mission.name,
+                    stages,
+                    metadata: this.config.metadata,
+                });
+                this.executionState.status = 'running';
+                await this.saveExecutionState();
+                this.log(`Started execution ${this.executionState.id}`);
+            }
+        }
+        // Emit onExecutionStart callback
+        this.config.progress?.onExecutionStart?.({
+            executionId: this.executionState?.id ?? 'ephemeral',
+            mission: mission.name,
+            stageCount: mission.pipeline.stages.length,
+            isResume,
+            metadata: this.config.metadata,
+        });
+    }
+    async saveExecutionState() {
+        if (this.executionStore && this.executionState) {
+            await this.executionStore.save(this.executionState);
+        }
+    }
+    updateStageState(stageIndex, updates) {
+        if (!this.executionState)
+            return;
+        const stage = this.executionState.stages[stageIndex];
+        if (!stage)
+            return;
+        if (updates.status === 'running' && !stage.startedAt) {
+            stage.startedAt = new Date();
+        }
+        if (updates.status === 'completed' || updates.status === 'failed') {
+            stage.completedAt = new Date();
+        }
+        if (updates.status) {
+            stage.status = updates.status;
+        }
+        if (updates.error) {
+            stage.error = updates.error;
+            this.executionState.errors.push({
+                stageIndex,
+                action: stage.action,
+                step: 'unknown',
+                message: updates.error,
+                timestamp: new Date(),
+                attempt: stage.attempt,
+            });
+        }
+    }
+    async executeMission(mission) {
+        this.log(`Executing mission: ${mission.name}`);
+        this.missionName = mission.name;
+        // Initialize sync store
+        this.syncStore = this.config.syncStore ?? new FileSyncStore(mission.name, `${this.config.dataDir ?? '.reqon-data'}/sync`);
+        // Initialize sources (HTTP clients)
+        for (const source of mission.sources) {
+            await this.initializeSource(source);
+        }
+        // Initialize stores
+        for (const store of mission.stores) {
+            await this.initializeStore(store);
+        }
+        // Initialize schemas (for match step schema matching)
+        for (const schema of mission.schemas) {
+            this.ctx.schemas.set(schema.name, schema);
+            this.log(`Registered schema: ${schema.name}`);
+        }
+        // Build action lookup
+        const actions = new Map();
+        for (const action of mission.actions) {
+            actions.set(action.name, action);
+        }
+        // Determine resume point
+        const resumeIndex = this.executionState ? findResumePoint(this.executionState) : 0;
+        if (resumeIndex > 0) {
+            const resumeStage = mission.pipeline.stages[resumeIndex];
+            const stageName = this.getStageName(resumeStage);
+            this.log(`Resuming from stage ${resumeIndex} (${stageName})`);
+        }
+        // Execute pipeline
+        for (let i = 0; i < mission.pipeline.stages.length; i++) {
+            const stage = mission.pipeline.stages[i];
+            // Skip already completed stages when resuming
+            if (i < resumeIndex) {
+                this.log(`Skipping ${this.getStageName(stage)} (already completed)`);
+                continue;
+            }
+            // Check condition if present
+            if (stage.condition) {
+                const shouldRun = evaluate(stage.condition, this.ctx);
+                if (!shouldRun) {
+                    this.log(`Skipping ${this.getStageName(stage)} (condition not met)`);
+                    this.updateStageState(i, { status: 'skipped' });
+                    await this.saveExecutionState();
+                    continue;
+                }
+            }
+            // Execute stage (parallel or sequential)
+            if (isParallelStage(stage)) {
+                await this.executeParallelStage(i, stage, actions, mission);
+            }
+            else if (stage.action) {
+                await this.executeSequentialStage(i, stage.action, actions, mission);
+            }
+        }
+    }
+    getStageName(stage) {
+        if (isParallelStage(stage)) {
+            return `[${stage.actions.join(', ')}]`;
+        }
+        return stage.action ?? 'unknown';
+    }
+    async executeSequentialStage(stageIndex, actionName, actions, mission) {
+        const action = actions.get(actionName);
+        if (!action) {
+            throw new Error(`Action not found: ${actionName}`);
+        }
+        // Update stage state to running
+        this.updateStageState(stageIndex, { status: 'running' });
+        await this.saveExecutionState();
+        const stageStartTime = Date.now();
+        // Emit onStageStart callback
+        this.config.progress?.onStageStart?.({
+            executionId: this.executionState?.id ?? 'ephemeral',
+            mission: mission.name,
+            stageIndex,
+            stageName: actionName,
+            totalStages: mission.pipeline.stages.length,
+        });
+        try {
+            await this.executeAction(action);
+            this.actionsRun.push(action.name);
+            // Mark stage as completed
+            this.updateStageState(stageIndex, { status: 'completed' });
+            await this.saveExecutionState();
+            // Emit onStageComplete callback (success)
+            this.config.progress?.onStageComplete?.({
+                executionId: this.executionState?.id ?? 'ephemeral',
+                mission: mission.name,
+                stageIndex,
+                stageName: actionName,
+                totalStages: mission.pipeline.stages.length,
+                success: true,
+                duration: Date.now() - stageStartTime,
+            });
+        }
+        catch (error) {
+            // Mark stage as failed
+            this.updateStageState(stageIndex, {
+                status: 'failed',
+                error: error.message,
+            });
+            await this.saveExecutionState();
+            // Emit onStageComplete callback (failure)
+            this.config.progress?.onStageComplete?.({
+                executionId: this.executionState?.id ?? 'ephemeral',
+                mission: mission.name,
+                stageIndex,
+                stageName: actionName,
+                totalStages: mission.pipeline.stages.length,
+                success: false,
+                duration: Date.now() - stageStartTime,
+                error: error.message,
+            });
+            throw error; // Re-throw to stop execution
+        }
+    }
+    async executeParallelStage(stageIndex, stage, actions, mission) {
+        const actionNames = stage.actions;
+        const stageName = `[${actionNames.join(', ')}]`;
+        // Validate all actions exist
+        const actionDefs = [];
+        for (const name of actionNames) {
+            const action = actions.get(name);
+            if (!action) {
+                throw new Error(`Action not found: ${name}`);
+            }
+            actionDefs.push(action);
+        }
+        // Update stage state to running
+        this.updateStageState(stageIndex, { status: 'running' });
+        await this.saveExecutionState();
+        const stageStartTime = Date.now();
+        // Emit onStageStart callback
+        this.config.progress?.onStageStart?.({
+            executionId: this.executionState?.id ?? 'ephemeral',
+            mission: mission.name,
+            stageIndex,
+            stageName,
+            totalStages: mission.pipeline.stages.length,
+        });
+        this.log(`Executing parallel stage: ${stageName}`);
+        try {
+            // Execute all actions in parallel
+            const results = await Promise.allSettled(actionDefs.map(action => this.executeAction(action)));
+            // Check for failures
+            const failures = [];
+            for (let i = 0; i < results.length; i++) {
+                const result = results[i];
+                if (result.status === 'fulfilled') {
+                    this.actionsRun.push(actionDefs[i].name);
+                }
+                else {
+                    failures.push({ name: actionDefs[i].name, error: result.reason });
+                }
+            }
+            if (failures.length > 0) {
+                const errorMsg = failures.map(f => `${f.name}: ${f.error.message}`).join('; ');
+                throw new Error(`Parallel stage failed: ${errorMsg}`);
+            }
+            // Mark stage as completed
+            this.updateStageState(stageIndex, { status: 'completed' });
+            await this.saveExecutionState();
+            // Emit onStageComplete callback (success)
+            this.config.progress?.onStageComplete?.({
+                executionId: this.executionState?.id ?? 'ephemeral',
+                mission: mission.name,
+                stageIndex,
+                stageName,
+                totalStages: mission.pipeline.stages.length,
+                success: true,
+                duration: Date.now() - stageStartTime,
+            });
+        }
+        catch (error) {
+            // Mark stage as failed
+            this.updateStageState(stageIndex, {
+                status: 'failed',
+                error: error.message,
+            });
+            await this.saveExecutionState();
+            // Emit onStageComplete callback (failure)
+            this.config.progress?.onStageComplete?.({
+                executionId: this.executionState?.id ?? 'ephemeral',
+                mission: mission.name,
+                stageIndex,
+                stageName,
+                totalStages: mission.pipeline.stages.length,
+                success: false,
+                duration: Date.now() - stageStartTime,
+                error: error.message,
+            });
+            throw error; // Re-throw to stop execution
+        }
+    }
+    async initializeSource(source) {
+        // Store source config for later reference
+        this.sourceConfigs.set(source.name, source);
+        const authConfig = this.config.auth?.[source.name];
+        let authProvider;
+        if (authConfig) {
+            if (authConfig.type === 'bearer' && authConfig.token) {
+                authProvider = new BearerAuthProvider(authConfig.token);
+            }
+            else if (authConfig.type === 'oauth2' && authConfig.accessToken) {
+                authProvider = new OAuth2AuthProvider({
+                    accessToken: authConfig.accessToken,
+                    refreshToken: authConfig.refreshToken,
+                    tokenEndpoint: authConfig.tokenEndpoint,
+                    clientId: authConfig.clientId,
+                    clientSecret: authConfig.clientSecret,
+                });
+            }
+        }
+        // If source has OAS spec, load it
+        let baseUrl = source.config.base;
+        if (source.specPath) {
+            try {
+                const oasSource = await loadOAS(source.specPath);
+                this.oasSources.set(source.name, oasSource);
+                // Use base URL from OAS if not explicitly provided
+                if (!baseUrl) {
+                    baseUrl = oasSource.baseUrl;
+                }
+                this.log(`Loaded OAS spec for ${source.name}: ${oasSource.operations.size} operations`);
+            }
+            catch (error) {
+                throw new Error(`Failed to load OAS spec for ${source.name}: ${error.message}`);
+            }
+        }
+        if (!baseUrl) {
+            throw new Error(`Source ${source.name} has no base URL (provide 'base' or OAS spec with servers)`);
+        }
+        // Configure rate limiter for this source
+        if (source.config.rateLimit) {
+            this.rateLimiter.configure(source.name, {
+                strategy: source.config.rateLimit.strategy,
+                maxWait: source.config.rateLimit.maxWait,
+                fallbackRpm: source.config.rateLimit.fallbackRpm,
+            });
+            this.log(`Rate limit config for ${source.name}: strategy=${source.config.rateLimit.strategy ?? 'pause'}, ` +
+                `maxWait=${source.config.rateLimit.maxWait ?? 300}s`);
+        }
+        // Configure circuit breaker for this source
+        if (source.config.circuitBreaker) {
+            this.circuitBreaker.configure(source.name, {
+                failureThreshold: source.config.circuitBreaker.failureThreshold,
+                // Convert seconds to milliseconds for the circuit breaker
+                resetTimeout: source.config.circuitBreaker.resetTimeout
+                    ? source.config.circuitBreaker.resetTimeout * 1000
+                    : undefined,
+                successThreshold: source.config.circuitBreaker.successThreshold,
+                failureWindow: source.config.circuitBreaker.failureWindow
+                    ? source.config.circuitBreaker.failureWindow * 1000
+                    : undefined,
+            });
+            this.log(`Circuit breaker config for ${source.name}: ` +
+                `failureThreshold=${source.config.circuitBreaker.failureThreshold ?? 5}, ` +
+                `resetTimeout=${source.config.circuitBreaker.resetTimeout ?? 30}s`);
+        }
+        const client = new HttpClient({
+            baseUrl,
+            auth: authProvider,
+            rateLimiter: this.rateLimiter,
+            circuitBreaker: this.circuitBreaker,
+            sourceName: source.name,
+        });
+        this.ctx.sources.set(source.name, client);
+        this.log(`Initialized source: ${source.name}`);
+    }
+    async initializeStore(store) {
+        // Check for custom store adapter
+        if (this.config.stores?.[store.name]) {
+            this.ctx.stores.set(store.name, this.config.stores[store.name]);
+            this.log(`Initialized store: ${store.name} (custom adapter)`);
+            return;
+        }
+        // Use store factory to create appropriate adapter
+        const developmentMode = this.config.developmentMode ?? true;
+        const storeType = resolveStoreType(store.storeType, developmentMode);
+        const adapter = createStore({
+            type: storeType,
+            name: store.target,
+            baseDir: this.config.dataDir,
+        });
+        this.ctx.stores.set(store.name, adapter);
+        this.log(`Initialized store: ${store.name} (${storeType}${storeType !== store.storeType ? ` <- ${store.storeType}` : ''})`);
+    }
+    async executeAction(action) {
+        this.log(`Executing action: ${action.name}`);
+        for (const step of action.steps) {
+            await this.executeStep(step, action.name);
+        }
+    }
+    async executeStep(step, actionName, ctx) {
+        // Use provided context or default to this.ctx
+        const execCtx = ctx ?? this.ctx;
+        const originalCtx = this.ctx;
+        // Temporarily use the provided context
+        if (ctx) {
+            this.ctx = ctx;
+        }
+        try {
+            switch (step.type) {
+                case 'FetchStep':
+                    await this.executeFetch(step);
+                    break;
+                case 'ForStep':
+                    await this.executeFor(step, actionName);
+                    break;
+                case 'MapStep':
+                    await this.executeMap(step);
+                    break;
+                case 'ValidateStep':
+                    await this.executeValidate(step);
+                    break;
+                case 'StoreStep':
+                    await this.executeStore(step);
+                    break;
+                case 'MatchStep':
+                    await this.executeMatch(step, actionName);
+                    break;
+                case 'LetStep':
+                    await this.executeLet(step);
+                    break;
+                case 'WebhookStep':
+                    await this.executeWebhook(step);
+                    break;
+                default:
+                    throw new Error(`Unknown step type: ${step.type}`);
+            }
+        }
+        catch (error) {
+            // Re-throw flow control signals without recording as errors
+            if (error instanceof SkipSignal ||
+                error instanceof RetrySignal ||
+                error instanceof JumpSignal ||
+                error instanceof QueueSignal) {
+                throw error;
+            }
+            // AbortError is a controlled abort, still record it
+            this.errors.push({
+                action: actionName,
+                step: step.type,
+                message: error.message,
+                details: error,
+            });
+            throw error;
+        }
+        finally {
+            // Restore original context
+            if (ctx) {
+                this.ctx = originalCtx;
+            }
+        }
+    }
+    async executeFetch(step) {
+        const fetchHandler = new FetchHandler({
+            ctx: this.ctx,
+            oasSources: this.oasSources,
+            sourceConfigs: this.sourceConfigs,
+            syncStore: this.syncStore,
+            missionName: this.missionName,
+            executionId: this.executionState?.id,
+            dryRun: this.config.dryRun,
+            log: (msg) => this.log(msg),
+        });
+        const result = await fetchHandler.execute(step);
+        this.ctx.response = result.data;
+        // Update sync checkpoint after successful fetch
+        if (result.checkpointKey && this.syncStore) {
+            await fetchHandler.recordCheckpoint(result.checkpointKey, step, result.data);
+        }
+    }
+    async executeFor(step, actionName) {
+        const handler = new ForHandler({
+            ctx: this.ctx,
+            log: (msg) => this.log(msg),
+            executeStep: (s, a, c) => this.executeStep(s, a, c),
+            actionName,
+        });
+        await handler.execute(step);
+    }
+    async executeMap(step) {
+        const handler = new MapHandler({
+            ctx: this.ctx,
+            log: (msg) => this.log(msg),
+        });
+        await handler.execute(step);
+    }
+    async executeValidate(step) {
+        const handler = new ValidateHandler({
+            ctx: this.ctx,
+            log: (msg) => this.log(msg),
+        });
+        await handler.execute(step);
+    }
+    async executeStore(step) {
+        const handler = new StoreHandler({
+            ctx: this.ctx,
+            log: (msg) => this.log(msg),
+        });
+        await handler.execute(step);
+    }
+    async executeMatch(step, actionName) {
+        const handler = new MatchHandler({
+            ctx: this.ctx,
+            log: (msg) => this.log(msg),
+            executeStep: (s, a, c) => this.executeStep(s, a, c),
+            actionName,
+        });
+        await handler.execute(step);
+        // Flow control signals (SkipSignal, RetrySignal, etc.) will propagate up
+    }
+    async executeLet(step) {
+        const value = evaluate(step.value, this.ctx);
+        setVariable(this.ctx, step.name, value);
+        this.log(`Set variable '${step.name}' = ${JSON.stringify(value)}`);
+    }
+    async executeWebhook(step) {
+        if (!this.config.webhookServer) {
+            throw new Error('Webhook server not configured. Use --webhook flag or configure webhookServer in executor config.');
+        }
+        const handler = new WebhookHandler({
+            ctx: this.ctx,
+            webhookServer: this.config.webhookServer,
+            executionId: this.executionState?.id ?? 'ephemeral',
+            log: (msg) => this.log(msg),
+        });
+        await handler.execute(step);
+    }
+    log(message) {
+        if (this.config.verbose) {
+            console.log(`[Reqon] ${message}`);
+        }
+    }
+}