reqon-dsl 0.2.0 → 0.4.0

package/dist/interpreter/evaluator.js

@@ -1,15 +1,63 @@
+/**
+ * ---
+ * purpose: Expression evaluator - evaluates Vague/Reqon expressions at runtime
+ * inputs:
+ *   - Expression AST node (literals, identifiers, binary ops, function calls)
+ *   - ExecutionContext with variables, response, stores
+ *   - Optional current record for iteration contexts
+ * outputs:
+ *   - Evaluated value (any JSON-compatible type)
+ * related:
+ *   - ./context.ts - provides variable bindings
+ *   - ./executor.ts - calls evaluate() for step expressions
+ *   - ../parser/expressions.ts - IsExpression, ObjectLiteralExpression types
+ * ---
+ */
 import { getVariable } from './context.js';
+import { isRecord } from '../utils/type-guards.js';
+import { EvaluatorError, UnsupportedOperationError } from '../errors/index.js';
+/**
+ * Evaluate a Reqon/Vague expression within an execution context.
+ *
+ * Supports all expression types from the Vague DSL including literals, identifiers,
+ * binary/logical/ternary expressions, function calls, and pattern matching.
+ *
+ * @param expr - The expression AST node to evaluate
+ * @param ctx - The execution context containing variables, response data, and stores
+ * @param current - Optional current record for iteration contexts (e.g., inside for/map)
+ * @returns The evaluated result, which can be any JSON-compatible value
+ *
+ * @example
+ * // Evaluate a simple identifier
+ * const result = evaluate({ type: 'Identifier', name: 'foo' }, ctx);
+ *
+ * @example
+ * // Evaluate with current record context
+ * const result = evaluate(expr, ctx, { id: 1, name: 'test' });
+ */
 export function evaluate(expr, ctx, current) {
+    // Handle IsExpression before the switch (custom Reqon type, not in vague-lang Expression union)
+    if (expr.type === 'IsExpression') {
+        const isExpr = expr;
+        const value = evaluate(isExpr.operand, ctx, current);
+        return checkType(value, isExpr.typeCheck);
+    }
+    // Handle ObjectLiteral before the switch (custom Reqon type)
+    if (expr.type === 'ObjectLiteral') {
+        const objExpr = expr;
+        const result = {};
+        for (const prop of objExpr.properties) {
+            result[prop.key] = evaluate(prop.value, ctx, current);
+        }
+        return result;
+    }
     switch (expr.type) {
         case 'Literal':
             return expr.value;
         case 'Identifier': {
             // Check if it's a field on current object
-            if (current && typeof current === 'object' && current !== null) {
-                const obj = current;
-                if (expr.name in obj) {
-                    return obj[expr.name];
-                }
+            if (isRecord(current) && expr.name in current) {
+                return current[expr.name];
             }
             // Check variables
             const value = getVariable(ctx, expr.name);
@@ -20,20 +68,31 @@ export function evaluate(expr, ctx, current) {
                 return ctx.response;
             }
             // Check if the identifier is a field on response
-            if (ctx.response && typeof ctx.response === 'object') {
-                const resp = ctx.response;
-                if (expr.name in resp)
-                    return resp[expr.name];
+            if (isRecord(ctx.response) && expr.name in ctx.response) {
+                return ctx.response[expr.name];
             }
             return undefined;
         }
         case 'QualifiedName': {
-            // Try from current/response first (common case)
+            // Handle 'response.x.y' specially - start from ctx.response
+            if (expr.parts.length > 0 && expr.parts[0] === 'response') {
+                let value = ctx.response;
+                for (let i = 1; i < expr.parts.length; i++) {
+                    if (isRecord(value)) {
+                        value = value[expr.parts[i]];
+                    }
+                    else {
+                        return undefined;
+                    }
+                }
+                return value;
+            }
+            // Try from current/response first (common case for .field paths)
             let value = current ?? ctx.response;
-            if (value && typeof value === 'object' && value !== null) {
+            if (isRecord(value)) {
                 let found = true;
                 for (const part of expr.parts) {
-                    if (value && typeof value === 'object' && value !== null) {
+                    if (isRecord(value)) {
                         value = value[part];
                     }
                     else {
@@ -50,7 +109,7 @@ export function evaluate(expr, ctx, current) {
                 value = getVariable(ctx, expr.parts[0]);
                 if (value !== undefined) {
                     for (let i = 1; i < expr.parts.length; i++) {
-                        if (value && typeof value === 'object') {
+                        if (isRecord(value)) {
                             value = value[expr.parts[i]];
                         }
                         else {
@@ -67,36 +126,48 @@ export function evaluate(expr, ctx, current) {
             const right = evaluate(expr.right, ctx, current);
             switch (expr.operator) {
                 case '+':
-                    return left + right;
+                    // Allow string concatenation
+                    if (typeof left === 'string' || typeof right === 'string') {
+                        return String(left ?? '') + String(right ?? '');
+                    }
+                    return toNumber(left, '+') + toNumber(right, '+');
                 case '-':
-                    return left - right;
+                    return toNumber(left, '-') - toNumber(right, '-');
                 case '*':
-                    return left * right;
-                case '/':
-                    return left / right;
+                    return toNumber(left, '*') * toNumber(right, '*');
+                case '/': {
+                    const divisor = toNumber(right, '/');
+                    if (divisor === 0) {
+                        throw new EvaluatorError('Division by zero', { expression: '/' });
+                    }
+                    return toNumber(left, '/') / divisor;
+                }
                 case '==':
                     return left === right;
                 case '!=':
                     return left !== right;
                 case '<':
-                    return left < right;
+                    return compareNumbers(left, right, '<');
                 case '>':
-                    return left > right;
+                    return compareNumbers(left, right, '>');
                 case '<=':
-                    return left <= right;
+                    return compareNumbers(left, right, '<=');
                 case '>=':
-                    return left >= right;
+                    return compareNumbers(left, right, '>=');
                 default:
-                    throw new Error(`Unknown operator: ${expr.operator}`);
+                    throw new UnsupportedOperationError(`operator: ${expr.operator}`, 'binary expression');
             }
         }
         case 'LogicalExpression': {
             const left = evaluate(expr.left, ctx, current);
+            // Return operand values (like JS &&/||), not coerced booleans, so
+            // idioms like `x or default` work. Short-circuit: only evaluate the
+            // right operand when the left doesn't already decide the result.
             if (expr.operator === 'and') {
-                return left ? evaluate(expr.right, ctx, current) : false;
+                return left ? evaluate(expr.right, ctx, current) : left;
             }
             else {
-                return left ? true : evaluate(expr.right, ctx, current);
+                return left ? left : evaluate(expr.right, ctx, current);
             }
         }
         case 'NotExpression':
@@ -104,9 +175,9 @@ export function evaluate(expr, ctx, current) {
         case 'UnaryExpression': {
             const operand = evaluate(expr.operand, ctx, current);
             if (expr.operator === '-')
-                return -operand;
+                return -toNumber(operand, 'unary -');
             if (expr.operator === '+')
-                return +operand;
+                return toNumber(operand, 'unary +');
             return operand;
         }
         case 'TernaryExpression': {
@@ -139,8 +210,12 @@ export function evaluate(expr, ctx, current) {
                     if (typeof args[0] === 'string')
                         return args[0].length;
                     return 0;
-                case 'sum':
-                    return args[0].reduce((a, b) => a + b, 0);
+                case 'sum': {
+                    if (!Array.isArray(args[0])) {
+                        throw new EvaluatorError('sum() requires an array argument', { expression: 'sum' });
+                    }
+                    return args[0].reduce((acc, val) => acc + toNumber(val, 'sum'), 0);
+                }
                 case 'count':
                     return Array.isArray(args[0]) ? args[0].length : 0;
                 case 'first':
@@ -148,13 +223,25 @@ export function evaluate(expr, ctx, current) {
                 case 'last':
                     return Array.isArray(args[0]) ? args[0][args[0].length - 1] : undefined;
                 case 'round':
-                    return Math.round(args[0]);
+                    return Math.round(toNumber(args[0], 'round'));
                 case 'floor':
-                    return Math.floor(args[0]);
+                    return Math.floor(toNumber(args[0], 'floor'));
                 case 'ceil':
-                    return Math.ceil(args[0]);
+                    return Math.ceil(toNumber(args[0], 'ceil'));
+                case 'now':
+                    return new Date().toISOString();
+                case 'env': {
+                    // The variable name must be a static string literal. A dynamic argument
+                    // (e.g. env(response.field)) would let fetched/untrusted data choose
+                    // which environment variable is read, exfiltrating secrets.
+                    const nameExpr = expr.arguments[0];
+                    if (!nameExpr || nameExpr.type !== 'Literal') {
+                        throw new EvaluatorError(`env() requires a string-literal variable name, not a dynamic expression`, { expression: 'env' });
+                    }
+                    return process.env[String(args[0])] ?? '';
+                }
                 default:
-                    throw new Error(`Unknown function: ${expr.callee}`);
+                    throw new UnsupportedOperationError(`function: ${expr.callee}`, 'call expression');
             }
         }
         case 'AnyOfExpression': {
@@ -166,36 +253,126 @@ export function evaluate(expr, ctx, current) {
             }
             return collection[Math.floor(Math.random() * collection.length)];
         }
-        case 'IsExpression': {
-            // Type checking expression: value is array, value is string, etc.
-            const isExpr = expr;
-            const value = evaluate(isExpr.operand, ctx, current);
-            return checkType(value, isExpr.typeCheck);
+        case 'OrderedSequenceType': {
+            // Inline array literal: [1, 2, 3]
+            const seqExpr = expr;
+            return seqExpr.elements.map((el) => evaluate(el, ctx, current));
         }
         default:
-            throw new Error(`Cannot evaluate expression type: ${expr.type}`);
+            throw new EvaluatorError(`Cannot evaluate expression type: ${expr.type}`, {
+                expression: expr.type,
+            });
     }
 }
+/**
+ * Evaluate an expression and convert the result to a string.
+ *
+ * @param expr - The expression to evaluate
+ * @param ctx - The execution context
+ * @param current - Optional current record for iteration contexts
+ * @returns The string representation of the evaluated value
+ */
 export function evaluateToString(expr, ctx, current) {
     const value = evaluate(expr, ctx, current);
     return String(value ?? '');
 }
+/**
+ * Interpolate variables in a path string using {variable.path} syntax.
+ *
+ * Replaces placeholders like {id} or {user.name} with values from the
+ * current record or execution context variables.
+ *
+ * @param path - The path string with {placeholder} syntax
+ * @param ctx - The execution context
+ * @param current - Optional current record for iteration contexts
+ * @returns The interpolated path string
+ *
+ * @example
+ * // Interpolate a simple variable
+ * interpolatePath('/users/{id}', ctx, { id: 123 }); // '/users/123'
+ *
+ * @example
+ * // Interpolate a nested path
+ * interpolatePath('/orgs/{org.id}/users', ctx, { org: { id: 'acme' } }); // '/orgs/acme/users'
+ */
 export function interpolatePath(path, ctx, current) {
     return path.replace(/\{([^}]+)\}/g, (_, expr) => {
         // Simple variable interpolation
         const parts = expr.split('.');
         let value = current;
         for (const part of parts) {
-            if (value && typeof value === 'object') {
+            if (isRecord(value)) {
                 value = value[part];
             }
             else {
                 value = getVariable(ctx, part);
             }
         }
-        return String(value ?? '');
+        // URL-encode each interpolated value so a path param can't inject path
+        // segments, query strings, or fragments into the request target. Path params
+        // are single segments, so encoding (which escapes '/', '?', '#', etc.) is the
+        // correct treatment.
+        return encodeURIComponent(String(value ?? ''));
+    });
+}
+/**
+ * Coerce a value to a number for arithmetic operations.
+ * Returns NaN for values that cannot be meaningfully converted.
+ */
+function toNumber(value, operator) {
+    if (typeof value === 'number') {
+        return value;
+    }
+    if (typeof value === 'string') {
+        const parsed = Number(value);
+        if (!isNaN(parsed)) {
+            return parsed;
+        }
+    }
+    if (typeof value === 'boolean') {
+        return value ? 1 : 0;
+    }
+    if (value === null || value === undefined) {
+        throw new EvaluatorError(`Cannot perform '${operator}' on ${value === null ? 'null' : 'undefined'}`, { expression: operator });
+    }
+    throw new EvaluatorError(`Cannot perform '${operator}' on ${typeof value} (expected number)`, {
+        expression: operator,
     });
 }
+/** Coerce for comparison, yielding NaN (not a throw) for nullish/non-numeric. */
+function toComparable(value) {
+    if (typeof value === 'number')
+        return value;
+    if (typeof value === 'boolean')
+        return value ? 1 : 0;
+    if (typeof value === 'string')
+        return Number(value);
+    return NaN;
+}
+/**
+ * Arithmetic comparison for `where`/`match`/`validate` guards. A missing or
+ * non-numeric operand makes the comparison false (the record is excluded) rather
+ * than throwing — real API data routinely omits fields, and one missing field
+ * shouldn't abort the whole stage.
+ */
+function compareNumbers(left, right, operator) {
+    const leftNum = toComparable(left);
+    const rightNum = toComparable(right);
+    if (Number.isNaN(leftNum) || Number.isNaN(rightNum))
+        return false;
+    switch (operator) {
+        case '<':
+            return leftNum < rightNum;
+        case '>':
+            return leftNum > rightNum;
+        case '<=':
+            return leftNum <= rightNum;
+        case '>=':
+            return leftNum >= rightNum;
+        default:
+            return false;
+    }
+}
 /** Type check functions map - more efficient than switch statement */
 const TYPE_CHECKERS = new Map([
     ['array', (v) => Array.isArray(v)],
@@ -217,7 +394,7 @@ const TYPE_CHECKERS = new Map([
 function checkType(value, typeName) {
     const checker = TYPE_CHECKERS.get(typeName.toLowerCase());
     if (!checker) {
-        throw new Error(`Unknown type for 'is' check: ${typeName}`);
+        throw new UnsupportedOperationError(`type check: ${typeName}`, "'is' expression");
     }
     return checker(value);
 }

package/dist/interpreter/executor.d.ts

@@ -1,10 +1,33 @@
+/**
+ * ---
+ * purpose: Mission executor - orchestrates pipeline execution
+ * inputs:
+ *   - ReqonProgram - parsed AST
+ *   - ExecutorConfig - auth, stores, callbacks, debug settings
+ * outputs:
+ *   - ExecutionResult - success/errors, stores, duration
+ * related:
+ *   - ./context.ts - execution state (variables, stores, sources)
+ *   - ./evaluator.ts - expression evaluation
+ *   - ./fetch-handler.ts - HTTP requests
+ *   - ./step-handlers/ - individual step type handlers
+ *   - ./source-manager.ts - auth provider management
+ * ---
+ */
 import type { ReqonProgram } from '../ast/nodes.js';
 import type { StoreAdapter } from '../stores/types.js';
+import { type AuthConfig } from './source-manager.js';
 import { type CircuitBreakerCallbacks } from '../auth/circuit-breaker.js';
 import type { RateLimitCallbacks } from '../auth/types.js';
 import { type ExecutionState, type ExecutionStore } from '../execution/index.js';
 import { type SyncStore } from '../sync/index.js';
 import type { WebhookServer } from '../webhook/index.js';
+import type { EventEmitter, StructuredLogger } from '../observability/index.js';
+import type { DebugController } from '../debug/index.js';
+import type { ControlServer } from '../control/index.js';
+import { type TraceStore } from '../trace/index.js';
+import { type PauseManager, type PauseStore } from '../pause/index.js';
+import type { ExecutionLogStore } from '../execution-log/index.js';
 export interface ExecutionResult {
     success: boolean;
     duration: number;
@@ -15,6 +38,10 @@ export interface ExecutionResult {
     executionId?: string;
     /** Execution state (if persistence enabled) */
     state?: ExecutionState;
+    /** Trace ID if tracing was enabled */
+    traceId?: string;
+    /** Pause ID if execution was paused */
+    pauseId?: string;
 }
 export interface ExecutionError {
     action: string;
@@ -71,6 +98,7 @@ export interface ExecutorConfig {
     stores?: Record<string, StoreAdapter>;
     dryRun?: boolean;
     verbose?: boolean;
+    missionDir?: string;
     rateLimitCallbacks?: RateLimitCallbacks;
     circuitBreakerCallbacks?: CircuitBreakerCallbacks;
     developmentMode?: boolean;
@@ -82,29 +110,56 @@ export interface ExecutorConfig {
     syncStore?: SyncStore;
     progress?: ProgressCallbacks;
     webhookServer?: WebhookServer;
+    eventEmitter?: EventEmitter;
+    logger?: StructuredLogger;
+    debugController?: DebugController;
+    controlServer?: ControlServer;
+    traceStore?: TraceStore;
+    pauseStore?: PauseStore;
+    pauseManager?: PauseManager;
+    executionLog?: ExecutionLogStore;
+    /**
+     * Max items a single run of a `backfill` paginated fetch accumulates before
+     * stopping cleanly (a later resume continues). Bounds memory per run for large
+     * backfills. Defaults to the handler's built-in cap.
+     */
+    backfillMaxItemsPerRun?: number;
 }
-interface AuthConfig {
-    type: 'bearer' | 'oauth2' | 'none';
-    token?: string;
-    accessToken?: string;
-    refreshToken?: string;
-    tokenEndpoint?: string;
-    clientId?: string;
-    clientSecret?: string;
-}
+export { type AuthConfig } from './source-manager.js';
 export declare class MissionExecutor {
     private config;
     private ctx;
     private errors;
     private actionsRun;
-    private oasSources;
-    private sourceConfigs;
+    /** Monotonic key generator for queued values lacking an id. */
+    private queueCounter;
+    private transforms;
     private rateLimiter;
     private circuitBreaker;
+    private sourceManager;
+    private storeManager;
     private executionStore?;
     private executionState?;
     private syncStore?;
     private missionName?;
+    private eventEmitter?;
+    private logger?;
+    private executionLog?;
+    /** Stable id used for the execution event log (independent of persistState). */
+    private logExecutionId?;
+    /** Effect ids already applied (from the log) — replay skips these. */
+    private appliedEffects;
+    /** Backfill page progress per step id (from the log) — resumes pagination. */
+    private pageProgress;
+    /** The pause being resumed on this run — its step replays past, not into, a pause. */
+    private resumingPause?;
+    private debugController?;
+    private traceRecorder?;
+    private traceStore?;
+    private pauseManager?;
+    private pauseStore?;
+    private currentStageIndex;
+    private currentPauseId?;
     constructor(config?: ExecutorConfig);
     execute(program: ReqonProgram): Promise<ExecutionResult>;
     private initializeExecutionState;
@@ -113,19 +168,82 @@ export declare class MissionExecutor {
     private executeMission;
     private getStageName;
     private executeSequentialStage;
+    /**
+     * Run all settled-style tasks with a bounded number in flight at once,
+     * preserving result order. Caps fan-out so a wide `run [...]` can't open an
+     * unbounded number of concurrent HTTP/store operations.
+     */
+    private settleWithLimit;
+    /**
+     * Execute a `run [A, B, ...]` stage.
+     *
+     * Failure semantics are **complete-then-fail**: every branch runs to
+     * completion (bounded by MAX_PARALLEL_ACTIONS in flight), then the stage
+     * fails if any branch failed. There is no cancellation of siblings and no
+     * rollback — a branch that committed store writes keeps them even if another
+     * branch failed. Each branch gets its own action scope (step counter +
+     * checkpoints); stores/sources/schemas are shared, so parallel branches that
+     * write the same key get last-writer-wins and should target disjoint keys.
+     */
     private executeParallelStage;
-    private initializeSource;
-    private initializeStore;
     private executeAction;
+    /** Compute a retry backoff delay from a RetrySignal's backoff config. */
+    private computeRetryDelay;
+    /** Push a queued value to its target store (queue directive). */
+    private handleQueue;
     private executeStep;
     private executeFetch;
+    /**
+     * Per-action mutable scope (step counter + deferred checkpoints). Lazily
+     * created so a step run with the bare mission context still works; normally
+     * executeAction installs a fresh scope that nested scopes inherit.
+     */
+    private scopeFor;
+    /**
+     * Append an event to the execution log. No-op (zero cost) when no log is
+     * configured. The executionId is supplied from this run's stable log id.
+     */
+    private logEvent;
+    /** Flush deferred sync checkpoints (called after a successful store / action). */
+    private flushPendingCheckpoints;
     private executeFor;
     private executeMap;
     private executeValidate;
     private executeStore;
+    /**
+     * A stable hash of the payload a store step will write, used to make the
+     * store-effect identity content-aware. Resolving the source is a pure read of
+     * the context (the same value the handler stores); on any evaluation failure we
+     * fall back to a constant so behaviour matches the old target-only identity
+     * rather than throwing inside the dedup path.
+     */
+    /** Load a pause's checkpoint into the shape the pause step uses to resume. */
+    private loadResumingPause;
+    private storeContentHash;
     private executeMatch;
     private executeLet;
+    private executeApply;
     private executeWebhook;
+    private executePause;
     private log;
+    /**
+     * Check if pause has been requested and handle it
+     * Should be called at safe pause points (between stages, loop iterations)
+     */
+    private checkPause;
+    /**
+     * Update control server with current state
+     */
+    private updateControlServerState;
+    private getStepType;
+    /** Get the event emitter (for external access) */
+    getEventEmitter(): EventEmitter | undefined;
+    /** Get the structured logger (for external access) */
+    getLogger(): StructuredLogger | undefined;
+    /** Get the debug controller (for external access) */
+    getDebugController(): DebugController | undefined;
+    /** Capture current execution state for debugging */
+    private captureDebugSnapshot;
+    /** Handle debug command and update state */
+    private handleDebugCommand;
 }
-export {};