reqon-dsl 0.2.0 → 0.4.0

package/dist/interpreter/step-handlers/pause-handler.d.ts

@@ -0,0 +1,52 @@
+/**
+ * Pause Step Handler - Handles resource-free long pauses
+ *
+ * Creates a durable pause that persists state and releases resources,
+ * allowing execution to be resumed after extended periods.
+ */
+import type { PauseStep } from '../../ast/nodes.js';
+import type { ExecutionContext } from '../context.js';
+import type { EventType } from '../../observability/index.js';
+import type { PauseManager } from '../../pause/index.js';
+/**
+ * Dependencies for pause handler
+ */
+export interface PauseHandlerDeps {
+    ctx: ExecutionContext;
+    log: (message: string) => void;
+    emit?: <T>(type: EventType, payload: T) => void;
+    /** Pause manager for creating and persisting pauses */
+    pauseManager: PauseManager;
+    /** Execution ID */
+    executionId: string;
+    /** Mission name */
+    mission: string;
+    /** Current action name */
+    actionName: string;
+    /** Current stage index */
+    stageIndex: number;
+    /** Current step index */
+    stepIndex: number;
+    /** For loop context (if inside a loop) */
+    loopItemIndex?: number;
+}
+/**
+ * Result of pause execution
+ */
+export interface PauseHandlerResult {
+    /** The pause ID */
+    pauseId: string;
+    /** Duration in milliseconds */
+    duration: number;
+    /** When the pause expires */
+    expiresAt: Date;
+}
+/**
+ * Handles pause step execution
+ */
+export declare class PauseHandler {
+    private deps;
+    constructor(deps: PauseHandlerDeps);
+    execute(step: PauseStep): Promise<PauseHandlerResult>;
+    private captureVariables;
+}

package/dist/interpreter/step-handlers/pause-handler.js

@@ -0,0 +1,87 @@
+/**
+ * Pause Step Handler - Handles resource-free long pauses
+ *
+ * Creates a durable pause that persists state and releases resources,
+ * allowing execution to be resumed after extended periods.
+ */
+import { parseDuration, formatDuration } from '../../pause/state.js';
+import { PauseSignal } from '../signals.js';
+/**
+ * Handles pause step execution
+ */
+export class PauseHandler {
+    deps;
+    constructor(deps) {
+        this.deps = deps;
+    }
+    async execute(step) {
+        const duration = parseDuration(step.duration);
+        this.deps.log(`Starting pause for ${formatDuration(duration)}`);
+        // Capture current context variables
+        const variables = this.captureVariables();
+        // Build checkpoint
+        const checkpoint = {
+            stageIndex: this.deps.stageIndex,
+            stepIndex: this.deps.stepIndex + 1, // Resume AFTER this pause step
+            action: this.deps.actionName,
+            itemIndex: this.deps.loopItemIndex,
+            variables,
+            response: this.deps.ctx.response,
+        };
+        // Build resume triggers
+        const resumeTriggers = [];
+        if (step.resumeOn && step.resumeOn.length > 0) {
+            for (const trigger of step.resumeOn) {
+                if (trigger.type === 'timeout') {
+                    resumeTriggers.push({ type: 'timeout' });
+                }
+                else if (trigger.type === 'webhook') {
+                    resumeTriggers.push({ type: 'webhook', path: trigger.path });
+                }
+            }
+        }
+        else {
+            // Default to timeout only
+            resumeTriggers.push({ type: 'timeout' });
+        }
+        // Create the pause via pause manager
+        const pause = await this.deps.pauseManager.createPause({
+            executionId: this.deps.executionId,
+            mission: this.deps.mission,
+            duration,
+            checkpoint,
+            resumeTriggers,
+        });
+        // Emit pause event
+        this.deps.emit?.('mission.paused', {
+            pauseId: pause.id,
+            duration,
+            expiresAt: pause.expiresAt,
+            resumeTriggers: resumeTriggers.map((t) => t.type),
+        });
+        this.deps.log(`Pause ${pause.id} created - will expire at ${pause.expiresAt.toISOString()}`);
+        // Throw PauseSignal to stop execution
+        // The executor will catch this and handle appropriately
+        throw new PauseSignal(pause.id);
+    }
+    captureVariables() {
+        const variables = {};
+        let current = this.deps.ctx;
+        while (current) {
+            for (const [key, value] of current.variables) {
+                if (!(key in variables)) {
+                    // Deep clone to ensure we capture the current state
+                    try {
+                        variables[key] = JSON.parse(JSON.stringify(value));
+                    }
+                    catch {
+                        // If value can't be serialized, skip it
+                        variables[key] = '[non-serializable]';
+                    }
+                }
+            }
+            current = current.parent;
+        }
+        return variables;
+    }
+}

package/dist/interpreter/step-handlers/store-handler.d.ts

@@ -1,13 +1,29 @@
 import type { StoreStep } from '../../ast/nodes.js';
 import type { StepHandler, StepHandlerDeps } from './types.js';
 /**
- * Handles store steps for persisting data
+ * Handles store steps for persisting data to configured store adapters.
+ * Supports bulk operations, upserts, and partial record updates.
  */
 export declare class StoreHandler implements StepHandler<StoreStep> {
     private deps;
     constructor(deps: StepHandlerDeps);
+    /**
+     * Compute the storage key for a record based on step options.
+     *
+     * Uses the key expression if provided, otherwise falls back to `record.id`.
+     * A missing or null/undefined key is an error: inventing a random key breaks
+     * dedup (re-runs duplicate everything), and stringifying undefined collapses
+     * every such record onto the literal key "undefined".
+     */
+    private getRecordKey;
+    /**
+     * Emit a data.store event with operation metadata.
+     */
+    private emitStoreEvent;
     execute(step: StoreStep): Promise<void>;
     private storeMany;
     private storeOne;
+    /** True when the record should be merged into an existing one (deep upsert). */
+    private shouldMerge;
     private storeRecord;
 }

package/dist/interpreter/step-handlers/store-handler.js

@@ -1,16 +1,49 @@
 import { evaluate } from '../evaluator.js';
+import { RuntimeError } from '../../errors/index.js';
 /**
- * Handles store steps for persisting data
+ * Handles store steps for persisting data to configured store adapters.
+ * Supports bulk operations, upserts, and partial record updates.
  */
 export class StoreHandler {
     deps;
     constructor(deps) {
         this.deps = deps;
     }
+    /**
+     * Compute the storage key for a record based on step options.
+     *
+     * Uses the key expression if provided, otherwise falls back to `record.id`.
+     * A missing or null/undefined key is an error: inventing a random key breaks
+     * dedup (re-runs duplicate everything), and stringifying undefined collapses
+     * every such record onto the literal key "undefined".
+     */
+    getRecordKey(step, record) {
+        const raw = step.options.key ? evaluate(step.options.key, this.deps.ctx, record) : record.id;
+        if (raw === undefined || raw === null || raw === '') {
+            const which = step.options.key ? 'key expression' : "record 'id'";
+            throw new RuntimeError(`Cannot store to '${step.target}': ${which} is missing or empty. ` +
+                `Provide a 'key:' option that resolves to a stable, non-empty value.`, { line: 1, column: 1 }, undefined, { stepType: 'store' });
+        }
+        return String(raw);
+    }
+    /**
+     * Emit a data.store event with operation metadata.
+     */
+    emitStoreEvent(step, operation, itemCount, key) {
+        this.deps.emit?.('data.store', {
+            storeName: step.target,
+            storeType: this.deps.ctx.storeTypes.get(step.target) ?? 'unknown',
+            operation,
+            itemCount,
+            ...(key !== undefined && { key }),
+        });
+    }
     async execute(step) {
         const store = this.deps.ctx.stores.get(step.target);
         if (!store) {
-            throw new Error(`Store not found: ${step.target}`);
+            throw new RuntimeError(`Store not found: ${step.target}`, { line: 1, column: 1 }, undefined, {
+                stepType: 'store',
+            });
         }
         const source = evaluate(step.source, this.deps.ctx);
         if (Array.isArray(source)) {
@@ -21,42 +54,50 @@ export class StoreHandler {
         }
     }
     async storeMany(step, store, items) {
-        // Use bulk operation if available and not doing upserts (which need individual handling)
-        if (store.bulkSet && !step.options.upsert) {
+        const merge = this.shouldMerge(step);
+        const operation = merge ? 'upsert' : 'set';
+        // Check if we can use bulk operations
+        const canBulkSet = store.bulkSet && !merge;
+        const canBulkUpsert = store.bulkUpsert && merge;
+        if (canBulkSet || canBulkUpsert) {
             const records = [];
             for (const item of items) {
                 const record = item;
-                const key = step.options.key
-                    ? String(evaluate(step.options.key, this.deps.ctx, record))
-                    : String(record.id ?? Math.random());
-                if (step.options.partial !== undefined) {
-                    record._partial = step.options.partial;
-                }
+                const key = this.getRecordKey(step, record);
                 records.push({ key, value: record });
             }
-            await store.bulkSet(records);
+            if (canBulkUpsert) {
+                await store.bulkUpsert(records);
+            }
+            else {
+                await store.bulkSet(records);
+            }
         }
         else {
-            // Fall back to individual operations for upserts or stores without bulkSet
+            // Fall back to individual operations for stores without bulk methods
             for (const item of items) {
                 const record = item;
                 await this.storeRecord(step, store, record);
             }
         }
-        this.deps.log(`Stored ${items.length} items to ${step.target}`);
+        this.deps.log(`Stored ${items.length} ${items.length === 1 ? 'item' : 'items'} to ${step.target}`);
+        this.emitStoreEvent(step, operation, items.length);
     }
     async storeOne(step, store, record) {
+        const key = this.getRecordKey(step, record);
+        const operation = this.shouldMerge(step) ? 'upsert' : 'set';
         await this.storeRecord(step, store, record);
         this.deps.log(`Stored item to ${step.target}`);
+        this.emitStoreEvent(step, operation, 1, key);
+    }
+    /** True when the record should be merged into an existing one (deep upsert). */
+    shouldMerge(step) {
+        return step.options.upsert === true || step.options.partial === true;
     }
     async storeRecord(step, store, record) {
-        const key = step.options.key
-            ? String(evaluate(step.options.key, this.deps.ctx, record))
-            : String(record.id ?? Math.random());
-        if (step.options.partial !== undefined) {
-            record._partial = step.options.partial;
-        }
-        if (step.options.upsert) {
+        const key = this.getRecordKey(step, record);
+        // Never mutate the caller's record or persist a storage-internal flag.
+        if (this.shouldMerge(step)) {
             await store.update(key, record);
         }
         else {

package/dist/interpreter/step-handlers/types.d.ts

@@ -1,11 +1,14 @@
 import type { ActionStep } from '../../ast/nodes.js';
 import type { ExecutionContext } from '../context.js';
+import type { EventType } from '../../observability/index.js';
 /**
  * Dependencies injected into step handlers
  */
 export interface StepHandlerDeps {
     ctx: ExecutionContext;
     log: (message: string) => void;
+    /** Optional event emitter for observability */
+    emit?: <T>(type: EventType, payload: T) => void;
 }
 /**
  * Base interface for step handlers

package/dist/interpreter/step-handlers/validate-handler.d.ts

@@ -1,7 +1,8 @@
 import type { ValidateStep } from '../../ast/nodes.js';
 import type { StepHandler, StepHandlerDeps } from './types.js';
 /**
- * Handles validate steps with assume constraints
+ * Handles validate steps with assume constraints.
+ * Evaluates constraint conditions and throws ValidationError on failure.
  */
 export declare class ValidateHandler implements StepHandler<ValidateStep> {
     private deps;

package/dist/interpreter/step-handlers/validate-handler.js

@@ -1,6 +1,8 @@
 import { evaluate } from '../evaluator.js';
+import { ValidationError } from '../../errors/index.js';
 /**
- * Handles validate steps with assume constraints
+ * Handles validate steps with assume constraints.
+ * Evaluates constraint conditions and throws ValidationError on failure.
  */
 export class ValidateHandler {
     deps;
@@ -14,7 +16,10 @@ export class ValidateHandler {
             if (!result) {
                 const message = constraint.message ?? `Validation failed: ${JSON.stringify(constraint.condition)}`;
                 if (constraint.severity === 'error') {
-                    throw new Error(message);
+                    throw new ValidationError(message, { line: 1, column: 1 }, undefined, {
+                        constraint: JSON.stringify(constraint.condition),
+                        severity: 'error',
+                    });
                 }
                 else {
                     this.deps.log(`Warning: ${message}`);

package/dist/interpreter/step-handlers/webhook-handler.d.ts

@@ -6,6 +6,7 @@
 import type { WebhookStep } from '../../ast/nodes.js';
 import type { ExecutionContext } from '../context.js';
 import type { WebhookServer, WebhookRegistration, WebhookEvent } from '../../webhook/index.js';
+import type { EventType } from '../../observability/index.js';
 /**
  * Dependencies for the webhook handler
  */
@@ -14,6 +15,8 @@ export interface WebhookHandlerDeps {
     webhookServer: WebhookServer;
     executionId: string;
     log: (message: string) => void;
+    /** Optional event emitter for observability */
+    emit?: <T>(type: EventType, payload: T) => void;
 }
 /**
  * Result of webhook handler execution
@@ -33,4 +36,5 @@ export declare class WebhookHandler {
      * Execute the wait step
      */
     execute(step: WebhookStep): Promise<WebhookHandlerResult>;
+    private waitAndProcess;
 }

package/dist/interpreter/step-handlers/webhook-handler.js

@@ -5,6 +5,7 @@
  */
 import { evaluate } from '../evaluator.js';
 import { RetrySignal } from '../signals.js';
+import { RuntimeError } from '../../errors/index.js';
 /**
  * Handler for webhook (wait) steps
  */
@@ -17,7 +18,7 @@ export class WebhookHandler {
      * Execute the wait step
      */
     async execute(step) {
-        const { ctx, webhookServer, executionId, log } = this.deps;
+        const { ctx, webhookServer, executionId, log, emit } = this.deps;
         // Register webhook endpoint
         const timeout = step.timeout ?? 300000; // 5 minutes default
         const expectedEvents = step.expectedEvents ?? 1;
@@ -29,13 +30,32 @@ export class WebhookHandler {
         });
         const webhookUrl = webhookServer.getWebhookUrl(registration);
         log(`Waiting for webhook: ${webhookUrl} (timeout: ${timeout}ms, expected: ${expectedEvents})`);
+        // Emit webhook.register event
+        emit?.('webhook.register', {
+            registrationId: registration.id,
+            path: registration.path,
+            webhookUrl,
+            timeout,
+            expectedEvents,
+        });
         // Set the webhook URL in context for use in subsequent steps
         ctx.response = {
             webhookId: registration.id,
             webhookUrl,
             webhookPath: registration.path,
         };
-        // Wait for webhook events
+        // Wait for webhook events. The registration must be torn down on every
+        // exit path (timeout, filter throw, store failure, retry signal) or it
+        // leaks the server-side handle — hence the try/finally below.
+        try {
+            return await this.waitAndProcess(step, registration, webhookUrl, timeout, ctx, log, emit);
+        }
+        finally {
+            await webhookServer.unregister(registration.id);
+        }
+    }
+    async waitAndProcess(step, registration, webhookUrl, timeout, ctx, log, emit) {
+        const webhookServer = this.deps.webhookServer;
         const result = await webhookServer.waitForEvents(registration.id, timeout);
         if (result.timedOut) {
             log(`Webhook timeout: ${webhookUrl}`);
@@ -45,7 +65,7 @@ export class WebhookHandler {
             }
             // If not retrying, still return partial results
             if (result.events.length === 0) {
-                throw new Error(`Webhook timeout: no events received within ${timeout}ms`);
+                throw new RuntimeError(`Webhook timeout: no events received within ${timeout}ms`, { line: 1, column: 1 }, undefined, { stepType: 'wait' });
             }
         }
         // Filter events if filter expression provided
@@ -93,8 +113,14 @@ export class WebhookHandler {
                 log(`Warning: Store '${step.storage.target}' not found for webhook storage`);
             }
         }
-        // Clean up registration
-        await webhookServer.unregister(registration.id);
+        // Emit webhook.complete event
+        emit?.('webhook.complete', {
+            registrationId: registration.id,
+            eventsReceived: events.length,
+            timedOut: result.timedOut ?? false,
+            storedTo: step.storage?.target,
+        });
+        // Registration teardown happens in the caller's finally block.
         return {
             registration,
             events,

package/dist/interpreter/store-manager.d.ts

@@ -0,0 +1,46 @@
+/**
+ * StoreManager handles store initialization and configuration.
+ *
+ * Extracted from MissionExecutor to improve separation of concerns.
+ * Responsible for:
+ * - Creating store adapters based on store definitions
+ * - Managing custom store adapters
+ * - Resolving store types for development/production modes
+ */
+import type { StoreDefinition } from '../ast/nodes.js';
+import type { ExecutionContext } from './context.js';
+import type { StoreAdapter } from '../stores/types.js';
+export interface StoreManagerConfig {
+    /** Custom store adapters by name */
+    customStores?: Record<string, StoreAdapter>;
+    /** Development mode - use file stores instead of sql/nosql (default: false) */
+    developmentMode?: boolean;
+    /** Base directory for file stores (default: '.reqon-data') */
+    dataDir?: string;
+    /** Logging function */
+    log?: (message: string) => void;
+}
+/**
+ * Manages store initialization and provides access to store adapters.
+ */
+export declare class StoreManager {
+    private config;
+    constructor(config?: StoreManagerConfig);
+    /**
+     * Initialize a store definition, creating the appropriate adapter.
+     */
+    initializeStore(store: StoreDefinition, ctx: ExecutionContext): Promise<void>;
+    /**
+     * Initialize multiple stores.
+     */
+    initializeStores(stores: StoreDefinition[], ctx: ExecutionContext): Promise<void>;
+    /**
+     * Get a store adapter from the context.
+     */
+    getStore(ctx: ExecutionContext, storeName: string): StoreAdapter | undefined;
+    /**
+     * Get the store type for a given store name.
+     */
+    getStoreType(ctx: ExecutionContext, storeName: string): string | undefined;
+    private log;
+}

package/dist/interpreter/store-manager.js

@@ -0,0 +1,70 @@
+/**
+ * StoreManager handles store initialization and configuration.
+ *
+ * Extracted from MissionExecutor to improve separation of concerns.
+ * Responsible for:
+ * - Creating store adapters based on store definitions
+ * - Managing custom store adapters
+ * - Resolving store types for development/production modes
+ */
+import { createStore, resolveStoreType } from '../stores/index.js';
+import { EXECUTION_DEFAULTS } from '../config/index.js';
+/**
+ * Manages store initialization and provides access to store adapters.
+ */
+export class StoreManager {
+    config;
+    constructor(config = {}) {
+        this.config = config;
+    }
+    /**
+     * Initialize a store definition, creating the appropriate adapter.
+     */
+    async initializeStore(store, ctx) {
+        // Check for custom store adapter
+        if (this.config.customStores?.[store.name]) {
+            ctx.stores.set(store.name, this.config.customStores[store.name]);
+            ctx.storeTypes.set(store.name, 'custom');
+            this.log(`Initialized store: ${store.name} (custom adapter)`);
+            return;
+        }
+        // Use store factory to create appropriate adapter
+        const developmentMode = this.config.developmentMode ?? EXECUTION_DEFAULTS.DEVELOPMENT_MODE;
+        const storeType = resolveStoreType(store.storeType, developmentMode);
+        const adapter = createStore({
+            type: storeType,
+            name: store.target,
+            baseDir: this.config.dataDir,
+            // In dev mode the type is already resolved to 'file'; this only matters
+            // if a raw sql/nosql type reaches the factory.
+            allowFileFallback: developmentMode,
+        });
+        ctx.stores.set(store.name, adapter);
+        ctx.storeTypes.set(store.name, storeType);
+        const typeInfo = storeType !== store.storeType ? ` <- ${store.storeType}` : '';
+        this.log(`Initialized store: ${store.name} (${storeType}${typeInfo})`);
+    }
+    /**
+     * Initialize multiple stores.
+     */
+    async initializeStores(stores, ctx) {
+        for (const store of stores) {
+            await this.initializeStore(store, ctx);
+        }
+    }
+    /**
+     * Get a store adapter from the context.
+     */
+    getStore(ctx, storeName) {
+        return ctx.stores.get(storeName);
+    }
+    /**
+     * Get the store type for a given store name.
+     */
+    getStoreType(ctx, storeName) {
+        return ctx.storeTypes.get(storeName);
+    }
+    log(message) {
+        this.config.log?.(message);
+    }
+}

package/dist/lexer/index.d.ts

@@ -1,8 +1,15 @@
 /**
- * Lexer module for Reqon.
- *
- * Uses Vague's Lexer with Reqon keywords registered via the plugin system.
- * Importing this module automatically registers the Reqon plugin.
+ * ---
+ * purpose: Lexer module - wraps Vague lexer with Reqon keywords
+ * note: Importing this module auto-registers Reqon plugin with Vague
+ * exports:
+ *   - ReqonLexer (alias for Vague Lexer)
+ *   - ReqonTokenType, REQON_KEYWORDS
+ * related:
+ *   - ./tokens.ts - Reqon-specific keywords (mission, action, fetch, etc.)
+ *   - ../plugin.ts - registers keywords with Vague
+ *   - ../parser/index.ts - consumes tokens
+ * ---
  */
 import '../plugin.js';
 export { Lexer, Lexer as ReqonLexer, TokenType, type Token } from 'vague-lang';

package/dist/lexer/index.js

@@ -1,8 +1,15 @@
 /**
- * Lexer module for Reqon.
- *
- * Uses Vague's Lexer with Reqon keywords registered via the plugin system.
- * Importing this module automatically registers the Reqon plugin.
+ * ---
+ * purpose: Lexer module - wraps Vague lexer with Reqon keywords
+ * note: Importing this module auto-registers Reqon plugin with Vague
+ * exports:
+ *   - ReqonLexer (alias for Vague Lexer)
+ *   - ReqonTokenType, REQON_KEYWORDS
+ * related:
+ *   - ./tokens.ts - Reqon-specific keywords (mission, action, fetch, etc.)
+ *   - ../plugin.ts - registers keywords with Vague
+ *   - ../parser/index.ts - consumes tokens
+ * ---
  */
 // Import plugin to auto-register Reqon keywords with Vague
 import '../plugin.js';

package/dist/lexer/tokens.d.ts

@@ -25,6 +25,8 @@ export declare enum ReqonTokenType {
     NOSQL = "NOSQL",
     SQL = "SQL",
     MEMORY = "MEMORY",
+    FILE = "FILE",
+    POSTGREST = "POSTGREST",
     OAUTH2 = "OAUTH2",
     BEARER = "BEARER",
     BASIC = "BASIC",
@@ -51,12 +53,26 @@ export declare enum ReqonTokenType {
     QUEUE = "QUEUE",
     JUMP = "JUMP",
     IS = "IS",
+    TRANSFORM = "TRANSFORM",
+    APPLY = "APPLY",
+    TO = "TO",
+    AS = "AS",
+    TRY = "TRY",
     WAIT = "WAIT",
     TIMEOUT = "TIMEOUT",
     PATH = "PATH",
     EXPECTED_EVENTS = "EXPECTED_EVENTS",
     EVENT_FILTER = "EVENT_FILTER",
-    STORAGE = "STORAGE"
+    STORAGE = "STORAGE",
+    CHECKPOINT = "CHECKPOINT",
+    TRACE = "TRACE",
+    PAUSE = "PAUSE",
+    PERSIST = "PERSIST",
+    DURATION = "DURATION",
+    RESUME_ON = "RESUME_ON",
+    AFTER_STEP = "AFTER_STEP",
+    ON_FAILURE = "ON_FAILURE",
+    FULL = "FULL"
 }
 export type TokenType = VagueTokenType | ReqonTokenType;
 export { TokenType as VagueTokenType } from 'vague-lang';