reqon-dsl 0.2.0

package/dist/execution/state.d.ts

@@ -0,0 +1,136 @@
+/**
+ * Execution State - Durable state for resumable missions
+ *
+ * Tracks progress through pipeline stages, enabling:
+ * - Resume from last successful step after failures
+ * - Idempotent re-execution
+ * - Progress visibility
+ */
+export type ExecutionStatus = 'pending' | 'running' | 'completed' | 'failed' | 'paused';
+export type StageStatus = 'pending' | 'running' | 'completed' | 'failed' | 'skipped';
+/**
+ * State for a single pipeline stage (action execution)
+ */
+export interface StageState {
+    /** Action name */
+    action: string;
+    /** Current status */
+    status: StageStatus;
+    /** When this stage started */
+    startedAt?: Date;
+    /** When this stage completed/failed */
+    completedAt?: Date;
+    /** Error message if failed */
+    error?: string;
+    /** Number of items processed (for actions with loops) */
+    itemsProcessed?: number;
+    /** Total items to process */
+    itemsTotal?: number;
+    /** Retry attempt number (0 = first attempt) */
+    attempt: number;
+}
+/**
+ * Checkpoint within an action (for resuming mid-action)
+ */
+export interface Checkpoint {
+    /** Stage index in pipeline */
+    stageIndex: number;
+    /** Step index within action */
+    stepIndex: number;
+    /** For loops: current item index */
+    itemIndex?: number;
+    /** Saved context variables */
+    variables?: Record<string, unknown>;
+    /** Timestamp */
+    createdAt: Date;
+    /** Webhook wait state (for resuming webhook waits) */
+    webhookWait?: WebhookWaitState;
+}
+/**
+ * State for waiting on webhook callbacks
+ */
+export interface WebhookWaitState {
+    /** Webhook registration ID */
+    registrationId: string;
+    /** Path for the webhook endpoint */
+    path: string;
+    /** Full webhook URL */
+    webhookUrl: string;
+    /** Number of expected events */
+    expectedEvents: number;
+    /** Number of events received so far */
+    receivedEvents: number;
+    /** When the wait started */
+    waitStartedAt: Date;
+    /** When the wait expires */
+    expiresAt: Date;
+}
+/**
+ * Complete execution state for a mission run
+ */
+export interface ExecutionState {
+    /** Unique execution ID */
+    id: string;
+    /** Mission name */
+    mission: string;
+    /** Overall status */
+    status: ExecutionStatus;
+    /** When execution started */
+    startedAt: Date;
+    /** When execution completed/failed */
+    completedAt?: Date;
+    /** Total duration in ms */
+    duration?: number;
+    /** State of each pipeline stage */
+    stages: StageState[];
+    /** Latest checkpoint for resume */
+    checkpoint?: Checkpoint;
+    /** Execution errors */
+    errors: ExecutionStateError[];
+    /** Metadata (user-provided context) */
+    metadata?: Record<string, unknown>;
+}
+export interface ExecutionStateError {
+    stageIndex: number;
+    action: string;
+    step: string;
+    message: string;
+    timestamp: Date;
+    attempt: number;
+}
+/**
+ * Options for creating a new execution
+ */
+export interface CreateExecutionOptions {
+    /** Mission name */
+    mission: string;
+    /** Pipeline stage names (actions) */
+    stages: string[];
+    /** Optional metadata */
+    metadata?: Record<string, unknown>;
+}
+/**
+ * Generate a unique execution ID
+ */
+export declare function generateExecutionId(): string;
+/**
+ * Create initial execution state
+ */
+export declare function createExecutionState(options: CreateExecutionOptions): ExecutionState;
+/**
+ * Find the stage to resume from
+ * Returns the index of the first non-completed stage, or -1 if all complete
+ */
+export declare function findResumePoint(state: ExecutionState): number;
+/**
+ * Check if execution can be resumed
+ */
+export declare function canResume(state: ExecutionState): boolean;
+/**
+ * Calculate execution progress as percentage
+ */
+export declare function getProgress(state: ExecutionState): number;
+/**
+ * Get a summary of the execution state
+ */
+export declare function getExecutionSummary(state: ExecutionState): string;

package/dist/execution/state.js

@@ -0,0 +1,82 @@
+/**
+ * Execution State - Durable state for resumable missions
+ *
+ * Tracks progress through pipeline stages, enabling:
+ * - Resume from last successful step after failures
+ * - Idempotent re-execution
+ * - Progress visibility
+ */
+/**
+ * Generate a unique execution ID
+ */
+export function generateExecutionId() {
+    const timestamp = Date.now().toString(36);
+    const random = Math.random().toString(36).substring(2, 8);
+    return `exec_${timestamp}_${random}`;
+}
+/**
+ * Create initial execution state
+ */
+export function createExecutionState(options) {
+    return {
+        id: generateExecutionId(),
+        mission: options.mission,
+        status: 'pending',
+        startedAt: new Date(),
+        stages: options.stages.map((action) => ({
+            action,
+            status: 'pending',
+            attempt: 0,
+        })),
+        errors: [],
+        metadata: options.metadata,
+    };
+}
+/**
+ * Find the stage to resume from
+ * Returns the index of the first non-completed stage, or -1 if all complete
+ */
+export function findResumePoint(state) {
+    // If there's a checkpoint, use it
+    if (state.checkpoint) {
+        return state.checkpoint.stageIndex;
+    }
+    // Otherwise, find first non-completed stage
+    for (let i = 0; i < state.stages.length; i++) {
+        const stage = state.stages[i];
+        if (stage.status !== 'completed' && stage.status !== 'skipped') {
+            return i;
+        }
+    }
+    return -1; // All stages complete
+}
+/**
+ * Check if execution can be resumed
+ */
+export function canResume(state) {
+    return state.status === 'failed' || state.status === 'paused';
+}
+/**
+ * Calculate execution progress as percentage
+ */
+export function getProgress(state) {
+    if (state.stages.length === 0)
+        return 100;
+    const completed = state.stages.filter((s) => s.status === 'completed' || s.status === 'skipped').length;
+    return Math.round((completed / state.stages.length) * 100);
+}
+/**
+ * Get a summary of the execution state
+ */
+export function getExecutionSummary(state) {
+    const progress = getProgress(state);
+    const completed = state.stages.filter((s) => s.status === 'completed').length;
+    const failed = state.stages.filter((s) => s.status === 'failed').length;
+    const pending = state.stages.filter((s) => s.status === 'pending').length;
+    let summary = `${state.mission} [${state.id}]: ${state.status} (${progress}%)`;
+    summary += ` - ${completed} completed, ${failed} failed, ${pending} pending`;
+    if (state.duration) {
+        summary += ` - ${Math.round(state.duration / 1000)}s`;
+    }
+    return summary;
+}

package/dist/execution/store.d.ts

@@ -0,0 +1,52 @@
+import type { ExecutionState } from './state.js';
+/**
+ * Execution state store interface
+ */
+export interface ExecutionStore {
+    /** Save execution state */
+    save(state: ExecutionState): Promise<void>;
+    /** Load execution state by ID */
+    load(id: string): Promise<ExecutionState | null>;
+    /** List all executions for a mission */
+    listByMission(mission: string): Promise<ExecutionState[]>;
+    /** List recent executions */
+    listRecent(limit?: number): Promise<ExecutionState[]>;
+    /** Delete execution state */
+    delete(id: string): Promise<void>;
+    /** Find the latest execution for a mission */
+    findLatest(mission: string): Promise<ExecutionState | null>;
+    /** Find resumable executions (failed/paused) */
+    findResumable(mission: string): Promise<ExecutionState[]>;
+}
+/**
+ * File-based execution state store
+ * Stores each execution as a JSON file in .reqon-data/executions/
+ */
+export declare class FileExecutionStore implements ExecutionStore {
+    private baseDir;
+    private initialized;
+    constructor(baseDir?: string);
+    private getFilePath;
+    private deserialize;
+    save(state: ExecutionState): Promise<void>;
+    load(id: string): Promise<ExecutionState | null>;
+    listByMission(mission: string): Promise<ExecutionState[]>;
+    listRecent(limit?: number): Promise<ExecutionState[]>;
+    delete(id: string): Promise<void>;
+    findLatest(mission: string): Promise<ExecutionState | null>;
+    findResumable(mission: string): Promise<ExecutionState[]>;
+}
+/**
+ * In-memory execution store (for testing)
+ */
+export declare class MemoryExecutionStore implements ExecutionStore {
+    private states;
+    save(state: ExecutionState): Promise<void>;
+    load(id: string): Promise<ExecutionState | null>;
+    listByMission(mission: string): Promise<ExecutionState[]>;
+    listRecent(limit?: number): Promise<ExecutionState[]>;
+    delete(id: string): Promise<void>;
+    findLatest(mission: string): Promise<ExecutionState | null>;
+    findResumable(mission: string): Promise<ExecutionState[]>;
+    clear(): void;
+}

package/dist/execution/store.js

@@ -0,0 +1,120 @@
+import { join } from 'node:path';
+import { ensureDirectory, writeJsonFile, readJsonFile, listFiles, deleteFile, restoreDates, restoreDatesInArray, } from '../utils/file.js';
+/**
+ * File-based execution state store
+ * Stores each execution as a JSON file in .reqon-data/executions/
+ */
+export class FileExecutionStore {
+    baseDir;
+    initialized;
+    constructor(baseDir = '.reqon-data/executions') {
+        this.baseDir = baseDir;
+        this.initialized = ensureDirectory(this.baseDir);
+    }
+    getFilePath(id) {
+        return join(this.baseDir, `${id}.json`);
+    }
+    deserialize(parsed) {
+        // Restore Date objects
+        restoreDates(parsed, ['startedAt', 'completedAt']);
+        if (parsed.checkpoint && typeof parsed.checkpoint === 'object') {
+            restoreDates(parsed.checkpoint, ['createdAt']);
+        }
+        restoreDatesInArray(parsed.stages, ['startedAt', 'completedAt']);
+        restoreDatesInArray(parsed.errors, ['timestamp']);
+        return parsed;
+    }
+    async save(state) {
+        await this.initialized;
+        const filePath = this.getFilePath(state.id);
+        await writeJsonFile(filePath, state);
+    }
+    async load(id) {
+        await this.initialized;
+        const filePath = this.getFilePath(id);
+        const parsed = await readJsonFile(filePath);
+        return parsed ? this.deserialize(parsed) : null;
+    }
+    async listByMission(mission) {
+        const all = await this.listRecent();
+        return all.filter((s) => s.mission === mission);
+    }
+    async listRecent(limit = 50) {
+        await this.initialized;
+        const files = await listFiles(this.baseDir, '.json');
+        const states = [];
+        for (const file of files) {
+            const parsed = await readJsonFile(file);
+            if (parsed) {
+                states.push(this.deserialize(parsed));
+            }
+        }
+        // Sort by start time, newest first
+        states.sort((a, b) => b.startedAt.getTime() - a.startedAt.getTime());
+        return states.slice(0, limit);
+    }
+    async delete(id) {
+        await this.initialized;
+        await deleteFile(this.getFilePath(id));
+    }
+    async findLatest(mission) {
+        const executions = await this.listByMission(mission);
+        return executions[0] ?? null;
+    }
+    async findResumable(mission) {
+        const executions = await this.listByMission(mission);
+        return executions.filter((e) => e.status === 'failed' || e.status === 'paused');
+    }
+}
+/**
+ * In-memory execution store (for testing)
+ */
+export class MemoryExecutionStore {
+    states = new Map();
+    async save(state) {
+        // Deep clone to avoid reference issues
+        this.states.set(state.id, JSON.parse(JSON.stringify(state)));
+    }
+    async load(id) {
+        const state = this.states.get(id);
+        if (!state)
+            return null;
+        // Restore Date objects
+        const restored = JSON.parse(JSON.stringify(state));
+        restored.startedAt = new Date(restored.startedAt);
+        if (restored.completedAt) {
+            restored.completedAt = new Date(restored.completedAt);
+        }
+        return restored;
+    }
+    async listByMission(mission) {
+        const all = await this.listRecent();
+        return all.filter((s) => s.mission === mission);
+    }
+    async listRecent(limit = 50) {
+        const states = Array.from(this.states.values()).map((s) => {
+            const restored = JSON.parse(JSON.stringify(s));
+            restored.startedAt = new Date(restored.startedAt);
+            if (restored.completedAt) {
+                restored.completedAt = new Date(restored.completedAt);
+            }
+            return restored;
+        });
+        states.sort((a, b) => b.startedAt.getTime() - a.startedAt.getTime());
+        return states.slice(0, limit);
+    }
+    async delete(id) {
+        this.states.delete(id);
+    }
+    async findLatest(mission) {
+        const executions = await this.listByMission(mission);
+        return executions[0] ?? null;
+    }
+    async findResumable(mission) {
+        const executions = await this.listByMission(mission);
+        return executions.filter((e) => e.status === 'failed' || e.status === 'paused');
+    }
+    clear() {
+        this.states.clear();
+    }
+}

package/dist/index.d.ts

@@ -0,0 +1,27 @@
+export { ReqonLexer, ReqonTokenType, type ReqonToken } from './lexer/index.js';
+export { reqonPlugin, registerReqonPlugin } from './plugin.js';
+export { ReqonParser } from './parser/index.js';
+export * from './ast/index.js';
+export { MissionExecutor, HttpClient, BearerAuthProvider, OAuth2AuthProvider, createContext, evaluate, type ExecutionResult, type ExecutionError, type ExecutorConfig, type ExecutionContext, type ProgressCallbacks, type ExecutionStartEvent, type ExecutionCompleteEvent, type StageStartEvent, type StageCompleteEvent, } from './interpreter/index.js';
+export { MemoryStore, FileStore, createStore, type StoreAdapter, type StoreFilter, type StoreConfig, } from './stores/index.js';
+export { createExecutionState, findResumePoint, canResume, getProgress, getExecutionSummary, FileExecutionStore, MemoryExecutionStore, type ExecutionState, type ExecutionStore, type StageState, } from './execution/index.js';
+export { Scheduler, parseCronExpression, getNextRunTime, intervalToMs, shouldRunNow, type ScheduledJob, type SchedulerState, type ScheduleEvent, type SchedulerCallbacks, type SchedulerConfig, type ScheduledMission, } from './scheduler/index.js';
+export { generateCheckpointKey, formatSinceDate, parseSinceDate, EPOCH, FileSyncStore, MemorySyncStore, type SyncCheckpoint, type SyncStore, } from './sync/index.js';
+export { ReqonError, ParseError, LexerError, RuntimeError, ValidationError, formatErrors, getSourceLine, getSourceContext, type SourceLocation, type ErrorContext, } from './errors/index.js';
+export { loadMission, isMissionFolder, getMissionName, type LoadResult, type LoadOptions, } from './loader/index.js';
+export { loadEnv, loadCredentials, resolveCredentials, resolveEnvString, hasEnvReference, credentialsFromEnv, type CredentialsConfig, type LoadEnvResult, type AuthCredentials, type SourceCredentials, } from './auth/credentials.js';
+export { WebhookServer, MemoryWebhookStore, FileWebhookStore, type WebhookStore, type WebhookServerConfig, type WebhookServerCallbacks, type WebhookRegistration, type WebhookEvent, type WaitResult, } from './webhook/index.js';
+import { type ExecutorConfig } from './interpreter/index.js';
+import type { ReqonProgram } from './ast/index.js';
+export declare function parse(source: string, filePath?: string): ReqonProgram;
+export declare function execute(source: string, config?: ExecutorConfig): Promise<import('./interpreter/index.js').ExecutionResult>;
+export declare function fromFile(filePath: string, config?: ExecutorConfig): Promise<import('./interpreter/index.js').ExecutionResult>;
+/**
+ * Load and execute a mission from a file or folder.
+ *
+ * Supports both:
+ * - Single file: ./sync-invoices.reqon
+ * - Folder: ./sync-invoices/ (with mission.reqon + action files)
+ */
+export declare function fromPath(path: string, config?: ExecutorConfig): Promise<import('./interpreter/index.js').ExecutionResult>;
+export declare function reqon(strings: TemplateStringsArray, ...values: unknown[]): ReqonProgram;

package/dist/index.js

@@ -0,0 +1,57 @@
+export { ReqonLexer, ReqonTokenType } from './lexer/index.js';
+export { reqonPlugin, registerReqonPlugin } from './plugin.js';
+export { ReqonParser } from './parser/index.js';
+export * from './ast/index.js';
+export { MissionExecutor, HttpClient, BearerAuthProvider, OAuth2AuthProvider, createContext, evaluate, } from './interpreter/index.js';
+export { MemoryStore, FileStore, createStore, } from './stores/index.js';
+export { createExecutionState, findResumePoint, canResume, getProgress, getExecutionSummary, FileExecutionStore, MemoryExecutionStore, } from './execution/index.js';
+export { Scheduler, parseCronExpression, getNextRunTime, intervalToMs, shouldRunNow, } from './scheduler/index.js';
+export { generateCheckpointKey, formatSinceDate, parseSinceDate, EPOCH, FileSyncStore, MemorySyncStore, } from './sync/index.js';
+export { ReqonError, ParseError, LexerError, RuntimeError, ValidationError, formatErrors, getSourceLine, getSourceContext, } from './errors/index.js';
+export { loadMission, isMissionFolder, getMissionName, } from './loader/index.js';
+export { loadEnv, loadCredentials, resolveCredentials, resolveEnvString, hasEnvReference, credentialsFromEnv, } from './auth/credentials.js';
+export { WebhookServer, MemoryWebhookStore, FileWebhookStore, } from './webhook/index.js';
+import { readFile } from 'node:fs/promises';
+import { resolve } from 'node:path';
+import { ReqonLexer } from './lexer/index.js';
+import { ReqonParser } from './parser/index.js';
+import { MissionExecutor } from './interpreter/index.js';
+import { loadMission } from './loader/index.js';
+export function parse(source, filePath) {
+    const lexer = new ReqonLexer(source);
+    const tokens = lexer.tokenize();
+    const parser = new ReqonParser(tokens, source, filePath);
+    return parser.parse();
+}
+export async function execute(source, config = {}) {
+    const program = parse(source);
+    const executor = new MissionExecutor(config);
+    return executor.execute(program);
+}
+export async function fromFile(filePath, config = {}) {
+    const absolutePath = resolve(filePath);
+    const source = await readFile(absolutePath, 'utf-8');
+    const program = parse(source, absolutePath);
+    const executor = new MissionExecutor(config);
+    return executor.execute(program);
+}
+/**
+ * Load and execute a mission from a file or folder.
+ *
+ * Supports both:
+ * - Single file: ./sync-invoices.reqon
+ * - Folder: ./sync-invoices/ (with mission.reqon + action files)
+ */
+export async function fromPath(path, config = {}) {
+    const { program } = await loadMission(path);
+    const executor = new MissionExecutor(config);
+    return executor.execute(program);
+}
+// Tagged template literal for inline missions
+export function reqon(strings, ...values) {
+    let source = strings[0];
+    for (let i = 0; i < values.length; i++) {
+        source += String(values[i]) + strings[i + 1];
+    }
+    return parse(source);
+}

package/dist/integration.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/integration.test.js

@@ -0,0 +1,168 @@
+import { describe, it, expect } from 'vitest';
+import { parse, execute } from './index.js';
+import { MemoryStore } from './stores/index.js';
+describe('Reqon Integration', () => {
+    it('parses and executes a simple mission with dry run', async () => {
+        const source = `
+      mission TestMission {
+        source API {
+          auth: bearer,
+          base: "https://api.example.com"
+        }
+
+        store items: memory("items")
+
+        action FetchItems {
+          fetch GET "/items"
+
+          store response -> items {
+            key: .id
+          }
+        }
+
+        run FetchItems
+      }
+    `;
+        const program = parse(source);
+        expect(program.type).toBe('ReqonProgram');
+        const result = await execute(source, { dryRun: true, verbose: false });
+        expect(result.success).toBe(true);
+        expect(result.actionsRun).toContain('FetchItems');
+    });
+    it('executes a mission with mock data flow', async () => {
+        const source = `
+      mission MockDataFlow {
+        source API {
+          auth: bearer,
+          base: "https://api.example.com"
+        }
+
+        store raw: memory("raw")
+        store processed: memory("processed")
+
+        action Process {
+          for item in raw {
+            map item -> Processed {
+              id: .id,
+              name: .title,
+              value: .amount
+            }
+
+            store response -> processed {
+              key: .id
+            }
+          }
+        }
+
+        run Process
+      }
+    `;
+        // Pre-populate the raw store
+        const rawStore = new MemoryStore('raw');
+        await rawStore.set('1', { id: '1', title: 'Item 1', amount: 100 });
+        await rawStore.set('2', { id: '2', title: 'Item 2', amount: 200 });
+        const result = await execute(source, {
+            dryRun: false,
+            verbose: false,
+            stores: { raw: rawStore },
+        });
+        expect(result.success).toBe(true);
+        expect(result.actionsRun).toContain('Process');
+        // Check processed store
+        const processedStore = result.stores.get('processed');
+        expect(processedStore).toBeDefined();
+        const items = await processedStore.list();
+        expect(items).toHaveLength(2);
+        expect(items[0]).toMatchObject({ id: '1', name: 'Item 1', value: 100 });
+    });
+    it('parses the Xero example file', async () => {
+        const fs = await import('node:fs/promises');
+        const source = await fs.readFile('./examples/xero/invoices.reqon', 'utf-8');
+        const program = parse(source);
+        expect(program.type).toBe('ReqonProgram');
+        expect(program.statements).toHaveLength(1);
+        const mission = program.statements[0];
+        if (mission.type === 'MissionDefinition') {
+            expect(mission.name).toBe('SyncXeroInvoices');
+            expect(mission.sources).toHaveLength(1);
+            expect(mission.stores).toHaveLength(2);
+            expect(mission.actions).toHaveLength(3);
+            expect(mission.pipeline.stages).toHaveLength(3);
+        }
+    });
+    it('handles validation failures gracefully', async () => {
+        const source = `
+      mission ValidationTest {
+        source API {
+          auth: bearer,
+          base: "https://api.example.com"
+        }
+
+        store items: memory("items")
+
+        action Validate {
+          for item in items {
+            validate item {
+              assume .value > 0
+            }
+          }
+        }
+
+        run Validate
+      }
+    `;
+        const itemsStore = new MemoryStore('items');
+        await itemsStore.set('1', { id: '1', value: -5 }); // Invalid: negative value
+        const result = await execute(source, {
+            stores: { items: itemsStore },
+        });
+        // Validation should fail
+        expect(result.success).toBe(false);
+        expect(result.errors.length).toBeGreaterThan(0);
+    });
+    it('processes match expressions in map steps', async () => {
+        const source = `
+      mission MatchTest {
+        source API {
+          auth: bearer,
+          base: "https://api.example.com"
+        }
+
+        store input: memory("input")
+        store output: memory("output")
+
+        action Transform {
+          for item in input {
+            map item -> Output {
+              id: .id,
+              status: match .state {
+                "A" => "active",
+                "I" => "inactive",
+                _ => "unknown"
+              }
+            }
+
+            store response -> output {
+              key: .id
+            }
+          }
+        }
+
+        run Transform
+      }
+    `;
+        const inputStore = new MemoryStore('input');
+        await inputStore.set('1', { id: '1', state: 'A' });
+        await inputStore.set('2', { id: '2', state: 'I' });
+        await inputStore.set('3', { id: '3', state: 'X' });
+        const result = await execute(source, {
+            stores: { input: inputStore },
+        });
+        expect(result.success).toBe(true);
+        const outputStore = result.stores.get('output');
+        const items = await outputStore.list();
+        expect(items).toContainEqual(expect.objectContaining({ id: '1', status: 'active' }));
+        expect(items).toContainEqual(expect.objectContaining({ id: '2', status: 'inactive' }));
+        expect(items).toContainEqual(expect.objectContaining({ id: '3', status: 'unknown' }));
+    });
+});

package/dist/interpreter/context.d.ts

@@ -0,0 +1,15 @@
+import type { SchemaDefinition } from 'vague-lang';
+import type { StoreAdapter } from '../stores/types.js';
+import type { HttpClient } from './http.js';
+export interface ExecutionContext {
+    stores: Map<string, StoreAdapter>;
+    sources: Map<string, HttpClient>;
+    schemas: Map<string, SchemaDefinition>;
+    variables: Map<string, unknown>;
+    response?: unknown;
+    parent?: ExecutionContext;
+}
+export declare function createContext(): ExecutionContext;
+export declare function childContext(parent: ExecutionContext): ExecutionContext;
+export declare function getVariable(ctx: ExecutionContext, name: string): unknown;
+export declare function setVariable(ctx: ExecutionContext, name: string, value: unknown): void;