reqon-dsl 0.2.0 → 0.4.0

package/dist/execution-log/store.js

@@ -0,0 +1,182 @@
+/**
+ * Append-only storage for execution events. The contract: events for a given
+ * execution are read back in append order with a contiguous `seq` starting at 0.
+ */
+import { appendFile, readFile, readdir, mkdir } from 'node:fs/promises';
+import { join } from 'node:path';
+/** True for the pause lifecycle events the pause read-model folds. */
+function isPauseEvent(event) {
+    return event.type === 'pause.created' || event.type === 'pause.resumed';
+}
+/**
+ * Fold `checkpoint.advanced` events into the latest checkpoint per key. "Latest"
+ * is the furthest-advanced sync: ordered by `syncedAt` (so a checkpoint never
+ * moves backwards across runs — incremental sync stays monotonic), ties broken
+ * by the recording time `at` then `seq`. Shared by every store backend so the
+ * read-model semantics stay identical across them.
+ */
+export function reduceCheckpoints(events, mission) {
+    const latest = new Map();
+    const isNewer = (a, prior) => {
+        if (a.syncedAt !== prior.syncedAt)
+            return a.syncedAt > prior.syncedAt;
+        if (a.at !== prior.at)
+            return a.at > prior.at;
+        return a.seq > prior.seq;
+    };
+    for (const event of events) {
+        if (event.type !== 'checkpoint.advanced')
+            continue;
+        if (mission !== undefined && event.mission !== mission)
+            continue;
+        const candidate = {
+            syncedAt: event.syncedAt,
+            at: event.at,
+            seq: event.seq,
+            record: {
+                key: event.key,
+                syncedAt: event.syncedAt,
+                recordCount: event.recordCount,
+                cursor: event.cursor,
+                mission: event.mission,
+                executionId: event.executionId,
+            },
+        };
+        const prior = latest.get(event.key);
+        if (!prior || isNewer(candidate, prior))
+            latest.set(event.key, candidate);
+    }
+    return Array.from(latest.values()).map((v) => v.record);
+}
+/** In-memory execution log — for tests and ephemeral runs. */
+export class MemoryExecutionLog {
+    events = new Map();
+    async append(event) {
+        const existing = this.events.get(event.executionId);
+        const log = existing ?? [];
+        const stored = { ...event, seq: log.length, at: new Date().toISOString() };
+        log.push(stored);
+        if (!existing)
+            this.events.set(event.executionId, log);
+        return stored;
+    }
+    async read(executionId) {
+        return [...(this.events.get(executionId) ?? [])];
+    }
+    async listCheckpoints(mission) {
+        const all = [];
+        for (const log of this.events.values())
+            all.push(...log);
+        return reduceCheckpoints(all, mission);
+    }
+    async listPauses() {
+        const pauses = [];
+        for (const log of this.events.values())
+            pauses.push(...log.filter(isPauseEvent));
+        return pauses;
+    }
+}
+/**
+ * Durable execution log: one append-only JSON-lines file per execution.
+ *
+ * Appends are O_APPEND single-line writes, so a crash mid-write can at worst
+ * leave a torn final line, which {@link read} skips. The log therefore survives
+ * a process restart — a resumed run reads its prior events back. Dev-grade: no
+ * atomic seq assignment or locking. For a transactional single-process backend
+ * use {@link SqliteExecutionLog}; Postgres (multi-node) is still to come.
+ */
+export class FileExecutionLog {
+    dir;
+    /** Cached next-seq per execution, lazily initialised from disk. */
+    counts = new Map();
+    constructor(dir = '.reqon-data/execution-log') {
+        this.dir = dir;
+    }
+    pathFor(executionId) {
+        // executionId is runtime-generated (uuid); keep the filename simple/safe.
+        const safe = executionId.replace(/[^A-Za-z0-9_.-]/g, '_');
+        return join(this.dir, `${safe}.jsonl`);
+    }
+    async append(event) {
+        await mkdir(this.dir, { recursive: true });
+        let seq = this.counts.get(event.executionId);
+        let prefix = '';
+        if (seq === undefined) {
+            // First append this instance: derive next seq from any persisted events,
+            // and detect an unterminated tail left by a crash mid-append. Writing
+            // straight after it would concatenate this event onto the torn line and
+            // corrupt it; a leading newline terminates the torn line (which read()
+            // still skips) and keeps this event a clean, parseable record.
+            const raw = await this.readRaw(event.executionId);
+            seq = this.parse(raw).length;
+            if (raw.length > 0 && !raw.endsWith('\n'))
+                prefix = '\n';
+        }
+        const stored = { ...event, seq, at: new Date().toISOString() };
+        // The log can carry resume state (e.g. a pause checkpoint's captured
+        // variables), so create files owner-only (0o600) — never world-readable.
+        // mode applies on creation; appends to an existing file keep its mode.
+        await appendFile(this.pathFor(event.executionId), `${prefix}${JSON.stringify(stored)}\n`, {
+            encoding: 'utf-8',
+            mode: 0o600,
+        });
+        this.counts.set(event.executionId, seq + 1);
+        return stored;
+    }
+    async read(executionId) {
+        return this.parse(await this.readRaw(executionId));
+    }
+    async listCheckpoints(mission) {
+        return reduceCheckpoints(await this.readAll(), mission);
+    }
+    async listPauses() {
+        return (await this.readAll()).filter(isPauseEvent);
+    }
+    /** Read and parse every execution's events in this directory. */
+    async readAll() {
+        let files;
+        try {
+            files = await readdir(this.dir);
+        }
+        catch (err) {
+            if (err.code === 'ENOENT')
+                return [];
+            throw err;
+        }
+        const all = [];
+        for (const file of files) {
+            if (!file.endsWith('.jsonl'))
+                continue;
+            all.push(...this.parse(await readFile(join(this.dir, file), 'utf-8')));
+        }
+        return all;
+    }
+    async readRaw(executionId) {
+        try {
+            return await readFile(this.pathFor(executionId), 'utf-8');
+        }
+        catch (err) {
+            if (err.code === 'ENOENT')
+                return '';
+            throw err;
+        }
+    }
+    parse(content) {
+        const events = [];
+        for (const line of content.split('\n')) {
+            if (line.trim() === '')
+                continue;
+            try {
+                events.push(JSON.parse(line));
+            }
+            catch {
+                // A torn line from a crash mid-append. Each event is an independent
+                // single-line record, so skip just this line rather than discarding
+                // everything after it (a healed torn line can sit mid-file, followed
+                // by valid events appended on resume).
+                continue;
+            }
+        }
+        return events;
+    }
+}

package/dist/index.d.ts

@@ -1,16 +1,37 @@
+/**
+ * ---
+ * purpose: Main library entry point - exports all public APIs
+ * exports:
+ *   - parse, execute, fromFile, fromPath, reqon - convenience functions
+ *   - ReqonLexer, ReqonParser - parsing infrastructure
+ *   - MissionExecutor - runtime execution
+ *   - Store adapters, Scheduler, Sync, Webhook, Observability
+ * related:
+ *   - ./cli.ts - CLI entry point using these exports
+ *   - ./plugin.ts - Vague plugin registration
+ *   - ./interpreter/executor.ts - core execution logic
+ * ---
+ */
 export { ReqonLexer, ReqonTokenType, type ReqonToken } from './lexer/index.js';
 export { reqonPlugin, registerReqonPlugin } from './plugin.js';
 export { ReqonParser } from './parser/index.js';
 export * from './ast/index.js';
 export { MissionExecutor, HttpClient, BearerAuthProvider, OAuth2AuthProvider, createContext, evaluate, type ExecutionResult, type ExecutionError, type ExecutorConfig, type ExecutionContext, type ProgressCallbacks, type ExecutionStartEvent, type ExecutionCompleteEvent, type StageStartEvent, type StageCompleteEvent, } from './interpreter/index.js';
 export { MemoryStore, FileStore, createStore, type StoreAdapter, type StoreFilter, type StoreConfig, } from './stores/index.js';
-export { createExecutionState, findResumePoint, canResume, getProgress, getExecutionSummary, FileExecutionStore, MemoryExecutionStore, type ExecutionState, type ExecutionStore, type StageState, } from './execution/index.js';
+export { createExecutionState, findResumePoint, canResume, getProgress, getExecutionSummary, FileExecutionStore, MemoryExecutionStore, type ExecutionState, type ExecutionStore, type StageState, type LiveProgress, } from './execution/index.js';
+export { MemoryExecutionLog, FileExecutionLog, SqliteExecutionLog, PostgresExecutionLog, foldLog, loadState, effectId, reduceCheckpoints, type ExecutionEvent, type StoredEvent, type ExecutionLogStore, type CheckpointRecord, type FoldedState, } from './execution-log/index.js';
 export { Scheduler, parseCronExpression, getNextRunTime, intervalToMs, shouldRunNow, type ScheduledJob, type SchedulerState, type ScheduleEvent, type SchedulerCallbacks, type SchedulerConfig, type ScheduledMission, } from './scheduler/index.js';
-export { generateCheckpointKey, formatSinceDate, parseSinceDate, EPOCH, FileSyncStore, MemorySyncStore, type SyncCheckpoint, type SyncStore, } from './sync/index.js';
+export { generateCheckpointKey, formatSinceDate, parseSinceDate, EPOCH, FileSyncStore, MemorySyncStore, LogBackedSyncStore, type SyncCheckpoint, type SyncStore, } from './sync/index.js';
 export { ReqonError, ParseError, LexerError, RuntimeError, ValidationError, formatErrors, getSourceLine, getSourceContext, type SourceLocation, type ErrorContext, } from './errors/index.js';
 export { loadMission, isMissionFolder, getMissionName, type LoadResult, type LoadOptions, } from './loader/index.js';
 export { loadEnv, loadCredentials, resolveCredentials, resolveEnvString, hasEnvReference, credentialsFromEnv, type CredentialsConfig, type LoadEnvResult, type AuthCredentials, type SourceCredentials, } from './auth/credentials.js';
 export { WebhookServer, MemoryWebhookStore, FileWebhookStore, type WebhookStore, type WebhookServerConfig, type WebhookServerCallbacks, type WebhookRegistration, type WebhookEvent, type WaitResult, } from './webhook/index.js';
+export { ControlServer, type ControlServerConfig, type ControlServerCallbacks, type StatusResponse, type ControlResponse, } from './control/index.js';
+export { ObservabilityEmitter, createEmitter, type ObservabilityEvent, type EventType, type EventHandler, type EventEmitter, type MissionStartPayload, type MissionCompletePayload, type StageStartPayload, type StageCompletePayload, type StepStartPayload, type StepCompletePayload, type FetchStartPayload, type FetchCompletePayload, type DataStorePayload, type LoopStartPayload, type LoopCompletePayload, type WebhookRegisterPayload, type WebhookCompletePayload, createStructuredLogger, ConsoleOutput, JsonLinesOutput, BufferOutput, type StructuredLogger, type LogEntry, type LogOutput, type Span, type CreateLoggerOptions, SpanBuilder, OTelEventAdapter, OTLPExporter, createOTelListener, type OTelSpan, type OTLPExporterConfig, } from './observability/index.js';
+export { MCP_SERVER_VERSION } from './mcp/index.js';
+export { BaseDebugController, CLIDebugger, type DebugController, type DebugMode, type DebugPauseReason, type DebugSnapshot, type DebugCommand, type DebugLocation, } from './debug/index.js';
+export { FileTraceStore, MemoryTraceStore, TraceRecorder, TraceReplayer, LogTraceView, traceTimelineFromLog, createTraceRecorder, createTraceReplayer, createExecutionTrace, safeClone, truncateForTrace, type LogTimelineEntry, type LogTraceSummary, type TraceStore, type TraceSnapshot, type StoreSnapshot as TraceStoreSnapshot, type LoopContext as TraceLoopContext, type ExecutionTrace, type TraceRecorderConfig, type ReplaySession, type ReplayStepResult, type TimelineEvent, type VariableChange, type SnapshotDiff, } from './trace/index.js';
+export { FilePauseStore, MemoryPauseStore, LogBackedPauseStore, PauseManager, createPauseManager, parseDuration, formatDuration, createPauseState, isPauseExpired, getRemainingTime, getPauseSummary, type PauseStore, type PauseState, type PauseCheckpoint, type PauseResumeTriggerState, type PauseManagerConfig, type CreatePauseOptions, type PauseStatus, } from './pause/index.js';
 import { type ExecutorConfig } from './interpreter/index.js';
 import type { ReqonProgram } from './ast/index.js';
 export declare function parse(source: string, filePath?: string): ReqonProgram;

package/dist/index.js

@@ -1,3 +1,17 @@
+/**
+ * ---
+ * purpose: Main library entry point - exports all public APIs
+ * exports:
+ *   - parse, execute, fromFile, fromPath, reqon - convenience functions
+ *   - ReqonLexer, ReqonParser - parsing infrastructure
+ *   - MissionExecutor - runtime execution
+ *   - Store adapters, Scheduler, Sync, Webhook, Observability
+ * related:
+ *   - ./cli.ts - CLI entry point using these exports
+ *   - ./plugin.ts - Vague plugin registration
+ *   - ./interpreter/executor.ts - core execution logic
+ * ---
+ */
 export { ReqonLexer, ReqonTokenType } from './lexer/index.js';
 export { reqonPlugin, registerReqonPlugin } from './plugin.js';
 export { ReqonParser } from './parser/index.js';
@@ -5,12 +19,30 @@ export * from './ast/index.js';
 export { MissionExecutor, HttpClient, BearerAuthProvider, OAuth2AuthProvider, createContext, evaluate, } from './interpreter/index.js';
 export { MemoryStore, FileStore, createStore, } from './stores/index.js';
 export { createExecutionState, findResumePoint, canResume, getProgress, getExecutionSummary, FileExecutionStore, MemoryExecutionStore, } from './execution/index.js';
+export { MemoryExecutionLog, FileExecutionLog, SqliteExecutionLog, PostgresExecutionLog, foldLog, loadState, effectId, reduceCheckpoints, } from './execution-log/index.js';
 export { Scheduler, parseCronExpression, getNextRunTime, intervalToMs, shouldRunNow, } from './scheduler/index.js';
-export { generateCheckpointKey, formatSinceDate, parseSinceDate, EPOCH, FileSyncStore, MemorySyncStore, } from './sync/index.js';
+export { generateCheckpointKey, formatSinceDate, parseSinceDate, EPOCH, FileSyncStore, MemorySyncStore, LogBackedSyncStore, } from './sync/index.js';
 export { ReqonError, ParseError, LexerError, RuntimeError, ValidationError, formatErrors, getSourceLine, getSourceContext, } from './errors/index.js';
 export { loadMission, isMissionFolder, getMissionName, } from './loader/index.js';
 export { loadEnv, loadCredentials, resolveCredentials, resolveEnvString, hasEnvReference, credentialsFromEnv, } from './auth/credentials.js';
 export { WebhookServer, MemoryWebhookStore, FileWebhookStore, } from './webhook/index.js';
+// Control server for pause/resume and status queries
+export { ControlServer, } from './control/index.js';
+// Observability
+export { 
+// Event system
+ObservabilityEmitter, createEmitter, 
+// Logger
+createStructuredLogger, ConsoleOutput, JsonLinesOutput, BufferOutput, 
+// OpenTelemetry
+SpanBuilder, OTelEventAdapter, OTLPExporter, createOTelListener, } from './observability/index.js';
+export { MCP_SERVER_VERSION } from './mcp/index.js';
+// Debug
+export { BaseDebugController, CLIDebugger, } from './debug/index.js';
+// Trace - time-travel debugging
+export { FileTraceStore, MemoryTraceStore, TraceRecorder, TraceReplayer, LogTraceView, traceTimelineFromLog, createTraceRecorder, createTraceReplayer, createExecutionTrace, safeClone, truncateForTrace, } from './trace/index.js';
+// Pause - resource-free long pauses
+export { FilePauseStore, MemoryPauseStore, LogBackedPauseStore, PauseManager, createPauseManager, parseDuration, formatDuration, createPauseState, isPauseExpired, getRemainingTime, getPauseSummary, } from './pause/index.js';
 import { readFile } from 'node:fs/promises';
 import { resolve } from 'node:path';
 import { ReqonLexer } from './lexer/index.js';
@@ -43,8 +75,8 @@ export async function fromFile(filePath, config = {}) {
  * - Folder: ./sync-invoices/ (with mission.reqon + action files)
  */
 export async function fromPath(path, config = {}) {
-    const { program } = await loadMission(path);
-    const executor = new MissionExecutor(config);
+    const { program, baseDir } = await loadMission(path);
+    const executor = new MissionExecutor({ ...config, missionDir: baseDir });
     return executor.execute(program);
 }
 // Tagged template literal for inline missions

package/dist/interpreter/context.d.ts

@@ -1,13 +1,42 @@
+/**
+ * ---
+ * purpose: Execution context - scoped state for variables, stores, sources
+ * exports:
+ *   - ExecutionContext - interface for execution state
+ *   - createContext, childContext - context factory functions
+ *   - getVariable, setVariable - scoped variable access
+ * related:
+ *   - ./executor.ts - creates and uses contexts
+ *   - ./evaluator.ts - reads from context during expression evaluation
+ *   - ../stores/types.ts - StoreAdapter interface
+ * ---
+ */
 import type { SchemaDefinition } from 'vague-lang';
 import type { StoreAdapter } from '../stores/types.js';
 import type { HttpClient } from './http.js';
+/**
+ * Mutable per-action execution state. Parallel actions (`run [A, B]`) must not
+ * share this — each gets its own scope so their step counters and deferred sync
+ * checkpoints can't corrupt each other. Nested scopes (for/match) inherit the
+ * same reference so the whole action shares one counter/checkpoint list.
+ */
+export interface ActionScope {
+    /** Monotonic step index within the action (for trace/debug positions). */
+    stepIndex: number;
+    /** Current retry attempt of the action (0-based), for effect identity. */
+    attempt: number;
+    /** Sync checkpoints deferred until the action's data is durably stored. */
+    pendingCheckpoints: Array<() => Promise<void>>;
+}
 export interface ExecutionContext {
     stores: Map<string, StoreAdapter>;
+    storeTypes: Map<string, string>;
     sources: Map<string, HttpClient>;
     schemas: Map<string, SchemaDefinition>;
     variables: Map<string, unknown>;
     response?: unknown;
     parent?: ExecutionContext;
+    actionScope?: ActionScope;
 }
 export declare function createContext(): ExecutionContext;
 export declare function childContext(parent: ExecutionContext): ExecutionContext;

package/dist/interpreter/context.js

@@ -1,6 +1,20 @@
+/**
+ * ---
+ * purpose: Execution context - scoped state for variables, stores, sources
+ * exports:
+ *   - ExecutionContext - interface for execution state
+ *   - createContext, childContext - context factory functions
+ *   - getVariable, setVariable - scoped variable access
+ * related:
+ *   - ./executor.ts - creates and uses contexts
+ *   - ./evaluator.ts - reads from context during expression evaluation
+ *   - ../stores/types.ts - StoreAdapter interface
+ * ---
+ */
 export function createContext() {
     return {
         stores: new Map(),
+        storeTypes: new Map(),
         sources: new Map(),
         schemas: new Map(),
         variables: new Map(),
@@ -9,10 +23,14 @@ export function createContext() {
 export function childContext(parent) {
     return {
         stores: parent.stores,
+        storeTypes: parent.storeTypes,
         sources: parent.sources,
         schemas: parent.schemas,
         variables: new Map(),
         parent,
+        // Nested scopes (for/match) share the action's scope so the step counter
+        // and checkpoint list stay coherent across the whole action.
+        actionScope: parent.actionScope,
     };
 }
 export function getVariable(ctx, name) {

package/dist/interpreter/evaluator.d.ts

@@ -1,5 +1,67 @@
+/**
+ * ---
+ * purpose: Expression evaluator - evaluates Vague/Reqon expressions at runtime
+ * inputs:
+ *   - Expression AST node (literals, identifiers, binary ops, function calls)
+ *   - ExecutionContext with variables, response, stores
+ *   - Optional current record for iteration contexts
+ * outputs:
+ *   - Evaluated value (any JSON-compatible type)
+ * related:
+ *   - ./context.ts - provides variable bindings
+ *   - ./executor.ts - calls evaluate() for step expressions
+ *   - ../parser/expressions.ts - IsExpression, ObjectLiteralExpression types
+ * ---
+ */
 import type { Expression } from 'vague-lang';
 import type { ExecutionContext } from './context.js';
-export declare function evaluate(expr: Expression, ctx: ExecutionContext, current?: unknown): unknown;
+import type { IsExpression, ObjectLiteralExpression } from '../parser/expressions.js';
+/**
+ * Evaluate a Reqon/Vague expression within an execution context.
+ *
+ * Supports all expression types from the Vague DSL including literals, identifiers,
+ * binary/logical/ternary expressions, function calls, and pattern matching.
+ *
+ * @param expr - The expression AST node to evaluate
+ * @param ctx - The execution context containing variables, response data, and stores
+ * @param current - Optional current record for iteration contexts (e.g., inside for/map)
+ * @returns The evaluated result, which can be any JSON-compatible value
+ *
+ * @example
+ * // Evaluate a simple identifier
+ * const result = evaluate({ type: 'Identifier', name: 'foo' }, ctx);
+ *
+ * @example
+ * // Evaluate with current record context
+ * const result = evaluate(expr, ctx, { id: 1, name: 'test' });
+ */
+export declare function evaluate(expr: Expression | IsExpression | ObjectLiteralExpression, ctx: ExecutionContext, current?: unknown): unknown;
+/**
+ * Evaluate an expression and convert the result to a string.
+ *
+ * @param expr - The expression to evaluate
+ * @param ctx - The execution context
+ * @param current - Optional current record for iteration contexts
+ * @returns The string representation of the evaluated value
+ */
 export declare function evaluateToString(expr: Expression, ctx: ExecutionContext, current?: unknown): string;
+/**
+ * Interpolate variables in a path string using {variable.path} syntax.
+ *
+ * Replaces placeholders like {id} or {user.name} with values from the
+ * current record or execution context variables.
+ *
+ * @param path - The path string with {placeholder} syntax
+ * @param ctx - The execution context
+ * @param current - Optional current record for iteration contexts
+ * @returns The interpolated path string
+ *
+ * @example
+ * // Interpolate a simple variable
+ * interpolatePath('/users/{id}', ctx, { id: 123 }); // '/users/123'
+ *
+ * @example
+ * // Interpolate a nested path
+ * interpolatePath('/orgs/{org.id}/users', ctx, { org: { id: 'acme' } }); // '/orgs/acme/users'
+ */
 export declare function interpolatePath(path: string, ctx: ExecutionContext, current?: unknown): string;