reqon-dsl 0.2.0 → 0.4.0

package/CHANGELOG.md

@@ -0,0 +1,7 @@
+# Changelog
+
+The changelog has moved to the documentation site:
+
+**[https://reqon.mcclowes.com/changelog](https://reqon.mcclowes.com/changelog)**
+
+This keeps the changelog alongside the rest of the documentation and ensures it stays up to date.

package/README.md

@@ -73,9 +73,9 @@ reqon sync-invoices.vague --dry-run
 ### Programmatic
 
 ```typescript
-import { parse, execute } from 'reqon';
+import { execute } from 'reqon';
 
-const program = parse(`
+const source = `
   mission Test {
     source API { auth: bearer, base: "https://api.example.com" }
     store items: memory("items")
@@ -85,7 +85,7 @@ const program = parse(`
     }
     run Fetch
   }
-`);
+`;
 
 const result = await execute(source, {
   auth: { API: { type: 'bearer', token: 'your-token' } }
@@ -191,6 +191,48 @@ run Step1 then Step2 then Step3
 run [Step1, Step2] then Step3  // Step1 and Step2 run in parallel, then Step3
 ```
 
+### Durability Features
+
+```vague
+mission DurablePipeline {
+  // Checkpoint after each step for resume-on-failure
+  checkpoint: afterStep  // or onFailure
+
+  // Enable time-travel debugging
+  trace: full  // or minimal
+
+  action WaitForApproval {
+    // Resource-free pause (days/weeks without holding resources)
+    pause {
+      duration: "7d",
+      resumeOn: timeout | webhook "/approved"
+    }
+  }
+
+  run WaitForApproval
+}
+```
+
+#### Durability guarantees
+
+Run a mission as a **durable execution** (`executionLog:`) and an append-only
+event log lets an interrupted run — crash, deploy, `kill -9` — resume exactly
+where it left off:
+
+- **Delivery**: at-least-once (a step is never silently dropped).
+- **Effects**: exactly-once where the API honours idempotency keys (mutating
+  fetches carry a stable `Idempotency-Key`), otherwise at-least-once + keyed
+  store dedup. Exactly-once on replay via recorded effect identity.
+- **Resume across restart**: replay + fold; effects already applied are skipped.
+- **Backends**: in-memory (tests), file (dev), `SqliteExecutionLog`
+  (transactional, fsync-backed) for single-process self-hosting, and
+  `PostgresExecutionLog` for multi-node.
+
+These guarantees are proven by a crash-injection suite that kills the run at
+every event boundary and asserts no lost record and no duplicated effect
+(`npm run test:crash`). See **[DURABILITY.md](./DURABILITY.md)** for the full
+statement and the guarantee → test map.
+
 ## OpenAPI Integration
 
 Reqon can consume OpenAPI specs directly, eliminating the need for handwritten SDK code:

package/dist/ast/nodes.d.ts

@@ -1,5 +1,19 @@
+/**
+ * ---
+ * purpose: AST node type definitions for Reqon programs
+ * exports:
+ *   - ReqonProgram, Statement - top-level types
+ *   - MissionDefinition, ActionDefinition - pipeline structure
+ *   - SourceDefinition, StoreDefinition - resource declarations
+ *   - FetchStep, ForStep, MapStep, ValidateStep, etc. - action steps
+ * related:
+ *   - ../parser/parser.ts - produces these AST nodes
+ *   - ../interpreter/executor.ts - executes AST
+ *   - vague-lang - Expression, SchemaDefinition from parent DSL
+ * ---
+ */
 import type { Expression, SchemaDefinition, Statement as VagueStatement } from 'vague-lang';
-export type Statement = VagueStatement | MissionDefinition | SourceDefinition | StoreDefinition | ActionDefinition;
+export type Statement = VagueStatement | MissionDefinition | SourceDefinition | StoreDefinition | ActionDefinition | TransformDefinition;
 export interface ReqonProgram {
     type: 'ReqonProgram';
     statements: Statement[];
@@ -39,7 +53,7 @@ export interface AuthConfig {
 export interface StoreDefinition {
     type: 'StoreDefinition';
     name: string;
-    storeType: 'nosql' | 'sql' | 'memory';
+    storeType: 'nosql' | 'sql' | 'memory' | 'file' | 'postgrest';
     target: string;
 }
 export interface ScheduleDefinition {
@@ -61,13 +75,32 @@ export interface ScheduleRetryConfig {
     maxRetries: number;
     delaySeconds: number;
 }
+export interface CheckpointConfig {
+    type: 'CheckpointConfig';
+    /** When to create checkpoints */
+    mode: 'after-step' | 'on-failure';
+    /** Optional: custom store for checkpoint data (defaults to FileStore) */
+    store?: string;
+}
+export interface TraceConfig {
+    type: 'TraceConfig';
+    /** Trace mode - 'full' captures state at each step */
+    mode: 'full' | 'minimal';
+    /** Optional: custom store for trace data */
+    store?: string;
+}
 export interface MissionDefinition {
     type: 'MissionDefinition';
     name: string;
     schedule?: ScheduleDefinition;
+    /** Checkpoint configuration for durable execution */
+    checkpoint?: CheckpointConfig;
+    /** Trace configuration for time-travel debugging */
+    trace?: TraceConfig;
     sources: SourceDefinition[];
     stores: StoreDefinition[];
     schemas: SchemaDefinition[];
+    transforms: TransformDefinition[];
     actions: ActionDefinition[];
     pipeline: PipelineDefinition;
 }
@@ -76,7 +109,7 @@ export interface ActionDefinition {
     name: string;
     steps: ActionStep[];
 }
-export type ActionStep = FetchStep | ForStep | MapStep | ValidateStep | StoreStep | MatchStep | LetStep | WebhookStep;
+export type ActionStep = FetchStep | ForStep | MapStep | ValidateStep | StoreStep | MatchStep | LetStep | ApplyStep | WebhookStep | PauseStep;
 export interface LetStep {
     type: 'LetStep';
     name: string;
@@ -103,6 +136,22 @@ export interface WebhookStorageConfig {
     /** Expression for extracting key from webhook payload */
     key?: Expression;
 }
+export interface PauseStep {
+    type: 'PauseStep';
+    /** Duration to pause (e.g., "7d", "1h", "30m", 86400000) */
+    duration: string | number;
+    /** Store to persist pause state (defaults to FileStore) */
+    persist?: string;
+    /** Resume triggers - timeout (default) and/or webhook */
+    resumeOn?: PauseResumeTrigger[];
+}
+export type PauseResumeTrigger = {
+    type: 'timeout';
+} | {
+    type: 'webhook';
+    path: string;
+    eventFilter?: Expression;
+};
 export type FlowDirective = {
     type: 'continue';
 } | {
@@ -129,6 +178,8 @@ export interface MatchStep {
 export interface MatchArm {
     /** Schema name to match against, or '_' for wildcard */
     schema: string;
+    /** Optional guard condition (when using 'where' clause) */
+    guard?: Expression;
     /** Steps to execute if matched */
     steps?: ActionStep[];
     /** Flow control directive (if no steps) */
@@ -146,14 +197,22 @@ export interface FetchStep {
     until?: Expression;
     retry?: RetryConfig;
     since?: SinceConfig;
+    /**
+     * Resumable backfill: persist per-page progress to the execution log so a
+     * large paginated fetch survives a restart/deploy and continues from the last
+     * page. Requires durable mode (an executionLog). Only meaningful with paginate.
+     */
+    backfill?: boolean;
 }
 export interface SinceConfig {
     /** How to resolve the "since" timestamp */
     type: 'lastSync' | 'expression';
     /** Custom checkpoint key (defaults to source:endpoint) */
     key?: string;
-    /** Query parameter name for the since value (default: varies by API) */
+    /** Query parameter name for the since value (default: 'since') */
     param?: string;
+    /** Header name for the since value (e.g., 'If-Modified-Since'). Mutually exclusive with param. */
+    header?: string;
     /** Date format for the since value */
     format?: 'iso' | 'unix' | 'unix-ms' | 'date-only';
     /** Expression to evaluate for 'expression' type */
@@ -176,6 +235,8 @@ export interface RetryConfig {
     backoff: 'exponential' | 'linear' | 'constant';
     initialDelay: number;
     maxDelay?: number;
+    /** Per-attempt request timeout in ms; aborts a request that hangs */
+    timeout?: number;
 }
 export interface ForStep {
     type: 'ForStep';
@@ -194,6 +255,32 @@ export interface FieldMapping {
     field: string;
     expression: Expression;
 }
+export interface TransformDefinition {
+    type: 'TransformDefinition';
+    name: string;
+    variants: TransformVariant[];
+}
+export interface TransformVariant {
+    /** Source schema name, or '_' for wildcard/default */
+    sourceSchema: string | '_';
+    /** Target schema name */
+    targetSchema: string;
+    /** Optional guard condition for additional matching */
+    guard?: Expression;
+    /** Field mappings for this variant */
+    mappings: FieldMapping[];
+}
+export interface ApplyStep {
+    type: 'ApplyStep';
+    transform: string;
+    source: Expression;
+    /** Optional: bind result to a variable instead of response */
+    as?: string;
+}
+export interface TryExpression {
+    type: 'TryExpression';
+    expressions: Expression[];
+}
 export interface ValidateStep {
     type: 'ValidateStep';
     target: Expression;

package/dist/ast/nodes.js

@@ -1,3 +1,17 @@
+/**
+ * ---
+ * purpose: AST node type definitions for Reqon programs
+ * exports:
+ *   - ReqonProgram, Statement - top-level types
+ *   - MissionDefinition, ActionDefinition - pipeline structure
+ *   - SourceDefinition, StoreDefinition - resource declarations
+ *   - FetchStep, ForStep, MapStep, ValidateStep, etc. - action steps
+ * related:
+ *   - ../parser/parser.ts - produces these AST nodes
+ *   - ../interpreter/executor.ts - executes AST
+ *   - vague-lang - Expression, SchemaDefinition from parent DSL
+ * ---
+ */
 /** Helper to check if a stage is parallel */
 export function isParallelStage(stage) {
     return Array.isArray(stage.actions) && stage.actions.length > 0;

package/dist/auth/circuit-breaker.d.ts

@@ -99,6 +99,7 @@ export declare class CircuitBreaker {
      * Get current status for a source/endpoint
      */
     getStatus(source: string, endpoint?: string): CircuitBreakerStatus;
+    private statusFromEntry;
     /**
      * Force reset a circuit to closed state
      */
@@ -107,8 +108,18 @@ export declare class CircuitBreaker {
      * Get all circuit statuses
      */
     getAllStatuses(): Map<string, CircuitBreakerStatus>;
+    /**
+     * Evict stale closed circuits to bound memory. Open and half-open circuits
+     * are always retained (they carry active backoff/recovery state).
+     *
+     * @param maxAgeMs Remove closed entries with no activity for this long.
+     * @returns Number of entries evicted.
+     */
+    evictStale(maxAgeMs?: number): number;
     private getKey;
     private createEntry;
+    /** Read-only lookup. Never inserts — used by status/probe-check paths. */
+    private getEntry;
     private getOrCreateEntry;
     private pruneOldFailures;
     private transitionTo;

package/dist/auth/circuit-breaker.js

@@ -9,13 +9,14 @@
  * - OPEN: Circuit tripped, requests fail immediately
  * - HALF_OPEN: Testing if service recovered, limited requests allowed
  */
+import { CIRCUIT_BREAKER_DEFAULTS } from '../config/index.js';
 const DEFAULT_CONFIG = {
-    failureThreshold: 5,
-    resetTimeout: 30000,
-    successThreshold: 2,
-    failureWindow: 60000,
-    failureStatusCodes: [500, 501, 502, 503, 504],
-    countNetworkErrors: true,
+    failureThreshold: CIRCUIT_BREAKER_DEFAULTS.FAILURE_THRESHOLD,
+    resetTimeout: CIRCUIT_BREAKER_DEFAULTS.RESET_TIMEOUT_MS,
+    successThreshold: CIRCUIT_BREAKER_DEFAULTS.SUCCESS_THRESHOLD,
+    failureWindow: CIRCUIT_BREAKER_DEFAULTS.FAILURE_WINDOW_MS,
+    failureStatusCodes: [...CIRCUIT_BREAKER_DEFAULTS.FAILURE_STATUS_CODES],
+    countNetworkErrors: CIRCUIT_BREAKER_DEFAULTS.COUNT_NETWORK_ERRORS,
 };
 /**
  * Error thrown when circuit breaker is open
@@ -66,7 +67,12 @@ export class CircuitBreaker {
      * Check if a request can proceed (throws if circuit is open)
      */
     canProceed(source, endpoint) {
-        const entry = this.getOrCreateEntry(source, endpoint);
+        // Read-only: an unknown key is implicitly closed and must not be inserted
+        // (otherwise every probed endpoint leaks an entry forever).
+        const entry = this.getEntry(source, endpoint);
+        if (!entry) {
+            return true;
+        }
         const now = Date.now();
         if (entry.state === 'closed') {
             return true;
@@ -74,13 +80,20 @@ export class CircuitBreaker {
         if (entry.state === 'open') {
             const timeSinceOpen = now - (entry.openedAt ?? now);
             if (timeSinceOpen >= entry.config.resetTimeout) {
-                // Transition to half-open
+                // Transition to half-open and let this single caller be the probe.
                 this.transitionTo(entry, 'half_open', source, endpoint);
+                entry.probeInFlight = true;
                 return true;
             }
             return false;
         }
-        // Half-open: allow the request through for testing
+        // Half-open: allow exactly one probe at a time. Every other request is
+        // denied until the in-flight probe resolves (success closes the circuit or
+        // advances the count; failure re-opens it), preventing a recovery stampede.
+        if (entry.probeInFlight) {
+            return false;
+        }
+        entry.probeInFlight = true;
         return true;
     }
     /**
@@ -88,9 +101,11 @@ export class CircuitBreaker {
      */
     ensureCanProceed(source, endpoint) {
         if (!this.canProceed(source, endpoint)) {
-            const entry = this.getOrCreateEntry(source, endpoint);
+            // canProceed only returns false for an existing open/half-open entry.
+            const entry = this.getEntry(source, endpoint);
+            const config = entry?.config ?? this.defaultConfig;
             const now = Date.now();
-            const nextAttemptIn = entry.config.resetTimeout - (now - (entry.openedAt ?? now));
+            const nextAttemptIn = config.resetTimeout - (now - (entry?.openedAt ?? now));
             this.callbacks.onRejected?.({
                 source,
                 endpoint,
@@ -103,13 +118,23 @@ export class CircuitBreaker {
      * Record a successful request
      */
     recordSuccess(source, endpoint) {
-        const entry = this.getOrCreateEntry(source, endpoint);
+        // Read-only lookup: a success on an untracked (implicitly closed) endpoint
+        // has nothing to record, so don't allocate an entry for it.
+        const entry = this.getEntry(source, endpoint);
+        if (!entry) {
+            return;
+        }
+        entry.lastActivity = Date.now();
         if (entry.state === 'half_open') {
             entry.successes++;
             if (entry.successes >= entry.config.successThreshold) {
                 // Recovery successful, close the circuit
                 this.transitionTo(entry, 'closed', source, endpoint);
             }
+            else {
+                // Probe succeeded but more are needed; release the slot for the next.
+                entry.probeInFlight = false;
+            }
         }
         else if (entry.state === 'closed') {
             // Clear old failures from window
@@ -120,15 +145,19 @@ export class CircuitBreaker {
      * Record a failed request
      */
     recordFailure(source, endpoint, statusCode, isNetworkError = false) {
-        const entry = this.getOrCreateEntry(source, endpoint);
-        const config = entry.config;
+        // Resolve config without forcing an entry — ignored failures (e.g. a 404)
+        // must not allocate a permanent circuit for an otherwise-untracked endpoint.
+        const existing = this.getEntry(source, endpoint);
+        const config = existing?.config ?? this.getEntry(source)?.config ?? this.defaultConfig;
         // Check if this failure type should be counted
         const isFailureStatus = statusCode !== undefined && config.failureStatusCodes.includes(statusCode);
         const shouldCount = isFailureStatus || (isNetworkError && config.countNetworkErrors);
         if (!shouldCount) {
             return;
         }
+        const entry = existing ?? this.getOrCreateEntry(source, endpoint);
         const now = Date.now();
+        entry.lastActivity = now;
         if (entry.state === 'half_open') {
             // Any failure in half-open immediately re-opens circuit
             this.transitionTo(entry, 'open', source, endpoint, 'Failure during recovery attempt');
@@ -150,8 +179,14 @@ export class CircuitBreaker {
      * Get current status for a source/endpoint
      */
     getStatus(source, endpoint) {
-        const entry = this.getOrCreateEntry(source, endpoint);
-        const now = Date.now();
+        // Read-only: querying an unknown circuit must not create one.
+        const entry = this.getEntry(source, endpoint);
+        if (!entry) {
+            return { state: 'closed', failures: 0, successes: 0, isOpen: false };
+        }
+        return this.statusFromEntry(entry);
+    }
+    statusFromEntry(entry) {
         let nextAttemptTime;
         if (entry.state === 'open' && entry.openedAt) {
             const nextAttemptMs = entry.openedAt + entry.config.resetTimeout;
@@ -180,6 +215,8 @@ export class CircuitBreaker {
             entry.failureTimestamps = [];
             entry.lastFailureTime = undefined;
             entry.openedAt = undefined;
+            entry.probeInFlight = false;
+            entry.lastActivity = Date.now();
             if (previousState !== 'closed') {
                 this.callbacks.onClose?.({
                     source,
@@ -197,12 +234,37 @@ export class CircuitBreaker {
      */
     getAllStatuses() {
         const result = new Map();
+        // Build the status from each entry directly. Splitting the key on ':' would
+        // corrupt URL-shaped keys (e.g. `https://x` → source `https`), and routing
+        // back through getStatus is needless. We never mutate `circuits` here.
         for (const [key, entry] of this.circuits) {
-            const [source, endpoint] = key.split(':');
-            result.set(key, this.getStatus(source, endpoint === '' ? undefined : endpoint));
+            result.set(key, this.statusFromEntry(entry));
         }
         return result;
     }
+    /**
+     * Evict stale closed circuits to bound memory. Open and half-open circuits
+     * are always retained (they carry active backoff/recovery state).
+     *
+     * @param maxAgeMs Remove closed entries with no activity for this long.
+     * @returns Number of entries evicted.
+     */
+    evictStale(maxAgeMs = this.defaultConfig.failureWindow) {
+        const cutoff = Date.now() - maxAgeMs;
+        let evicted = 0;
+        // Collect first, then delete — never mutate the map mid-iteration.
+        const toDelete = [];
+        for (const [key, entry] of this.circuits) {
+            if (entry.state === 'closed' && entry.lastActivity < cutoff) {
+                toDelete.push(key);
+            }
+        }
+        for (const key of toDelete) {
+            this.circuits.delete(key);
+            evicted++;
+        }
+        return evicted;
+    }
     getKey(source, endpoint) {
         return endpoint ? `${source}:${endpoint}` : source;
     }
@@ -212,9 +274,15 @@ export class CircuitBreaker {
             failures: 0,
             successes: 0,
             failureTimestamps: [],
+            probeInFlight: false,
+            lastActivity: Date.now(),
             config,
         };
     }
+    /** Read-only lookup. Never inserts — used by status/probe-check paths. */
+    getEntry(source, endpoint) {
+        return this.circuits.get(this.getKey(source, endpoint));
+    }
     getOrCreateEntry(source, endpoint) {
         const key = this.getKey(source, endpoint);
         let entry = this.circuits.get(key);
@@ -236,6 +304,10 @@ export class CircuitBreaker {
     transitionTo(entry, newState, source, endpoint, reason) {
         const previousState = entry.state;
         entry.state = newState;
+        entry.lastActivity = Date.now();
+        // Any state change clears the half-open probe slot; canProceed re-claims it
+        // when it admits the next probe.
+        entry.probeInFlight = false;
         const event = {
             source,
             endpoint,

package/dist/auth/credentials.d.ts

@@ -33,12 +33,17 @@ export interface LoadEnvResult {
  */
 export declare function loadEnv(options?: CredentialsConfig): LoadEnvResult;
 /**
- * Resolve environment variable references in a string
+ * Resolve environment variable references in a string.
  *
  * Supports:
  * - $VAR_NAME
  * - ${VAR_NAME}
  * - ${VAR_NAME:-default}
+ *
+ * A reference with no value and no default is treated as *required* and throws,
+ * rather than silently resolving to '' (which previously let requests go out
+ * with an empty Authorization header). Use the `${VAR:-}` form to opt a
+ * reference into being optional with an empty fallback.
  */
 export declare function resolveEnvString(value: string): string;
 /**

package/dist/auth/credentials.js

@@ -55,12 +55,17 @@ export function loadEnv(options = {}) {
     };
 }
 /**
- * Resolve environment variable references in a string
+ * Resolve environment variable references in a string.
  *
  * Supports:
  * - $VAR_NAME
  * - ${VAR_NAME}
  * - ${VAR_NAME:-default}
+ *
+ * A reference with no value and no default is treated as *required* and throws,
+ * rather than silently resolving to '' (which previously let requests go out
+ * with an empty Authorization header). Use the `${VAR:-}` form to opt a
+ * reference into being optional with an empty fallback.
  */
 export function resolveEnvString(value) {
     // Pattern for ${VAR:-default} or ${VAR}
@@ -75,15 +80,18 @@ export function resolveEnvString(value) {
             return envValue;
         }
         if (defaultValue !== undefined) {
+            // Includes the empty-default escape hatch `${VAR:-}`.
             return defaultValue;
         }
-        // Return empty string if no value and no default
-        return '';
+        throw new Error(`unresolved required credential ${varName}`);
     });
     // Then resolve $VAR patterns (only if not already resolved)
     result = result.replace(simplePattern, (_match, varName) => {
         const envValue = process.env[varName];
-        return envValue !== undefined ? envValue : '';
+        if (envValue !== undefined) {
+            return envValue;
+        }
+        throw new Error(`unresolved required credential ${varName}`);
     });
     return result;
 }

package/dist/auth/oauth2-provider.js

@@ -60,10 +60,16 @@ export class OAuth2AuthProvider {
             body,
         });
         if (!response.ok) {
-            const error = await response.text();
-            throw new Error(`Token refresh failed: ${response.status} ${error}`);
+            // Do not include the response body: token endpoints can echo the
+            // submitted client secret / refresh token in error responses.
+            throw new Error(`Token refresh failed: ${response.status} ${response.statusText}`);
         }
         const data = (await response.json());
+        // A 2xx with no access_token would otherwise propagate as `Bearer undefined`
+        // on every subsequent request. Fail loudly instead.
+        if (!data.access_token) {
+            throw new Error(`Token refresh for connection "${this.connectionId}" returned no access_token`);
+        }
         const newTokens = {
             accessToken: data.access_token,
             refreshToken: data.refresh_token ?? tokens.refreshToken, // Keep old if not rotated
@@ -80,7 +86,11 @@ export class OAuth2AuthProvider {
     }
     shouldRefresh(tokens) {
         if (!tokens.expiresAt) {
-            return false; // No expiry info, assume valid
+            // No expiry metadata: we cannot know when the token lapses, so we treat it
+            // as still valid and let a downstream 401 trigger an explicit
+            // refreshToken() call. Proactively refreshing here would burn the refresh
+            // token (and break non-expiring tokens) on every single request.
+            return false;
         }
         const now = Date.now();
         const expiresAt = tokens.expiresAt.getTime();

package/dist/auth/rate-limiter.d.ts

@@ -16,6 +16,10 @@ export declare class RateLimitError extends Error {
     resetAt?: Date | undefined;
     constructor(source: string, resetAt?: Date | undefined);
 }
+/**
+ * Adaptive rate limiter that learns from response headers
+ * Supports pause, throttle, and fail strategies
+ */
 export declare class AdaptiveRateLimiter implements RateLimiter {
     private defaultConfig;
     private state;
@@ -28,7 +32,14 @@ export declare class AdaptiveRateLimiter implements RateLimiter {
         maxStaleAgeMs?: number;
     });
     /**
-     * Clean up stale entries from the state map to prevent memory leaks
+     * Clean up stale entries from the state map to prevent memory leaks.
+     *
+     * Eviction is driven by `lastRequestAt`, not `resetAt`. Most of the leak came
+     * from keys whose API sends no reset header: `resetAt` is never set, so the
+     * old reset-passed condition could never fire and those keys lived forever.
+     * Now any key idle longer than the stale threshold is evicted regardless of
+     * `resetAt`, except while an active `retryAfter` backoff is still pending
+     * (that state must survive so we keep honouring the 429).
      */
     private cleanup;
     /**