reqon-dsl 0.2.0 → 0.4.0

package/dist/auth/rate-limiter.js

@@ -1,15 +1,10 @@
 import { sleep } from '../utils/async.js';
-/** Max age for stale entries (default: 1 hour) */
-const DEFAULT_MAX_STALE_AGE_MS = 60 * 60 * 1000;
-/** Cleanup interval (default: 5 minutes) */
-const CLEANUP_INTERVAL_MS = 5 * 60 * 1000;
-/** Max number of entries before forced cleanup */
-const MAX_ENTRIES_BEFORE_CLEANUP = 1000;
+import { RATE_LIMIT_DEFAULTS } from '../config/index.js';
 const DEFAULT_CONFIG = {
-    strategy: 'pause',
-    maxWait: 300,
-    notifyAt: 10,
-    fallbackRpm: 60,
+    strategy: RATE_LIMIT_DEFAULTS.STRATEGY,
+    maxWait: RATE_LIMIT_DEFAULTS.MAX_WAIT_SECONDS,
+    notifyAt: RATE_LIMIT_DEFAULTS.NOTIFY_AT_SECONDS,
+    fallbackRpm: RATE_LIMIT_DEFAULTS.FALLBACK_RPM,
 };
 /**
  * Rate limit timeout error - thrown when maxWait is exceeded
@@ -44,8 +39,6 @@ export class RateLimitError extends Error {
  * Adaptive rate limiter that learns from response headers
  * Supports pause, throttle, and fail strategies
  */
-/** Number of responses between cleanup checks */
-const CLEANUP_CHECK_INTERVAL = 100;
 export class AdaptiveRateLimiter {
     defaultConfig;
     state = new Map();
@@ -56,29 +49,37 @@ export class AdaptiveRateLimiter {
     responsesSinceCleanup = 0;
     constructor(defaultConfig = {}, options = {}) {
         this.defaultConfig = defaultConfig;
-        this.maxStaleAgeMs = options.maxStaleAgeMs ?? DEFAULT_MAX_STALE_AGE_MS;
+        this.maxStaleAgeMs = options.maxStaleAgeMs ?? RATE_LIMIT_DEFAULTS.MAX_STALE_AGE_MS;
     }
     /**
-     * Clean up stale entries from the state map to prevent memory leaks
+     * Clean up stale entries from the state map to prevent memory leaks.
+     *
+     * Eviction is driven by `lastRequestAt`, not `resetAt`. Most of the leak came
+     * from keys whose API sends no reset header: `resetAt` is never set, so the
+     * old reset-passed condition could never fire and those keys lived forever.
+     * Now any key idle longer than the stale threshold is evicted regardless of
+     * `resetAt`, except while an active `retryAfter` backoff is still pending
+     * (that state must survive so we keep honouring the 429).
      */
     cleanup() {
         const now = Date.now();
         // Skip if cleanup was done recently and we're under the entry limit
-        if (now - this.lastCleanup < CLEANUP_INTERVAL_MS &&
-            this.state.size < MAX_ENTRIES_BEFORE_CLEANUP) {
+        if (now - this.lastCleanup < RATE_LIMIT_DEFAULTS.CLEANUP_INTERVAL_MS &&
+            this.state.size < RATE_LIMIT_DEFAULTS.MAX_ENTRIES_BEFORE_CLEANUP) {
             return;
         }
         this.lastCleanup = now;
         const nowDate = new Date(now);
         const staleThreshold = new Date(now - this.maxStaleAgeMs);
         for (const [key, state] of this.state) {
-            // Remove if:
-            // 1. Reset time has passed AND no recent requests
-            // 2. Last request is older than max stale age
-            const isResetPassed = state.resetAt && state.resetAt < nowDate;
-            const isRetryPassed = !state.retryAfter || state.retryAfter < nowDate;
-            const isStale = state.lastRequestAt && state.lastRequestAt < staleThreshold;
-            if ((isResetPassed && isRetryPassed && isStale) || (!state.lastRequestAt && isResetPassed)) {
+            // A pending retry-after backoff must never be evicted out from under us.
+            const isRetryPending = state.retryAfter !== undefined && state.retryAfter >= nowDate;
+            if (isRetryPending)
+                continue;
+            // Stale if the last request is older than the threshold, or if no request
+            // was ever recorded for this key.
+            const isStale = !state.lastRequestAt || state.lastRequestAt < staleThreshold;
+            if (isStale) {
                 this.state.delete(key);
             }
         }
@@ -258,9 +259,18 @@ export class AdaptiveRateLimiter {
         }
         state.lastRequestAt = now;
         this.state.set(key, state);
-        // Periodically clean up stale entries (every N responses)
+        // Trigger a stale-entry sweep every N responses. The sweep itself is rate-
+        // limited by CLEANUP_INTERVAL_MS / MAX_ENTRIES_BEFORE_CLEANUP inside
+        // cleanup(), so this counter only decides how often we *consider* sweeping.
+        //
+        // TODO: this limiter is purely reactive — it learns limits from response
+        // headers after the fact. Add proactive in-flight token accounting (track
+        // tokens consumed by requests that have been issued but not yet returned a
+        // response) so concurrent callers can't collectively blow past the limit
+        // between header updates. That is a larger redesign; the eviction leak fix
+        // above is independent of it.
         this.responsesSinceCleanup++;
-        if (this.responsesSinceCleanup >= CLEANUP_CHECK_INTERVAL) {
+        if (this.responsesSinceCleanup >= RATE_LIMIT_DEFAULTS.CLEANUP_CHECK_INTERVAL) {
             this.responsesSinceCleanup = 0;
             this.cleanup();
         }
@@ -273,7 +283,10 @@ export class AdaptiveRateLimiter {
             return { isLimited: false };
         }
         const isLimited = (state.retryAfter && state.retryAfter > now) ||
-            (state.remaining !== undefined && state.remaining <= 0 && state.resetAt && state.resetAt > now);
+            (state.remaining !== undefined &&
+                state.remaining <= 0 &&
+                state.resetAt &&
+                state.resetAt > now);
         let resetInSeconds;
         if (isLimited && state.resetAt) {
             resetInSeconds = Math.ceil((state.resetAt.getTime() - now.getTime()) / 1000);

package/dist/auth/token-store.js

@@ -103,7 +103,14 @@ export class FileTokenStore {
         for (const [key, value] of this.cache) {
             data[key] = value;
         }
-        await fs.writeFile(this.filePath, JSON.stringify(data, null, 2), 'utf-8');
+        // Owner-only (0o600): the file holds OAuth tokens and must not be
+        // world-readable. `mode` only applies on creation, so chmod an existing
+        // file too (in case it was created before this fix or under a loose umask).
+        await fs.writeFile(this.filePath, JSON.stringify(data, null, 2), {
+            encoding: 'utf-8',
+            mode: 0o600,
+        });
+        await fs.chmod(this.filePath, 0o600).catch(() => { });
     }
     async get(connectionId) {
         const map = await this.load();

package/dist/cli.d.ts

@@ -1,2 +1,25 @@
 #!/usr/bin/env node
-export {};
+/**
+ * ---
+ * purpose: CLI entry point - parses args and runs missions
+ * inputs:
+ *   - file.reqon or folder path
+ *   - --dry-run, --verbose, --auth, --env, --output, --daemon, --webhook, --debug
+ * related:
+ *   - ./index.ts - library APIs used by CLI
+ *   - ./scheduler/index.ts - daemon mode scheduling
+ *   - ./webhook/index.ts - webhook server for wait steps
+ *   - ./debug/cli-debugger.ts - interactive debugging
+ * ---
+ */
+/**
+ * Load and resolve an --auth credentials file, turning low-level fs/JSON
+ * failures into clean, user-facing messages instead of raw stack traces.
+ */
+export declare function loadAuthFile(rawPath: string): Promise<Record<string, unknown>>;
+/**
+ * Print an error in a clean, user-facing form (no stack trace). ReqonErrors
+ * get their rich source-context formatting; everything else gets a one-line
+ * message.
+ */
+export declare function reportFatalError(error: unknown): void;

package/dist/cli.js

@@ -1,10 +1,70 @@
 #!/usr/bin/env node
+/**
+ * ---
+ * purpose: CLI entry point - parses args and runs missions
+ * inputs:
+ *   - file.reqon or folder path
+ *   - --dry-run, --verbose, --auth, --env, --output, --daemon, --webhook, --debug
+ * related:
+ *   - ./index.ts - library APIs used by CLI
+ *   - ./scheduler/index.ts - daemon mode scheduling
+ *   - ./webhook/index.ts - webhook server for wait steps
+ *   - ./debug/cli-debugger.ts - interactive debugging
+ * ---
+ */
 import { readFile, writeFile, mkdir } from 'node:fs/promises';
 import { resolve, dirname } from 'node:path';
+import { pathToFileURL } from 'node:url';
 import { fromPath, Scheduler, loadMission } from './index.js';
 import { ReqonError } from './errors/index.js';
 import { loadEnv, loadCredentials } from './auth/credentials.js';
 import { WebhookServer } from './webhook/index.js';
+import { ControlServer } from './control/index.js';
+/**
+ * Load and resolve an --auth credentials file, turning low-level fs/JSON
+ * failures into clean, user-facing messages instead of raw stack traces.
+ */
+export async function loadAuthFile(rawPath) {
+    const authPath = resolve(rawPath);
+    let authContent;
+    try {
+        authContent = await readFile(authPath, 'utf-8');
+    }
+    catch (err) {
+        if (err.code === 'ENOENT') {
+            throw new Error(`auth file not found: ${authPath}`);
+        }
+        if (err.code === 'EISDIR') {
+            throw new Error(`auth path is a directory, not a file: ${authPath}`);
+        }
+        throw new Error(`could not read auth file ${authPath}: ${err.message}`);
+    }
+    let rawAuth;
+    try {
+        rawAuth = JSON.parse(authContent);
+    }
+    catch {
+        throw new Error(`auth file is not valid JSON: ${authPath}`);
+    }
+    // Resolve env var references in the auth config
+    return loadCredentials(rawAuth);
+}
+/**
+ * Print an error in a clean, user-facing form (no stack trace). ReqonErrors
+ * get their rich source-context formatting; everything else gets a one-line
+ * message.
+ */
+export function reportFatalError(error) {
+    if (error instanceof ReqonError) {
+        console.error(error.format());
+    }
+    else if (error instanceof Error) {
+        console.error(`Error: ${error.message}`);
+    }
+    else {
+        console.error(`Error: ${String(error)}`);
+    }
+}
 async function main() {
     const args = process.argv.slice(2);
     if (args.length === 0 || args.includes('--help') || args.includes('-h')) {
@@ -17,6 +77,7 @@ Usage:
 Options:
   --dry-run            Run without making actual HTTP requests
   --verbose            Enable verbose logging
+  --dev                Development mode: let sql/nosql stores fall back to local JSON files
   --auth <file>        JSON file with auth credentials (supports env var interpolation)
   --env <file>         Path to .env file (default: .env in current directory)
   --output <path>      Export stores to JSON (file or directory)
@@ -25,6 +86,10 @@ Options:
   --webhook            Enable webhook server for 'wait' steps
   --webhook-port <n>   Port for webhook server (default: 3000)
   --webhook-url <url>  Base URL for webhook endpoints (default: http://localhost:3000)
+  --control            Enable control server for pause/resume and status queries
+  --control-port <n>   Port for control server (default: 3001)
+  --debug              Enable step-through debugging
+  --resume <id>        Resume a paused or failed execution by ID
   --help, -h           Show this help message
 
 Environment Variables:
@@ -44,15 +109,27 @@ Examples:
   reqon sync-invoices.reqon --output ./output.json
   reqon sync-invoices.reqon --daemon --verbose
   reqon sync-invoices.reqon --webhook --webhook-port 8080 --verbose
+  reqon sync-invoices.reqon --control --verbose   # enable control server
+  reqon sync-invoices.reqon --resume exec_abc123  # resume paused execution
+
+Control Server:
+  When --control is enabled, the server exposes these endpoints:
+    POST /pause    Request graceful pause at next safe point
+    POST /resume   Clear pause request
+    GET  /status   Get current execution state and progress
+    GET  /health   Health check
 `);
         process.exit(0);
     }
     const filePath = args[0];
     const dryRun = args.includes('--dry-run');
     const verbose = args.includes('--verbose');
+    const devMode = args.includes('--dev');
     const daemon = args.includes('--daemon');
     const once = args.includes('--once');
     const webhookEnabled = args.includes('--webhook');
+    const debugEnabled = args.includes('--debug');
+    const controlEnabled = args.includes('--control');
     // Parse webhook options
     let webhookPort = 3000;
     const webhookPortIndex = args.indexOf('--webhook-port');
@@ -64,6 +141,18 @@ Examples:
     if (webhookUrlIndex !== -1 && args[webhookUrlIndex + 1]) {
         webhookUrl = args[webhookUrlIndex + 1];
     }
+    // Parse control server options
+    let controlPort = 3001;
+    const controlPortIndex = args.indexOf('--control-port');
+    if (controlPortIndex !== -1 && args[controlPortIndex + 1]) {
+        controlPort = parseInt(args[controlPortIndex + 1], 10);
+    }
+    // Parse resume option
+    let resumeFrom;
+    const resumeIndex = args.indexOf('--resume');
+    if (resumeIndex !== -1 && args[resumeIndex + 1]) {
+        resumeFrom = args[resumeIndex + 1];
+    }
     // Load .env file(s)
     let envFile;
     const envIndex = args.indexOf('--env');
@@ -78,11 +167,7 @@ Examples:
     let auth;
     const authIndex = args.indexOf('--auth');
     if (authIndex !== -1 && args[authIndex + 1]) {
-        const authPath = resolve(args[authIndex + 1]);
-        const authContent = await readFile(authPath, 'utf-8');
-        const rawAuth = JSON.parse(authContent);
-        // Resolve env var references in the auth config
-        auth = loadCredentials(rawAuth);
+        auth = await loadAuthFile(args[authIndex + 1]);
     }
     let outputPath;
     const outputIndex = args.indexOf('--output');
@@ -112,14 +197,44 @@ Examples:
         await webhookServer.start();
         console.log(`Webhook server started on port ${webhookPort}`);
     }
+    // Start control server if enabled
+    let controlServer;
+    if (controlEnabled) {
+        controlServer = new ControlServer({ port: controlPort, verbose });
+        await controlServer.start();
+        console.log(`Control server started on port ${controlPort}`);
+        console.log(`  POST http://localhost:${controlPort}/pause  - request pause`);
+        console.log(`  GET  http://localhost:${controlPort}/status - check status`);
+    }
+    // Initialize debug controller if enabled
+    let debugController;
+    if (debugEnabled) {
+        const { CLIDebugger } = await import('./debug/cli-debugger.js');
+        debugController = new CLIDebugger();
+        console.log('Debug mode enabled. Commands: c(ontinue), s(tep), si(step-into), so(step-over), q(uit)');
+        console.log('Type "help" at the debug prompt for more commands.\n');
+    }
     try {
         const result = await fromPath(filePath, {
             dryRun,
             verbose,
+            developmentMode: devMode,
             auth: auth,
             webhookServer,
+            debugController,
+            controlServer,
+            resumeFrom,
         });
-        if (result.success) {
+        // Check if execution was paused
+        const isPaused = result.state?.status === 'paused';
+        if (isPaused) {
+            console.log(`\n⏸ Mission paused`);
+            console.log(`  Duration: ${result.duration}ms`);
+            console.log(`  Actions run: ${result.actionsRun.join(' → ')}`);
+            console.log(`  Execution ID: ${result.executionId}`);
+            console.log(`\n  To resume: reqon ${filePath} --resume ${result.executionId}`);
+        }
+        else if (result.success) {
             console.log(`\n✓ Mission completed successfully`);
             console.log(`  Duration: ${result.duration}ms`);
             console.log(`  Actions run: ${result.actionsRun.join(' → ')}`);
@@ -127,7 +242,7 @@ Examples:
             const storeData = {};
             for (const [name, store] of result.stores) {
                 const items = await store.list();
-                console.log(`  Store "${name}": ${items.length} items`);
+                console.log(`  Store "${name}": ${items.length} ${items.length === 1 ? 'item' : 'items'}`);
                 storeData[name] = items;
             }
             // Export to JSON if requested
@@ -142,6 +257,10 @@ Examples:
             for (const error of result.errors) {
                 console.error(`  [${error.action}/${error.step}] ${error.message}`);
             }
+            if (result.executionId) {
+                console.log(`\n  Execution ID: ${result.executionId}`);
+                console.log(`  To resume: reqon ${filePath} --resume ${result.executionId}`);
+            }
             process.exit(1);
         }
     }
@@ -159,16 +278,22 @@ Examples:
         if (webhookServer) {
             await webhookServer.stop();
         }
+        // Stop control server if it was started
+        if (controlServer) {
+            await controlServer.stop();
+        }
+        // Close debug controller if it was started
+        if (debugController?.close) {
+            debugController.close();
+        }
     }
 }
 async function runDaemon(filePath, options) {
     const absolutePath = resolve(filePath);
     let program;
-    let baseDir;
     try {
         const result = await loadMission(absolutePath);
         program = result.program;
-        baseDir = result.baseDir;
     }
     catch (error) {
         if (error instanceof ReqonError) {
@@ -267,4 +392,18 @@ async function runDaemon(filePath, options) {
 function formatTime(date) {
     return date.toISOString().replace('T', ' ').substring(0, 19);
 }
-main();
+// Only auto-run when invoked directly as the CLI entry point, so the module can
+// be imported in tests without executing main() against the test runner's argv.
+const isMain = typeof process.argv[1] === 'string' && import.meta.url === pathToFileURL(process.argv[1]).href;
+if (isMain) {
+    // A rejected promise anywhere (including outside main's try/catch) should
+    // surface as a clean message and a non-zero exit, never a raw stack trace.
+    process.on('unhandledRejection', (reason) => {
+        reportFatalError(reason);
+        process.exit(1);
+    });
+    main().catch((error) => {
+        reportFatalError(error);
+        process.exit(1);
+    });
+}

package/dist/config/constants.d.ts

@@ -0,0 +1,152 @@
+/**
+ * Centralized Configuration Constants
+ *
+ * This module contains all default values and configuration constants used
+ * throughout the Reqon framework. Centralizing these values makes it easier
+ * to understand, tune, and override defaults.
+ */
+/**
+ * Default retry configuration for HTTP requests
+ */
+export declare const HTTP_RETRY_DEFAULTS: {
+    /** Maximum number of retry attempts */
+    readonly MAX_ATTEMPTS: 3;
+    /** Initial delay between retries in milliseconds */
+    readonly INITIAL_DELAY_MS: 1000;
+    /** Maximum delay between retries in milliseconds */
+    readonly MAX_DELAY_MS: 30000;
+    /** Backoff strategy: 'exponential', 'linear', or 'constant' */
+    readonly BACKOFF: "exponential";
+    /** Per-attempt request timeout in milliseconds (aborts a hung request) */
+    readonly TIMEOUT_MS: 30000;
+};
+/**
+ * Default HTTP headers
+ */
+export declare const HTTP_DEFAULT_HEADERS: {
+    readonly CONTENT_TYPE: "application/json";
+    readonly ACCEPT: "application/json";
+};
+/**
+ * Default rate limiter configuration
+ */
+export declare const RATE_LIMIT_DEFAULTS: {
+    /** Default strategy when rate limited: 'pause', 'throttle', or 'fail' */
+    readonly STRATEGY: "pause";
+    /** Maximum wait time in seconds before timing out */
+    readonly MAX_WAIT_SECONDS: 300;
+    /** Seconds before starting to notify about ongoing waits */
+    readonly NOTIFY_AT_SECONDS: 10;
+    /** Fallback requests per minute when no rate limit info available */
+    readonly FALLBACK_RPM: 60;
+    /** Maximum age for stale rate limit entries (1 hour in ms) */
+    readonly MAX_STALE_AGE_MS: number;
+    /** Cleanup interval for stale entries (5 minutes in ms) */
+    readonly CLEANUP_INTERVAL_MS: number;
+    /** Number of responses between cleanup checks */
+    readonly CLEANUP_CHECK_INTERVAL: 100;
+    /** Maximum entries before forced cleanup */
+    readonly MAX_ENTRIES_BEFORE_CLEANUP: 1000;
+};
+/**
+ * Default circuit breaker configuration
+ */
+export declare const CIRCUIT_BREAKER_DEFAULTS: {
+    /** Number of failures before opening circuit */
+    readonly FAILURE_THRESHOLD: 5;
+    /** Time in milliseconds before attempting recovery */
+    readonly RESET_TIMEOUT_MS: 30000;
+    /** Number of successful requests in half-open to close circuit */
+    readonly SUCCESS_THRESHOLD: 2;
+    /** Time window in milliseconds for counting failures */
+    readonly FAILURE_WINDOW_MS: 60000;
+    /** HTTP status codes to count as failures */
+    readonly FAILURE_STATUS_CODES: readonly number[];
+    /** Whether to count network errors as failures */
+    readonly COUNT_NETWORK_ERRORS: true;
+};
+/**
+ * Default webhook server configuration
+ */
+export declare const WEBHOOK_DEFAULTS: {
+    /** Default port for webhook server */
+    readonly PORT: 3000;
+    /** Default host binding — loopback only; opt into 0.0.0.0 explicitly. */
+    readonly HOST: "127.0.0.1";
+    /** Default timeout for wait steps (5 minutes in ms) */
+    readonly DEFAULT_TIMEOUT_MS: 300000;
+    /** Cleanup interval for expired registrations (1 minute in ms) */
+    readonly CLEANUP_INTERVAL_MS: 60000;
+    /** Maximum accepted request body size in bytes (1 MiB) */
+    readonly MAX_BODY_BYTES: number;
+    /** Idle socket timeout in ms; drops slow-drip connections */
+    readonly SOCKET_TIMEOUT_MS: 30000;
+};
+/**
+ * Default store configuration
+ */
+export declare const STORE_DEFAULTS: {
+    /** Default base directory for file stores */
+    readonly DATA_DIR: ".reqon-data";
+    /** Default subdirectory for executions */
+    readonly EXECUTIONS_DIR: "executions";
+    /** Default subdirectory for sync state */
+    readonly SYNC_DIR: "sync";
+};
+/**
+ * Default scheduler configuration
+ */
+export declare const SCHEDULER_DEFAULTS: {
+    /** Default retry configuration for failed scheduled missions */
+    readonly MAX_RETRIES: 3;
+    /** Default delay between retries in seconds */
+    readonly RETRY_DELAY_SECONDS: 60;
+};
+/**
+ * Default execution configuration
+ */
+export declare const EXECUTION_DEFAULTS: {
+    /**
+     * Whether development mode is enabled by default. When true, sql/nosql
+     * stores fall back to local JSON files; default false so a mission that
+     * declares `store sql`/`nosql` errors loudly instead of silently writing
+     * to disk.
+     */
+    readonly DEVELOPMENT_MODE: false;
+    /** Whether to persist execution state by default */
+    readonly PERSIST_STATE: false;
+};
+/** Type for rate limit strategy */
+export type RateLimitStrategy = 'pause' | 'throttle' | 'fail';
+/** Type for retry backoff strategy */
+export type BackoffStrategy = 'exponential' | 'linear' | 'constant';
+/**
+ * Merged configuration type utilities
+ */
+export type HttpRetryConfig = {
+    maxAttempts?: number;
+    initialDelay?: number;
+    maxDelay?: number;
+    backoff?: BackoffStrategy;
+};
+export type RateLimitConfig = {
+    strategy?: RateLimitStrategy;
+    maxWait?: number;
+    notifyAt?: number;
+    fallbackRpm?: number;
+};
+export type CircuitBreakerConfig = {
+    failureThreshold?: number;
+    resetTimeout?: number;
+    successThreshold?: number;
+    failureWindow?: number;
+    failureStatusCodes?: number[];
+    countNetworkErrors?: boolean;
+};
+export type WebhookConfig = {
+    port?: number;
+    host?: string;
+    baseUrl?: string;
+    defaultTimeout?: number;
+    verbose?: boolean;
+};