reqon-dsl 0.2.0

package/dist/auth/token-store.js

@@ -0,0 +1,148 @@
+/**
+ * In-memory token store - useful for development and single-process deployments
+ */
+export class InMemoryTokenStore {
+    tokens = new Map();
+    async get(connectionId) {
+        const stored = this.tokens.get(connectionId);
+        if (!stored)
+            return null;
+        return {
+            accessToken: stored.accessToken,
+            refreshToken: stored.refreshToken,
+            expiresAt: stored.expiresAt,
+            refreshExpiresAt: stored.refreshExpiresAt,
+            tokenType: stored.tokenType,
+            scope: stored.scope,
+        };
+    }
+    async set(connectionId, tokens) {
+        const existing = this.tokens.get(connectionId);
+        this.tokens.set(connectionId, {
+            ...tokens,
+            lastUsedAt: existing?.lastUsedAt ?? new Date(),
+        });
+    }
+    async delete(connectionId) {
+        this.tokens.delete(connectionId);
+    }
+    async touch(connectionId) {
+        const stored = this.tokens.get(connectionId);
+        if (stored) {
+            stored.lastUsedAt = new Date();
+        }
+    }
+    async list() {
+        return Array.from(this.tokens.keys());
+    }
+    /** Get tokens that need proactive refresh (approaching expiry or non-use expiry) */
+    async getTokensNeedingRefresh(bufferSeconds = 300) {
+        const now = new Date();
+        const buffer = bufferSeconds * 1000;
+        const needsRefresh = [];
+        for (const [connectionId, stored] of this.tokens) {
+            // Check access token expiry
+            if (stored.expiresAt) {
+                const expiresIn = stored.expiresAt.getTime() - now.getTime();
+                if (expiresIn < buffer) {
+                    needsRefresh.push(connectionId);
+                    continue;
+                }
+            }
+            // Check refresh token expiry from non-use
+            if (stored.refreshExpiresAt && stored.lastUsedAt) {
+                const refreshExpiresIn = stored.refreshExpiresAt.getTime() - now.getTime();
+                if (refreshExpiresIn < buffer) {
+                    needsRefresh.push(connectionId);
+                }
+            }
+        }
+        return needsRefresh;
+    }
+}
+/**
+ * File-based token store - persists tokens to a JSON file
+ * Suitable for CLI tools and single-user scenarios
+ */
+export class FileTokenStore {
+    filePath;
+    cache = null;
+    constructor(filePath) {
+        this.filePath = filePath;
+    }
+    async load() {
+        if (this.cache)
+            return this.cache;
+        try {
+            const fs = await import('node:fs/promises');
+            const content = await fs.readFile(this.filePath, 'utf-8');
+            const data = JSON.parse(content);
+            // Revive dates
+            const map = new Map();
+            for (const [key, value] of Object.entries(data)) {
+                map.set(key, {
+                    ...value,
+                    expiresAt: value.expiresAt ? new Date(value.expiresAt) : undefined,
+                    refreshExpiresAt: value.refreshExpiresAt ? new Date(value.refreshExpiresAt) : undefined,
+                    lastUsedAt: value.lastUsedAt ? new Date(value.lastUsedAt) : undefined,
+                });
+            }
+            this.cache = map;
+            return map;
+        }
+        catch {
+            this.cache = new Map();
+            return this.cache;
+        }
+    }
+    async save() {
+        if (!this.cache)
+            return;
+        const fs = await import('node:fs/promises');
+        const data = {};
+        for (const [key, value] of this.cache) {
+            data[key] = value;
+        }
+        await fs.writeFile(this.filePath, JSON.stringify(data, null, 2), 'utf-8');
+    }
+    async get(connectionId) {
+        const map = await this.load();
+        const stored = map.get(connectionId);
+        if (!stored)
+            return null;
+        return {
+            accessToken: stored.accessToken,
+            refreshToken: stored.refreshToken,
+            expiresAt: stored.expiresAt,
+            refreshExpiresAt: stored.refreshExpiresAt,
+            tokenType: stored.tokenType,
+            scope: stored.scope,
+        };
+    }
+    async set(connectionId, tokens) {
+        const map = await this.load();
+        const existing = map.get(connectionId);
+        map.set(connectionId, {
+            ...tokens,
+            lastUsedAt: existing?.lastUsedAt ?? new Date(),
+        });
+        await this.save();
+    }
+    async delete(connectionId) {
+        const map = await this.load();
+        map.delete(connectionId);
+        await this.save();
+    }
+    async touch(connectionId) {
+        const map = await this.load();
+        const stored = map.get(connectionId);
+        if (stored) {
+            stored.lastUsedAt = new Date();
+            await this.save();
+        }
+    }
+    async list() {
+        const map = await this.load();
+        return Array.from(map.keys());
+    }
+}

package/dist/auth/types.d.ts

@@ -0,0 +1,142 @@
+/**
+ * Auth provider interface - abstracts token retrieval and refresh
+ */
+export interface AuthProvider {
+    /** Get a valid access token (refreshing if needed) */
+    getToken(): Promise<string>;
+    /** Force a token refresh */
+    refreshToken?(): Promise<string>;
+    /** Get token metadata (for monitoring) */
+    getTokenInfo?(): TokenInfo;
+}
+export interface TokenInfo {
+    /** When the access token expires */
+    expiresAt?: Date;
+    /** When the refresh token expires (for non-use expiry tracking) */
+    refreshExpiresAt?: Date;
+    /** Last time the token was successfully used */
+    lastUsedAt?: Date;
+    /** Connection/tenant identifier */
+    connectionId?: string;
+}
+/**
+ * OAuth2 token set - what gets stored
+ */
+export interface OAuth2Tokens {
+    accessToken: string;
+    refreshToken?: string;
+    expiresAt?: Date;
+    refreshExpiresAt?: Date;
+    tokenType?: string;
+    scope?: string;
+}
+/**
+ * Token store interface - pluggable storage backend
+ */
+export interface TokenStore {
+    /** Get tokens for a connection */
+    get(connectionId: string): Promise<OAuth2Tokens | null>;
+    /** Store tokens for a connection */
+    set(connectionId: string, tokens: OAuth2Tokens): Promise<void>;
+    /** Delete tokens for a connection */
+    delete(connectionId: string): Promise<void>;
+    /** Update last used timestamp */
+    touch(connectionId: string): Promise<void>;
+    /** List all connections (for proactive refresh) */
+    list(): Promise<string[]>;
+}
+/**
+ * OAuth2 configuration for token refresh
+ */
+export interface OAuth2Config {
+    tokenEndpoint: string;
+    clientId: string;
+    clientSecret?: string;
+    /** Seconds before expiry to trigger refresh (default: 300 = 5 min) */
+    refreshBuffer?: number;
+}
+/**
+ * Rate limit information extracted from response headers
+ */
+export interface RateLimitInfo {
+    /** Requests remaining in current window */
+    remaining?: number;
+    /** Total requests allowed in window */
+    limit?: number;
+    /** When the rate limit resets (Unix timestamp or Date) */
+    resetAt?: Date;
+    /** Retry after N seconds (from 429 response) */
+    retryAfter?: number;
+}
+/**
+ * Rate limit strategy
+ */
+export type RateLimitStrategy = 'pause' | 'throttle' | 'fail';
+/**
+ * Rate limit configuration
+ */
+export interface RateLimitConfig {
+    /** Strategy when rate limited (default: 'pause') */
+    strategy?: RateLimitStrategy;
+    /** Max seconds to wait before failing (default: 300) */
+    maxWait?: number;
+    /** Log warning after waiting this many seconds (default: 10) */
+    notifyAt?: number;
+    /** Fallback rate limit if no headers (requests per minute) */
+    fallbackRpm?: number;
+}
+/**
+ * Rate limit event - emitted when rate limiting occurs
+ */
+export interface RateLimitEvent {
+    source: string;
+    endpoint?: string;
+    waitSeconds: number;
+    remaining?: number;
+    resetAt?: Date;
+    strategy: RateLimitStrategy;
+}
+/**
+ * Rate limit event handlers
+ */
+export interface RateLimitCallbacks {
+    /** Called when rate limited and waiting */
+    onRateLimited?: (event: RateLimitEvent) => void;
+    /** Called when rate limit wait is complete */
+    onResumed?: (event: {
+        source: string;
+        endpoint?: string;
+        waitedSeconds: number;
+    }) => void;
+    /** Called periodically during long waits */
+    onWaiting?: (event: RateLimitEvent & {
+        elapsedSeconds: number;
+    }) => void;
+}
+/**
+ * Rate limiter interface - tracks and enforces rate limits
+ */
+export interface RateLimiter {
+    /** Check if we should proceed with a request */
+    canProceed(source: string, endpoint?: string): Promise<boolean>;
+    /** Wait until we can proceed (blocks if rate limited) */
+    waitForCapacity(source: string, endpoint?: string): Promise<void>;
+    /** Record rate limit info from a response */
+    recordResponse(source: string, info: RateLimitInfo, endpoint?: string): void;
+    /** Get current rate limit status */
+    getStatus(source: string, endpoint?: string): RateLimitStatus;
+    /** Configure rate limiting for a source */
+    configure(source: string, config: RateLimitConfig): void;
+    /** Set event callbacks */
+    setCallbacks(callbacks: RateLimitCallbacks): void;
+    /** Get delay for throttle mode (returns 0 if no throttling needed) */
+    getThrottleDelay(source: string, endpoint?: string): number;
+}
+export interface RateLimitStatus {
+    remaining?: number;
+    limit?: number;
+    resetAt?: Date;
+    isLimited: boolean;
+    /** Seconds until reset (if limited) */
+    resetInSeconds?: number;
+}

package/dist/auth/types.js

@@ -0,0 +1 @@
+export {};

package/dist/cli.d.ts

@@ -0,0 +1,2 @@
+#!/usr/bin/env node
+export {};

package/dist/cli.js

@@ -0,0 +1,270 @@
+#!/usr/bin/env node
+import { readFile, writeFile, mkdir } from 'node:fs/promises';
+import { resolve, dirname } from 'node:path';
+import { fromPath, Scheduler, loadMission } from './index.js';
+import { ReqonError } from './errors/index.js';
+import { loadEnv, loadCredentials } from './auth/credentials.js';
+import { WebhookServer } from './webhook/index.js';
+async function main() {
+    const args = process.argv.slice(2);
+    if (args.length === 0 || args.includes('--help') || args.includes('-h')) {
+        console.log(`
+Reqon - A DSL for fetch, map, validate pipelines
+
+Usage:
+  reqon <file.reqon|folder> [options]
+
+Options:
+  --dry-run            Run without making actual HTTP requests
+  --verbose            Enable verbose logging
+  --auth <file>        JSON file with auth credentials (supports env var interpolation)
+  --env <file>         Path to .env file (default: .env in current directory)
+  --output <path>      Export stores to JSON (file or directory)
+  --daemon             Run as daemon, executing scheduled missions
+  --once               Run scheduled missions once immediately, then exit
+  --webhook            Enable webhook server for 'wait' steps
+  --webhook-port <n>   Port for webhook server (default: 3000)
+  --webhook-url <url>  Base URL for webhook endpoints (default: http://localhost:3000)
+  --help, -h           Show this help message
+
+Environment Variables:
+  Credentials in --auth files support env var interpolation:
+    $VAR_NAME, \${VAR_NAME}, \${VAR_NAME:-default}
+
+  Auto-discovery from env vars (no --auth file needed):
+    REQON_{SOURCE}_TOKEN      Bearer token for source
+    REQON_{SOURCE}_TYPE       Auth type (bearer, oauth2, api_key, basic)
+    REQON_{SOURCE}_API_KEY    API key for source
+
+Examples:
+  reqon sync-invoices.reqon --verbose
+  reqon ./sync-invoices/ --verbose        # folder with mission.reqon + action files
+  reqon sync-invoices.reqon --auth ./credentials.json
+  reqon sync-invoices.reqon --env .env.production --auth ./credentials.json
+  reqon sync-invoices.reqon --output ./output.json
+  reqon sync-invoices.reqon --daemon --verbose
+  reqon sync-invoices.reqon --webhook --webhook-port 8080 --verbose
+`);
+        process.exit(0);
+    }
+    const filePath = args[0];
+    const dryRun = args.includes('--dry-run');
+    const verbose = args.includes('--verbose');
+    const daemon = args.includes('--daemon');
+    const once = args.includes('--once');
+    const webhookEnabled = args.includes('--webhook');
+    // Parse webhook options
+    let webhookPort = 3000;
+    const webhookPortIndex = args.indexOf('--webhook-port');
+    if (webhookPortIndex !== -1 && args[webhookPortIndex + 1]) {
+        webhookPort = parseInt(args[webhookPortIndex + 1], 10);
+    }
+    let webhookUrl;
+    const webhookUrlIndex = args.indexOf('--webhook-url');
+    if (webhookUrlIndex !== -1 && args[webhookUrlIndex + 1]) {
+        webhookUrl = args[webhookUrlIndex + 1];
+    }
+    // Load .env file(s)
+    let envFile;
+    const envIndex = args.indexOf('--env');
+    if (envIndex !== -1 && args[envIndex + 1]) {
+        envFile = args[envIndex + 1];
+    }
+    const envResult = loadEnv({ envFile });
+    if (verbose && envResult.loaded) {
+        console.log(`Loaded ${envResult.count} env vars from: ${envResult.files.join(', ')}`);
+    }
+    // Load and resolve auth credentials
+    let auth;
+    const authIndex = args.indexOf('--auth');
+    if (authIndex !== -1 && args[authIndex + 1]) {
+        const authPath = resolve(args[authIndex + 1]);
+        const authContent = await readFile(authPath, 'utf-8');
+        const rawAuth = JSON.parse(authContent);
+        // Resolve env var references in the auth config
+        auth = loadCredentials(rawAuth);
+    }
+    let outputPath;
+    const outputIndex = args.indexOf('--output');
+    if (outputIndex !== -1 && args[outputIndex + 1]) {
+        outputPath = resolve(args[outputIndex + 1]);
+    }
+    // Daemon mode: run scheduled missions
+    if (daemon || once) {
+        await runDaemon(filePath, {
+            verbose,
+            dryRun,
+            auth: auth,
+            once,
+        });
+        return;
+    }
+    // Single run mode (default)
+    console.log(`Running: ${filePath}`);
+    // Start webhook server if enabled
+    let webhookServer;
+    if (webhookEnabled) {
+        webhookServer = new WebhookServer({
+            port: webhookPort,
+            baseUrl: webhookUrl ?? `http://localhost:${webhookPort}`,
+            verbose,
+        });
+        await webhookServer.start();
+        console.log(`Webhook server started on port ${webhookPort}`);
+    }
+    try {
+        const result = await fromPath(filePath, {
+            dryRun,
+            verbose,
+            auth: auth,
+            webhookServer,
+        });
+        if (result.success) {
+            console.log(`\n✓ Mission completed successfully`);
+            console.log(`  Duration: ${result.duration}ms`);
+            console.log(`  Actions run: ${result.actionsRun.join(' → ')}`);
+            // Print store stats and optionally export
+            const storeData = {};
+            for (const [name, store] of result.stores) {
+                const items = await store.list();
+                console.log(`  Store "${name}": ${items.length} items`);
+                storeData[name] = items;
+            }
+            // Export to JSON if requested
+            if (outputPath) {
+                await mkdir(dirname(outputPath), { recursive: true });
+                await writeFile(outputPath, JSON.stringify(storeData, null, 2));
+                console.log(`  Output written to: ${outputPath}`);
+            }
+        }
+        else {
+            console.log(`\n✗ Mission failed`);
+            for (const error of result.errors) {
+                console.error(`  [${error.action}/${error.step}] ${error.message}`);
+            }
+            process.exit(1);
+        }
+    }
+    catch (error) {
+        if (error instanceof ReqonError) {
+            console.error(error.format());
+        }
+        else {
+            console.error(`Error: ${error.message}`);
+        }
+        process.exit(1);
+    }
+    finally {
+        // Stop webhook server if it was started
+        if (webhookServer) {
+            await webhookServer.stop();
+        }
+    }
+}
+async function runDaemon(filePath, options) {
+    const absolutePath = resolve(filePath);
+    let program;
+    let baseDir;
+    try {
+        const result = await loadMission(absolutePath);
+        program = result.program;
+        baseDir = result.baseDir;
+    }
+    catch (error) {
+        if (error instanceof ReqonError) {
+            console.error(error.format());
+        }
+        else {
+            console.error(`Error loading mission: ${error.message}`);
+        }
+        process.exit(1);
+    }
+    // Find scheduled missions
+    const scheduledMissions = program.statements.filter((s) => s.type === 'MissionDefinition' && s.schedule);
+    if (scheduledMissions.length === 0) {
+        console.error('No scheduled missions found in the file');
+        console.error('Add a schedule to your mission, e.g.:');
+        console.error('  schedule: every 6 hours');
+        console.error('  schedule: cron "0 */6 * * *"');
+        process.exit(1);
+    }
+    console.log(`Found ${scheduledMissions.length} scheduled mission(s)`);
+    // Create scheduler with callbacks
+    const scheduler = new Scheduler({
+        verbose: options.verbose,
+        callbacks: {
+            onJobStarted: (event) => {
+                console.log(`[${formatTime(event.timestamp)}] Starting: ${event.missionName}`);
+            },
+            onJobCompleted: (event) => {
+                console.log(`[${formatTime(event.timestamp)}] Completed: ${event.missionName} (${event.duration}ms)`);
+            },
+            onJobFailed: (event) => {
+                console.error(`[${formatTime(event.timestamp)}] Failed: ${event.missionName} - ${event.error}`);
+            },
+            onJobSkipped: (event) => {
+                console.log(`[${formatTime(event.timestamp)}] Skipped: ${event.missionName} - ${event.reason}`);
+            },
+        },
+    }, {
+        dryRun: options.dryRun,
+        verbose: options.verbose,
+        auth: options.auth,
+    });
+    // Register scheduled missions
+    scheduler.registerProgram(program, absolutePath);
+    // Print job info
+    const jobs = scheduler.getJobs();
+    console.log('\nScheduled jobs:');
+    for (const job of jobs) {
+        const schedule = job.schedule;
+        let scheduleStr;
+        if (schedule.scheduleType === 'interval') {
+            scheduleStr = `every ${schedule.interval.value} ${schedule.interval.unit}`;
+        }
+        else if (schedule.scheduleType === 'cron') {
+            scheduleStr = `cron "${schedule.cronExpression}"`;
+        }
+        else {
+            scheduleStr = `at "${schedule.runAt}"`;
+        }
+        console.log(`  - ${job.missionName}: ${scheduleStr}`);
+        if (job.nextRun) {
+            console.log(`    Next run: ${job.nextRun.toISOString()}`);
+        }
+    }
+    console.log('');
+    // Handle --once mode
+    if (options.once) {
+        console.log('Running all scheduled missions once...\n');
+        for (const job of jobs) {
+            await scheduler.trigger(job.missionName);
+        }
+        console.log('\nAll missions completed.');
+        return;
+    }
+    // Start daemon mode
+    console.log('Starting scheduler daemon (Ctrl+C to stop)...\n');
+    // Handle graceful shutdown
+    let shuttingDown = false;
+    const shutdown = async () => {
+        if (shuttingDown)
+            return;
+        shuttingDown = true;
+        console.log('\nShutting down scheduler...');
+        await scheduler.stop();
+        process.exit(0);
+    };
+    process.on('SIGINT', shutdown);
+    process.on('SIGTERM', shutdown);
+    // Start the scheduler
+    await scheduler.start();
+    // Keep the process running
+    await new Promise(() => {
+        // This promise never resolves - we wait for SIGINT/SIGTERM
+    });
+}
+function formatTime(date) {
+    return date.toISOString().replace('T', ' ').substring(0, 19);
+}
+main();

package/dist/errors/errors.test.d.ts

@@ -0,0 +1 @@
+export {};