reqon-dsl 0.2.0

package/dist/ast/nodes.d.ts

@@ -0,0 +1,237 @@
+import type { Expression, SchemaDefinition, Statement as VagueStatement } from 'vague-lang';
+export type Statement = VagueStatement | MissionDefinition | SourceDefinition | StoreDefinition | ActionDefinition;
+export interface ReqonProgram {
+    type: 'ReqonProgram';
+    statements: Statement[];
+}
+export interface SourceDefinition {
+    type: 'SourceDefinition';
+    name: string;
+    specPath?: string;
+    config: SourceConfig;
+}
+export interface SourceConfig {
+    auth: AuthConfig;
+    base?: string;
+    headers?: Record<string, Expression>;
+    validateResponses?: boolean;
+    rateLimit?: RateLimitSourceConfig;
+    circuitBreaker?: CircuitBreakerSourceConfig;
+}
+export interface CircuitBreakerSourceConfig {
+    /** Number of failures before opening circuit (default: 5) */
+    failureThreshold?: number;
+    /** Time in seconds before attempting recovery (default: 30) */
+    resetTimeout?: number;
+    /** Number of successful requests in half-open to close circuit (default: 2) */
+    successThreshold?: number;
+    /** Time window in seconds for counting failures (default: 60) */
+    failureWindow?: number;
+}
+export interface RateLimitSourceConfig {
+    strategy?: 'pause' | 'throttle' | 'fail';
+    maxWait?: number;
+    fallbackRpm?: number;
+}
+export interface AuthConfig {
+    type: 'oauth2' | 'bearer' | 'basic' | 'api_key' | 'none';
+}
+export interface StoreDefinition {
+    type: 'StoreDefinition';
+    name: string;
+    storeType: 'nosql' | 'sql' | 'memory';
+    target: string;
+}
+export interface ScheduleDefinition {
+    type: 'ScheduleDefinition';
+    scheduleType: 'interval' | 'cron' | 'once';
+    interval?: IntervalSchedule;
+    cronExpression?: string;
+    runAt?: string;
+    timezone?: string;
+    maxConcurrency?: number;
+    skipIfRunning?: boolean;
+    retryOnFailure?: ScheduleRetryConfig;
+}
+export interface IntervalSchedule {
+    value: number;
+    unit: 'seconds' | 'minutes' | 'hours' | 'days' | 'weeks';
+}
+export interface ScheduleRetryConfig {
+    maxRetries: number;
+    delaySeconds: number;
+}
+export interface MissionDefinition {
+    type: 'MissionDefinition';
+    name: string;
+    schedule?: ScheduleDefinition;
+    sources: SourceDefinition[];
+    stores: StoreDefinition[];
+    schemas: SchemaDefinition[];
+    actions: ActionDefinition[];
+    pipeline: PipelineDefinition;
+}
+export interface ActionDefinition {
+    type: 'ActionDefinition';
+    name: string;
+    steps: ActionStep[];
+}
+export type ActionStep = FetchStep | ForStep | MapStep | ValidateStep | StoreStep | MatchStep | LetStep | WebhookStep;
+export interface LetStep {
+    type: 'LetStep';
+    name: string;
+    value: Expression;
+}
+export interface WebhookStep {
+    type: 'WebhookStep';
+    /** Timeout in milliseconds to wait for webhook (default: 300000 = 5 minutes) */
+    timeout?: number;
+    /** Path for the webhook endpoint (e.g., "/webhooks/callback") */
+    path?: string;
+    /** Number of webhook events to collect before continuing (default: 1) */
+    expectedEvents?: number;
+    /** Filter expression for matching webhook events */
+    eventFilter?: Expression;
+    /** Retry configuration if timeout occurs */
+    retryOnTimeout?: RetryConfig;
+    /** Store configuration for saving webhook payloads */
+    storage?: WebhookStorageConfig;
+}
+export interface WebhookStorageConfig {
+    /** Store name to save webhook payloads */
+    target: string;
+    /** Expression for extracting key from webhook payload */
+    key?: Expression;
+}
+export type FlowDirective = {
+    type: 'continue';
+} | {
+    type: 'skip';
+} | {
+    type: 'abort';
+    message?: string;
+} | {
+    type: 'retry';
+    backoff?: RetryConfig;
+} | {
+    type: 'queue';
+    target?: string;
+} | {
+    type: 'jump';
+    action: string;
+    then?: 'retry' | 'continue';
+};
+export interface MatchStep {
+    type: 'MatchStep';
+    target: Expression;
+    arms: MatchArm[];
+}
+export interface MatchArm {
+    /** Schema name to match against, or '_' for wildcard */
+    schema: string;
+    /** Steps to execute if matched */
+    steps?: ActionStep[];
+    /** Flow control directive (if no steps) */
+    flow?: FlowDirective;
+}
+export interface FetchStep {
+    type: 'FetchStep';
+    method?: 'GET' | 'POST' | 'PUT' | 'PATCH' | 'DELETE';
+    path?: Expression;
+    operationRef?: OperationRef;
+    source?: string;
+    body?: Expression;
+    headers?: Record<string, Expression>;
+    paginate?: PaginationConfig;
+    until?: Expression;
+    retry?: RetryConfig;
+    since?: SinceConfig;
+}
+export interface SinceConfig {
+    /** How to resolve the "since" timestamp */
+    type: 'lastSync' | 'expression';
+    /** Custom checkpoint key (defaults to source:endpoint) */
+    key?: string;
+    /** Query parameter name for the since value (default: varies by API) */
+    param?: string;
+    /** Date format for the since value */
+    format?: 'iso' | 'unix' | 'unix-ms' | 'date-only';
+    /** Expression to evaluate for 'expression' type */
+    expression?: Expression;
+    /** Field in response to use as the new "since" value for next sync */
+    updateFrom?: string;
+}
+export interface OperationRef {
+    source: string;
+    operationId: string;
+}
+export interface PaginationConfig {
+    type: 'offset' | 'cursor' | 'page';
+    param: string;
+    pageSize: number;
+    cursorPath?: string;
+}
+export interface RetryConfig {
+    maxAttempts: number;
+    backoff: 'exponential' | 'linear' | 'constant';
+    initialDelay: number;
+    maxDelay?: number;
+}
+export interface ForStep {
+    type: 'ForStep';
+    variable: string;
+    collection: Expression;
+    condition?: Expression;
+    steps: ActionStep[];
+}
+export interface MapStep {
+    type: 'MapStep';
+    source: Expression;
+    targetSchema: string;
+    mappings: FieldMapping[];
+}
+export interface FieldMapping {
+    field: string;
+    expression: Expression;
+}
+export interface ValidateStep {
+    type: 'ValidateStep';
+    target: Expression;
+    constraints: ValidationConstraint[];
+}
+export interface ValidationConstraint {
+    type: 'ValidationConstraint';
+    condition: Expression;
+    message?: string;
+    severity: 'error' | 'warning';
+}
+export interface StoreStep {
+    type: 'StoreStep';
+    source: Expression;
+    target: string;
+    options: StoreOptions;
+}
+export interface StoreOptions {
+    key?: Expression;
+    partial?: boolean;
+    upsert?: boolean;
+}
+export interface PipelineDefinition {
+    type: 'PipelineDefinition';
+    stages: PipelineStage[];
+}
+export interface PipelineStage {
+    /** Single action name (for sequential stages) */
+    action?: string;
+    /** Multiple action names (for parallel stages with bracket syntax) */
+    actions?: string[];
+    /** Optional: only run if condition true */
+    condition?: Expression;
+}
+/** Helper to check if a stage is parallel */
+export declare function isParallelStage(stage: PipelineStage): stage is PipelineStage & {
+    actions: string[];
+};
+/** Helper to get action names from a stage (works for both single and parallel) */
+export declare function getStageActions(stage: PipelineStage): string[];
+export type { Expression, FieldDefinition, SchemaDefinition } from 'vague-lang';

package/dist/ast/nodes.js

@@ -0,0 +1,12 @@
+/** Helper to check if a stage is parallel */
+export function isParallelStage(stage) {
+    return Array.isArray(stage.actions) && stage.actions.length > 0;
+}
+/** Helper to get action names from a stage (works for both single and parallel) */
+export function getStageActions(stage) {
+    if (stage.actions)
+        return stage.actions;
+    if (stage.action)
+        return [stage.action];
+    return [];
+}

package/dist/auth/auth.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/auth/auth.test.js

@@ -0,0 +1,255 @@
+import { describe, it, expect, beforeEach, vi } from 'vitest';
+import { AdaptiveRateLimiter, parseRateLimitHeaders, RateLimitError, RateLimitTimeoutError, } from './rate-limiter.js';
+import { InMemoryTokenStore } from './token-store.js';
+describe('parseRateLimitHeaders', () => {
+    it('parses X-RateLimit headers', () => {
+        const headers = {
+            'X-RateLimit-Limit': '100',
+            'X-RateLimit-Remaining': '42',
+            'X-RateLimit-Reset': '1700000000',
+        };
+        const info = parseRateLimitHeaders(headers);
+        expect(info.limit).toBe(100);
+        expect(info.remaining).toBe(42);
+        expect(info.resetAt).toBeInstanceOf(Date);
+        expect(info.resetAt?.getTime()).toBe(1700000000 * 1000);
+    });
+    it('parses lowercase ratelimit headers', () => {
+        const headers = {
+            'ratelimit-limit': '60',
+            'ratelimit-remaining': '5',
+        };
+        const info = parseRateLimitHeaders(headers);
+        expect(info.limit).toBe(60);
+        expect(info.remaining).toBe(5);
+    });
+    it('parses Retry-After header (seconds)', () => {
+        const headers = {
+            'Retry-After': '30',
+        };
+        const info = parseRateLimitHeaders(headers);
+        expect(info.retryAfter).toBe(30);
+    });
+    it('handles missing headers gracefully', () => {
+        const info = parseRateLimitHeaders({});
+        expect(info.limit).toBeUndefined();
+        expect(info.remaining).toBeUndefined();
+        expect(info.resetAt).toBeUndefined();
+        expect(info.retryAfter).toBeUndefined();
+    });
+});
+describe('AdaptiveRateLimiter', () => {
+    let limiter;
+    beforeEach(() => {
+        limiter = new AdaptiveRateLimiter();
+    });
+    it('allows requests when no limits recorded', async () => {
+        const canProceed = await limiter.canProceed('TestAPI');
+        expect(canProceed).toBe(true);
+    });
+    it('blocks requests when remaining is 0', async () => {
+        const futureReset = new Date(Date.now() + 60000);
+        limiter.recordResponse('TestAPI', {
+            remaining: 0,
+            limit: 100,
+            resetAt: futureReset,
+        });
+        const canProceed = await limiter.canProceed('TestAPI');
+        expect(canProceed).toBe(false);
+    });
+    it('allows requests after reset time passes', async () => {
+        const pastReset = new Date(Date.now() - 1000);
+        limiter.recordResponse('TestAPI', {
+            remaining: 0,
+            limit: 100,
+            resetAt: pastReset,
+        });
+        const canProceed = await limiter.canProceed('TestAPI');
+        expect(canProceed).toBe(true);
+    });
+    it('blocks during retry-after period', async () => {
+        limiter.recordResponse('TestAPI', {
+            retryAfter: 60, // 60 seconds
+        });
+        const canProceed = await limiter.canProceed('TestAPI');
+        expect(canProceed).toBe(false);
+    });
+    it('tracks limits per source', async () => {
+        limiter.recordResponse('API1', { remaining: 0, resetAt: new Date(Date.now() + 60000) });
+        limiter.recordResponse('API2', { remaining: 50, limit: 100 });
+        expect(await limiter.canProceed('API1')).toBe(false);
+        expect(await limiter.canProceed('API2')).toBe(true);
+    });
+    it('tracks limits per endpoint within source', async () => {
+        limiter.recordResponse('API', { remaining: 0, resetAt: new Date(Date.now() + 60000) }, '/invoices');
+        limiter.recordResponse('API', { remaining: 50, limit: 100 }, '/contacts');
+        expect(await limiter.canProceed('API', '/invoices')).toBe(false);
+        expect(await limiter.canProceed('API', '/contacts')).toBe(true);
+    });
+    it('provides accurate status', () => {
+        limiter.recordResponse('TestAPI', {
+            remaining: 42,
+            limit: 100,
+            resetAt: new Date(Date.now() + 60000),
+        });
+        const status = limiter.getStatus('TestAPI');
+        expect(status.remaining).toBe(42);
+        expect(status.limit).toBe(100);
+        expect(status.isLimited).toBe(false);
+    });
+    describe('fail strategy', () => {
+        it('throws RateLimitError immediately when rate limited', async () => {
+            limiter.configure('FailAPI', { strategy: 'fail' });
+            limiter.recordResponse('FailAPI', {
+                remaining: 0,
+                resetAt: new Date(Date.now() + 60000),
+            });
+            await expect(limiter.waitForCapacity('FailAPI')).rejects.toThrow(RateLimitError);
+        });
+        it('includes reset time in error', async () => {
+            limiter.configure('FailAPI', { strategy: 'fail' });
+            const resetAt = new Date(Date.now() + 60000);
+            limiter.recordResponse('FailAPI', { remaining: 0, resetAt });
+            try {
+                await limiter.waitForCapacity('FailAPI');
+                expect.fail('Should have thrown');
+            }
+            catch (error) {
+                expect(error).toBeInstanceOf(RateLimitError);
+                expect(error.resetAt).toEqual(resetAt);
+            }
+        });
+    });
+    describe('pause strategy', () => {
+        it('throws RateLimitTimeoutError when maxWait exceeded', async () => {
+            limiter.configure('PauseAPI', { strategy: 'pause', maxWait: 1 });
+            limiter.recordResponse('PauseAPI', {
+                remaining: 0,
+                resetAt: new Date(Date.now() + 60000), // 60s in future
+            });
+            await expect(limiter.waitForCapacity('PauseAPI')).rejects.toThrow(RateLimitTimeoutError);
+        });
+        it('proceeds after reset time passes', async () => {
+            limiter.configure('PauseAPI', { strategy: 'pause', maxWait: 5 });
+            // Reset in 50ms
+            limiter.recordResponse('PauseAPI', {
+                remaining: 0,
+                resetAt: new Date(Date.now() + 50),
+            });
+            // Should complete without throwing
+            await limiter.waitForCapacity('PauseAPI');
+        });
+    });
+    describe('throttle strategy', () => {
+        it('calculates delay based on remaining requests and reset time', () => {
+            limiter.configure('ThrottleAPI', { strategy: 'throttle' });
+            limiter.recordResponse('ThrottleAPI', {
+                remaining: 10,
+                limit: 100,
+                resetAt: new Date(Date.now() + 10000), // 10s left
+            });
+            const delay = limiter.getThrottleDelay('ThrottleAPI');
+            // 10s / 10 remaining = 1s per request = 1000ms
+            expect(delay).toBeGreaterThanOrEqual(900);
+            expect(delay).toBeLessThanOrEqual(1100);
+        });
+        it('returns 0 delay when not in throttle mode', () => {
+            limiter.configure('PauseAPI', { strategy: 'pause' });
+            limiter.recordResponse('PauseAPI', { remaining: 10, limit: 100 });
+            const delay = limiter.getThrottleDelay('PauseAPI');
+            expect(delay).toBe(0);
+        });
+        it('uses fallback RPM when no rate limit headers', () => {
+            limiter.configure('FallbackAPI', { strategy: 'throttle', fallbackRpm: 60 });
+            limiter.recordResponse('FallbackAPI', {}); // No rate limit info
+            const delay = limiter.getThrottleDelay('FallbackAPI');
+            // 60 RPM = 1 per second = 1000ms intervals
+            expect(delay).toBeLessThanOrEqual(1000);
+        });
+    });
+    describe('callbacks', () => {
+        it('calls onRateLimited when rate limited', async () => {
+            const onRateLimited = vi.fn();
+            limiter.setCallbacks({ onRateLimited });
+            // Use maxWait of 10s so it doesn't timeout immediately (wait is only 50ms)
+            limiter.configure('CallbackAPI', { strategy: 'pause', maxWait: 10 });
+            limiter.recordResponse('CallbackAPI', {
+                remaining: 0,
+                resetAt: new Date(Date.now() + 50), // Resets in 50ms
+            });
+            await limiter.waitForCapacity('CallbackAPI');
+            expect(onRateLimited).toHaveBeenCalledTimes(1);
+            expect(onRateLimited).toHaveBeenCalledWith(expect.objectContaining({
+                source: 'CallbackAPI',
+                strategy: 'pause',
+            }));
+        });
+        it('calls onResumed after waiting completes', async () => {
+            const onResumed = vi.fn();
+            limiter.setCallbacks({ onResumed });
+            limiter.configure('ResumeAPI', { strategy: 'pause', maxWait: 5 });
+            // Reset in 50ms
+            limiter.recordResponse('ResumeAPI', {
+                remaining: 0,
+                resetAt: new Date(Date.now() + 50),
+            });
+            await limiter.waitForCapacity('ResumeAPI');
+            expect(onResumed).toHaveBeenCalledTimes(1);
+            expect(onResumed).toHaveBeenCalledWith(expect.objectContaining({
+                source: 'ResumeAPI',
+            }));
+        });
+    });
+});
+describe('InMemoryTokenStore', () => {
+    let store;
+    beforeEach(() => {
+        store = new InMemoryTokenStore();
+    });
+    it('stores and retrieves tokens', async () => {
+        const tokens = {
+            accessToken: 'access123',
+            refreshToken: 'refresh456',
+            expiresAt: new Date(Date.now() + 3600000),
+        };
+        await store.set('connection-1', tokens);
+        const retrieved = await store.get('connection-1');
+        expect(retrieved?.accessToken).toBe('access123');
+        expect(retrieved?.refreshToken).toBe('refresh456');
+    });
+    it('returns null for unknown connections', async () => {
+        const result = await store.get('unknown');
+        expect(result).toBeNull();
+    });
+    it('deletes tokens', async () => {
+        await store.set('connection-1', { accessToken: 'test' });
+        await store.delete('connection-1');
+        const result = await store.get('connection-1');
+        expect(result).toBeNull();
+    });
+    it('lists all connections', async () => {
+        await store.set('conn-1', { accessToken: 'a' });
+        await store.set('conn-2', { accessToken: 'b' });
+        await store.set('conn-3', { accessToken: 'c' });
+        const connections = await store.list();
+        expect(connections).toHaveLength(3);
+        expect(connections).toContain('conn-1');
+        expect(connections).toContain('conn-2');
+        expect(connections).toContain('conn-3');
+    });
+    it('identifies tokens needing refresh', async () => {
+        // Token expiring in 10 seconds (within 5 min buffer)
+        await store.set('expiring-soon', {
+            accessToken: 'test',
+            expiresAt: new Date(Date.now() + 10000),
+        });
+        // Token not expiring soon
+        await store.set('valid', {
+            accessToken: 'test',
+            expiresAt: new Date(Date.now() + 3600000),
+        });
+        const needsRefresh = await store.getTokensNeedingRefresh(300);
+        expect(needsRefresh).toContain('expiring-soon');
+        expect(needsRefresh).not.toContain('valid');
+    });
+});

package/dist/auth/circuit-breaker.d.ts

@@ -0,0 +1,115 @@
+/**
+ * Circuit Breaker implementation for HTTP requests.
+ *
+ * Prevents repeated failures from cascading by automatically detecting
+ * failure patterns and "opening" the circuit to fail fast.
+ *
+ * States:
+ * - CLOSED: Normal operation, requests pass through
+ * - OPEN: Circuit tripped, requests fail immediately
+ * - HALF_OPEN: Testing if service recovered, limited requests allowed
+ */
+export type CircuitState = 'closed' | 'open' | 'half_open';
+export interface CircuitBreakerConfig {
+    /** Number of failures before opening circuit (default: 5) */
+    failureThreshold?: number;
+    /** Time in ms before attempting recovery (default: 30000 = 30s) */
+    resetTimeout?: number;
+    /** Number of successful requests in half-open to close circuit (default: 2) */
+    successThreshold?: number;
+    /** Time window in ms for counting failures (default: 60000 = 60s) */
+    failureWindow?: number;
+    /** HTTP status codes to count as failures (default: 500-599) */
+    failureStatusCodes?: number[];
+    /** Whether to count network errors as failures (default: true) */
+    countNetworkErrors?: boolean;
+}
+export interface CircuitBreakerStatus {
+    state: CircuitState;
+    failures: number;
+    successes: number;
+    lastFailureTime?: Date;
+    nextAttemptTime?: Date;
+    isOpen: boolean;
+}
+export interface CircuitBreakerEvent {
+    source: string;
+    endpoint?: string;
+    state: CircuitState;
+    previousState: CircuitState;
+    failures: number;
+    reason?: string;
+}
+export interface CircuitBreakerCallbacks {
+    /** Called when circuit opens (too many failures) */
+    onOpen?: (event: CircuitBreakerEvent) => void;
+    /** Called when circuit closes (recovery successful) */
+    onClose?: (event: CircuitBreakerEvent) => void;
+    /** Called when circuit enters half-open state */
+    onHalfOpen?: (event: CircuitBreakerEvent) => void;
+    /** Called when a request is rejected due to open circuit */
+    onRejected?: (event: {
+        source: string;
+        endpoint?: string;
+        nextAttemptIn: number;
+    }) => void;
+}
+/**
+ * Error thrown when circuit breaker is open
+ */
+export declare class CircuitBreakerError extends Error {
+    readonly source: string;
+    readonly endpoint: string | undefined;
+    readonly nextAttemptIn: number;
+    constructor(source: string, endpoint: string | undefined, nextAttemptIn: number);
+}
+/**
+ * Circuit breaker for managing failure detection and recovery
+ */
+export declare class CircuitBreaker {
+    private circuits;
+    private callbacks;
+    private defaultConfig;
+    constructor(defaultConfig?: CircuitBreakerConfig);
+    /**
+     * Configure circuit breaker for a specific source
+     */
+    configure(source: string, config: CircuitBreakerConfig): void;
+    /**
+     * Set event callbacks
+     */
+    setCallbacks(callbacks: CircuitBreakerCallbacks): void;
+    /**
+     * Check if a request can proceed (throws if circuit is open)
+     */
+    canProceed(source: string, endpoint?: string): boolean;
+    /**
+     * Ensure request can proceed, throwing CircuitBreakerError if not
+     */
+    ensureCanProceed(source: string, endpoint?: string): void;
+    /**
+     * Record a successful request
+     */
+    recordSuccess(source: string, endpoint?: string): void;
+    /**
+     * Record a failed request
+     */
+    recordFailure(source: string, endpoint?: string, statusCode?: number, isNetworkError?: boolean): void;
+    /**
+     * Get current status for a source/endpoint
+     */
+    getStatus(source: string, endpoint?: string): CircuitBreakerStatus;
+    /**
+     * Force reset a circuit to closed state
+     */
+    reset(source: string, endpoint?: string): void;
+    /**
+     * Get all circuit statuses
+     */
+    getAllStatuses(): Map<string, CircuitBreakerStatus>;
+    private getKey;
+    private createEntry;
+    private getOrCreateEntry;
+    private pruneOldFailures;
+    private transitionTo;
+}