reqon-dsl 0.2.0

package/src/ast/index.ts

@@ -0,0 +1 @@
+export * from './nodes.js';

package/src/ast/nodes.ts

@@ -0,0 +1,310 @@
+import type {
+  Expression,
+  FieldDefinition,
+  SchemaDefinition,
+  Statement as VagueStatement,
+} from 'vague-lang';
+
+// Reqon extends Vague's statements
+export type Statement = VagueStatement | MissionDefinition | SourceDefinition | StoreDefinition | ActionDefinition;
+
+export interface ReqonProgram {
+  type: 'ReqonProgram';
+  statements: Statement[];
+}
+
+// source Xero { auth: oauth2, base: "https://api.xero.com/..." }
+// source Xero from "./xero-openapi.yaml" { auth: oauth2 }
+export interface SourceDefinition {
+  type: 'SourceDefinition';
+  name: string;
+  specPath?: string; // OAS spec path (URL or file path)
+  config: SourceConfig;
+}
+
+export interface SourceConfig {
+  auth: AuthConfig;
+  base?: string; // Optional if using OAS (derived from spec)
+  headers?: Record<string, Expression>;
+  validateResponses?: boolean; // Validate responses against OAS schema
+  rateLimit?: RateLimitSourceConfig; // Rate limiting configuration
+  circuitBreaker?: CircuitBreakerSourceConfig; // Circuit breaker configuration
+}
+
+export interface CircuitBreakerSourceConfig {
+  /** Number of failures before opening circuit (default: 5) */
+  failureThreshold?: number;
+  /** Time in seconds before attempting recovery (default: 30) */
+  resetTimeout?: number;
+  /** Number of successful requests in half-open to close circuit (default: 2) */
+  successThreshold?: number;
+  /** Time window in seconds for counting failures (default: 60) */
+  failureWindow?: number;
+}
+
+export interface RateLimitSourceConfig {
+  strategy?: 'pause' | 'throttle' | 'fail'; // Default: 'pause'
+  maxWait?: number; // Max seconds to wait (default: 300)
+  fallbackRpm?: number; // Fallback requests per minute if no headers
+}
+
+export interface AuthConfig {
+  type: 'oauth2' | 'bearer' | 'basic' | 'api_key' | 'none';
+  // Details resolved at runtime from environment/config
+}
+
+// store invoices_cache: nosql("invoices")
+// store invoices_sql: sql("accounting.invoices")
+export interface StoreDefinition {
+  type: 'StoreDefinition';
+  name: string;
+  storeType: 'nosql' | 'sql' | 'memory';
+  target: string; // collection/table name
+}
+
+// Schedule configuration for missions
+// schedule: every 6 hours
+// schedule: cron "0 */6 * * *"
+// schedule: at "2025-01-20 09:00 UTC"
+export interface ScheduleDefinition {
+  type: 'ScheduleDefinition';
+  scheduleType: 'interval' | 'cron' | 'once';
+  // For interval-based scheduling
+  interval?: IntervalSchedule;
+  // For cron-based scheduling
+  cronExpression?: string;
+  // For one-time scheduling
+  runAt?: string; // ISO 8601 datetime or parseable date string
+  // Optional timezone (defaults to UTC)
+  timezone?: string;
+  // Maximum concurrent executions (default: 1)
+  maxConcurrency?: number;
+  // Skip execution if previous run is still running (default: true)
+  skipIfRunning?: boolean;
+  // Retry configuration for failed scheduled runs
+  retryOnFailure?: ScheduleRetryConfig;
+}
+
+export interface IntervalSchedule {
+  value: number;
+  unit: 'seconds' | 'minutes' | 'hours' | 'days' | 'weeks';
+}
+
+export interface ScheduleRetryConfig {
+  maxRetries: number;
+  delaySeconds: number;
+}
+
+// mission SyncXeroInvoices { ... }
+export interface MissionDefinition {
+  type: 'MissionDefinition';
+  name: string;
+  schedule?: ScheduleDefinition;
+  sources: SourceDefinition[];
+  stores: StoreDefinition[];
+  schemas: SchemaDefinition[];
+  actions: ActionDefinition[];
+  pipeline: PipelineDefinition;
+}
+
+// action FetchInvoiceList { ... }
+export interface ActionDefinition {
+  type: 'ActionDefinition';
+  name: string;
+  steps: ActionStep[];
+}
+
+export type ActionStep = FetchStep | ForStep | MapStep | ValidateStep | StoreStep | MatchStep | LetStep | WebhookStep;
+
+// let myVar = expression
+export interface LetStep {
+  type: 'LetStep';
+  name: string;
+  value: Expression;
+}
+
+// wait { timeout: 60000, path: "/webhooks/callback", ... }
+// Waits for an external webhook callback before continuing execution
+export interface WebhookStep {
+  type: 'WebhookStep';
+  /** Timeout in milliseconds to wait for webhook (default: 300000 = 5 minutes) */
+  timeout?: number;
+  /** Path for the webhook endpoint (e.g., "/webhooks/callback") */
+  path?: string;
+  /** Number of webhook events to collect before continuing (default: 1) */
+  expectedEvents?: number;
+  /** Filter expression for matching webhook events */
+  eventFilter?: Expression;
+  /** Retry configuration if timeout occurs */
+  retryOnTimeout?: RetryConfig;
+  /** Store configuration for saving webhook payloads */
+  storage?: WebhookStorageConfig;
+}
+
+export interface WebhookStorageConfig {
+  /** Store name to save webhook payloads */
+  target: string;
+  /** Expression for extracting key from webhook payload */
+  key?: Expression;
+}
+
+// Flow control directives for match arms
+export type FlowDirective =
+  | { type: 'continue' }                          // proceed to next pipeline stage
+  | { type: 'skip' }                              // skip remaining steps, move to next stage
+  | { type: 'abort'; message?: string }           // halt mission with error
+  | { type: 'retry'; backoff?: RetryConfig }      // retry current action
+  | { type: 'queue'; target?: string }            // queue for later processing
+  | { type: 'jump'; action: string; then?: 'retry' | 'continue' };  // jump to action
+
+// match response { Schema1 -> steps..., Schema2 -> flow directive }
+export interface MatchStep {
+  type: 'MatchStep';
+  target: Expression;  // what to match (usually 'response')
+  arms: MatchArm[];
+}
+
+export interface MatchArm {
+  /** Schema name to match against, or '_' for wildcard */
+  schema: string;
+  /** Steps to execute if matched */
+  steps?: ActionStep[];
+  /** Flow control directive (if no steps) */
+  flow?: FlowDirective;
+}
+
+// get "/Invoices" { paginate: ..., until: ... }
+// call Xero.getInvoices { paginate: ... }  -- OAS operationId reference
+// get "/Invoices" { since: lastSync }  -- Incremental sync
+export interface FetchStep {
+  type: 'FetchStep';
+  // Traditional: explicit method + path
+  method?: 'GET' | 'POST' | 'PUT' | 'PATCH' | 'DELETE';
+  path?: Expression; // Can contain interpolations like "/Invoices/{id}"
+  // OAS-based: source.operationId reference
+  operationRef?: OperationRef;
+  source?: string; // Which source to use (defaults to first defined)
+  body?: Expression;
+  headers?: Record<string, Expression>;
+  paginate?: PaginationConfig;
+  until?: Expression; // Condition to stop pagination
+  retry?: RetryConfig;
+  // Incremental sync
+  since?: SinceConfig;
+}
+
+// Configuration for incremental sync
+export interface SinceConfig {
+  /** How to resolve the "since" timestamp */
+  type: 'lastSync' | 'expression';
+  /** Custom checkpoint key (defaults to source:endpoint) */
+  key?: string;
+  /** Query parameter name for the since value (default: varies by API) */
+  param?: string;
+  /** Date format for the since value */
+  format?: 'iso' | 'unix' | 'unix-ms' | 'date-only';
+  /** Expression to evaluate for 'expression' type */
+  expression?: Expression;
+  /** Field in response to use as the new "since" value for next sync */
+  updateFrom?: string;
+}
+
+export interface OperationRef {
+  source: string; // Source name (e.g., "Xero")
+  operationId: string; // OAS operationId (e.g., "getInvoices")
+}
+
+export interface PaginationConfig {
+  type: 'offset' | 'cursor' | 'page';
+  param: string; // Query param name: "page", "offset", "cursor"
+  pageSize: number;
+  cursorPath?: string; // For cursor pagination: where to find next cursor in response
+}
+
+export interface RetryConfig {
+  maxAttempts: number;
+  backoff: 'exponential' | 'linear' | 'constant';
+  initialDelay: number; // ms
+  maxDelay?: number; // ms
+}
+
+// for invoice in invoices_cache where .partial == true { ... }
+export interface ForStep {
+  type: 'ForStep';
+  variable: string;
+  collection: Expression;
+  condition?: Expression; // where clause
+  steps: ActionStep[];
+}
+
+// map invoice -> StandardInvoice { id: .InvoiceID, ... }
+export interface MapStep {
+  type: 'MapStep';
+  source: Expression;
+  targetSchema: string;
+  mappings: FieldMapping[];
+}
+
+export interface FieldMapping {
+  field: string;
+  expression: Expression;
+}
+
+// validate response { assume .Total is decimal, ... }
+export interface ValidateStep {
+  type: 'ValidateStep';
+  target: Expression;
+  constraints: ValidationConstraint[];
+}
+
+export interface ValidationConstraint {
+  type: 'ValidationConstraint';
+  condition: Expression;
+  message?: string;
+  severity: 'error' | 'warning';
+}
+
+// store response -> invoices_cache { key: .InvoiceID, partial: false }
+export interface StoreStep {
+  type: 'StoreStep';
+  source: Expression;
+  target: string; // store name
+  options: StoreOptions;
+}
+
+export interface StoreOptions {
+  key?: Expression; // Primary key field
+  partial?: boolean; // Mark as partial entity
+  upsert?: boolean; // Update if exists
+}
+
+// run FetchInvoiceList then HydrateInvoices then NormalizeInvoices
+// run [FetchOrders, FetchPayments] then ReconcileData  -- parallel stages
+export interface PipelineDefinition {
+  type: 'PipelineDefinition';
+  stages: PipelineStage[];
+}
+
+export interface PipelineStage {
+  /** Single action name (for sequential stages) */
+  action?: string;
+  /** Multiple action names (for parallel stages with bracket syntax) */
+  actions?: string[];
+  /** Optional: only run if condition true */
+  condition?: Expression;
+}
+
+/** Helper to check if a stage is parallel */
+export function isParallelStage(stage: PipelineStage): stage is PipelineStage & { actions: string[] } {
+  return Array.isArray(stage.actions) && stage.actions.length > 0;
+}
+
+/** Helper to get action names from a stage (works for both single and parallel) */
+export function getStageActions(stage: PipelineStage): string[] {
+  if (stage.actions) return stage.actions;
+  if (stage.action) return [stage.action];
+  return [];
+}
+
+// Re-export Vague types for convenience
+export type { Expression, FieldDefinition, SchemaDefinition } from 'vague-lang';

package/src/auth/auth.test.ts

@@ -0,0 +1,326 @@
+import { describe, it, expect, beforeEach, vi } from 'vitest';
+import {
+  AdaptiveRateLimiter,
+  parseRateLimitHeaders,
+  RateLimitError,
+  RateLimitTimeoutError,
+} from './rate-limiter.js';
+import { InMemoryTokenStore } from './token-store.js';
+import type { OAuth2Tokens, RateLimitEvent } from './types.js';
+
+describe('parseRateLimitHeaders', () => {
+  it('parses X-RateLimit headers', () => {
+    const headers = {
+      'X-RateLimit-Limit': '100',
+      'X-RateLimit-Remaining': '42',
+      'X-RateLimit-Reset': '1700000000',
+    };
+
+    const info = parseRateLimitHeaders(headers);
+
+    expect(info.limit).toBe(100);
+    expect(info.remaining).toBe(42);
+    expect(info.resetAt).toBeInstanceOf(Date);
+    expect(info.resetAt?.getTime()).toBe(1700000000 * 1000);
+  });
+
+  it('parses lowercase ratelimit headers', () => {
+    const headers = {
+      'ratelimit-limit': '60',
+      'ratelimit-remaining': '5',
+    };
+
+    const info = parseRateLimitHeaders(headers);
+
+    expect(info.limit).toBe(60);
+    expect(info.remaining).toBe(5);
+  });
+
+  it('parses Retry-After header (seconds)', () => {
+    const headers = {
+      'Retry-After': '30',
+    };
+
+    const info = parseRateLimitHeaders(headers);
+
+    expect(info.retryAfter).toBe(30);
+  });
+
+  it('handles missing headers gracefully', () => {
+    const info = parseRateLimitHeaders({});
+
+    expect(info.limit).toBeUndefined();
+    expect(info.remaining).toBeUndefined();
+    expect(info.resetAt).toBeUndefined();
+    expect(info.retryAfter).toBeUndefined();
+  });
+});
+
+describe('AdaptiveRateLimiter', () => {
+  let limiter: AdaptiveRateLimiter;
+
+  beforeEach(() => {
+    limiter = new AdaptiveRateLimiter();
+  });
+
+  it('allows requests when no limits recorded', async () => {
+    const canProceed = await limiter.canProceed('TestAPI');
+    expect(canProceed).toBe(true);
+  });
+
+  it('blocks requests when remaining is 0', async () => {
+    const futureReset = new Date(Date.now() + 60000);
+    limiter.recordResponse('TestAPI', {
+      remaining: 0,
+      limit: 100,
+      resetAt: futureReset,
+    });
+
+    const canProceed = await limiter.canProceed('TestAPI');
+    expect(canProceed).toBe(false);
+  });
+
+  it('allows requests after reset time passes', async () => {
+    const pastReset = new Date(Date.now() - 1000);
+    limiter.recordResponse('TestAPI', {
+      remaining: 0,
+      limit: 100,
+      resetAt: pastReset,
+    });
+
+    const canProceed = await limiter.canProceed('TestAPI');
+    expect(canProceed).toBe(true);
+  });
+
+  it('blocks during retry-after period', async () => {
+    limiter.recordResponse('TestAPI', {
+      retryAfter: 60, // 60 seconds
+    });
+
+    const canProceed = await limiter.canProceed('TestAPI');
+    expect(canProceed).toBe(false);
+  });
+
+  it('tracks limits per source', async () => {
+    limiter.recordResponse('API1', { remaining: 0, resetAt: new Date(Date.now() + 60000) });
+    limiter.recordResponse('API2', { remaining: 50, limit: 100 });
+
+    expect(await limiter.canProceed('API1')).toBe(false);
+    expect(await limiter.canProceed('API2')).toBe(true);
+  });
+
+  it('tracks limits per endpoint within source', async () => {
+    limiter.recordResponse('API', { remaining: 0, resetAt: new Date(Date.now() + 60000) }, '/invoices');
+    limiter.recordResponse('API', { remaining: 50, limit: 100 }, '/contacts');
+
+    expect(await limiter.canProceed('API', '/invoices')).toBe(false);
+    expect(await limiter.canProceed('API', '/contacts')).toBe(true);
+  });
+
+  it('provides accurate status', () => {
+    limiter.recordResponse('TestAPI', {
+      remaining: 42,
+      limit: 100,
+      resetAt: new Date(Date.now() + 60000),
+    });
+
+    const status = limiter.getStatus('TestAPI');
+
+    expect(status.remaining).toBe(42);
+    expect(status.limit).toBe(100);
+    expect(status.isLimited).toBe(false);
+  });
+
+  describe('fail strategy', () => {
+    it('throws RateLimitError immediately when rate limited', async () => {
+      limiter.configure('FailAPI', { strategy: 'fail' });
+      limiter.recordResponse('FailAPI', {
+        remaining: 0,
+        resetAt: new Date(Date.now() + 60000),
+      });
+
+      await expect(limiter.waitForCapacity('FailAPI')).rejects.toThrow(RateLimitError);
+    });
+
+    it('includes reset time in error', async () => {
+      limiter.configure('FailAPI', { strategy: 'fail' });
+      const resetAt = new Date(Date.now() + 60000);
+      limiter.recordResponse('FailAPI', { remaining: 0, resetAt });
+
+      try {
+        await limiter.waitForCapacity('FailAPI');
+        expect.fail('Should have thrown');
+      } catch (error) {
+        expect(error).toBeInstanceOf(RateLimitError);
+        expect((error as RateLimitError).resetAt).toEqual(resetAt);
+      }
+    });
+  });
+
+  describe('pause strategy', () => {
+    it('throws RateLimitTimeoutError when maxWait exceeded', async () => {
+      limiter.configure('PauseAPI', { strategy: 'pause', maxWait: 1 });
+      limiter.recordResponse('PauseAPI', {
+        remaining: 0,
+        resetAt: new Date(Date.now() + 60000), // 60s in future
+      });
+
+      await expect(limiter.waitForCapacity('PauseAPI')).rejects.toThrow(RateLimitTimeoutError);
+    });
+
+    it('proceeds after reset time passes', async () => {
+      limiter.configure('PauseAPI', { strategy: 'pause', maxWait: 5 });
+      // Reset in 50ms
+      limiter.recordResponse('PauseAPI', {
+        remaining: 0,
+        resetAt: new Date(Date.now() + 50),
+      });
+
+      // Should complete without throwing
+      await limiter.waitForCapacity('PauseAPI');
+    });
+  });
+
+  describe('throttle strategy', () => {
+    it('calculates delay based on remaining requests and reset time', () => {
+      limiter.configure('ThrottleAPI', { strategy: 'throttle' });
+      limiter.recordResponse('ThrottleAPI', {
+        remaining: 10,
+        limit: 100,
+        resetAt: new Date(Date.now() + 10000), // 10s left
+      });
+
+      const delay = limiter.getThrottleDelay('ThrottleAPI');
+      // 10s / 10 remaining = 1s per request = 1000ms
+      expect(delay).toBeGreaterThanOrEqual(900);
+      expect(delay).toBeLessThanOrEqual(1100);
+    });
+
+    it('returns 0 delay when not in throttle mode', () => {
+      limiter.configure('PauseAPI', { strategy: 'pause' });
+      limiter.recordResponse('PauseAPI', { remaining: 10, limit: 100 });
+
+      const delay = limiter.getThrottleDelay('PauseAPI');
+      expect(delay).toBe(0);
+    });
+
+    it('uses fallback RPM when no rate limit headers', () => {
+      limiter.configure('FallbackAPI', { strategy: 'throttle', fallbackRpm: 60 });
+      limiter.recordResponse('FallbackAPI', {}); // No rate limit info
+
+      const delay = limiter.getThrottleDelay('FallbackAPI');
+      // 60 RPM = 1 per second = 1000ms intervals
+      expect(delay).toBeLessThanOrEqual(1000);
+    });
+  });
+
+  describe('callbacks', () => {
+    it('calls onRateLimited when rate limited', async () => {
+      const onRateLimited = vi.fn();
+      limiter.setCallbacks({ onRateLimited });
+      // Use maxWait of 10s so it doesn't timeout immediately (wait is only 50ms)
+      limiter.configure('CallbackAPI', { strategy: 'pause', maxWait: 10 });
+      limiter.recordResponse('CallbackAPI', {
+        remaining: 0,
+        resetAt: new Date(Date.now() + 50), // Resets in 50ms
+      });
+
+      await limiter.waitForCapacity('CallbackAPI');
+
+      expect(onRateLimited).toHaveBeenCalledTimes(1);
+      expect(onRateLimited).toHaveBeenCalledWith(
+        expect.objectContaining({
+          source: 'CallbackAPI',
+          strategy: 'pause',
+        })
+      );
+    });
+
+    it('calls onResumed after waiting completes', async () => {
+      const onResumed = vi.fn();
+      limiter.setCallbacks({ onResumed });
+      limiter.configure('ResumeAPI', { strategy: 'pause', maxWait: 5 });
+      // Reset in 50ms
+      limiter.recordResponse('ResumeAPI', {
+        remaining: 0,
+        resetAt: new Date(Date.now() + 50),
+      });
+
+      await limiter.waitForCapacity('ResumeAPI');
+
+      expect(onResumed).toHaveBeenCalledTimes(1);
+      expect(onResumed).toHaveBeenCalledWith(
+        expect.objectContaining({
+          source: 'ResumeAPI',
+        })
+      );
+    });
+  });
+});
+
+describe('InMemoryTokenStore', () => {
+  let store: InMemoryTokenStore;
+
+  beforeEach(() => {
+    store = new InMemoryTokenStore();
+  });
+
+  it('stores and retrieves tokens', async () => {
+    const tokens: OAuth2Tokens = {
+      accessToken: 'access123',
+      refreshToken: 'refresh456',
+      expiresAt: new Date(Date.now() + 3600000),
+    };
+
+    await store.set('connection-1', tokens);
+    const retrieved = await store.get('connection-1');
+
+    expect(retrieved?.accessToken).toBe('access123');
+    expect(retrieved?.refreshToken).toBe('refresh456');
+  });
+
+  it('returns null for unknown connections', async () => {
+    const result = await store.get('unknown');
+    expect(result).toBeNull();
+  });
+
+  it('deletes tokens', async () => {
+    await store.set('connection-1', { accessToken: 'test' });
+    await store.delete('connection-1');
+
+    const result = await store.get('connection-1');
+    expect(result).toBeNull();
+  });
+
+  it('lists all connections', async () => {
+    await store.set('conn-1', { accessToken: 'a' });
+    await store.set('conn-2', { accessToken: 'b' });
+    await store.set('conn-3', { accessToken: 'c' });
+
+    const connections = await store.list();
+
+    expect(connections).toHaveLength(3);
+    expect(connections).toContain('conn-1');
+    expect(connections).toContain('conn-2');
+    expect(connections).toContain('conn-3');
+  });
+
+  it('identifies tokens needing refresh', async () => {
+    // Token expiring in 10 seconds (within 5 min buffer)
+    await store.set('expiring-soon', {
+      accessToken: 'test',
+      expiresAt: new Date(Date.now() + 10000),
+    });
+
+    // Token not expiring soon
+    await store.set('valid', {
+      accessToken: 'test',
+      expiresAt: new Date(Date.now() + 3600000),
+    });
+
+    const needsRefresh = await store.getTokensNeedingRefresh(300);
+
+    expect(needsRefresh).toContain('expiring-soon');
+    expect(needsRefresh).not.toContain('valid');
+  });
+});