reqon-dsl 0.2.0 → 0.4.0

package/src/ast/index.ts

@@ -1 +0,0 @@
-export * from './nodes.js';

package/src/ast/nodes.ts

@@ -1,310 +0,0 @@
-import type {
-  Expression,
-  FieldDefinition,
-  SchemaDefinition,
-  Statement as VagueStatement,
-} from 'vague-lang';
-
-// Reqon extends Vague's statements
-export type Statement = VagueStatement | MissionDefinition | SourceDefinition | StoreDefinition | ActionDefinition;
-
-export interface ReqonProgram {
-  type: 'ReqonProgram';
-  statements: Statement[];
-}
-
-// source Xero { auth: oauth2, base: "https://api.xero.com/..." }
-// source Xero from "./xero-openapi.yaml" { auth: oauth2 }
-export interface SourceDefinition {
-  type: 'SourceDefinition';
-  name: string;
-  specPath?: string; // OAS spec path (URL or file path)
-  config: SourceConfig;
-}
-
-export interface SourceConfig {
-  auth: AuthConfig;
-  base?: string; // Optional if using OAS (derived from spec)
-  headers?: Record<string, Expression>;
-  validateResponses?: boolean; // Validate responses against OAS schema
-  rateLimit?: RateLimitSourceConfig; // Rate limiting configuration
-  circuitBreaker?: CircuitBreakerSourceConfig; // Circuit breaker configuration
-}
-
-export interface CircuitBreakerSourceConfig {
-  /** Number of failures before opening circuit (default: 5) */
-  failureThreshold?: number;
-  /** Time in seconds before attempting recovery (default: 30) */
-  resetTimeout?: number;
-  /** Number of successful requests in half-open to close circuit (default: 2) */
-  successThreshold?: number;
-  /** Time window in seconds for counting failures (default: 60) */
-  failureWindow?: number;
-}
-
-export interface RateLimitSourceConfig {
-  strategy?: 'pause' | 'throttle' | 'fail'; // Default: 'pause'
-  maxWait?: number; // Max seconds to wait (default: 300)
-  fallbackRpm?: number; // Fallback requests per minute if no headers
-}
-
-export interface AuthConfig {
-  type: 'oauth2' | 'bearer' | 'basic' | 'api_key' | 'none';
-  // Details resolved at runtime from environment/config
-}
-
-// store invoices_cache: nosql("invoices")
-// store invoices_sql: sql("accounting.invoices")
-export interface StoreDefinition {
-  type: 'StoreDefinition';
-  name: string;
-  storeType: 'nosql' | 'sql' | 'memory';
-  target: string; // collection/table name
-}
-
-// Schedule configuration for missions
-// schedule: every 6 hours
-// schedule: cron "0 */6 * * *"
-// schedule: at "2025-01-20 09:00 UTC"
-export interface ScheduleDefinition {
-  type: 'ScheduleDefinition';
-  scheduleType: 'interval' | 'cron' | 'once';
-  // For interval-based scheduling
-  interval?: IntervalSchedule;
-  // For cron-based scheduling
-  cronExpression?: string;
-  // For one-time scheduling
-  runAt?: string; // ISO 8601 datetime or parseable date string
-  // Optional timezone (defaults to UTC)
-  timezone?: string;
-  // Maximum concurrent executions (default: 1)
-  maxConcurrency?: number;
-  // Skip execution if previous run is still running (default: true)
-  skipIfRunning?: boolean;
-  // Retry configuration for failed scheduled runs
-  retryOnFailure?: ScheduleRetryConfig;
-}
-
-export interface IntervalSchedule {
-  value: number;
-  unit: 'seconds' | 'minutes' | 'hours' | 'days' | 'weeks';
-}
-
-export interface ScheduleRetryConfig {
-  maxRetries: number;
-  delaySeconds: number;
-}
-
-// mission SyncXeroInvoices { ... }
-export interface MissionDefinition {
-  type: 'MissionDefinition';
-  name: string;
-  schedule?: ScheduleDefinition;
-  sources: SourceDefinition[];
-  stores: StoreDefinition[];
-  schemas: SchemaDefinition[];
-  actions: ActionDefinition[];
-  pipeline: PipelineDefinition;
-}
-
-// action FetchInvoiceList { ... }
-export interface ActionDefinition {
-  type: 'ActionDefinition';
-  name: string;
-  steps: ActionStep[];
-}
-
-export type ActionStep = FetchStep | ForStep | MapStep | ValidateStep | StoreStep | MatchStep | LetStep | WebhookStep;
-
-// let myVar = expression
-export interface LetStep {
-  type: 'LetStep';
-  name: string;
-  value: Expression;
-}
-
-// wait { timeout: 60000, path: "/webhooks/callback", ... }
-// Waits for an external webhook callback before continuing execution
-export interface WebhookStep {
-  type: 'WebhookStep';
-  /** Timeout in milliseconds to wait for webhook (default: 300000 = 5 minutes) */
-  timeout?: number;
-  /** Path for the webhook endpoint (e.g., "/webhooks/callback") */
-  path?: string;
-  /** Number of webhook events to collect before continuing (default: 1) */
-  expectedEvents?: number;
-  /** Filter expression for matching webhook events */
-  eventFilter?: Expression;
-  /** Retry configuration if timeout occurs */
-  retryOnTimeout?: RetryConfig;
-  /** Store configuration for saving webhook payloads */
-  storage?: WebhookStorageConfig;
-}
-
-export interface WebhookStorageConfig {
-  /** Store name to save webhook payloads */
-  target: string;
-  /** Expression for extracting key from webhook payload */
-  key?: Expression;
-}
-
-// Flow control directives for match arms
-export type FlowDirective =
-  | { type: 'continue' }                          // proceed to next pipeline stage
-  | { type: 'skip' }                              // skip remaining steps, move to next stage
-  | { type: 'abort'; message?: string }           // halt mission with error
-  | { type: 'retry'; backoff?: RetryConfig }      // retry current action
-  | { type: 'queue'; target?: string }            // queue for later processing
-  | { type: 'jump'; action: string; then?: 'retry' | 'continue' };  // jump to action
-
-// match response { Schema1 -> steps..., Schema2 -> flow directive }
-export interface MatchStep {
-  type: 'MatchStep';
-  target: Expression;  // what to match (usually 'response')
-  arms: MatchArm[];
-}
-
-export interface MatchArm {
-  /** Schema name to match against, or '_' for wildcard */
-  schema: string;
-  /** Steps to execute if matched */
-  steps?: ActionStep[];
-  /** Flow control directive (if no steps) */
-  flow?: FlowDirective;
-}
-
-// get "/Invoices" { paginate: ..., until: ... }
-// call Xero.getInvoices { paginate: ... }  -- OAS operationId reference
-// get "/Invoices" { since: lastSync }  -- Incremental sync
-export interface FetchStep {
-  type: 'FetchStep';
-  // Traditional: explicit method + path
-  method?: 'GET' | 'POST' | 'PUT' | 'PATCH' | 'DELETE';
-  path?: Expression; // Can contain interpolations like "/Invoices/{id}"
-  // OAS-based: source.operationId reference
-  operationRef?: OperationRef;
-  source?: string; // Which source to use (defaults to first defined)
-  body?: Expression;
-  headers?: Record<string, Expression>;
-  paginate?: PaginationConfig;
-  until?: Expression; // Condition to stop pagination
-  retry?: RetryConfig;
-  // Incremental sync
-  since?: SinceConfig;
-}
-
-// Configuration for incremental sync
-export interface SinceConfig {
-  /** How to resolve the "since" timestamp */
-  type: 'lastSync' | 'expression';
-  /** Custom checkpoint key (defaults to source:endpoint) */
-  key?: string;
-  /** Query parameter name for the since value (default: varies by API) */
-  param?: string;
-  /** Date format for the since value */
-  format?: 'iso' | 'unix' | 'unix-ms' | 'date-only';
-  /** Expression to evaluate for 'expression' type */
-  expression?: Expression;
-  /** Field in response to use as the new "since" value for next sync */
-  updateFrom?: string;
-}
-
-export interface OperationRef {
-  source: string; // Source name (e.g., "Xero")
-  operationId: string; // OAS operationId (e.g., "getInvoices")
-}
-
-export interface PaginationConfig {
-  type: 'offset' | 'cursor' | 'page';
-  param: string; // Query param name: "page", "offset", "cursor"
-  pageSize: number;
-  cursorPath?: string; // For cursor pagination: where to find next cursor in response
-}
-
-export interface RetryConfig {
-  maxAttempts: number;
-  backoff: 'exponential' | 'linear' | 'constant';
-  initialDelay: number; // ms
-  maxDelay?: number; // ms
-}
-
-// for invoice in invoices_cache where .partial == true { ... }
-export interface ForStep {
-  type: 'ForStep';
-  variable: string;
-  collection: Expression;
-  condition?: Expression; // where clause
-  steps: ActionStep[];
-}
-
-// map invoice -> StandardInvoice { id: .InvoiceID, ... }
-export interface MapStep {
-  type: 'MapStep';
-  source: Expression;
-  targetSchema: string;
-  mappings: FieldMapping[];
-}
-
-export interface FieldMapping {
-  field: string;
-  expression: Expression;
-}
-
-// validate response { assume .Total is decimal, ... }
-export interface ValidateStep {
-  type: 'ValidateStep';
-  target: Expression;
-  constraints: ValidationConstraint[];
-}
-
-export interface ValidationConstraint {
-  type: 'ValidationConstraint';
-  condition: Expression;
-  message?: string;
-  severity: 'error' | 'warning';
-}
-
-// store response -> invoices_cache { key: .InvoiceID, partial: false }
-export interface StoreStep {
-  type: 'StoreStep';
-  source: Expression;
-  target: string; // store name
-  options: StoreOptions;
-}
-
-export interface StoreOptions {
-  key?: Expression; // Primary key field
-  partial?: boolean; // Mark as partial entity
-  upsert?: boolean; // Update if exists
-}
-
-// run FetchInvoiceList then HydrateInvoices then NormalizeInvoices
-// run [FetchOrders, FetchPayments] then ReconcileData  -- parallel stages
-export interface PipelineDefinition {
-  type: 'PipelineDefinition';
-  stages: PipelineStage[];
-}
-
-export interface PipelineStage {
-  /** Single action name (for sequential stages) */
-  action?: string;
-  /** Multiple action names (for parallel stages with bracket syntax) */
-  actions?: string[];
-  /** Optional: only run if condition true */
-  condition?: Expression;
-}
-
-/** Helper to check if a stage is parallel */
-export function isParallelStage(stage: PipelineStage): stage is PipelineStage & { actions: string[] } {
-  return Array.isArray(stage.actions) && stage.actions.length > 0;
-}
-
-/** Helper to get action names from a stage (works for both single and parallel) */
-export function getStageActions(stage: PipelineStage): string[] {
-  if (stage.actions) return stage.actions;
-  if (stage.action) return [stage.action];
-  return [];
-}
-
-// Re-export Vague types for convenience
-export type { Expression, FieldDefinition, SchemaDefinition } from 'vague-lang';

package/src/auth/auth.test.ts

@@ -1,326 +0,0 @@
-import { describe, it, expect, beforeEach, vi } from 'vitest';
-import {
-  AdaptiveRateLimiter,
-  parseRateLimitHeaders,
-  RateLimitError,
-  RateLimitTimeoutError,
-} from './rate-limiter.js';
-import { InMemoryTokenStore } from './token-store.js';
-import type { OAuth2Tokens, RateLimitEvent } from './types.js';
-
-describe('parseRateLimitHeaders', () => {
-  it('parses X-RateLimit headers', () => {
-    const headers = {
-      'X-RateLimit-Limit': '100',
-      'X-RateLimit-Remaining': '42',
-      'X-RateLimit-Reset': '1700000000',
-    };
-
-    const info = parseRateLimitHeaders(headers);
-
-    expect(info.limit).toBe(100);
-    expect(info.remaining).toBe(42);
-    expect(info.resetAt).toBeInstanceOf(Date);
-    expect(info.resetAt?.getTime()).toBe(1700000000 * 1000);
-  });
-
-  it('parses lowercase ratelimit headers', () => {
-    const headers = {
-      'ratelimit-limit': '60',
-      'ratelimit-remaining': '5',
-    };
-
-    const info = parseRateLimitHeaders(headers);
-
-    expect(info.limit).toBe(60);
-    expect(info.remaining).toBe(5);
-  });
-
-  it('parses Retry-After header (seconds)', () => {
-    const headers = {
-      'Retry-After': '30',
-    };
-
-    const info = parseRateLimitHeaders(headers);
-
-    expect(info.retryAfter).toBe(30);
-  });
-
-  it('handles missing headers gracefully', () => {
-    const info = parseRateLimitHeaders({});
-
-    expect(info.limit).toBeUndefined();
-    expect(info.remaining).toBeUndefined();
-    expect(info.resetAt).toBeUndefined();
-    expect(info.retryAfter).toBeUndefined();
-  });
-});
-
-describe('AdaptiveRateLimiter', () => {
-  let limiter: AdaptiveRateLimiter;
-
-  beforeEach(() => {
-    limiter = new AdaptiveRateLimiter();
-  });
-
-  it('allows requests when no limits recorded', async () => {
-    const canProceed = await limiter.canProceed('TestAPI');
-    expect(canProceed).toBe(true);
-  });
-
-  it('blocks requests when remaining is 0', async () => {
-    const futureReset = new Date(Date.now() + 60000);
-    limiter.recordResponse('TestAPI', {
-      remaining: 0,
-      limit: 100,
-      resetAt: futureReset,
-    });
-
-    const canProceed = await limiter.canProceed('TestAPI');
-    expect(canProceed).toBe(false);
-  });
-
-  it('allows requests after reset time passes', async () => {
-    const pastReset = new Date(Date.now() - 1000);
-    limiter.recordResponse('TestAPI', {
-      remaining: 0,
-      limit: 100,
-      resetAt: pastReset,
-    });
-
-    const canProceed = await limiter.canProceed('TestAPI');
-    expect(canProceed).toBe(true);
-  });
-
-  it('blocks during retry-after period', async () => {
-    limiter.recordResponse('TestAPI', {
-      retryAfter: 60, // 60 seconds
-    });
-
-    const canProceed = await limiter.canProceed('TestAPI');
-    expect(canProceed).toBe(false);
-  });
-
-  it('tracks limits per source', async () => {
-    limiter.recordResponse('API1', { remaining: 0, resetAt: new Date(Date.now() + 60000) });
-    limiter.recordResponse('API2', { remaining: 50, limit: 100 });
-
-    expect(await limiter.canProceed('API1')).toBe(false);
-    expect(await limiter.canProceed('API2')).toBe(true);
-  });
-
-  it('tracks limits per endpoint within source', async () => {
-    limiter.recordResponse('API', { remaining: 0, resetAt: new Date(Date.now() + 60000) }, '/invoices');
-    limiter.recordResponse('API', { remaining: 50, limit: 100 }, '/contacts');
-
-    expect(await limiter.canProceed('API', '/invoices')).toBe(false);
-    expect(await limiter.canProceed('API', '/contacts')).toBe(true);
-  });
-
-  it('provides accurate status', () => {
-    limiter.recordResponse('TestAPI', {
-      remaining: 42,
-      limit: 100,
-      resetAt: new Date(Date.now() + 60000),
-    });
-
-    const status = limiter.getStatus('TestAPI');
-
-    expect(status.remaining).toBe(42);
-    expect(status.limit).toBe(100);
-    expect(status.isLimited).toBe(false);
-  });
-
-  describe('fail strategy', () => {
-    it('throws RateLimitError immediately when rate limited', async () => {
-      limiter.configure('FailAPI', { strategy: 'fail' });
-      limiter.recordResponse('FailAPI', {
-        remaining: 0,
-        resetAt: new Date(Date.now() + 60000),
-      });
-
-      await expect(limiter.waitForCapacity('FailAPI')).rejects.toThrow(RateLimitError);
-    });
-
-    it('includes reset time in error', async () => {
-      limiter.configure('FailAPI', { strategy: 'fail' });
-      const resetAt = new Date(Date.now() + 60000);
-      limiter.recordResponse('FailAPI', { remaining: 0, resetAt });
-
-      try {
-        await limiter.waitForCapacity('FailAPI');
-        expect.fail('Should have thrown');
-      } catch (error) {
-        expect(error).toBeInstanceOf(RateLimitError);
-        expect((error as RateLimitError).resetAt).toEqual(resetAt);
-      }
-    });
-  });
-
-  describe('pause strategy', () => {
-    it('throws RateLimitTimeoutError when maxWait exceeded', async () => {
-      limiter.configure('PauseAPI', { strategy: 'pause', maxWait: 1 });
-      limiter.recordResponse('PauseAPI', {
-        remaining: 0,
-        resetAt: new Date(Date.now() + 60000), // 60s in future
-      });
-
-      await expect(limiter.waitForCapacity('PauseAPI')).rejects.toThrow(RateLimitTimeoutError);
-    });
-
-    it('proceeds after reset time passes', async () => {
-      limiter.configure('PauseAPI', { strategy: 'pause', maxWait: 5 });
-      // Reset in 50ms
-      limiter.recordResponse('PauseAPI', {
-        remaining: 0,
-        resetAt: new Date(Date.now() + 50),
-      });
-
-      // Should complete without throwing
-      await limiter.waitForCapacity('PauseAPI');
-    });
-  });
-
-  describe('throttle strategy', () => {
-    it('calculates delay based on remaining requests and reset time', () => {
-      limiter.configure('ThrottleAPI', { strategy: 'throttle' });
-      limiter.recordResponse('ThrottleAPI', {
-        remaining: 10,
-        limit: 100,
-        resetAt: new Date(Date.now() + 10000), // 10s left
-      });
-
-      const delay = limiter.getThrottleDelay('ThrottleAPI');
-      // 10s / 10 remaining = 1s per request = 1000ms
-      expect(delay).toBeGreaterThanOrEqual(900);
-      expect(delay).toBeLessThanOrEqual(1100);
-    });
-
-    it('returns 0 delay when not in throttle mode', () => {
-      limiter.configure('PauseAPI', { strategy: 'pause' });
-      limiter.recordResponse('PauseAPI', { remaining: 10, limit: 100 });
-
-      const delay = limiter.getThrottleDelay('PauseAPI');
-      expect(delay).toBe(0);
-    });
-
-    it('uses fallback RPM when no rate limit headers', () => {
-      limiter.configure('FallbackAPI', { strategy: 'throttle', fallbackRpm: 60 });
-      limiter.recordResponse('FallbackAPI', {}); // No rate limit info
-
-      const delay = limiter.getThrottleDelay('FallbackAPI');
-      // 60 RPM = 1 per second = 1000ms intervals
-      expect(delay).toBeLessThanOrEqual(1000);
-    });
-  });
-
-  describe('callbacks', () => {
-    it('calls onRateLimited when rate limited', async () => {
-      const onRateLimited = vi.fn();
-      limiter.setCallbacks({ onRateLimited });
-      // Use maxWait of 10s so it doesn't timeout immediately (wait is only 50ms)
-      limiter.configure('CallbackAPI', { strategy: 'pause', maxWait: 10 });
-      limiter.recordResponse('CallbackAPI', {
-        remaining: 0,
-        resetAt: new Date(Date.now() + 50), // Resets in 50ms
-      });
-
-      await limiter.waitForCapacity('CallbackAPI');
-
-      expect(onRateLimited).toHaveBeenCalledTimes(1);
-      expect(onRateLimited).toHaveBeenCalledWith(
-        expect.objectContaining({
-          source: 'CallbackAPI',
-          strategy: 'pause',
-        })
-      );
-    });
-
-    it('calls onResumed after waiting completes', async () => {
-      const onResumed = vi.fn();
-      limiter.setCallbacks({ onResumed });
-      limiter.configure('ResumeAPI', { strategy: 'pause', maxWait: 5 });
-      // Reset in 50ms
-      limiter.recordResponse('ResumeAPI', {
-        remaining: 0,
-        resetAt: new Date(Date.now() + 50),
-      });
-
-      await limiter.waitForCapacity('ResumeAPI');
-
-      expect(onResumed).toHaveBeenCalledTimes(1);
-      expect(onResumed).toHaveBeenCalledWith(
-        expect.objectContaining({
-          source: 'ResumeAPI',
-        })
-      );
-    });
-  });
-});
-
-describe('InMemoryTokenStore', () => {
-  let store: InMemoryTokenStore;
-
-  beforeEach(() => {
-    store = new InMemoryTokenStore();
-  });
-
-  it('stores and retrieves tokens', async () => {
-    const tokens: OAuth2Tokens = {
-      accessToken: 'access123',
-      refreshToken: 'refresh456',
-      expiresAt: new Date(Date.now() + 3600000),
-    };
-
-    await store.set('connection-1', tokens);
-    const retrieved = await store.get('connection-1');
-
-    expect(retrieved?.accessToken).toBe('access123');
-    expect(retrieved?.refreshToken).toBe('refresh456');
-  });
-
-  it('returns null for unknown connections', async () => {
-    const result = await store.get('unknown');
-    expect(result).toBeNull();
-  });
-
-  it('deletes tokens', async () => {
-    await store.set('connection-1', { accessToken: 'test' });
-    await store.delete('connection-1');
-
-    const result = await store.get('connection-1');
-    expect(result).toBeNull();
-  });
-
-  it('lists all connections', async () => {
-    await store.set('conn-1', { accessToken: 'a' });
-    await store.set('conn-2', { accessToken: 'b' });
-    await store.set('conn-3', { accessToken: 'c' });
-
-    const connections = await store.list();
-
-    expect(connections).toHaveLength(3);
-    expect(connections).toContain('conn-1');
-    expect(connections).toContain('conn-2');
-    expect(connections).toContain('conn-3');
-  });
-
-  it('identifies tokens needing refresh', async () => {
-    // Token expiring in 10 seconds (within 5 min buffer)
-    await store.set('expiring-soon', {
-      accessToken: 'test',
-      expiresAt: new Date(Date.now() + 10000),
-    });
-
-    // Token not expiring soon
-    await store.set('valid', {
-      accessToken: 'test',
-      expiresAt: new Date(Date.now() + 3600000),
-    });
-
-    const needsRefresh = await store.getTokensNeedingRefresh(300);
-
-    expect(needsRefresh).toContain('expiring-soon');
-    expect(needsRefresh).not.toContain('valid');
-  });
-});