reqon-dsl 0.2.0

package/src/auth/rate-limiter.ts

@@ -0,0 +1,459 @@
+import type {
+  RateLimiter,
+  RateLimitInfo,
+  RateLimitStatus,
+  RateLimitConfig,
+  RateLimitCallbacks,
+  RateLimitEvent,
+} from './types.js';
+import { sleep } from '../utils/async.js';
+
+interface RateLimitState {
+  remaining?: number;
+  limit?: number;
+  resetAt?: Date;
+  retryAfter?: Date;
+  lastRequestAt?: Date;
+}
+
+/** Max age for stale entries (default: 1 hour) */
+const DEFAULT_MAX_STALE_AGE_MS = 60 * 60 * 1000;
+/** Cleanup interval (default: 5 minutes) */
+const CLEANUP_INTERVAL_MS = 5 * 60 * 1000;
+/** Max number of entries before forced cleanup */
+const MAX_ENTRIES_BEFORE_CLEANUP = 1000;
+
+const DEFAULT_CONFIG: Required<RateLimitConfig> = {
+  strategy: 'pause',
+  maxWait: 300,
+  notifyAt: 10,
+  fallbackRpm: 60,
+};
+
+/**
+ * Rate limit timeout error - thrown when maxWait is exceeded
+ */
+export class RateLimitTimeoutError extends Error {
+  constructor(
+    public source: string,
+    public waitedSeconds: number,
+    public maxWait: number
+  ) {
+    super(`Rate limit timeout: waited ${waitedSeconds}s (max: ${maxWait}s) for ${source}`);
+    this.name = 'RateLimitTimeoutError';
+  }
+}
+
+/**
+ * Rate limit error - thrown when strategy is 'fail'
+ */
+export class RateLimitError extends Error {
+  constructor(
+    public source: string,
+    public resetAt?: Date
+  ) {
+    const resetIn = resetAt ? Math.ceil((resetAt.getTime() - Date.now()) / 1000) : undefined;
+    super(`Rate limited on ${source}${resetIn ? ` - resets in ${resetIn}s` : ''}`);
+    this.name = 'RateLimitError';
+  }
+}
+
+/**
+ * Adaptive rate limiter that learns from response headers
+ * Supports pause, throttle, and fail strategies
+ */
+/** Number of responses between cleanup checks */
+const CLEANUP_CHECK_INTERVAL = 100;
+
+export class AdaptiveRateLimiter implements RateLimiter {
+  private state: Map<string, RateLimitState> = new Map();
+  private configs: Map<string, RateLimitConfig> = new Map();
+  private callbacks: RateLimitCallbacks = {};
+  private lastCleanup: number = Date.now();
+  private maxStaleAgeMs: number;
+  private responsesSinceCleanup: number = 0;
+
+  constructor(
+    private defaultConfig: Partial<RateLimitConfig> = {},
+    options: { maxStaleAgeMs?: number } = {}
+  ) {
+    this.maxStaleAgeMs = options.maxStaleAgeMs ?? DEFAULT_MAX_STALE_AGE_MS;
+  }
+
+  /**
+   * Clean up stale entries from the state map to prevent memory leaks
+   */
+  private cleanup(): void {
+    const now = Date.now();
+
+    // Skip if cleanup was done recently and we're under the entry limit
+    if (
+      now - this.lastCleanup < CLEANUP_INTERVAL_MS &&
+      this.state.size < MAX_ENTRIES_BEFORE_CLEANUP
+    ) {
+      return;
+    }
+
+    this.lastCleanup = now;
+    const nowDate = new Date(now);
+    const staleThreshold = new Date(now - this.maxStaleAgeMs);
+
+    for (const [key, state] of this.state) {
+      // Remove if:
+      // 1. Reset time has passed AND no recent requests
+      // 2. Last request is older than max stale age
+      const isResetPassed = state.resetAt && state.resetAt < nowDate;
+      const isRetryPassed = !state.retryAfter || state.retryAfter < nowDate;
+      const isStale = state.lastRequestAt && state.lastRequestAt < staleThreshold;
+
+      if ((isResetPassed && isRetryPassed && isStale) || (!state.lastRequestAt && isResetPassed)) {
+        this.state.delete(key);
+      }
+    }
+  }
+
+  /**
+   * Get the number of tracked endpoints (for monitoring)
+   */
+  getTrackedEndpointCount(): number {
+    return this.state.size;
+  }
+
+  /**
+   * Force cleanup of stale entries
+   */
+  forceCleanup(): void {
+    this.lastCleanup = 0;
+    this.cleanup();
+  }
+
+  private getKey(source: string, endpoint?: string): string {
+    return endpoint ? `${source}:${endpoint}` : source;
+  }
+
+  private getConfig(source: string): Required<RateLimitConfig> {
+    const sourceConfig = this.configs.get(source) ?? {};
+    return { ...DEFAULT_CONFIG, ...this.defaultConfig, ...sourceConfig };
+  }
+
+  configure(source: string, config: RateLimitConfig): void {
+    this.configs.set(source, config);
+  }
+
+  setCallbacks(callbacks: RateLimitCallbacks): void {
+    this.callbacks = callbacks;
+  }
+
+  async canProceed(source: string, endpoint?: string): Promise<boolean> {
+    const key = this.getKey(source, endpoint);
+    const state = this.state.get(key);
+
+    if (!state) return true;
+
+    const now = new Date();
+
+    // Check retry-after (from 429)
+    if (state.retryAfter && state.retryAfter > now) {
+      return false;
+    }
+
+    // Check if we've reset
+    if (state.resetAt && state.resetAt <= now) {
+      // Reset has passed, clear the limit
+      this.state.delete(key);
+      return true;
+    }
+
+    // Check remaining quota
+    if (state.remaining !== undefined && state.remaining <= 0) {
+      return false;
+    }
+
+    return true;
+  }
+
+  async waitForCapacity(source: string, endpoint?: string): Promise<void> {
+    const config = this.getConfig(source);
+    const key = this.getKey(source, endpoint);
+
+    // Check if we can proceed immediately
+    if (await this.canProceed(source, endpoint)) {
+      // In throttle mode, add delay between requests
+      if (config.strategy === 'throttle') {
+        const delay = this.getThrottleDelay(source, endpoint);
+        if (delay > 0) {
+          await sleep(delay);
+        }
+      }
+      return;
+    }
+
+    // We're rate limited
+    const state = this.state.get(key);
+    const now = Date.now();
+
+    // Calculate wait time
+    let waitUntil: Date;
+    if (state?.retryAfter) {
+      waitUntil = state.retryAfter;
+    } else if (state?.resetAt) {
+      waitUntil = state.resetAt;
+    } else {
+      // No reset info, use fallback
+      waitUntil = new Date(now + 60000);
+    }
+
+    const totalWaitMs = waitUntil.getTime() - now;
+    const totalWaitSeconds = Math.ceil(totalWaitMs / 1000);
+
+    // Check strategy
+    if (config.strategy === 'fail') {
+      throw new RateLimitError(source, state?.resetAt);
+    }
+
+    // Check if wait exceeds max
+    if (totalWaitSeconds > config.maxWait) {
+      throw new RateLimitTimeoutError(source, 0, config.maxWait);
+    }
+
+    // Emit rate limited event
+    const event: RateLimitEvent = {
+      source,
+      endpoint,
+      waitSeconds: totalWaitSeconds,
+      remaining: state?.remaining,
+      resetAt: state?.resetAt,
+      strategy: config.strategy,
+    };
+    this.callbacks.onRateLimited?.(event);
+
+    // Wait with periodic updates
+    const startTime = Date.now();
+    let lastNotify = 0;
+
+    while (!(await this.canProceed(source, endpoint))) {
+      const elapsed = Math.floor((Date.now() - startTime) / 1000);
+
+      // Check timeout
+      if (elapsed >= config.maxWait) {
+        throw new RateLimitTimeoutError(source, elapsed, config.maxWait);
+      }
+
+      // Periodic notification
+      if (elapsed >= config.notifyAt && elapsed - lastNotify >= 10) {
+        lastNotify = elapsed;
+        this.callbacks.onWaiting?.({
+          ...event,
+          elapsedSeconds: elapsed,
+          waitSeconds: Math.max(0, totalWaitSeconds - elapsed),
+        });
+      }
+
+      // Sleep in chunks (check every 1-5 seconds)
+      const remainingMs = waitUntil.getTime() - Date.now();
+      const sleepMs = Math.min(Math.max(remainingMs, 1000), 5000);
+      await sleep(sleepMs);
+    }
+
+    // Emit resumed event
+    const waitedSeconds = Math.floor((Date.now() - startTime) / 1000);
+    this.callbacks.onResumed?.({ source, endpoint, waitedSeconds });
+  }
+
+  getThrottleDelay(source: string, endpoint?: string): number {
+    const key = this.getKey(source, endpoint);
+    const state = this.state.get(key);
+    const config = this.getConfig(source);
+
+    if (config.strategy !== 'throttle') return 0;
+    if (!state) return 0;
+
+    const now = Date.now();
+
+    // If we have remaining count and reset time, calculate optimal spacing
+    if (
+      state.remaining !== undefined &&
+      state.remaining > 0 &&
+      state.resetAt &&
+      state.resetAt.getTime() > now
+    ) {
+      const msUntilReset = state.resetAt.getTime() - now;
+      // Space requests evenly across remaining time
+      const optimalInterval = msUntilReset / state.remaining;
+
+      // Check time since last request
+      if (state.lastRequestAt) {
+        const msSinceLastRequest = now - state.lastRequestAt.getTime();
+        const delay = Math.max(0, optimalInterval - msSinceLastRequest);
+        return Math.round(delay);
+      }
+
+      return Math.round(optimalInterval);
+    }
+
+    // Fallback: use configured RPM
+    const intervalMs = 60000 / config.fallbackRpm;
+    if (state.lastRequestAt) {
+      const msSinceLastRequest = now - state.lastRequestAt.getTime();
+      return Math.max(0, Math.round(intervalMs - msSinceLastRequest));
+    }
+
+    return 0;
+  }
+
+  recordResponse(source: string, info: RateLimitInfo, endpoint?: string): void {
+    const key = this.getKey(source, endpoint);
+    const now = new Date();
+
+    const state: RateLimitState = this.state.get(key) ?? {};
+
+    if (info.remaining !== undefined) {
+      state.remaining = info.remaining;
+    }
+
+    if (info.limit !== undefined) {
+      state.limit = info.limit;
+    }
+
+    if (info.resetAt) {
+      state.resetAt = info.resetAt;
+    }
+
+    if (info.retryAfter !== undefined) {
+      state.retryAfter = new Date(now.getTime() + info.retryAfter * 1000);
+    }
+
+    state.lastRequestAt = now;
+
+    this.state.set(key, state);
+
+    // Periodically clean up stale entries (every N responses)
+    this.responsesSinceCleanup++;
+    if (this.responsesSinceCleanup >= CLEANUP_CHECK_INTERVAL) {
+      this.responsesSinceCleanup = 0;
+      this.cleanup();
+    }
+  }
+
+  getStatus(source: string, endpoint?: string): RateLimitStatus {
+    const key = this.getKey(source, endpoint);
+    const state = this.state.get(key);
+    const now = new Date();
+
+    if (!state) {
+      return { isLimited: false };
+    }
+
+    const isLimited =
+      (state.retryAfter && state.retryAfter > now) ||
+      (state.remaining !== undefined && state.remaining <= 0 && state.resetAt && state.resetAt > now);
+
+    let resetInSeconds: number | undefined;
+    if (isLimited && state.resetAt) {
+      resetInSeconds = Math.ceil((state.resetAt.getTime() - now.getTime()) / 1000);
+    } else if (isLimited && state.retryAfter) {
+      resetInSeconds = Math.ceil((state.retryAfter.getTime() - now.getTime()) / 1000);
+    }
+
+    return {
+      remaining: state.remaining,
+      limit: state.limit,
+      resetAt: state.resetAt,
+      isLimited: Boolean(isLimited),
+      resetInSeconds,
+    };
+  }
+}
+
+/**
+ * Parse rate limit info from HTTP response headers
+ *
+ * Supports common header patterns:
+ * - X-RateLimit-Remaining, X-RateLimit-Limit, X-RateLimit-Reset
+ * - RateLimit-Remaining, RateLimit-Limit, RateLimit-Reset
+ * - X-Rate-Limit-Remaining, etc.
+ * - Retry-After (from 429 responses)
+ */
+export function parseRateLimitHeaders(headers: Record<string, string>): RateLimitInfo {
+  const info: RateLimitInfo = {};
+
+  // Normalize headers to lowercase for matching
+  const normalized: Record<string, string> = {};
+  for (const [key, value] of Object.entries(headers)) {
+    normalized[key.toLowerCase()] = value;
+  }
+
+  // Parse remaining
+  const remainingKey = findHeader(normalized, [
+    'x-ratelimit-remaining',
+    'ratelimit-remaining',
+    'x-rate-limit-remaining',
+  ]);
+  if (remainingKey) {
+    info.remaining = parseInt(normalized[remainingKey], 10);
+  }
+
+  // Parse limit
+  const limitKey = findHeader(normalized, [
+    'x-ratelimit-limit',
+    'ratelimit-limit',
+    'x-rate-limit-limit',
+  ]);
+  if (limitKey) {
+    info.limit = parseInt(normalized[limitKey], 10);
+  }
+
+  // Parse reset
+  const resetKey = findHeader(normalized, [
+    'x-ratelimit-reset',
+    'ratelimit-reset',
+    'x-rate-limit-reset',
+  ]);
+  if (resetKey) {
+    const resetValue = normalized[resetKey];
+    // Could be Unix timestamp or ISO date
+    const asNumber = parseInt(resetValue, 10);
+    if (!isNaN(asNumber)) {
+      // Unix timestamp (seconds)
+      if (asNumber > 1000000000000) {
+        // Already in milliseconds
+        info.resetAt = new Date(asNumber);
+      } else {
+        // In seconds
+        info.resetAt = new Date(asNumber * 1000);
+      }
+    } else {
+      // Try as date string
+      const parsed = new Date(resetValue);
+      if (!isNaN(parsed.getTime())) {
+        info.resetAt = parsed;
+      }
+    }
+  }
+
+  // Parse Retry-After (usually from 429 responses)
+  const retryAfter = normalized['retry-after'];
+  if (retryAfter) {
+    const asNumber = parseInt(retryAfter, 10);
+    if (!isNaN(asNumber)) {
+      info.retryAfter = asNumber;
+    } else {
+      // HTTP-date format
+      const parsed = new Date(retryAfter);
+      if (!isNaN(parsed.getTime())) {
+        info.retryAfter = Math.ceil((parsed.getTime() - Date.now()) / 1000);
+      }
+    }
+  }
+
+  return info;
+}
+
+function findHeader(headers: Record<string, string>, candidates: string[]): string | undefined {
+  for (const candidate of candidates) {
+    if (candidate in headers) {
+      return candidate;
+    }
+  }
+  return undefined;
+}

package/src/auth/token-store.ts

@@ -0,0 +1,177 @@
+import type { TokenStore, OAuth2Tokens } from './types.js';
+
+interface StoredToken extends OAuth2Tokens {
+  lastUsedAt?: Date;
+}
+
+/**
+ * In-memory token store - useful for development and single-process deployments
+ */
+export class InMemoryTokenStore implements TokenStore {
+  private tokens: Map<string, StoredToken> = new Map();
+
+  async get(connectionId: string): Promise<OAuth2Tokens | null> {
+    const stored = this.tokens.get(connectionId);
+    if (!stored) return null;
+
+    return {
+      accessToken: stored.accessToken,
+      refreshToken: stored.refreshToken,
+      expiresAt: stored.expiresAt,
+      refreshExpiresAt: stored.refreshExpiresAt,
+      tokenType: stored.tokenType,
+      scope: stored.scope,
+    };
+  }
+
+  async set(connectionId: string, tokens: OAuth2Tokens): Promise<void> {
+    const existing = this.tokens.get(connectionId);
+    this.tokens.set(connectionId, {
+      ...tokens,
+      lastUsedAt: existing?.lastUsedAt ?? new Date(),
+    });
+  }
+
+  async delete(connectionId: string): Promise<void> {
+    this.tokens.delete(connectionId);
+  }
+
+  async touch(connectionId: string): Promise<void> {
+    const stored = this.tokens.get(connectionId);
+    if (stored) {
+      stored.lastUsedAt = new Date();
+    }
+  }
+
+  async list(): Promise<string[]> {
+    return Array.from(this.tokens.keys());
+  }
+
+  /** Get tokens that need proactive refresh (approaching expiry or non-use expiry) */
+  async getTokensNeedingRefresh(bufferSeconds = 300): Promise<string[]> {
+    const now = new Date();
+    const buffer = bufferSeconds * 1000;
+    const needsRefresh: string[] = [];
+
+    for (const [connectionId, stored] of this.tokens) {
+      // Check access token expiry
+      if (stored.expiresAt) {
+        const expiresIn = stored.expiresAt.getTime() - now.getTime();
+        if (expiresIn < buffer) {
+          needsRefresh.push(connectionId);
+          continue;
+        }
+      }
+
+      // Check refresh token expiry from non-use
+      if (stored.refreshExpiresAt && stored.lastUsedAt) {
+        const refreshExpiresIn = stored.refreshExpiresAt.getTime() - now.getTime();
+        if (refreshExpiresIn < buffer) {
+          needsRefresh.push(connectionId);
+        }
+      }
+    }
+
+    return needsRefresh;
+  }
+}
+
+/**
+ * File-based token store - persists tokens to a JSON file
+ * Suitable for CLI tools and single-user scenarios
+ */
+export class FileTokenStore implements TokenStore {
+  private filePath: string;
+  private cache: Map<string, StoredToken> | null = null;
+
+  constructor(filePath: string) {
+    this.filePath = filePath;
+  }
+
+  private async load(): Promise<Map<string, StoredToken>> {
+    if (this.cache) return this.cache;
+
+    try {
+      const fs = await import('node:fs/promises');
+      const content = await fs.readFile(this.filePath, 'utf-8');
+      const data = JSON.parse(content) as Record<string, StoredToken>;
+
+      // Revive dates
+      const map = new Map<string, StoredToken>();
+      for (const [key, value] of Object.entries(data)) {
+        map.set(key, {
+          ...value,
+          expiresAt: value.expiresAt ? new Date(value.expiresAt) : undefined,
+          refreshExpiresAt: value.refreshExpiresAt ? new Date(value.refreshExpiresAt) : undefined,
+          lastUsedAt: value.lastUsedAt ? new Date(value.lastUsedAt) : undefined,
+        });
+      }
+
+      this.cache = map;
+      return map;
+    } catch {
+      this.cache = new Map();
+      return this.cache;
+    }
+  }
+
+  private async save(): Promise<void> {
+    if (!this.cache) return;
+
+    const fs = await import('node:fs/promises');
+    const data: Record<string, StoredToken> = {};
+
+    for (const [key, value] of this.cache) {
+      data[key] = value;
+    }
+
+    await fs.writeFile(this.filePath, JSON.stringify(data, null, 2), 'utf-8');
+  }
+
+  async get(connectionId: string): Promise<OAuth2Tokens | null> {
+    const map = await this.load();
+    const stored = map.get(connectionId);
+    if (!stored) return null;
+
+    return {
+      accessToken: stored.accessToken,
+      refreshToken: stored.refreshToken,
+      expiresAt: stored.expiresAt,
+      refreshExpiresAt: stored.refreshExpiresAt,
+      tokenType: stored.tokenType,
+      scope: stored.scope,
+    };
+  }
+
+  async set(connectionId: string, tokens: OAuth2Tokens): Promise<void> {
+    const map = await this.load();
+    const existing = map.get(connectionId);
+
+    map.set(connectionId, {
+      ...tokens,
+      lastUsedAt: existing?.lastUsedAt ?? new Date(),
+    });
+
+    await this.save();
+  }
+
+  async delete(connectionId: string): Promise<void> {
+    const map = await this.load();
+    map.delete(connectionId);
+    await this.save();
+  }
+
+  async touch(connectionId: string): Promise<void> {
+    const map = await this.load();
+    const stored = map.get(connectionId);
+    if (stored) {
+      stored.lastUsedAt = new Date();
+      await this.save();
+    }
+  }
+
+  async list(): Promise<string[]> {
+    const map = await this.load();
+    return Array.from(map.keys());
+  }
+}