reqon-dsl 0.2.0

package/dist/interpreter/fetch-handler.d.ts

@@ -0,0 +1,43 @@
+import type { FetchStep } from '../ast/nodes.js';
+import type { ExecutionContext } from './context.js';
+import type { OASSource } from '../oas/index.js';
+import { type SyncStore } from '../sync/index.js';
+export interface FetchHandlerDeps {
+    ctx: ExecutionContext;
+    oasSources: Map<string, OASSource>;
+    sourceConfigs: Map<string, {
+        config: {
+            validateResponses?: boolean;
+        };
+    }>;
+    syncStore?: SyncStore;
+    missionName?: string;
+    executionId?: string;
+    dryRun?: boolean;
+    log: (message: string) => void;
+}
+export interface FetchResult {
+    data: unknown;
+    checkpointKey?: string;
+}
+/**
+ * Handles HTTP fetch operations including pagination and incremental sync.
+ * Extracted from MissionExecutor for better separation of concerns.
+ */
+export declare class FetchHandler {
+    private deps;
+    constructor(deps: FetchHandlerDeps);
+    /**
+     * Execute a fetch step, handling OAS resolution, pagination, and sync checkpoints.
+     */
+    execute(step: FetchStep): Promise<FetchResult>;
+    /**
+     * Record a sync checkpoint after successful fetch.
+     */
+    recordCheckpoint(key: string, step: FetchStep, data: unknown): Promise<void>;
+    private resolveFetchTarget;
+    private resolveSinceParams;
+    private executePaginated;
+    private countRecords;
+    private validateOASResponse;
+}

package/dist/interpreter/fetch-handler.js

@@ -0,0 +1,203 @@
+import { evaluate, interpolatePath } from './evaluator.js';
+import { resolveOperation, getResponseSchema, validateResponse } from '../oas/index.js';
+import { generateCheckpointKey, formatSinceDate } from '../sync/index.js';
+import { extractNestedValue } from '../utils/path.js';
+import { createPaginationStrategy } from './pagination.js';
+/** Maximum pages to fetch to prevent infinite loops */
+const MAX_PAGINATION_PAGES = 100;
+/**
+ * Handles HTTP fetch operations including pagination and incremental sync.
+ * Extracted from MissionExecutor for better separation of concerns.
+ */
+export class FetchHandler {
+    deps;
+    constructor(deps) {
+        this.deps = deps;
+    }
+    /**
+     * Execute a fetch step, handling OAS resolution, pagination, and sync checkpoints.
+     */
+    async execute(step) {
+        const resolved = this.resolveFetchTarget(step);
+        const client = this.deps.ctx.sources.get(resolved.sourceName);
+        if (!client) {
+            throw new Error(`Source not found: ${resolved.sourceName}`);
+        }
+        // Resolve "since" query parameter for incremental sync
+        const { query: sinceQuery, checkpointKey } = await this.resolveSinceParams(step, resolved.sourceName, resolved.operationId, resolved.path);
+        if (this.deps.dryRun) {
+            this.deps.log('(dry run - skipping actual request)');
+            return { data: { dryRun: true }, checkpointKey };
+        }
+        // Execute with or without pagination
+        let data;
+        if (step.paginate) {
+            data = await this.executePaginated(step, client, resolved.path, resolved.method, resolved.sourceName, resolved.operationId, sinceQuery);
+        }
+        else {
+            const response = await client.request({
+                method: resolved.method,
+                path: resolved.path,
+                query: Object.keys(sinceQuery).length > 0 ? sinceQuery : undefined,
+                body: step.body ? evaluate(step.body, this.deps.ctx) : undefined,
+            }, step.retry);
+            await this.validateOASResponse(resolved.sourceName, resolved.operationId, response.data);
+            data = response.data;
+        }
+        return { data, checkpointKey };
+    }
+    /**
+     * Record a sync checkpoint after successful fetch.
+     */
+    async recordCheckpoint(key, step, data) {
+        if (!this.deps.syncStore)
+            return;
+        let syncedAt = new Date();
+        // If updateFrom is specified, extract the timestamp from response
+        if (step.since?.updateFrom && data && typeof data === 'object') {
+            const extracted = extractNestedValue(data, step.since.updateFrom);
+            if (extracted instanceof Date) {
+                syncedAt = extracted;
+            }
+            else if (typeof extracted === 'string' || typeof extracted === 'number') {
+                const parsed = new Date(extracted);
+                if (!isNaN(parsed.getTime())) {
+                    syncedAt = parsed;
+                }
+            }
+        }
+        // Count records if response is an array
+        const recordCount = this.countRecords(data);
+        await this.deps.syncStore.recordSync({
+            key,
+            syncedAt,
+            recordCount,
+            mission: this.deps.missionName,
+            executionId: this.deps.executionId,
+        });
+        this.deps.log(`Recorded sync checkpoint: ${key} at ${syncedAt.toISOString()}`);
+    }
+    resolveFetchTarget(step) {
+        if (step.operationRef) {
+            // OAS-style: resolve from operationId
+            const sourceName = step.operationRef.source;
+            const operationId = step.operationRef.operationId;
+            const oasSource = this.deps.oasSources.get(sourceName);
+            if (!oasSource) {
+                throw new Error(`Source '${sourceName}' does not have an OAS spec. Use 'source ${sourceName} from "spec.yaml"'`);
+            }
+            const operation = resolveOperation(oasSource, operationId);
+            const path = interpolatePath(operation.path, this.deps.ctx);
+            this.deps.log(`Fetching: ${sourceName}.${operationId} -> ${operation.method} ${path}`);
+            return { sourceName, method: operation.method, path, operationId };
+        }
+        // Traditional: explicit method + path
+        const sourceName = step.source ?? this.deps.ctx.sources.keys().next().value;
+        const method = step.method;
+        let path;
+        if (step.path.type === 'Literal' && step.path.dataType === 'string') {
+            path = interpolatePath(step.path.value, this.deps.ctx);
+        }
+        else {
+            path = String(evaluate(step.path, this.deps.ctx));
+        }
+        this.deps.log(`Fetching: ${method} ${path}`);
+        return { sourceName, method, path };
+    }
+    async resolveSinceParams(step, sourceName, operationId, path) {
+        const query = {};
+        if (!step.since || !this.deps.syncStore) {
+            return { query };
+        }
+        const checkpointKey = step.since.key ?? generateCheckpointKey(sourceName, operationId, path);
+        if (step.since.type === 'lastSync') {
+            const lastSync = await this.deps.syncStore.getLastSync(checkpointKey);
+            const paramName = step.since.param ?? 'since';
+            const format = step.since.format ?? 'iso';
+            query[paramName] = formatSinceDate(lastSync, format);
+            this.deps.log(`Incremental sync: ${paramName}=${query[paramName]} (key: ${checkpointKey})`);
+        }
+        else if (step.since.type === 'expression' && step.since.expression) {
+            const value = evaluate(step.since.expression, this.deps.ctx);
+            const paramName = step.since.param ?? 'since';
+            query[paramName] = String(value);
+        }
+        return { query, checkpointKey };
+    }
+    async executePaginated(step, client, basePath, method, sourceName, operationId, sinceQuery = {}) {
+        const allResults = [];
+        const paginate = step.paginate;
+        const strategy = createPaginationStrategy(paginate);
+        const ctx = {
+            page: 0,
+            pageSize: paginate.pageSize,
+        };
+        let hasMore = true;
+        while (hasMore) {
+            // Build query with pagination params
+            const paginationQuery = strategy.buildQuery(ctx);
+            const query = { ...sinceQuery, ...paginationQuery };
+            this.deps.log(`Fetching page ${ctx.page + 1}...`);
+            const response = await client.request({ method, path: basePath, query }, step.retry);
+            await this.validateOASResponse(sourceName, operationId, response.data);
+            // Temporarily set response for until condition evaluation
+            this.deps.ctx.response = response.data;
+            // Check until condition
+            if (step.until) {
+                const shouldStop = evaluate(step.until, this.deps.ctx);
+                if (shouldStop) {
+                    break;
+                }
+            }
+            // Extract results using strategy
+            const pageResult = strategy.extractResults(response.data, ctx);
+            allResults.push(...pageResult.items);
+            hasMore = pageResult.hasMore;
+            // Update cursor for next iteration
+            if (pageResult.nextCursor) {
+                ctx.cursor = pageResult.nextCursor;
+            }
+            ctx.page++;
+            // Safety limit
+            if (ctx.page >= MAX_PAGINATION_PAGES) {
+                this.deps.log(`Warning: pagination limit (${MAX_PAGINATION_PAGES}) reached`);
+                hasMore = false;
+            }
+        }
+        this.deps.log(`Fetched ${allResults.length} total items`);
+        return allResults;
+    }
+    countRecords(data) {
+        if (Array.isArray(data)) {
+            return data.length;
+        }
+        if (data && typeof data === 'object') {
+            for (const val of Object.values(data)) {
+                if (Array.isArray(val)) {
+                    return val.length;
+                }
+            }
+        }
+        return undefined;
+    }
+    async validateOASResponse(sourceName, operationId, data) {
+        if (!operationId)
+            return;
+        const sourceConfig = this.deps.sourceConfigs.get(sourceName);
+        if (!sourceConfig?.config.validateResponses)
+            return;
+        const oasSource = this.deps.oasSources.get(sourceName);
+        if (!oasSource)
+            return;
+        const schema = getResponseSchema(oasSource, operationId);
+        if (!schema) {
+            this.deps.log(`No response schema found for ${operationId}`);
+            return;
+        }
+        const result = validateResponse(data, schema);
+        if (!result.valid) {
+            const errorMessages = result.errors.map((e) => `  ${e.path}: ${e.message}`).join('\n');
+            this.deps.log(`Response validation warnings for ${operationId}:\n${errorMessages}`);
+        }
+    }
+}

package/dist/interpreter/http.d.ts

@@ -0,0 +1,57 @@
+import type { RetryConfig } from '../ast/nodes.js';
+import type { RateLimiter } from '../auth/types.js';
+import { CircuitBreaker } from '../auth/circuit-breaker.js';
+export interface HttpClientConfig {
+    baseUrl: string;
+    headers?: Record<string, string>;
+    auth?: AuthProvider;
+    rateLimiter?: RateLimiter;
+    circuitBreaker?: CircuitBreaker;
+    /** Source name for rate limit and circuit breaker tracking */
+    sourceName?: string;
+}
+export interface AuthProvider {
+    getToken(): Promise<string>;
+    refreshToken?(): Promise<string>;
+}
+export interface HttpRequest {
+    method: 'GET' | 'POST' | 'PUT' | 'PATCH' | 'DELETE';
+    path: string;
+    body?: unknown;
+    headers?: Record<string, string>;
+    query?: Record<string, string>;
+}
+export interface HttpResponse<T = unknown> {
+    status: number;
+    data: T;
+    headers: Record<string, string>;
+}
+export declare class HttpClient {
+    private config;
+    constructor(config: HttpClientConfig);
+    request<T = unknown>(req: HttpRequest, retry?: RetryConfig): Promise<HttpResponse<T>>;
+    private buildUrl;
+    private buildHeaders;
+    private calculateDelay;
+}
+export declare class BearerAuthProvider implements AuthProvider {
+    private token;
+    constructor(token: string);
+    getToken(): Promise<string>;
+}
+export declare class OAuth2AuthProvider implements AuthProvider {
+    private accessToken;
+    private refreshTokenValue?;
+    private tokenEndpoint?;
+    private clientId?;
+    private clientSecret?;
+    constructor(config: {
+        accessToken: string;
+        refreshToken?: string;
+        tokenEndpoint?: string;
+        clientId?: string;
+        clientSecret?: string;
+    });
+    getToken(): Promise<string>;
+    refreshToken(): Promise<string>;
+}

package/dist/interpreter/http.js

@@ -0,0 +1,210 @@
+import { parseRateLimitHeaders } from '../auth/rate-limiter.js';
+import { CircuitBreakerError } from '../auth/circuit-breaker.js';
+import { sleep } from '../utils/async.js';
+/** Default retry configuration values */
+const DEFAULT_RETRY = {
+    MAX_ATTEMPTS: 3,
+    INITIAL_DELAY_MS: 1000,
+    MAX_DELAY_MS: 30000,
+    BACKOFF: 'exponential',
+};
+export class HttpClient {
+    config;
+    constructor(config) {
+        this.config = config;
+    }
+    async request(req, retry) {
+        const url = this.buildUrl(req.path, req.query);
+        const headers = await this.buildHeaders(req.headers);
+        const fetchOptions = {
+            method: req.method,
+            headers,
+            body: req.body ? JSON.stringify(req.body) : undefined,
+        };
+        const maxAttempts = retry?.maxAttempts ?? DEFAULT_RETRY.MAX_ATTEMPTS;
+        const backoff = retry?.backoff ?? DEFAULT_RETRY.BACKOFF;
+        const initialDelay = retry?.initialDelay ?? DEFAULT_RETRY.INITIAL_DELAY_MS;
+        const maxDelay = retry?.maxDelay ?? DEFAULT_RETRY.MAX_DELAY_MS;
+        let lastError = null;
+        // Check circuit breaker before attempting requests
+        if (this.config.circuitBreaker && this.config.sourceName) {
+            // This will throw CircuitBreakerError if circuit is open
+            this.config.circuitBreaker.ensureCanProceed(this.config.sourceName, req.path);
+        }
+        for (let attempt = 1; attempt <= maxAttempts; attempt++) {
+            try {
+                // Re-check circuit breaker on retries (state may have changed)
+                if (attempt > 1 && this.config.circuitBreaker && this.config.sourceName) {
+                    if (!this.config.circuitBreaker.canProceed(this.config.sourceName, req.path)) {
+                        // Circuit opened during retries, fail fast
+                        throw new CircuitBreakerError(this.config.sourceName, req.path, this.config.circuitBreaker.getStatus(this.config.sourceName, req.path).nextAttemptTime?.getTime() ?? 0 - Date.now());
+                    }
+                }
+                // Wait for rate limit capacity if we have a rate limiter
+                if (this.config.rateLimiter && this.config.sourceName) {
+                    await this.config.rateLimiter.waitForCapacity(this.config.sourceName, req.path);
+                }
+                const response = await fetch(url, fetchOptions);
+                // Extract and record rate limit info from response headers
+                const responseHeaders = {};
+                response.headers.forEach((value, key) => {
+                    responseHeaders[key] = value;
+                });
+                if (this.config.rateLimiter && this.config.sourceName) {
+                    const rateLimitInfo = parseRateLimitHeaders(responseHeaders);
+                    // Add retry-after from 429 responses
+                    if (response.status === 429) {
+                        const retryAfter = response.headers.get('Retry-After');
+                        if (retryAfter) {
+                            rateLimitInfo.retryAfter = parseInt(retryAfter, 10);
+                        }
+                    }
+                    this.config.rateLimiter.recordResponse(this.config.sourceName, rateLimitInfo, req.path);
+                }
+                // Handle rate limiting
+                if (response.status === 429) {
+                    const retryAfter = response.headers.get('Retry-After');
+                    const delay = retryAfter ? parseInt(retryAfter, 10) * 1000 : this.calculateDelay(attempt, backoff, initialDelay, maxDelay);
+                    await sleep(delay);
+                    continue;
+                }
+                // Handle server errors with retry
+                if (response.status >= 500) {
+                    // Record failure in circuit breaker
+                    if (this.config.circuitBreaker && this.config.sourceName) {
+                        this.config.circuitBreaker.recordFailure(this.config.sourceName, req.path, response.status);
+                    }
+                    if (attempt < maxAttempts) {
+                        const delay = this.calculateDelay(attempt, backoff, initialDelay, maxDelay);
+                        await sleep(delay);
+                        continue;
+                    }
+                }
+                // Handle 401 - try token refresh
+                if (response.status === 401 && this.config.auth?.refreshToken && attempt < maxAttempts) {
+                    await this.config.auth.refreshToken();
+                    // Rebuild headers with new token
+                    const newHeaders = await this.buildHeaders(req.headers);
+                    fetchOptions.headers = newHeaders;
+                    continue;
+                }
+                const data = await response.json();
+                // Record success in circuit breaker
+                if (this.config.circuitBreaker && this.config.sourceName && response.status < 500) {
+                    this.config.circuitBreaker.recordSuccess(this.config.sourceName, req.path);
+                }
+                return {
+                    status: response.status,
+                    data,
+                    headers: responseHeaders,
+                };
+            }
+            catch (error) {
+                lastError = error;
+                // Re-throw circuit breaker errors immediately
+                if (error instanceof CircuitBreakerError) {
+                    throw error;
+                }
+                // Record network errors in circuit breaker
+                if (this.config.circuitBreaker && this.config.sourceName) {
+                    this.config.circuitBreaker.recordFailure(this.config.sourceName, req.path, undefined, true);
+                }
+                if (attempt < maxAttempts) {
+                    const delay = this.calculateDelay(attempt, backoff, initialDelay, maxDelay);
+                    await sleep(delay);
+                }
+            }
+        }
+        throw lastError ?? new Error('Request failed after all retries');
+    }
+    buildUrl(path, query) {
+        const base = this.config.baseUrl.replace(/\/$/, '');
+        const cleanPath = path.startsWith('/') ? path : `/${path}`;
+        let url = `${base}${cleanPath}`;
+        if (query && Object.keys(query).length > 0) {
+            const params = new URLSearchParams(query);
+            url += `?${params.toString()}`;
+        }
+        return url;
+    }
+    async buildHeaders(requestHeaders) {
+        const headers = {
+            'Content-Type': 'application/json',
+            Accept: 'application/json',
+            ...this.config.headers,
+            ...requestHeaders,
+        };
+        if (this.config.auth) {
+            const token = await this.config.auth.getToken();
+            headers['Authorization'] = `Bearer ${token}`;
+        }
+        return headers;
+    }
+    calculateDelay(attempt, backoff, initialDelay, maxDelay) {
+        let delay;
+        switch (backoff) {
+            case 'exponential':
+                delay = initialDelay * Math.pow(2, attempt - 1);
+                break;
+            case 'linear':
+                delay = initialDelay * attempt;
+                break;
+            case 'constant':
+            default:
+                delay = initialDelay;
+        }
+        // Add jitter (±10%)
+        const jitter = delay * 0.1 * (Math.random() * 2 - 1);
+        delay = Math.min(delay + jitter, maxDelay);
+        return Math.round(delay);
+    }
+}
+// Simple token-based auth provider
+export class BearerAuthProvider {
+    token;
+    constructor(token) {
+        this.token = token;
+    }
+    async getToken() {
+        return this.token;
+    }
+}
+// OAuth2 auth provider (simplified)
+export class OAuth2AuthProvider {
+    accessToken;
+    refreshTokenValue;
+    tokenEndpoint;
+    clientId;
+    clientSecret;
+    constructor(config) {
+        this.accessToken = config.accessToken;
+        this.refreshTokenValue = config.refreshToken;
+        this.tokenEndpoint = config.tokenEndpoint;
+        this.clientId = config.clientId;
+        this.clientSecret = config.clientSecret;
+    }
+    async getToken() {
+        return this.accessToken;
+    }
+    async refreshToken() {
+        if (!this.refreshTokenValue || !this.tokenEndpoint) {
+            throw new Error('Cannot refresh token: missing refresh token or endpoint');
+        }
+        const response = await fetch(this.tokenEndpoint, {
+            method: 'POST',
+            headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
+            body: new URLSearchParams({
+                grant_type: 'refresh_token',
+                refresh_token: this.refreshTokenValue,
+                client_id: this.clientId ?? '',
+                client_secret: this.clientSecret ?? '',
+            }),
+        });
+        const data = await response.json();
+        this.accessToken = data.access_token;
+        if (data.refresh_token) {
+            this.refreshTokenValue = data.refresh_token;
+        }
+        return this.accessToken;
+    }
+}

package/dist/interpreter/http.test.d.ts

@@ -0,0 +1 @@
+export {};