reqon-dsl 0.2.0 → 0.4.0

package/dist/interpreter/fetch-handler.d.ts

@@ -2,6 +2,7 @@ import type { FetchStep } from '../ast/nodes.js';
 import type { ExecutionContext } from './context.js';
 import type { OASSource } from '../oas/index.js';
 import { type SyncStore } from '../sync/index.js';
+import type { EventType } from '../observability/index.js';
 export interface FetchHandlerDeps {
     ctx: ExecutionContext;
     oasSources: Map<string, OASSource>;
@@ -15,6 +16,37 @@ export interface FetchHandlerDeps {
     executionId?: string;
     dryRun?: boolean;
     log: (message: string) => void;
+    /** Optional event emitter for observability */
+    emit?: <T>(type: EventType, payload: T) => void;
+    /**
+     * When set (durable mode), mutating requests carry a stable Idempotency-Key
+     * derived from (executionId, stepId, request signature) so retries and
+     * replays don't double-apply server-side where the API honours the header.
+     */
+    idempotency?: {
+        executionId: string;
+        stepId: string;
+    };
+    /**
+     * Resumable (backfill) pagination, wired in durable mode. `resume` seeds the
+     * starting page/cursor from the folded log; `onPage` records each completed
+     * page back to the log; `maxItemsPerRun` bounds memory per run (a backfill
+     * that exceeds it stops cleanly and continues on the next resume).
+     */
+    pagination?: {
+        resume?: {
+            page: number;
+            cursor?: string;
+            done: boolean;
+        };
+        onPage?: (progress: {
+            page: number;
+            cursor?: string;
+            recordCount: number;
+            done: boolean;
+        }) => Promise<void>;
+        maxItemsPerRun?: number;
+    };
 }
 export interface FetchResult {
     data: unknown;
@@ -31,13 +63,28 @@ export declare class FetchHandler {
      * Execute a fetch step, handling OAS resolution, pagination, and sync checkpoints.
      */
     execute(step: FetchStep): Promise<FetchResult>;
+    private isRetryableError;
     /**
      * Record a sync checkpoint after successful fetch.
      */
-    recordCheckpoint(key: string, step: FetchStep, data: unknown): Promise<void>;
+    recordCheckpoint(key: string, step: FetchStep, data: unknown, fallbackTime?: Date): Promise<Date | undefined>;
     private resolveFetchTarget;
     private resolveSinceParams;
+    /**
+     * Idempotency-Key header for a mutating request, when durable mode is on.
+     *
+     * GET is safe and gets no key. The key is a hash of
+     * (executionId, stepId, method, path, body) so it is stable across retries
+     * and replays of the same request, yet distinct per loop iteration (different
+     * path/body) — letting a cooperating API dedupe re-issued effects.
+     */
+    private idempotencyKeyFor;
     private executePaginated;
     private countRecords;
     private validateOASResponse;
+    /**
+     * Generate mock data for dry run mode.
+     * Uses OAS response schema if available, otherwise returns a simple placeholder.
+     */
+    private generateDryRunMockData;
 }

package/dist/interpreter/fetch-handler.js

@@ -1,10 +1,17 @@
+import { createHash } from 'node:crypto';
 import { evaluate, interpolatePath } from './evaluator.js';
-import { resolveOperation, getResponseSchema, validateResponse } from '../oas/index.js';
+import { resolveOperation, getResponseSchema, validateResponse, generateMockData, } from '../oas/index.js';
 import { generateCheckpointKey, formatSinceDate } from '../sync/index.js';
 import { extractNestedValue } from '../utils/path.js';
 import { createPaginationStrategy } from './pagination.js';
 /** Maximum pages to fetch to prevent infinite loops */
 const MAX_PAGINATION_PAGES = 100;
+/**
+ * Maximum items to accumulate in memory across all pages. Results are buffered
+ * in one array (no per-page streaming yet), so this caps memory regardless of
+ * page size. Hitting it stops pagination with a warning.
+ */
+const MAX_PAGINATION_ITEMS = 100_000;
 /**
  * Handles HTTP fetch operations including pagination and incremental sync.
  * Extracted from MissionExecutor for better separation of concerns.
@@ -23,36 +30,92 @@ export class FetchHandler {
         if (!client) {
             throw new Error(`Source not found: ${resolved.sourceName}`);
         }
-        // Resolve "since" query parameter for incremental sync
-        const { query: sinceQuery, checkpointKey } = await this.resolveSinceParams(step, resolved.sourceName, resolved.operationId, resolved.path);
+        // Resolve "since" query parameter and/or header for incremental sync
+        const { query: sinceQuery, headers: sinceHeaders, checkpointKey, } = await this.resolveSinceParams(step, resolved.sourceName, resolved.operationId, resolved.path);
+        // Emit fetch.start event
+        this.deps.emit?.('fetch.start', {
+            source: resolved.sourceName,
+            method: resolved.method,
+            path: resolved.path,
+            isOAS: !!resolved.operationId,
+            operationId: resolved.operationId,
+            hasPagination: !!step.paginate,
+            hasSince: !!step.since,
+        });
+        const _fetchStartTime = Date.now(); // Reserved for fetch duration tracking
         if (this.deps.dryRun) {
+            const mockData = this.generateDryRunMockData(resolved.sourceName, resolved.operationId);
             this.deps.log('(dry run - skipping actual request)');
-            return { data: { dryRun: true }, checkpointKey };
+            return { data: mockData, checkpointKey };
         }
         // Execute with or without pagination
         let data;
-        if (step.paginate) {
-            data = await this.executePaginated(step, client, resolved.path, resolved.method, resolved.sourceName, resolved.operationId, sinceQuery);
-        }
-        else {
-            const response = await client.request({
+        let pagesFetched;
+        let statusCode = 200;
+        try {
+            if (step.paginate) {
+                const result = await this.executePaginated(step, client, resolved.path, resolved.method, resolved.sourceName, resolved.operationId, sinceQuery, sinceHeaders);
+                data = result;
+                pagesFetched = Array.isArray(result) ? undefined : 1; // Will be set by executePaginated
+            }
+            else {
+                const body = step.body ? evaluate(step.body, this.deps.ctx) : undefined;
+                const response = await client.request({
+                    method: resolved.method,
+                    path: resolved.path,
+                    query: Object.keys(sinceQuery).length > 0 ? sinceQuery : undefined,
+                    headers: Object.keys(sinceHeaders).length > 0 ? sinceHeaders : undefined,
+                    body,
+                    idempotencyKey: this.idempotencyKeyFor(resolved.method, resolved.path, body),
+                }, step.retry);
+                await this.validateOASResponse(resolved.sourceName, resolved.operationId, response.data);
+                data = response.data;
+                statusCode = response.status ?? 200;
+            }
+            // Emit fetch.complete event
+            this.deps.emit?.('fetch.complete', {
+                source: resolved.sourceName,
                 method: resolved.method,
                 path: resolved.path,
-                query: Object.keys(sinceQuery).length > 0 ? sinceQuery : undefined,
-                body: step.body ? evaluate(step.body, this.deps.ctx) : undefined,
-            }, step.retry);
-            await this.validateOASResponse(resolved.sourceName, resolved.operationId, response.data);
-            data = response.data;
+                statusCode,
+                recordCount: this.countRecords(data) ?? 0,
+                pagesFetched,
+            });
+        }
+        catch (error) {
+            // Emit fetch.error event
+            this.deps.emit?.('fetch.error', {
+                source: resolved.sourceName,
+                path: resolved.path,
+                error: error.message,
+                retryable: this.isRetryableError(error),
+            });
+            throw error;
         }
         return { data, checkpointKey };
     }
+    isRetryableError(error) {
+        if (error instanceof Error) {
+            const message = error.message.toLowerCase();
+            return (message.includes('timeout') ||
+                message.includes('network') ||
+                message.includes('rate limit') ||
+                message.includes('429') ||
+                message.includes('503') ||
+                message.includes('502'));
+        }
+        return false;
+    }
     /**
      * Record a sync checkpoint after successful fetch.
      */
-    async recordCheckpoint(key, step, data) {
+    async recordCheckpoint(key, step, data, fallbackTime) {
         if (!this.deps.syncStore)
-            return;
-        let syncedAt = new Date();
+            return undefined;
+        // Prefer the data's own clock; otherwise fall back to when the request was
+        // issued (not now), so records written server-side during the fetch are
+        // re-fetched next run rather than skipped on clock skew.
+        let syncedAt = fallbackTime ?? new Date();
         // If updateFrom is specified, extract the timestamp from response
         if (step.since?.updateFrom && data && typeof data === 'object') {
             const extracted = extractNestedValue(data, step.since.updateFrom);
@@ -76,6 +139,14 @@ export class FetchHandler {
             executionId: this.deps.executionId,
         });
         this.deps.log(`Recorded sync checkpoint: ${key} at ${syncedAt.toISOString()}`);
+        // Emit sync.checkpoint event
+        this.deps.emit?.('sync.checkpoint', {
+            checkpointKey: key,
+            lastSyncTime: syncedAt.toISOString(),
+            recordsFetched: recordCount ?? 0,
+            isIncremental: true,
+        });
+        return syncedAt;
     }
     resolveFetchTarget(step) {
         if (step.operationRef) {
@@ -92,7 +163,15 @@ export class FetchHandler {
             return { sourceName, method: operation.method, path, operationId };
         }
         // Traditional: explicit method + path
-        const sourceName = step.source ?? this.deps.ctx.sources.keys().next().value;
+        let sourceName = step.source;
+        if (!sourceName) {
+            // Use the first available source as default
+            const firstSource = this.deps.ctx.sources.keys().next();
+            if (firstSource.done) {
+                throw new Error('No sources defined. Add a source to your mission before making fetch requests.');
+            }
+            sourceName = firstSource.value;
+        }
         const method = step.method;
         let path;
         if (step.path.type === 'Literal' && step.path.dataType === 'string') {
@@ -106,25 +185,55 @@ export class FetchHandler {
     }
     async resolveSinceParams(step, sourceName, operationId, path) {
         const query = {};
+        const headers = {};
         if (!step.since || !this.deps.syncStore) {
-            return { query };
+            return { query, headers };
         }
         const checkpointKey = step.since.key ?? generateCheckpointKey(sourceName, operationId, path);
         if (step.since.type === 'lastSync') {
             const lastSync = await this.deps.syncStore.getLastSync(checkpointKey);
-            const paramName = step.since.param ?? 'since';
             const format = step.since.format ?? 'iso';
-            query[paramName] = formatSinceDate(lastSync, format);
-            this.deps.log(`Incremental sync: ${paramName}=${query[paramName]} (key: ${checkpointKey})`);
+            const formattedDate = formatSinceDate(lastSync, format);
+            if (step.since.header) {
+                // Send as header (e.g., If-Modified-Since)
+                headers[step.since.header] = formattedDate;
+                this.deps.log(`Incremental sync: header ${step.since.header}=${formattedDate} (key: ${checkpointKey})`);
+            }
+            else {
+                // Send as query parameter (default behavior)
+                const paramName = step.since.param ?? 'since';
+                query[paramName] = formattedDate;
+                this.deps.log(`Incremental sync: ${paramName}=${formattedDate} (key: ${checkpointKey})`);
+            }
         }
         else if (step.since.type === 'expression' && step.since.expression) {
             const value = evaluate(step.since.expression, this.deps.ctx);
-            const paramName = step.since.param ?? 'since';
-            query[paramName] = String(value);
+            if (step.since.header) {
+                headers[step.since.header] = String(value);
+            }
+            else {
+                const paramName = step.since.param ?? 'since';
+                query[paramName] = String(value);
+            }
         }
-        return { query, checkpointKey };
+        return { query, headers, checkpointKey };
+    }
+    /**
+     * Idempotency-Key header for a mutating request, when durable mode is on.
+     *
+     * GET is safe and gets no key. The key is a hash of
+     * (executionId, stepId, method, path, body) so it is stable across retries
+     * and replays of the same request, yet distinct per loop iteration (different
+     * path/body) — letting a cooperating API dedupe re-issued effects.
+     */
+    idempotencyKeyFor(method, path, body) {
+        if (!this.deps.idempotency || method === 'GET')
+            return undefined;
+        const { executionId, stepId } = this.deps.idempotency;
+        const signature = `${executionId}${stepId}${method}${path}${JSON.stringify(body ?? null)}`;
+        return createHash('sha256').update(signature).digest('hex').slice(0, 32);
     }
-    async executePaginated(step, client, basePath, method, sourceName, operationId, sinceQuery = {}) {
+    async executePaginated(step, client, basePath, method, sourceName, operationId, sinceQuery = {}, sinceHeaders = {}) {
         const allResults = [];
         const paginate = step.paginate;
         const strategy = createPaginationStrategy(paginate);
@@ -132,32 +241,82 @@ export class FetchHandler {
             page: 0,
             pageSize: paginate.pageSize,
         };
+        // Resumable (backfill) pagination: seed the cursor/page from the last
+        // persisted position so a restart continues from where it stopped.
+        const backfill = this.deps.pagination;
+        if (step.backfill && backfill?.resume) {
+            if (backfill.resume.done) {
+                this.deps.log('Backfill already complete; nothing to fetch.');
+                return [];
+            }
+            ctx.page = backfill.resume.page;
+            ctx.cursor = backfill.resume.cursor;
+            this.deps.log(`Resuming backfill from page ${ctx.page + 1}`);
+        }
+        // Per-run item cap. For a backfill this bounds memory per run: when hit, the
+        // run stops *incomplete* and a later resume continues — so an arbitrarily
+        // large backfill completes across runs without buffering it all at once.
+        const itemCap = backfill?.maxItemsPerRun ?? MAX_PAGINATION_ITEMS;
         let hasMore = true;
+        let stoppedByCap = false;
         while (hasMore) {
             // Build query with pagination params
             const paginationQuery = strategy.buildQuery(ctx);
             const query = { ...sinceQuery, ...paginationQuery };
+            const headers = Object.keys(sinceHeaders).length > 0 ? sinceHeaders : undefined;
             this.deps.log(`Fetching page ${ctx.page + 1}...`);
-            const response = await client.request({ method, path: basePath, query }, step.retry);
+            const response = await client.request({ method, path: basePath, query, headers }, step.retry);
             await this.validateOASResponse(sourceName, operationId, response.data);
             // Temporarily set response for until condition evaluation
             this.deps.ctx.response = response.data;
-            // Check until condition
-            if (step.until) {
-                const shouldStop = evaluate(step.until, this.deps.ctx);
-                if (shouldStop) {
-                    break;
-                }
-            }
-            // Extract results using strategy
+            // Extract and append the current page BEFORE deciding whether to stop, so
+            // the page that satisfies `until` is not silently dropped.
             const pageResult = strategy.extractResults(response.data, ctx);
             allResults.push(...pageResult.items);
             hasMore = pageResult.hasMore;
-            // Update cursor for next iteration
-            if (pageResult.nextCursor) {
-                ctx.cursor = pageResult.nextCursor;
+            // Advance the cursor. If the API echoes the cursor we just used, stop
+            // instead of looping to the page cap and duplicating items each round.
+            if (pageResult.nextCursor !== undefined) {
+                if (pageResult.nextCursor === ctx.cursor) {
+                    this.deps.log(`Pagination cursor did not advance ('${pageResult.nextCursor}'); stopping.`);
+                    hasMore = false;
+                }
+                else {
+                    ctx.cursor = pageResult.nextCursor;
+                }
             }
             ctx.page++;
+            // `until` is checked after the current page has been appended.
+            if (hasMore && step.until && evaluate(step.until, this.deps.ctx)) {
+                hasMore = false;
+            }
+            // Memory safety: cap total accumulated items, not just page count. For a
+            // backfill this is a clean stop-and-resume boundary, not an end of data.
+            if (allResults.length >= itemCap) {
+                this.deps.log(`Pagination item limit (${itemCap}) reached`);
+                if (hasMore)
+                    stoppedByCap = true;
+                hasMore = false;
+            }
+            // Record this page's position for a resumable backfill. `done` is the
+            // natural end of pagination — never set when we stopped on the item cap,
+            // so a resume knows to continue rather than treat the backfill as finished.
+            if (step.backfill && backfill?.onPage) {
+                await backfill.onPage({
+                    page: ctx.page,
+                    cursor: ctx.cursor,
+                    recordCount: pageResult.items.length,
+                    done: !hasMore && !stoppedByCap,
+                });
+            }
+            // Emit heartbeat after each page
+            this.deps.emit?.('fetch.heartbeat', {
+                source: sourceName,
+                path: basePath,
+                pagesProcessed: ctx.page,
+                itemsFetched: allResults.length,
+                hasMore,
+            });
             // Safety limit
             if (ctx.page >= MAX_PAGINATION_PAGES) {
                 this.deps.log(`Warning: pagination limit (${MAX_PAGINATION_PAGES}) reached`);
@@ -200,4 +359,23 @@ export class FetchHandler {
             this.deps.log(`Response validation warnings for ${operationId}:\n${errorMessages}`);
         }
     }
+    /**
+     * Generate mock data for dry run mode.
+     * Uses OAS response schema if available, otherwise returns a simple placeholder.
+     */
+    generateDryRunMockData(sourceName, operationId) {
+        // Try to get schema from OAS source
+        if (operationId) {
+            const oasSource = this.deps.oasSources.get(sourceName);
+            if (oasSource) {
+                const schema = getResponseSchema(oasSource, operationId);
+                if (schema) {
+                    this.deps.log(`Generating mock data from OAS schema for ${operationId}`);
+                    return generateMockData(schema);
+                }
+            }
+        }
+        // Fallback to simple placeholder
+        return { _dryRun: true, _message: 'No OAS schema available for mock generation' };
+    }
 }

package/dist/interpreter/http.d.ts

@@ -1,6 +1,14 @@
 import type { RetryConfig } from '../ast/nodes.js';
 import type { RateLimiter } from '../auth/types.js';
 import { CircuitBreaker } from '../auth/circuit-breaker.js';
+/**
+ * Parse a `Retry-After` header into a delay in ms, clamped to `maxDelayMs`.
+ * The header may be delta-seconds (`120`) or an HTTP-date; a date in the past or
+ * an unparseable value yields 0 and `undefined` respectively. Clamping stops a
+ * hostile/broken server from pinning the client for hours, and the date branch
+ * stops `parseInt` from turning a date into `NaN` → `sleep(NaN)` → a tight loop.
+ */
+export declare function parseRetryAfterMs(value: string | null | undefined, maxDelayMs: number): number | undefined;
 export interface HttpClientConfig {
     baseUrl: string;
     headers?: Record<string, string>;
@@ -9,6 +17,8 @@ export interface HttpClientConfig {
     circuitBreaker?: CircuitBreaker;
     /** Source name for rate limit and circuit breaker tracking */
     sourceName?: string;
+    /** Default per-request timeout in ms (overridden by RetryConfig.timeout) */
+    timeout?: number;
 }
 export interface AuthProvider {
     getToken(): Promise<string>;
@@ -20,6 +30,12 @@ export interface HttpRequest {
     body?: unknown;
     headers?: Record<string, string>;
     query?: Record<string, string>;
+    /**
+     * Opt a non-idempotent request (POST/PATCH) into automatic retries by
+     * supplying an idempotency key. Sent as the `Idempotency-Key` header so the
+     * server can dedup a re-sent write.
+     */
+    idempotencyKey?: string;
 }
 export interface HttpResponse<T = unknown> {
     status: number;
@@ -30,6 +46,21 @@ export declare class HttpClient {
     private config;
     constructor(config: HttpClientConfig);
     request<T = unknown>(req: HttpRequest, retry?: RetryConfig): Promise<HttpResponse<T>>;
+    /**
+     * Run a fetch with a per-attempt timeout. Aborts the request (freeing the
+     * connection and rate-limiter slot) if it exceeds `timeoutMs`, surfacing a
+     * retryable FetchError rather than hanging forever.
+     */
+    private fetchWithTimeout;
+    /**
+     * Parse a successful (2xx) response body. Handles empty/204 responses,
+     * returns non-JSON content as raw text, and caps the buffered size.
+     */
+    private parseResponseBody;
+    /** Read a response body to text, rejecting once it exceeds MAX_RESPONSE_BYTES. */
+    private readCappedText;
+    /** Read a short snippet of a body for an error message (best-effort). */
+    private safeReadSnippet;
     private buildUrl;
     private buildHeaders;
     private calculateDelay;
@@ -45,6 +76,8 @@ export declare class OAuth2AuthProvider implements AuthProvider {
     private tokenEndpoint?;
     private clientId?;
     private clientSecret?;
+    /** Single-flight guard: coalesces concurrent refreshes into one in-flight request */
+    private refreshPromise;
     constructor(config: {
         accessToken: string;
         refreshToken?: string;
@@ -54,4 +87,5 @@ export declare class OAuth2AuthProvider implements AuthProvider {
     });
     getToken(): Promise<string>;
     refreshToken(): Promise<string>;
+    private doRefresh;
 }