reqon-dsl 0.2.0 → 0.4.0

package/dist/oas/mock-generator.js

@@ -0,0 +1,196 @@
+/**
+ * Generate mock data from an OpenAPI schema.
+ * Used for dry-run mode to produce realistic mock responses.
+ */
+export function generateMockData(schema, options = {}) {
+    const ctx = {
+        depth: 0,
+        maxDepth: options.maxDepth ?? 3,
+        arrayLength: options.arrayLength ?? 2,
+        seenRefs: new Set(),
+        counter: { value: 0 },
+    };
+    return generateValue(schema, ctx);
+}
+function generateValue(schema, ctx) {
+    // Depth guard: after dereferencing, a self-referential schema becomes a
+    // circular object reference (not a $ref), so the only thing that stops
+    // infinite recursion is the depth cap. Enforce it here for every path,
+    // including allOf/oneOf/anyOf below.
+    if (ctx.depth > ctx.maxDepth) {
+        return null;
+    }
+    // Handle nullable - only return null if explicitly marked nullable
+    // and we're past the top level (depth > 0). Use strict equality check
+    // to avoid false positives from undefined/truthy coercion.
+    if (schema.nullable === true && ctx.depth > 1) {
+        // Return null ~20% of the time for nullable fields deep in the tree
+        if (Math.random() < 0.2) {
+            return null;
+        }
+    }
+    // Handle enum - pick first value
+    if (schema.enum && schema.enum.length > 0) {
+        return schema.enum[0];
+    }
+    // Handle example if provided
+    if (schema.example !== undefined) {
+        return schema.example;
+    }
+    // Handle default if provided
+    if (schema.default !== undefined) {
+        return schema.default;
+    }
+    // Check for allOf/oneOf/anyOf. Each recursion must increment depth so the
+    // depth cap can terminate self-referential combinator schemas.
+    const deeper = { ...ctx, depth: ctx.depth + 1 };
+    if (schema.allOf && schema.allOf.length > 0) {
+        // Merge all schemas
+        const merged = {};
+        for (const subSchema of schema.allOf) {
+            const subValue = generateValue(subSchema, deeper);
+            if (typeof subValue === 'object' && subValue !== null) {
+                Object.assign(merged, subValue);
+            }
+        }
+        return merged;
+    }
+    if (schema.oneOf && schema.oneOf.length > 0) {
+        return generateValue(schema.oneOf[0], deeper);
+    }
+    if (schema.anyOf && schema.anyOf.length > 0) {
+        return generateValue(schema.anyOf[0], deeper);
+    }
+    // Generate based on type
+    switch (schema.type) {
+        case 'string':
+            return generateString(schema, ctx);
+        case 'number':
+            return generateNumber(schema);
+        case 'integer':
+            return generateInteger(schema);
+        case 'boolean':
+            return true;
+        case 'array':
+            return generateArray(schema, ctx);
+        case 'object':
+            return generateObject(schema, ctx);
+        default: {
+            // No type specified - try to infer from properties
+            // Cast to SchemaObject to access properties on untyped schemas
+            const untyped = schema;
+            if (untyped.properties) {
+                return generateObject(untyped, ctx);
+            }
+            // Return empty object as fallback
+            return {};
+        }
+    }
+}
+function generateString(schema, ctx) {
+    // Handle format
+    switch (schema.format) {
+        case 'date':
+            return '2024-01-15';
+        case 'date-time':
+            return '2024-01-15T10:30:00Z';
+        case 'email':
+            return `user${ctx.counter.value++}@example.com`;
+        case 'uri':
+        case 'url':
+            return 'https://example.com';
+        case 'uuid': {
+            // Generate unique UUIDs using counter
+            const id = ctx.counter.value++;
+            const hex = id.toString(16).padStart(12, '0');
+            return `550e8400-e29b-41d4-a716-${hex}`;
+        }
+        case 'hostname':
+            return 'example.com';
+        case 'ipv4':
+            return '192.168.1.1';
+        case 'ipv6':
+            return '2001:0db8:85a3:0000:0000:8a2e:0370:7334';
+        case 'byte':
+            return 'SGVsbG8gV29ybGQ='; // Base64 encoded "Hello World"
+        case 'binary':
+            return '<binary>';
+        case 'password':
+            return '********';
+        case 'phone':
+            return '+1-555-555-5555';
+    }
+    // Respect minLength/maxLength
+    const minLen = schema.minLength ?? 0;
+    const maxLen = schema.maxLength ?? 50;
+    const targetLen = Math.max(minLen, Math.min(8, maxLen));
+    // Generate a simple mock string
+    const base = 'mock_value';
+    if (base.length >= targetLen) {
+        return base.substring(0, targetLen);
+    }
+    return base.padEnd(targetLen, '_');
+}
+function generateNumber(schema) {
+    const min = schema.minimum ?? 0;
+    const max = schema.maximum ?? 100;
+    // Generate a value in range
+    const value = min + (max - min) / 2;
+    // Handle multipleOf
+    if (schema.multipleOf) {
+        return Math.round(value / schema.multipleOf) * schema.multipleOf;
+    }
+    return Math.round(value * 100) / 100;
+}
+function generateInteger(schema) {
+    const min = schema.minimum ?? 0;
+    const max = schema.maximum ?? 100;
+    // Generate integer in range
+    const value = Math.floor(min + (max - min) / 2);
+    // Handle multipleOf
+    if (schema.multipleOf) {
+        return Math.round(value / schema.multipleOf) * schema.multipleOf;
+    }
+    return value;
+}
+function generateArray(schema, ctx) {
+    // Check depth limit
+    if (ctx.depth >= ctx.maxDepth) {
+        return [];
+    }
+    // Determine array length
+    const minItems = schema.minItems ?? 0;
+    const maxItems = schema.maxItems ?? ctx.arrayLength;
+    const length = Math.max(minItems, Math.min(ctx.arrayLength, maxItems));
+    if (!schema.items) {
+        return Array(length).fill({});
+    }
+    const itemSchema = schema.items;
+    const items = [];
+    for (let i = 0; i < length; i++) {
+        items.push(generateValue(itemSchema, {
+            ...ctx,
+            depth: ctx.depth + 1,
+        }));
+    }
+    return items;
+}
+function generateObject(schema, ctx) {
+    // Check depth limit
+    if (ctx.depth >= ctx.maxDepth) {
+        return {};
+    }
+    const result = {};
+    const properties = schema.properties;
+    if (!properties) {
+        return result;
+    }
+    // Include all properties at any depth to generate complete mock data
+    for (const [key, propSchema] of Object.entries(properties)) {
+        result[key] = generateValue(propSchema, {
+            ...ctx,
+            depth: ctx.depth + 1,
+        });
+    }
+    return result;
+}

package/dist/oas/validator.js

@@ -1,3 +1,32 @@
+/**
+ * Upper bound on the length of a string we will test against a `pattern`
+ * regex. Both the schema pattern and the response value are untrusted, so a
+ * pathological pattern (e.g. `(a+)+$`) over a long string can trigger
+ * catastrophic backtracking. Anything longer is rejected without running the
+ * regex, so validation always returns promptly.
+ */
+const MAX_PATTERN_INPUT_LENGTH = 10_000;
+/**
+ * Cache of compiled pattern regexes. Avoids recompiling the same (untrusted)
+ * pattern per value, and stores `null` for patterns that fail to compile so we
+ * skip them consistently.
+ */
+const patternRegexCache = new Map();
+function getPatternRegex(pattern) {
+    const cached = patternRegexCache.get(pattern);
+    if (cached !== undefined)
+        return cached;
+    let regex;
+    try {
+        regex = new RegExp(pattern);
+    }
+    catch {
+        // Invalid regex: don't crash validation, just skip the pattern check.
+        regex = null;
+    }
+    patternRegexCache.set(pattern, regex);
+    return regex;
+}
 export function validateResponse(data, schema, path = '') {
     const errors = [];
     validateValue(data, schema, path, errors);
@@ -79,15 +108,26 @@ function validateString(value, schema, path, errors) {
         });
     }
     if (schema.pattern) {
-        const regex = new RegExp(schema.pattern);
-        if (!regex.test(value)) {
+        if (value.length > MAX_PATTERN_INPUT_LENGTH) {
+            // Refuse to run an untrusted regex over an oversized string (ReDoS guard).
             errors.push({
                 path,
-                message: `String does not match pattern`,
-                expected: schema.pattern,
-                actual: value,
+                message: `String too long to validate against pattern`,
+                expected: `<= ${MAX_PATTERN_INPUT_LENGTH} chars`,
+                actual: `${value.length} chars`,
             });
         }
+        else {
+            const regex = getPatternRegex(schema.pattern);
+            if (regex && !regex.test(value)) {
+                errors.push({
+                    path,
+                    message: `String does not match pattern`,
+                    expected: schema.pattern,
+                    actual: value,
+                });
+            }
+        }
     }
 }
 function validateNumber(value, schema, path, errors) {

package/dist/observability/events.d.ts

@@ -0,0 +1,248 @@
+/**
+ * Observability Events - Typed events for mission execution
+ *
+ * Provides a comprehensive event system for tracking execution progress,
+ * performance metrics, and debugging information.
+ */
+/** Log levels for structured logging */
+export type LogLevel = 'debug' | 'info' | 'warn' | 'error';
+/** Base event structure for all observability events */
+export interface ObservabilityEvent<T = unknown> {
+    /** Event type identifier */
+    type: string;
+    /** Execution context ID */
+    executionId: string;
+    /** Mission name */
+    mission: string;
+    /** ISO timestamp */
+    timestamp: string;
+    /**
+     * Duration in milliseconds of the operation this event reports, when the
+     * payload carries one. Not set by the emitter itself: a generic "time since
+     * the previous event of any type" value was misleading, so operation timing
+     * now lives in the typed payloads instead.
+     */
+    duration?: number;
+    /** Event-specific payload */
+    payload: T;
+}
+export interface MissionStartPayload {
+    stageCount: number;
+    isResume: boolean;
+    resumeFromStage?: number;
+    metadata?: Record<string, unknown>;
+}
+export interface MissionCompletePayload {
+    success: boolean;
+    stagesCompleted: number;
+    stagesFailed: number;
+    stagesSkipped: number;
+    errorCount: number;
+}
+export interface MissionFailedPayload {
+    error: string;
+    failedStage?: string;
+    stagesCompleted: number;
+}
+export interface StageStartPayload {
+    stageIndex: number;
+    stageName: string;
+    totalStages: number;
+    isParallel: boolean;
+    parallelActions?: string[];
+}
+export interface StageCompletePayload {
+    stageIndex: number;
+    stageName: string;
+    success: boolean;
+    error?: string;
+    itemsProcessed?: number;
+}
+export type StepType = 'fetch' | 'for' | 'map' | 'validate' | 'store' | 'match' | 'let' | 'webhook' | 'pause';
+export interface StepStartPayload {
+    actionName: string;
+    stepIndex: number;
+    stepType: StepType;
+}
+export interface StepCompletePayload {
+    actionName: string;
+    stepIndex: number;
+    stepType: StepType;
+    success: boolean;
+    error?: string;
+}
+export interface FetchStartPayload {
+    source: string;
+    method: string;
+    path: string;
+    isOAS: boolean;
+    operationId?: string;
+    hasPagination: boolean;
+    hasSince: boolean;
+}
+export interface FetchCompletePayload {
+    source: string;
+    method: string;
+    path: string;
+    statusCode: number;
+    recordCount: number;
+    pagesFetched?: number;
+    bytesReceived?: number;
+    fromCache?: boolean;
+}
+export interface FetchRetryPayload {
+    source: string;
+    path: string;
+    attempt: number;
+    maxAttempts: number;
+    reason: string;
+    waitMs: number;
+}
+export interface FetchErrorPayload {
+    source: string;
+    path: string;
+    statusCode?: number;
+    error: string;
+    retryable: boolean;
+}
+export interface FetchHeartbeatPayload {
+    source: string;
+    path: string;
+    pagesProcessed: number;
+    itemsFetched: number;
+    hasMore: boolean;
+}
+export interface DataTransformPayload {
+    targetSchema?: string;
+    inputCount: number;
+    outputCount: number;
+    fieldsMapping: number;
+}
+export interface DataValidatePayload {
+    target: string;
+    passed: boolean;
+    warningCount: number;
+    errorCount: number;
+    rules: string[];
+}
+export interface DataStorePayload {
+    storeName: string;
+    storeType: string;
+    operation: 'set' | 'update' | 'upsert' | 'append';
+    itemCount: number;
+    key?: string;
+}
+export interface LoopStartPayload {
+    variable: string;
+    collectionSize: number;
+    hasFilter: boolean;
+}
+export interface LoopIterationPayload {
+    variable: string;
+    itemIndex: number;
+    totalItems: number;
+}
+export interface LoopCompletePayload {
+    variable: string;
+    totalItems: number;
+    itemsProcessed: number;
+    itemsSkipped: number;
+    itemsFailed: number;
+}
+export interface LoopHeartbeatPayload {
+    variable: string;
+    current: number;
+    total: number;
+    processedCount: number;
+}
+export interface MatchAttemptPayload {
+    schemas: string[];
+    hasDefault: boolean;
+}
+export interface MatchResultPayload {
+    matchedSchema?: string;
+    isDefault: boolean;
+    flowDirective?: 'skip' | 'retry' | 'abort';
+}
+export interface WebhookRegisterPayload {
+    registrationId: string;
+    path: string;
+    webhookUrl: string;
+    timeout: number;
+    expectedEvents: number;
+}
+export interface WebhookEventPayload {
+    registrationId: string;
+    eventIndex: number;
+    totalExpected: number;
+    filtered: boolean;
+}
+export interface WebhookCompletePayload {
+    registrationId: string;
+    eventsReceived: number;
+    timedOut: boolean;
+    storedTo?: string;
+}
+export interface CheckpointSavePayload {
+    stageIndex: number;
+    stepIndex: number;
+    reason: 'manual' | 'webhook' | 'error' | 'pause';
+    variableCount: number;
+}
+export interface CheckpointResumePayload {
+    stageIndex: number;
+    stepIndex: number;
+    originalExecutionId: string;
+    stagesSkipped: number;
+}
+export interface SyncCheckpointPayload {
+    checkpointKey: string;
+    lastSyncTime: string;
+    recordsFetched: number;
+    isIncremental: boolean;
+}
+export interface RateLimitPayload {
+    source: string;
+    endpoint?: string;
+    waitSeconds: number;
+    strategy: string;
+}
+export interface CircuitBreakerPayload {
+    source: string;
+    endpoint?: string;
+    state: 'open' | 'half-open' | 'closed';
+    failures: number;
+    reason?: string;
+}
+export type EventType = 'mission.start' | 'mission.complete' | 'mission.failed' | 'mission.paused' | 'stage.start' | 'stage.complete' | 'step.start' | 'step.complete' | 'fetch.start' | 'fetch.complete' | 'fetch.retry' | 'fetch.error' | 'fetch.heartbeat' | 'data.transform' | 'data.validate' | 'data.store' | 'loop.start' | 'loop.iteration' | 'loop.complete' | 'loop.heartbeat' | 'match.attempt' | 'match.result' | 'webhook.register' | 'webhook.event' | 'webhook.complete' | 'checkpoint.save' | 'checkpoint.resume' | 'sync.checkpoint' | 'ratelimit.wait' | 'ratelimit.resume' | 'circuit.open' | 'circuit.halfopen' | 'circuit.close';
+export type EventHandler<T = unknown> = (event: ObservabilityEvent<T>) => void;
+export interface EventEmitter {
+    /** Emit an event */
+    emit<T>(type: EventType, payload: T): void;
+    /** Subscribe to a specific event type */
+    on<T>(type: EventType, handler: EventHandler<T>): () => void;
+    /** Subscribe to all events */
+    onAll(handler: EventHandler): () => void;
+    /** Remove all handlers */
+    clear(): void;
+}
+export declare class ObservabilityEmitter implements EventEmitter {
+    private executionId;
+    private mission;
+    private handlers;
+    private allHandlers;
+    private startTime;
+    constructor(executionId: string, mission: string);
+    emit<T>(type: EventType, payload: T): void;
+    on<T>(type: EventType, handler: EventHandler<T>): () => void;
+    onAll(handler: EventHandler): () => void;
+    clear(): void;
+    /** Get total elapsed time since emitter creation */
+    getElapsedTime(): number;
+    /** Update execution context (for resume scenarios) */
+    setContext(executionId: string, mission: string): void;
+}
+/**
+ * Create a new observability emitter
+ */
+export declare function createEmitter(executionId: string, mission: string): ObservabilityEmitter;

package/dist/observability/events.js

@@ -0,0 +1,85 @@
+/**
+ * Observability Events - Typed events for mission execution
+ *
+ * Provides a comprehensive event system for tracking execution progress,
+ * performance metrics, and debugging information.
+ */
+// ============================================================================
+// Event Emitter Implementation
+// ============================================================================
+export class ObservabilityEmitter {
+    executionId;
+    mission;
+    handlers = new Map();
+    allHandlers = new Set();
+    startTime;
+    constructor(executionId, mission) {
+        this.executionId = executionId;
+        this.mission = mission;
+        this.startTime = Date.now();
+    }
+    emit(type, payload) {
+        const event = {
+            type,
+            executionId: this.executionId,
+            mission: this.mission,
+            timestamp: new Date().toISOString(),
+            payload,
+        };
+        // Notify specific handlers
+        const typeHandlers = this.handlers.get(type);
+        if (typeHandlers) {
+            for (const handler of typeHandlers) {
+                try {
+                    handler(event);
+                }
+                catch {
+                    // Swallow handler errors to prevent breaking execution
+                }
+            }
+        }
+        // Notify all-event handlers
+        for (const handler of this.allHandlers) {
+            try {
+                handler(event);
+            }
+            catch {
+                // Swallow handler errors
+            }
+        }
+    }
+    on(type, handler) {
+        if (!this.handlers.has(type)) {
+            this.handlers.set(type, new Set());
+        }
+        this.handlers.get(type).add(handler);
+        return () => {
+            this.handlers.get(type)?.delete(handler);
+        };
+    }
+    onAll(handler) {
+        this.allHandlers.add(handler);
+        return () => {
+            this.allHandlers.delete(handler);
+        };
+    }
+    clear() {
+        this.handlers.clear();
+        this.allHandlers.clear();
+    }
+    /** Get total elapsed time since emitter creation */
+    getElapsedTime() {
+        return Date.now() - this.startTime;
+    }
+    /** Update execution context (for resume scenarios) */
+    setContext(executionId, mission) {
+        this.executionId = executionId;
+        this.mission = mission;
+    }
+}
+/**
+ * Create a new observability emitter
+ */
+export function createEmitter(executionId, mission) {
+    return new ObservabilityEmitter(executionId, mission);
+}

package/dist/observability/index.d.ts

@@ -0,0 +1,15 @@
+/**
+ * Observability Module - Mission execution monitoring
+ *
+ * Provides comprehensive observability for Reqon missions:
+ * - Structured event system
+ * - Enhanced logging with context
+ * - OpenTelemetry integration
+ * - Multiple output adapters
+ */
+export type { LogLevel, ObservabilityEvent, EventType, EventHandler, EventEmitter, MissionStartPayload, MissionCompletePayload, MissionFailedPayload, StageStartPayload, StageCompletePayload, StepType, StepStartPayload, StepCompletePayload, FetchStartPayload, FetchCompletePayload, FetchRetryPayload, FetchErrorPayload, FetchHeartbeatPayload, DataTransformPayload, DataValidatePayload, DataStorePayload, LoopStartPayload, LoopIterationPayload, LoopCompletePayload, LoopHeartbeatPayload, MatchAttemptPayload, MatchResultPayload, WebhookRegisterPayload, WebhookEventPayload, WebhookCompletePayload, CheckpointSavePayload, CheckpointResumePayload, SyncCheckpointPayload, RateLimitPayload, CircuitBreakerPayload, } from './events.js';
+export { ObservabilityEmitter, createEmitter } from './events.js';
+export type { LogEntry, Span, LogOutput, StructuredLogger, CreateLoggerOptions } from './logger.js';
+export { ConsoleOutput, JsonLinesOutput, BufferOutput, EventOutput, createStructuredLogger, } from './logger.js';
+export type { OTelSpan, OTelAttribute, OTelSpanEvent, TraceContext, OTLPExporterConfig, } from './otel.js';
+export { generateTraceId, generateSpanId, SpanBuilder, OTelEventAdapter, OTLPExporter, OTelLogOutput, createOTelListener, } from './otel.js';

package/dist/observability/index.js

@@ -0,0 +1,12 @@
+/**
+ * Observability Module - Mission execution monitoring
+ *
+ * Provides comprehensive observability for Reqon missions:
+ * - Structured event system
+ * - Enhanced logging with context
+ * - OpenTelemetry integration
+ * - Multiple output adapters
+ */
+export { ObservabilityEmitter, createEmitter } from './events.js';
+export { ConsoleOutput, JsonLinesOutput, BufferOutput, EventOutput, createStructuredLogger, } from './logger.js';
+export { generateTraceId, generateSpanId, SpanBuilder, OTelEventAdapter, OTLPExporter, OTelLogOutput, createOTelListener, } from './otel.js';