reqon-dsl 0.2.0 → 0.3.0

package/dist/trace/store.js

@@ -0,0 +1,225 @@
+/**
+ * Trace Store - Persistence for execution traces
+ *
+ * Stores complete execution traces for time-travel debugging,
+ * enabling replay and step-by-step inspection of past runs.
+ */
+import { join } from 'node:path';
+import { ensureDirectory, writeJsonFile, readJsonFile, listFiles, deleteFile, restoreDates, restoreDatesInArray, } from '../utils/file.js';
+/**
+ * File-based trace store
+ * Stores traces in .reqon-data/traces/ with separate files for metadata and snapshots
+ */
+export class FileTraceStore {
+    baseDir;
+    initialized;
+    constructor(baseDir = '.reqon-data/traces') {
+        this.baseDir = baseDir;
+        this.initialized = ensureDirectory(this.baseDir);
+    }
+    getTraceDir(id) {
+        return join(this.baseDir, id);
+    }
+    getMetadataPath(id) {
+        return join(this.getTraceDir(id), 'metadata.json');
+    }
+    getSnapshotPath(id, index) {
+        return join(this.getTraceDir(id), `snapshot_${index.toString().padStart(6, '0')}.json`);
+    }
+    async deserializeTrace(parsed) {
+        restoreDates(parsed, ['startedAt', 'completedAt']);
+        if (parsed.snapshots && Array.isArray(parsed.snapshots)) {
+            restoreDatesInArray(parsed.snapshots, ['timestamp']);
+        }
+        return parsed;
+    }
+    deserializeSnapshot(parsed) {
+        restoreDates(parsed, ['timestamp']);
+        return parsed;
+    }
+    async save(trace) {
+        await this.initialized;
+        const traceDir = this.getTraceDir(trace.id);
+        await ensureDirectory(traceDir);
+        // Save metadata separately from snapshots for efficient listing
+        const metadata = {
+            id: trace.id,
+            mission: trace.mission,
+            mode: trace.mode,
+            startedAt: trace.startedAt,
+            completedAt: trace.completedAt,
+            duration: trace.duration,
+            success: trace.success,
+            metadata: trace.metadata,
+            snapshotCount: trace.snapshots.length,
+        };
+        await writeJsonFile(this.getMetadataPath(trace.id), metadata);
+        // Save each snapshot separately for efficient random access
+        for (let i = 0; i < trace.snapshots.length; i++) {
+            await writeJsonFile(this.getSnapshotPath(trace.id, i), trace.snapshots[i]);
+        }
+    }
+    async appendSnapshot(traceId, snapshot) {
+        await this.initialized;
+        const traceDir = this.getTraceDir(traceId);
+        await ensureDirectory(traceDir);
+        // Load current metadata to get snapshot count
+        const metadataPath = this.getMetadataPath(traceId);
+        const metadata = await readJsonFile(metadataPath);
+        let snapshotCount = 0;
+        if (metadata && typeof metadata.snapshotCount === 'number') {
+            snapshotCount = metadata.snapshotCount;
+        }
+        // Save the new snapshot
+        await writeJsonFile(this.getSnapshotPath(traceId, snapshotCount), snapshot);
+        // Update metadata with new count
+        if (metadata) {
+            metadata.snapshotCount = snapshotCount + 1;
+            await writeJsonFile(metadataPath, metadata);
+        }
+    }
+    async load(id) {
+        await this.initialized;
+        const metadata = await this.getMetadata(id);
+        if (!metadata)
+            return null;
+        // Load all snapshots
+        const snapshots = [];
+        const snapshotFiles = await listFiles(this.getTraceDir(id), '.json');
+        for (const file of snapshotFiles) {
+            if (file.includes('snapshot_')) {
+                const parsed = await readJsonFile(file);
+                if (parsed) {
+                    snapshots.push(this.deserializeSnapshot(parsed));
+                }
+            }
+        }
+        // Sort by index
+        snapshots.sort((a, b) => a.index - b.index);
+        return {
+            ...metadata,
+            snapshots,
+        };
+    }
+    async loadSnapshot(traceId, snapshotIndex) {
+        await this.initialized;
+        const snapshotPath = this.getSnapshotPath(traceId, snapshotIndex);
+        const parsed = await readJsonFile(snapshotPath);
+        return parsed ? this.deserializeSnapshot(parsed) : null;
+    }
+    async listByMission(mission, limit = 50) {
+        const all = await this.listRecent(limit * 2); // Load more to filter
+        return all.filter((t) => t.mission === mission).slice(0, limit);
+    }
+    async listRecent(limit = 50) {
+        await this.initialized;
+        const entries = await listFiles(this.baseDir, '');
+        const traces = [];
+        // Read metadata files from subdirectories
+        for (const entry of entries) {
+            if (entry.endsWith('metadata.json')) {
+                const parsed = await readJsonFile(entry);
+                if (parsed) {
+                    const trace = await this.deserializeTrace(parsed);
+                    // Return without snapshots for listing efficiency
+                    traces.push({ ...trace, snapshots: [] });
+                }
+            }
+        }
+        // Sort by start time, newest first
+        traces.sort((a, b) => b.startedAt.getTime() - a.startedAt.getTime());
+        return traces.slice(0, limit);
+    }
+    async delete(id) {
+        await this.initialized;
+        const traceDir = this.getTraceDir(id);
+        const files = await listFiles(traceDir, '.json');
+        for (const file of files) {
+            await deleteFile(file);
+        }
+        // Note: Directory cleanup would require additional fs.rmdir
+    }
+    async findLatest(mission) {
+        const traces = await this.listByMission(mission, 1);
+        return traces[0] ?? null;
+    }
+    async getMetadata(id) {
+        await this.initialized;
+        const parsed = await readJsonFile(this.getMetadataPath(id));
+        if (!parsed)
+            return null;
+        restoreDates(parsed, ['startedAt', 'completedAt']);
+        const { snapshotCount: _snapshotCount, ...rest } = parsed;
+        return rest;
+    }
+}
+/**
+ * In-memory trace store (for testing)
+ */
+export class MemoryTraceStore {
+    traces = new Map();
+    async save(trace) {
+        this.traces.set(trace.id, JSON.parse(JSON.stringify(trace)));
+    }
+    async appendSnapshot(traceId, snapshot) {
+        const trace = this.traces.get(traceId);
+        if (trace) {
+            trace.snapshots.push(JSON.parse(JSON.stringify(snapshot)));
+        }
+    }
+    async load(id) {
+        const trace = this.traces.get(id);
+        if (!trace)
+            return null;
+        const restored = JSON.parse(JSON.stringify(trace));
+        restored.startedAt = new Date(restored.startedAt);
+        if (restored.completedAt) {
+            restored.completedAt = new Date(restored.completedAt);
+        }
+        for (const snapshot of restored.snapshots) {
+            snapshot.timestamp = new Date(snapshot.timestamp);
+        }
+        return restored;
+    }
+    async loadSnapshot(traceId, snapshotIndex) {
+        const trace = this.traces.get(traceId);
+        if (!trace || snapshotIndex >= trace.snapshots.length)
+            return null;
+        const snapshot = JSON.parse(JSON.stringify(trace.snapshots[snapshotIndex]));
+        snapshot.timestamp = new Date(snapshot.timestamp);
+        return snapshot;
+    }
+    async listByMission(mission, limit = 50) {
+        const all = await this.listRecent();
+        return all.filter((t) => t.mission === mission).slice(0, limit);
+    }
+    async listRecent(limit = 50) {
+        const traces = Array.from(this.traces.values()).map((t) => {
+            const restored = JSON.parse(JSON.stringify(t));
+            restored.startedAt = new Date(restored.startedAt);
+            if (restored.completedAt) {
+                restored.completedAt = new Date(restored.completedAt);
+            }
+            return { ...restored, snapshots: [] };
+        });
+        traces.sort((a, b) => b.startedAt.getTime() - a.startedAt.getTime());
+        return traces.slice(0, limit);
+    }
+    async delete(id) {
+        this.traces.delete(id);
+    }
+    async findLatest(mission) {
+        const traces = await this.listByMission(mission, 1);
+        return traces[0] ?? null;
+    }
+    async getMetadata(id) {
+        const trace = await this.load(id);
+        if (!trace)
+            return null;
+        const { snapshots: _snapshots, ...metadata } = trace;
+        return metadata;
+    }
+    clear() {
+        this.traces.clear();
+    }
+}

package/dist/utils/index.d.ts

@@ -2,3 +2,4 @@ export { sleep } from './async.js';
 export { extractNestedValue, traversePath } from './path.js';
 export { type Logger, ConsoleLogger, SilentLogger, createLogger } from './logger.js';
 export { ensureDirectory, ensureParentDirectory, serialize, writeJsonFile, readJsonFile, listFiles, deleteFile, restoreDates, restoreDatesInArray, } from './file.js';
+export { isRecord, isObject, isArrayOf, isString, isNumber, isBoolean, isDefined, isPresent, getProperty, getNestedProperty, hasProperty, hasTypedProperty, } from './type-guards.js';

package/dist/utils/index.js

@@ -2,3 +2,4 @@ export { sleep } from './async.js';
 export { extractNestedValue, traversePath } from './path.js';
 export { ConsoleLogger, SilentLogger, createLogger } from './logger.js';
 export { ensureDirectory, ensureParentDirectory, serialize, writeJsonFile, readJsonFile, listFiles, deleteFile, restoreDates, restoreDatesInArray, } from './file.js';
+export { isRecord, isObject, isArrayOf, isString, isNumber, isBoolean, isDefined, isPresent, getProperty, getNestedProperty, hasProperty, hasTypedProperty, } from './type-guards.js';

package/dist/utils/type-guards.d.ts

@@ -0,0 +1,58 @@
+/**
+ * Type guard utilities to reduce type assertions and improve type safety.
+ * These guards provide runtime type checking with TypeScript type narrowing.
+ */
+/**
+ * Type guard to check if a value is a non-null object (excluding arrays).
+ * Narrows the type to Record<string, unknown> for safe property access.
+ */
+export declare function isRecord(value: unknown): value is Record<string, unknown>;
+/**
+ * Type guard to check if a value is a non-null object (including arrays).
+ * Useful when you need to access properties on any object-like value.
+ */
+export declare function isObject(value: unknown): value is object;
+/**
+ * Type guard to check if a value is an array of a specific type.
+ * Performs runtime check on each element using the provided guard.
+ */
+export declare function isArrayOf<T>(value: unknown, guard: (item: unknown) => item is T): value is T[];
+/**
+ * Type guard to check if a value is a string.
+ */
+export declare function isString(value: unknown): value is string;
+/**
+ * Type guard to check if a value is a number.
+ */
+export declare function isNumber(value: unknown): value is number;
+/**
+ * Type guard to check if a value is a boolean.
+ */
+export declare function isBoolean(value: unknown): value is boolean;
+/**
+ * Type guard to check if a value is defined (not undefined).
+ */
+export declare function isDefined<T>(value: T | undefined): value is T;
+/**
+ * Type guard to check if a value is not null or undefined.
+ */
+export declare function isPresent<T>(value: T | null | undefined): value is T;
+/**
+ * Safely access a property on an unknown value.
+ * Returns undefined if the value is not an object or the property doesn't exist.
+ */
+export declare function getProperty(value: unknown, key: string): unknown;
+/**
+ * Safely access a nested property path on an unknown value.
+ * Returns undefined if any part of the path is not an object.
+ */
+export declare function getNestedProperty(value: unknown, path: string[]): unknown;
+/**
+ * Check if an object has a specific property.
+ * Narrows the type to include that property.
+ */
+export declare function hasProperty<K extends string>(value: unknown, key: K): value is Record<K, unknown>;
+/**
+ * Check if an object has a specific property with a specific type.
+ */
+export declare function hasTypedProperty<K extends string, T>(value: unknown, key: K, guard: (v: unknown) => v is T): value is Record<K, T>;

package/dist/utils/type-guards.js

@@ -0,0 +1,92 @@
+/**
+ * Type guard utilities to reduce type assertions and improve type safety.
+ * These guards provide runtime type checking with TypeScript type narrowing.
+ */
+/**
+ * Type guard to check if a value is a non-null object (excluding arrays).
+ * Narrows the type to Record<string, unknown> for safe property access.
+ */
+export function isRecord(value) {
+    return value !== null && typeof value === 'object' && !Array.isArray(value);
+}
+/**
+ * Type guard to check if a value is a non-null object (including arrays).
+ * Useful when you need to access properties on any object-like value.
+ */
+export function isObject(value) {
+    return value !== null && typeof value === 'object';
+}
+/**
+ * Type guard to check if a value is an array of a specific type.
+ * Performs runtime check on each element using the provided guard.
+ */
+export function isArrayOf(value, guard) {
+    return Array.isArray(value) && value.every(guard);
+}
+/**
+ * Type guard to check if a value is a string.
+ */
+export function isString(value) {
+    return typeof value === 'string';
+}
+/**
+ * Type guard to check if a value is a number.
+ */
+export function isNumber(value) {
+    return typeof value === 'number';
+}
+/**
+ * Type guard to check if a value is a boolean.
+ */
+export function isBoolean(value) {
+    return typeof value === 'boolean';
+}
+/**
+ * Type guard to check if a value is defined (not undefined).
+ */
+export function isDefined(value) {
+    return value !== undefined;
+}
+/**
+ * Type guard to check if a value is not null or undefined.
+ */
+export function isPresent(value) {
+    return value !== null && value !== undefined;
+}
+/**
+ * Safely access a property on an unknown value.
+ * Returns undefined if the value is not an object or the property doesn't exist.
+ */
+export function getProperty(value, key) {
+    if (isRecord(value)) {
+        return value[key];
+    }
+    return undefined;
+}
+/**
+ * Safely access a nested property path on an unknown value.
+ * Returns undefined if any part of the path is not an object.
+ */
+export function getNestedProperty(value, path) {
+    let current = value;
+    for (const key of path) {
+        if (!isRecord(current)) {
+            return undefined;
+        }
+        current = current[key];
+    }
+    return current;
+}
+/**
+ * Check if an object has a specific property.
+ * Narrows the type to include that property.
+ */
+export function hasProperty(value, key) {
+    return isRecord(value) && key in value;
+}
+/**
+ * Check if an object has a specific property with a specific type.
+ */
+export function hasTypedProperty(value, key, guard) {
+    return hasProperty(value, key) && guard(value[key]);
+}

package/dist/webhook/server.js

@@ -8,6 +8,7 @@ import { createServer } from 'node:http';
 import { parse as parseUrl } from 'node:url';
 import { randomUUID } from 'node:crypto';
 import { MemoryWebhookStore } from './store.js';
+import { WEBHOOK_DEFAULTS } from '../config/index.js';
 /**
  * Webhook Server
  *
@@ -23,10 +24,10 @@ export class WebhookServer {
     running = false;
     constructor(config = {}, store, callbacks = {}) {
         this.config = {
-            port: config.port ?? 3000,
-            host: config.host ?? '0.0.0.0',
-            baseUrl: config.baseUrl ?? `http://localhost:${config.port ?? 3000}`,
-            defaultTimeout: config.defaultTimeout ?? 300000, // 5 minutes
+            port: config.port ?? WEBHOOK_DEFAULTS.PORT,
+            host: config.host ?? WEBHOOK_DEFAULTS.HOST,
+            baseUrl: config.baseUrl ?? `http://localhost:${config.port ?? WEBHOOK_DEFAULTS.PORT}`,
+            defaultTimeout: config.defaultTimeout ?? WEBHOOK_DEFAULTS.DEFAULT_TIMEOUT_MS,
             verbose: config.verbose ?? false,
         };
         this.store = store ?? new MemoryWebhookStore();
@@ -46,8 +47,8 @@ export class WebhookServer {
             this.server.listen(this.config.port, this.config.host, () => {
                 this.running = true;
                 this.log(`Webhook server listening on ${this.config.host}:${this.config.port}`);
-                // Start cleanup interval (every minute)
-                this.cleanupInterval = setInterval(() => this.cleanup(), 60000);
+                // Start cleanup interval
+                this.cleanupInterval = setInterval(() => this.cleanup(), WEBHOOK_DEFAULTS.CLEANUP_INTERVAL_MS);
                 resolve();
             });
         });

package/package.json

@@ -1,12 +1,13 @@
 {
   "name": "reqon-dsl",
-  "version": "0.2.0",
+  "version": "0.3.0",
   "description": "A DSL framework for fetch, map, validate pipelines - built on Vague",
   "type": "module",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
   "bin": {
-    "reqon": "dist/cli.js"
+    "reqon": "dist/cli.js",
+    "reqon-mcp": "dist/mcp/server.js"
   },
   "scripts": {
     "build": "tsc",
@@ -14,27 +15,75 @@
     "test": "vitest",
     "test:run": "vitest run",
     "test:coverage": "vitest run --coverage",
+    "lint": "eslint src/",
+    "lint:fix": "eslint src/ --fix",
+    "format": "prettier --write src/",
+    "format:check": "prettier --check src/",
     "bench": "npx tsx src/benchmark/index.ts",
     "bench:lexer": "npx tsx src/benchmark/index.ts --suite lexer",
     "bench:parser": "npx tsx src/benchmark/index.ts --suite parser",
     "bench:evaluator": "npx tsx src/benchmark/index.ts --suite evaluator",
     "bench:store": "npx tsx src/benchmark/index.ts --suite store",
     "bench:resilience": "npx tsx src/benchmark/index.ts --suite resilience",
-    "bench:e2e": "npx tsx src/benchmark/index.ts --suite e2e"
+    "bench:e2e": "npx tsx src/benchmark/index.ts --suite e2e",
+    "prerelease": "npm run test:run && npm run build",
+    "release": "node scripts/release.js",
+    "release:dry-run": "npm publish --dry-run",
+    "version:patch": "npm version patch --no-git-tag-version",
+    "version:minor": "npm version minor --no-git-tag-version",
+    "version:major": "npm version major --no-git-tag-version",
+    "prepare": "husky"
   },
+  "files": [
+    "dist",
+    "CHANGELOG.md",
+    "README.md"
+  ],
   "engines": {
     "node": ">=18"
   },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/mcclowes/reqon.git"
+  },
+  "bugs": {
+    "url": "https://github.com/mcclowes/reqon/issues"
+  },
+  "homepage": "https://github.com/mcclowes/reqon#readme",
+  "keywords": [
+    "dsl",
+    "data-pipeline",
+    "fetch",
+    "api",
+    "declarative",
+    "etl",
+    "vague"
+  ],
   "author": "Max Clayton Clowes <max@mcclowes.com>",
   "license": "ISC",
   "dependencies": {
     "@apidevtools/swagger-parser": "^12.1.0",
-    "dotenv": "^16.4.7",
-    "vague-lang": "file:../vague"
+    "@modelcontextprotocol/sdk": "^1.12.0",
+    "dotenv": "^17.2.3",
+    "vague-lang": "^3.3.0",
+    "zod": "^4.2.1"
+  },
+  "lint-staged": {
+    "src/**/*.{ts,tsx}": [
+      "eslint --fix",
+      "prettier --write"
+    ]
   },
   "devDependencies": {
-    "@types/node": "^22.0.0",
+    "@eslint/js": "^9.17.0",
+    "@types/node": "^25.0.3",
+    "@vitest/coverage-v8": "^4.0.16",
+    "eslint": "^9.17.0",
+    "husky": "^9.1.7",
+    "lint-staged": "^16.2.7",
+    "prettier": "^3.4.2",
     "typescript": "^5.9.3",
+    "typescript-eslint": "^8.18.2",
     "vitest": "^4.0.15"
   }
 }

package/.claude/settings.local.json

@@ -1,31 +0,0 @@
-{
-  "permissions": {
-    "allow": [
-      "Bash(npm init -y)",
-      "Bash(npm install)",
-      "Bash(npm run build:*)",
-      "Bash(npm run test:run:*)",
-      "Bash(npx vitest run src/debug.test.ts)",
-      "Bash(npx vitest:*)",
-      "Bash(git init:*)",
-      "Bash(git add:*)",
-      "Bash(git commit:*)",
-      "Bash(git push:*)",
-      "Bash(npm install:*)",
-      "Bash(node dist/cli.js:*)",
-      "Bash(ls:*)",
-      "Bash(node:*)",
-      "Bash(cat:*)",
-      "Bash(git checkout:*)",
-      "Bash(git rebase:*)",
-      "Bash(find:*)",
-      "Bash(git fetch:*)",
-      "Bash(xargs sed:*)",
-      "Bash(for f in examples/**/*.reqon)",
-      "Bash(do mv $f $f%.reqon.vague)",
-      "Bash(done)",
-      "Bash(npx:*)",
-      "Skill(reqon)"
-    ]
-  }
-}

package/.claude/skills/api-integration.md

@@ -1,125 +0,0 @@
-# API Integration Patterns Skill
-
-Use this skill when implementing API client functionality, authentication flows, rate limiting, or pagination in Reqon.
-
-## Capabilities
-
-### OAuth2 Token Refresh Flows
-- Implement automatic token refresh on 401 responses
-- Handle refresh token rotation
-- Store and retrieve tokens securely
-- Support authorization_code, client_credentials, and refresh_token grants
-
-### Rate Limiting with Exponential Backoff
-- Parse rate limit headers (X-RateLimit-*, Retry-After)
-- Implement exponential backoff with jitter
-- Queue requests when rate limited
-- Respect per-endpoint rate limits
-
-### Pagination Pattern Implementations
-- **Cursor-based**: Use `next_cursor` or `after` parameters
-- **Offset-based**: Use `offset` and `limit` parameters
-- **Page-based**: Use `page` and `per_page` parameters
-- **Link header**: Parse RFC 5988 Link headers
-
-## Context Files
-When using this skill, read:
-- `src/interpreter/http.ts` - HTTP client implementation
-- `src/interpreter/executor.ts` - Fetch execution logic
-- `src/ast/nodes.ts` - FetchNode, PaginationConfig types
-
-## Implementation Patterns
-
-### OAuth2 Token Refresh
-```typescript
-async function fetchWithAuth(url: string, auth: OAuth2Auth): Promise<Response> {
-  let response = await fetch(url, {
-    headers: { Authorization: `Bearer ${auth.accessToken}` }
-  });
-
-  if (response.status === 401 && auth.refreshToken) {
-    const newTokens = await refreshAccessToken(auth);
-    auth.accessToken = newTokens.access_token;
-    response = await fetch(url, {
-      headers: { Authorization: `Bearer ${auth.accessToken}` }
-    });
-  }
-
-  return response;
-}
-```
-
-### Exponential Backoff
-```typescript
-async function fetchWithBackoff(
-  url: string,
-  maxRetries = 5,
-  baseDelay = 1000
-): Promise<Response> {
-  for (let attempt = 0; attempt < maxRetries; attempt++) {
-    const response = await fetch(url);
-
-    if (response.status === 429) {
-      const retryAfter = response.headers.get('Retry-After');
-      const delay = retryAfter
-        ? parseInt(retryAfter) * 1000
-        : baseDelay * Math.pow(2, attempt) + Math.random() * 1000;
-      await sleep(delay);
-      continue;
-    }
-
-    return response;
-  }
-  throw new Error('Max retries exceeded');
-}
-```
-
-### Pagination Patterns
-
-#### Cursor-based
-```typescript
-async function* paginateCursor(baseUrl: string) {
-  let cursor: string | undefined;
-  do {
-    const url = cursor ? `${baseUrl}?cursor=${cursor}` : baseUrl;
-    const response = await fetch(url);
-    const data = await response.json();
-    yield data.items;
-    cursor = data.next_cursor;
-  } while (cursor);
-}
-```
-
-#### Offset-based
-```typescript
-async function* paginateOffset(baseUrl: string, limit = 100) {
-  let offset = 0;
-  let hasMore = true;
-  while (hasMore) {
-    const response = await fetch(`${baseUrl}?offset=${offset}&limit=${limit}`);
-    const data = await response.json();
-    yield data.items;
-    offset += limit;
-    hasMore = data.items.length === limit;
-  }
-}
-```
-
-#### Link Header
-```typescript
-function parseLinks(header: string): Record<string, string> {
-  const links: Record<string, string> = {};
-  header.split(',').forEach(part => {
-    const match = part.match(/<([^>]+)>;\s*rel="([^"]+)"/);
-    if (match) links[match[2]] = match[1];
-  });
-  return links;
-}
-```
-
-## Rate Limit Headers
-Common headers to parse:
-- `X-RateLimit-Limit` - Request limit per window
-- `X-RateLimit-Remaining` - Requests remaining
-- `X-RateLimit-Reset` - Unix timestamp when limit resets
-- `Retry-After` - Seconds to wait (on 429/503)