reqon-dsl 0.2.0 → 0.4.0

package/package.json

@@ -1,40 +1,107 @@
 {
   "name": "reqon-dsl",
-  "version": "0.2.0",
+  "version": "0.4.0",
   "description": "A DSL framework for fetch, map, validate pipelines - built on Vague",
   "type": "module",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
   "bin": {
-    "reqon": "dist/cli.js"
+    "reqon": "dist/cli.js",
+    "reqon-mcp": "dist/mcp/server.js"
   },
   "scripts": {
-    "build": "tsc",
-    "dev": "tsc --watch",
+    "build": "tsc -p tsconfig.build.json",
+    "dev": "tsc -p tsconfig.build.json --watch",
+    "typecheck": "tsc --noEmit",
     "test": "vitest",
     "test:run": "vitest run",
     "test:coverage": "vitest run --coverage",
+    "test:integration": "RUN_NETWORK_TESTS=1 vitest run src/api-integration.test.ts",
+    "test:crash": "vitest run src/durability/",
+    "test:pg": "vitest run src/execution-log/postgres-store.test.ts",
+    "lint": "eslint src/",
+    "lint:fix": "eslint src/ --fix",
+    "format": "prettier --write src/",
+    "format:check": "prettier --check src/",
     "bench": "npx tsx src/benchmark/index.ts",
     "bench:lexer": "npx tsx src/benchmark/index.ts --suite lexer",
     "bench:parser": "npx tsx src/benchmark/index.ts --suite parser",
     "bench:evaluator": "npx tsx src/benchmark/index.ts --suite evaluator",
     "bench:store": "npx tsx src/benchmark/index.ts --suite store",
     "bench:resilience": "npx tsx src/benchmark/index.ts --suite resilience",
-    "bench:e2e": "npx tsx src/benchmark/index.ts --suite e2e"
+    "bench:e2e": "npx tsx src/benchmark/index.ts --suite e2e",
+    "prerelease": "npm run test:run && npm run build",
+    "release": "node scripts/release.js",
+    "release:dry-run": "npm publish --dry-run",
+    "version:patch": "npm version patch --no-git-tag-version",
+    "version:minor": "npm version minor --no-git-tag-version",
+    "version:major": "npm version major --no-git-tag-version",
+    "prepare": "husky"
   },
+  "files": [
+    "dist",
+    "CHANGELOG.md",
+    "README.md"
+  ],
   "engines": {
-    "node": ">=18"
+    "node": ">=20"
   },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/mcclowes/reqon.git"
+  },
+  "bugs": {
+    "url": "https://github.com/mcclowes/reqon/issues"
+  },
+  "homepage": "https://github.com/mcclowes/reqon#readme",
+  "keywords": [
+    "dsl",
+    "data-pipeline",
+    "fetch",
+    "api",
+    "declarative",
+    "etl",
+    "vague"
+  ],
   "author": "Max Clayton Clowes <max@mcclowes.com>",
   "license": "ISC",
   "dependencies": {
     "@apidevtools/swagger-parser": "^12.1.0",
-    "dotenv": "^16.4.7",
-    "vague-lang": "file:../vague"
+    "@modelcontextprotocol/sdk": "^1.12.0",
+    "dotenv": "^17.2.3",
+    "vague-lang": "^3.3.0",
+    "zod": "^4.2.1"
+  },
+  "peerDependencies": {
+    "better-sqlite3": ">=11",
+    "pg": ">=8"
+  },
+  "peerDependenciesMeta": {
+    "better-sqlite3": {
+      "optional": true
+    },
+    "pg": {
+      "optional": true
+    }
+  },
+  "lint-staged": {
+    "src/**/*.{ts,tsx}": [
+      "eslint --fix",
+      "prettier --write"
+    ]
   },
   "devDependencies": {
-    "@types/node": "^22.0.0",
+    "@eslint/js": "^9.17.0",
+    "@types/node": "^25.0.3",
+    "@vitest/coverage-v8": "^4.0.16",
+    "better-sqlite3": "^12.11.1",
+    "eslint": "^9.17.0",
+    "husky": "^9.1.7",
+    "lint-staged": "^16.2.7",
+    "pg": "^8.22.0",
+    "prettier": "^3.4.2",
     "typescript": "^5.9.3",
+    "typescript-eslint": "^8.18.2",
     "vitest": "^4.0.15"
   }
 }

package/.claude/settings.local.json

@@ -1,31 +0,0 @@
-{
-  "permissions": {
-    "allow": [
-      "Bash(npm init -y)",
-      "Bash(npm install)",
-      "Bash(npm run build:*)",
-      "Bash(npm run test:run:*)",
-      "Bash(npx vitest run src/debug.test.ts)",
-      "Bash(npx vitest:*)",
-      "Bash(git init:*)",
-      "Bash(git add:*)",
-      "Bash(git commit:*)",
-      "Bash(git push:*)",
-      "Bash(npm install:*)",
-      "Bash(node dist/cli.js:*)",
-      "Bash(ls:*)",
-      "Bash(node:*)",
-      "Bash(cat:*)",
-      "Bash(git checkout:*)",
-      "Bash(git rebase:*)",
-      "Bash(find:*)",
-      "Bash(git fetch:*)",
-      "Bash(xargs sed:*)",
-      "Bash(for f in examples/**/*.reqon)",
-      "Bash(do mv $f $f%.reqon.vague)",
-      "Bash(done)",
-      "Bash(npx:*)",
-      "Skill(reqon)"
-    ]
-  }
-}

package/.claude/skills/api-integration.md

@@ -1,125 +0,0 @@
-# API Integration Patterns Skill
-
-Use this skill when implementing API client functionality, authentication flows, rate limiting, or pagination in Reqon.
-
-## Capabilities
-
-### OAuth2 Token Refresh Flows
-- Implement automatic token refresh on 401 responses
-- Handle refresh token rotation
-- Store and retrieve tokens securely
-- Support authorization_code, client_credentials, and refresh_token grants
-
-### Rate Limiting with Exponential Backoff
-- Parse rate limit headers (X-RateLimit-*, Retry-After)
-- Implement exponential backoff with jitter
-- Queue requests when rate limited
-- Respect per-endpoint rate limits
-
-### Pagination Pattern Implementations
-- **Cursor-based**: Use `next_cursor` or `after` parameters
-- **Offset-based**: Use `offset` and `limit` parameters
-- **Page-based**: Use `page` and `per_page` parameters
-- **Link header**: Parse RFC 5988 Link headers
-
-## Context Files
-When using this skill, read:
-- `src/interpreter/http.ts` - HTTP client implementation
-- `src/interpreter/executor.ts` - Fetch execution logic
-- `src/ast/nodes.ts` - FetchNode, PaginationConfig types
-
-## Implementation Patterns
-
-### OAuth2 Token Refresh
-```typescript
-async function fetchWithAuth(url: string, auth: OAuth2Auth): Promise<Response> {
-  let response = await fetch(url, {
-    headers: { Authorization: `Bearer ${auth.accessToken}` }
-  });
-
-  if (response.status === 401 && auth.refreshToken) {
-    const newTokens = await refreshAccessToken(auth);
-    auth.accessToken = newTokens.access_token;
-    response = await fetch(url, {
-      headers: { Authorization: `Bearer ${auth.accessToken}` }
-    });
-  }
-
-  return response;
-}
-```
-
-### Exponential Backoff
-```typescript
-async function fetchWithBackoff(
-  url: string,
-  maxRetries = 5,
-  baseDelay = 1000
-): Promise<Response> {
-  for (let attempt = 0; attempt < maxRetries; attempt++) {
-    const response = await fetch(url);
-
-    if (response.status === 429) {
-      const retryAfter = response.headers.get('Retry-After');
-      const delay = retryAfter
-        ? parseInt(retryAfter) * 1000
-        : baseDelay * Math.pow(2, attempt) + Math.random() * 1000;
-      await sleep(delay);
-      continue;
-    }
-
-    return response;
-  }
-  throw new Error('Max retries exceeded');
-}
-```
-
-### Pagination Patterns
-
-#### Cursor-based
-```typescript
-async function* paginateCursor(baseUrl: string) {
-  let cursor: string | undefined;
-  do {
-    const url = cursor ? `${baseUrl}?cursor=${cursor}` : baseUrl;
-    const response = await fetch(url);
-    const data = await response.json();
-    yield data.items;
-    cursor = data.next_cursor;
-  } while (cursor);
-}
-```
-
-#### Offset-based
-```typescript
-async function* paginateOffset(baseUrl: string, limit = 100) {
-  let offset = 0;
-  let hasMore = true;
-  while (hasMore) {
-    const response = await fetch(`${baseUrl}?offset=${offset}&limit=${limit}`);
-    const data = await response.json();
-    yield data.items;
-    offset += limit;
-    hasMore = data.items.length === limit;
-  }
-}
-```
-
-#### Link Header
-```typescript
-function parseLinks(header: string): Record<string, string> {
-  const links: Record<string, string> = {};
-  header.split(',').forEach(part => {
-    const match = part.match(/<([^>]+)>;\s*rel="([^"]+)"/);
-    if (match) links[match[2]] = match[1];
-  });
-  return links;
-}
-```
-
-## Rate Limit Headers
-Common headers to parse:
-- `X-RateLimit-Limit` - Request limit per window
-- `X-RateLimit-Remaining` - Requests remaining
-- `X-RateLimit-Reset` - Unix timestamp when limit resets
-- `Retry-After` - Seconds to wait (on 429/503)

package/.claude/skills/database-schema.md

@@ -1,51 +0,0 @@
-# Database Schema Skill
-
-Use this skill when working on SQL/NoSQL store adapters or database-related functionality in Reqon.
-
-## Capabilities
-
-### Generating Store Implementations
-- Create PostgreSQL store adapters implementing the `StoreAdapter` interface from `src/stores/types.ts`
-- Create MySQL store adapters with appropriate driver usage
-- Create MongoDB/DynamoDB NoSQL adapters
-
-### Creating Migration Files
-- Generate SQL migration files for schema creation
-- Create up/down migration pairs for reversibility
-- Handle incremental schema changes
-
-### Type Mapping
-Map Reqon/Vague DSL types to database column types:
-
-| DSL Type | PostgreSQL | MySQL | MongoDB |
-|----------|------------|-------|---------|
-| string | TEXT/VARCHAR | VARCHAR | String |
-| number | NUMERIC/INTEGER | DECIMAL/INT | Number |
-| boolean | BOOLEAN | TINYINT(1) | Boolean |
-| array | JSONB | JSON | Array |
-| object | JSONB | JSON | Object |
-| date | TIMESTAMP | DATETIME | Date |
-
-## Context Files
-When using this skill, read:
-- `src/stores/types.ts` - StoreAdapter interface
-- `src/stores/memory.ts` - Reference implementation
-- `src/ast/nodes.ts` - Schema and type definitions
-
-## Implementation Patterns
-
-### Store Adapter Structure
-```typescript
-import { StoreAdapter, StoreRecord } from './types';
-
-export class PostgresStore implements StoreAdapter {
-  async get(key: string): Promise<StoreRecord | undefined> { }
-  async set(key: string, value: StoreRecord): Promise<void> { }
-  async delete(key: string): Promise<boolean> { }
-  async query(filter: Record<string, unknown>): Promise<StoreRecord[]> { }
-  async upsert(key: string, value: StoreRecord): Promise<void> { }
-}
-```
-
-### Migration File Naming
-Use timestamp-based naming: `YYYYMMDDHHMMSS_description.sql`

package/.claude/skills/dsl-design.md

@@ -1,80 +0,0 @@
-# Language/DSL Design Skill
-
-Use this skill when extending Reqon's DSL syntax, implementing new language constructs, or working on the lexer/parser.
-
-## Architecture Overview
-
-Reqon extends Vague's language infrastructure:
-- **Vague** provides: lexer base, expression syntax, match expressions, core AST
-- **Reqon** adds: mission/action/fetch/store constructs, execution semantics
-
-## Capabilities
-
-### Implementing New DSL Constructs
-- Add new keywords to `src/lexer/tokens.ts`
-- Extend the lexer in `src/lexer/lexer.ts`
-- Add parser rules in `src/parser/parser.ts`
-- Define AST nodes in `src/ast/nodes.ts`
-
-### Writing Parser Tests
-- Create comprehensive test cases in `src/parser/parser.test.ts`
-- Test edge cases and error conditions
-- Use Vitest's describe/it/expect patterns
-
-### Generating Better Error Messages
-- Include line and column numbers in parse errors
-- Provide context about what was expected
-- Suggest fixes for common mistakes
-
-## Context Files
-When using this skill, read:
-- `src/lexer/tokens.ts` - Token definitions
-- `src/lexer/lexer.ts` - Lexer implementation
-- `src/parser/parser.ts` - Main parser
-- `src/parser/base.ts` - Parser utilities
-- `src/ast/nodes.ts` - AST node types
-
-## Implementation Patterns
-
-### Adding a New Keyword
-1. Add token type to `ReqonTokenType` enum in `tokens.ts`
-2. Add keyword mapping in `REQON_KEYWORDS` in `lexer.ts`
-3. Add parsing logic in `parser.ts`
-4. Define AST node in `nodes.ts`
-
-### Token Definition Pattern
-```typescript
-export enum ReqonTokenType {
-  // ... existing tokens
-  NEW_KEYWORD = 'NEW_KEYWORD',
-}
-
-export const REQON_KEYWORDS: Record<string, ReqonTokenType> = {
-  // ... existing keywords
-  'newkeyword': ReqonTokenType.NEW_KEYWORD,
-};
-```
-
-### Parser Rule Pattern
-```typescript
-private parseNewConstruct(): NewConstructNode {
-  this.expect(ReqonTokenType.NEW_KEYWORD);
-  const name = this.parseIdentifier();
-  // ... parse body
-  return { type: 'NewConstruct', name, ... };
-}
-```
-
-### Error Message Pattern
-```typescript
-throw new ParseError(
-  `Expected ${expected} but found ${actual}`,
-  { line: token.line, column: token.column }
-);
-```
-
-## Planned Constructs (from TODO.md)
-- `is` type checking: `assume .items is array`
-- Parallel execution: `run Step1, Step2 then Step3`
-- Conditional actions: `run Step1 then Step2 if condition`
-- Variables/let bindings: `let x = expression`

package/.claude/skills/property-testing.md

@@ -1,143 +0,0 @@
-# Property-Based Testing Skill
-
-Use this skill when implementing property-based tests, fuzzing the parser, or generating mock API responses.
-
-## Capabilities
-
-### Generating Fuzzing Inputs for the Parser
-- Generate random valid Reqon programs
-- Generate edge-case inputs (empty strings, unicode, deeply nested)
-- Generate semi-valid inputs to test error recovery
-- Mutate valid programs to find parser bugs
-
-### Writing Vitest Property-Based Tests
-- Use fast-check or similar libraries
-- Define properties that should hold for all inputs
-- Shrink failing cases to minimal reproductions
-- Combine with traditional example-based tests
-
-### Creating Mock API Responses
-- Generate responses matching Reqon schemas
-- Create varied test data (edge cases, nulls, arrays)
-- Simulate pagination responses
-- Mock error responses (4xx, 5xx)
-
-## Context Files
-When using this skill, read:
-- `src/parser/parser.test.ts` - Existing parser tests
-- `src/integration.test.ts` - Integration test patterns
-- `src/ast/nodes.ts` - AST structure for generation
-- `src/lexer/tokens.ts` - Valid tokens for fuzzing
-
-## Implementation Patterns
-
-### Property-Based Test Setup
-```typescript
-import { describe, it, expect } from 'vitest';
-import * as fc from 'fast-check';
-import { parse } from './parser';
-import { lex } from '../lexer';
-
-describe('Parser Properties', () => {
-  it('should parse any valid mission without throwing', () => {
-    fc.assert(
-      fc.property(validMissionArb, (mission) => {
-        expect(() => parse(lex(mission))).not.toThrow();
-      })
-    );
-  });
-});
-```
-
-### Arbitrary Generators for Reqon
-
-```typescript
-// Generate valid identifiers
-const identifierArb = fc.stringOf(
-  fc.constantFrom(...'abcdefghijklmnopqrstuvwxyz_'),
-  { minLength: 1, maxLength: 20 }
-).filter(s => /^[a-z_][a-z0-9_]*$/i.test(s));
-
-// Generate valid string literals
-const stringLiteralArb = fc.string().map(s => `"${s.replace(/"/g, '\\"')}"`);
-
-// Generate valid fetch paths
-const pathArb = fc.array(identifierArb, { minLength: 1, maxLength: 5 })
-  .map(parts => '/' + parts.join('/'));
-
-// Generate valid actions
-const actionArb = fc.record({
-  name: identifierArb,
-  path: pathArb,
-}).map(({ name, path }) => `action ${name} { fetch ${path} }`);
-
-// Generate valid missions
-const validMissionArb = fc.record({
-  name: identifierArb,
-  url: fc.webUrl(),
-  actions: fc.array(actionArb, { minLength: 1, maxLength: 5 }),
-}).map(({ name, url, actions }) => `
-  mission ${name} {
-    source api { url: "${url}" }
-    ${actions.join('\n')}
-  }
-`);
-```
-
-### Mock Response Generation
-```typescript
-// Generate mock response matching schema
-const mockResponseArb = (schema: SchemaNode): fc.Arbitrary<unknown> => {
-  switch (schema.type) {
-    case 'string': return fc.string();
-    case 'number': return fc.double();
-    case 'boolean': return fc.boolean();
-    case 'array': return fc.array(mockResponseArb(schema.items));
-    case 'object': return fc.record(
-      Object.fromEntries(
-        Object.entries(schema.properties).map(
-          ([k, v]) => [k, mockResponseArb(v)]
-        )
-      )
-    );
-  }
-};
-
-// Generate paginated response
-const paginatedResponseArb = <T>(itemArb: fc.Arbitrary<T>) =>
-  fc.record({
-    items: fc.array(itemArb, { minLength: 0, maxLength: 100 }),
-    next_cursor: fc.option(fc.hexaString({ minLength: 16, maxLength: 16 })),
-    total: fc.nat(),
-  });
-```
-
-### Parser Fuzzing Strategies
-```typescript
-// Mutation-based fuzzing
-const mutateProgram = (valid: string): fc.Arbitrary<string> =>
-  fc.oneof(
-    // Delete random character
-    fc.nat({ max: valid.length - 1 }).map(i =>
-      valid.slice(0, i) + valid.slice(i + 1)
-    ),
-    // Insert random character
-    fc.tuple(fc.nat({ max: valid.length }), fc.char()).map(([i, c]) =>
-      valid.slice(0, i) + c + valid.slice(i)
-    ),
-    // Replace random character
-    fc.tuple(fc.nat({ max: valid.length - 1 }), fc.char()).map(([i, c]) =>
-      valid.slice(0, i) + c + valid.slice(i + 1)
-    )
-  );
-```
-
-## Dependencies
-Add to `package.json`:
-```json
-{
-  "devDependencies": {
-    "fast-check": "^3.15.0"
-  }
-}
-```

package/.claude/skills/reqon/SKILL.md

@@ -1,44 +0,0 @@
----
-name: reqon
-# prettier-ignore
-description: Use when writing or editing .vague files for Reqon declarative API data pipelines
----
-
-# Reqon
-
-Declarative DSL for fetch, map, validate pipelines. File extension: `.vague`
-
-## Quick Start
-
-```
-mission SyncData {
-  source API { auth: bearer, base: "https://api.example.com" }
-  store items: memory("items")
-
-  action Fetch {
-    get "/items" { paginate: page(page, 100), until: length(response) == 0 }
-    store response -> items { key: .id }
-  }
-
-  run Fetch
-}
-```
-
-## Core Constructs
-
-- `mission` - Pipeline container (sources, stores, schemas, actions)
-- `source` - API: auth (bearer/basic/api_key/oauth2), base, headers, rateLimit
-- `store` - Storage: `memory("name")`, `file("path")`, `sql("table")`
-- `action` - Pipeline step: fetch, map, validate, store
-- `run [A, B] then C` - Parallel then sequential execution
-- `match response { Schema -> ..., _ -> skip }` - Pattern matching
-- `for item in store where .active { ... }` - Iteration with filter
-
-## Flow Control
-
-`continue`, `skip`, `abort "msg"`, `retry {...}`, `queue dlq`, `jump Action then retry`
-
-## Reference Files
-
-- [references/syntax.md](references/syntax.md) - Full DSL syntax
-- [references/examples.md](references/examples.md) - Complete examples