reqon-dsl 0.2.0

package/.claude/settings.local.json

@@ -0,0 +1,31 @@
+{
+  "permissions": {
+    "allow": [
+      "Bash(npm init -y)",
+      "Bash(npm install)",
+      "Bash(npm run build:*)",
+      "Bash(npm run test:run:*)",
+      "Bash(npx vitest run src/debug.test.ts)",
+      "Bash(npx vitest:*)",
+      "Bash(git init:*)",
+      "Bash(git add:*)",
+      "Bash(git commit:*)",
+      "Bash(git push:*)",
+      "Bash(npm install:*)",
+      "Bash(node dist/cli.js:*)",
+      "Bash(ls:*)",
+      "Bash(node:*)",
+      "Bash(cat:*)",
+      "Bash(git checkout:*)",
+      "Bash(git rebase:*)",
+      "Bash(find:*)",
+      "Bash(git fetch:*)",
+      "Bash(xargs sed:*)",
+      "Bash(for f in examples/**/*.reqon)",
+      "Bash(do mv $f $f%.reqon.vague)",
+      "Bash(done)",
+      "Bash(npx:*)",
+      "Skill(reqon)"
+    ]
+  }
+}

package/.claude/skills/api-integration.md

@@ -0,0 +1,125 @@
+# API Integration Patterns Skill
+
+Use this skill when implementing API client functionality, authentication flows, rate limiting, or pagination in Reqon.
+
+## Capabilities
+
+### OAuth2 Token Refresh Flows
+- Implement automatic token refresh on 401 responses
+- Handle refresh token rotation
+- Store and retrieve tokens securely
+- Support authorization_code, client_credentials, and refresh_token grants
+
+### Rate Limiting with Exponential Backoff
+- Parse rate limit headers (X-RateLimit-*, Retry-After)
+- Implement exponential backoff with jitter
+- Queue requests when rate limited
+- Respect per-endpoint rate limits
+
+### Pagination Pattern Implementations
+- **Cursor-based**: Use `next_cursor` or `after` parameters
+- **Offset-based**: Use `offset` and `limit` parameters
+- **Page-based**: Use `page` and `per_page` parameters
+- **Link header**: Parse RFC 5988 Link headers
+
+## Context Files
+When using this skill, read:
+- `src/interpreter/http.ts` - HTTP client implementation
+- `src/interpreter/executor.ts` - Fetch execution logic
+- `src/ast/nodes.ts` - FetchNode, PaginationConfig types
+
+## Implementation Patterns
+
+### OAuth2 Token Refresh
+```typescript
+async function fetchWithAuth(url: string, auth: OAuth2Auth): Promise<Response> {
+  let response = await fetch(url, {
+    headers: { Authorization: `Bearer ${auth.accessToken}` }
+  });
+
+  if (response.status === 401 && auth.refreshToken) {
+    const newTokens = await refreshAccessToken(auth);
+    auth.accessToken = newTokens.access_token;
+    response = await fetch(url, {
+      headers: { Authorization: `Bearer ${auth.accessToken}` }
+    });
+  }
+
+  return response;
+}
+```
+
+### Exponential Backoff
+```typescript
+async function fetchWithBackoff(
+  url: string,
+  maxRetries = 5,
+  baseDelay = 1000
+): Promise<Response> {
+  for (let attempt = 0; attempt < maxRetries; attempt++) {
+    const response = await fetch(url);
+
+    if (response.status === 429) {
+      const retryAfter = response.headers.get('Retry-After');
+      const delay = retryAfter
+        ? parseInt(retryAfter) * 1000
+        : baseDelay * Math.pow(2, attempt) + Math.random() * 1000;
+      await sleep(delay);
+      continue;
+    }
+
+    return response;
+  }
+  throw new Error('Max retries exceeded');
+}
+```
+
+### Pagination Patterns
+
+#### Cursor-based
+```typescript
+async function* paginateCursor(baseUrl: string) {
+  let cursor: string | undefined;
+  do {
+    const url = cursor ? `${baseUrl}?cursor=${cursor}` : baseUrl;
+    const response = await fetch(url);
+    const data = await response.json();
+    yield data.items;
+    cursor = data.next_cursor;
+  } while (cursor);
+}
+```
+
+#### Offset-based
+```typescript
+async function* paginateOffset(baseUrl: string, limit = 100) {
+  let offset = 0;
+  let hasMore = true;
+  while (hasMore) {
+    const response = await fetch(`${baseUrl}?offset=${offset}&limit=${limit}`);
+    const data = await response.json();
+    yield data.items;
+    offset += limit;
+    hasMore = data.items.length === limit;
+  }
+}
+```
+
+#### Link Header
+```typescript
+function parseLinks(header: string): Record<string, string> {
+  const links: Record<string, string> = {};
+  header.split(',').forEach(part => {
+    const match = part.match(/<([^>]+)>;\s*rel="([^"]+)"/);
+    if (match) links[match[2]] = match[1];
+  });
+  return links;
+}
+```
+
+## Rate Limit Headers
+Common headers to parse:
+- `X-RateLimit-Limit` - Request limit per window
+- `X-RateLimit-Remaining` - Requests remaining
+- `X-RateLimit-Reset` - Unix timestamp when limit resets
+- `Retry-After` - Seconds to wait (on 429/503)

package/.claude/skills/database-schema.md

@@ -0,0 +1,51 @@
+# Database Schema Skill
+
+Use this skill when working on SQL/NoSQL store adapters or database-related functionality in Reqon.
+
+## Capabilities
+
+### Generating Store Implementations
+- Create PostgreSQL store adapters implementing the `StoreAdapter` interface from `src/stores/types.ts`
+- Create MySQL store adapters with appropriate driver usage
+- Create MongoDB/DynamoDB NoSQL adapters
+
+### Creating Migration Files
+- Generate SQL migration files for schema creation
+- Create up/down migration pairs for reversibility
+- Handle incremental schema changes
+
+### Type Mapping
+Map Reqon/Vague DSL types to database column types:
+
+| DSL Type | PostgreSQL | MySQL | MongoDB |
+|----------|------------|-------|---------|
+| string | TEXT/VARCHAR | VARCHAR | String |
+| number | NUMERIC/INTEGER | DECIMAL/INT | Number |
+| boolean | BOOLEAN | TINYINT(1) | Boolean |
+| array | JSONB | JSON | Array |
+| object | JSONB | JSON | Object |
+| date | TIMESTAMP | DATETIME | Date |
+
+## Context Files
+When using this skill, read:
+- `src/stores/types.ts` - StoreAdapter interface
+- `src/stores/memory.ts` - Reference implementation
+- `src/ast/nodes.ts` - Schema and type definitions
+
+## Implementation Patterns
+
+### Store Adapter Structure
+```typescript
+import { StoreAdapter, StoreRecord } from './types';
+
+export class PostgresStore implements StoreAdapter {
+  async get(key: string): Promise<StoreRecord | undefined> { }
+  async set(key: string, value: StoreRecord): Promise<void> { }
+  async delete(key: string): Promise<boolean> { }
+  async query(filter: Record<string, unknown>): Promise<StoreRecord[]> { }
+  async upsert(key: string, value: StoreRecord): Promise<void> { }
+}
+```
+
+### Migration File Naming
+Use timestamp-based naming: `YYYYMMDDHHMMSS_description.sql`

package/.claude/skills/dsl-design.md

@@ -0,0 +1,80 @@
+# Language/DSL Design Skill
+
+Use this skill when extending Reqon's DSL syntax, implementing new language constructs, or working on the lexer/parser.
+
+## Architecture Overview
+
+Reqon extends Vague's language infrastructure:
+- **Vague** provides: lexer base, expression syntax, match expressions, core AST
+- **Reqon** adds: mission/action/fetch/store constructs, execution semantics
+
+## Capabilities
+
+### Implementing New DSL Constructs
+- Add new keywords to `src/lexer/tokens.ts`
+- Extend the lexer in `src/lexer/lexer.ts`
+- Add parser rules in `src/parser/parser.ts`
+- Define AST nodes in `src/ast/nodes.ts`
+
+### Writing Parser Tests
+- Create comprehensive test cases in `src/parser/parser.test.ts`
+- Test edge cases and error conditions
+- Use Vitest's describe/it/expect patterns
+
+### Generating Better Error Messages
+- Include line and column numbers in parse errors
+- Provide context about what was expected
+- Suggest fixes for common mistakes
+
+## Context Files
+When using this skill, read:
+- `src/lexer/tokens.ts` - Token definitions
+- `src/lexer/lexer.ts` - Lexer implementation
+- `src/parser/parser.ts` - Main parser
+- `src/parser/base.ts` - Parser utilities
+- `src/ast/nodes.ts` - AST node types
+
+## Implementation Patterns
+
+### Adding a New Keyword
+1. Add token type to `ReqonTokenType` enum in `tokens.ts`
+2. Add keyword mapping in `REQON_KEYWORDS` in `lexer.ts`
+3. Add parsing logic in `parser.ts`
+4. Define AST node in `nodes.ts`
+
+### Token Definition Pattern
+```typescript
+export enum ReqonTokenType {
+  // ... existing tokens
+  NEW_KEYWORD = 'NEW_KEYWORD',
+}
+
+export const REQON_KEYWORDS: Record<string, ReqonTokenType> = {
+  // ... existing keywords
+  'newkeyword': ReqonTokenType.NEW_KEYWORD,
+};
+```
+
+### Parser Rule Pattern
+```typescript
+private parseNewConstruct(): NewConstructNode {
+  this.expect(ReqonTokenType.NEW_KEYWORD);
+  const name = this.parseIdentifier();
+  // ... parse body
+  return { type: 'NewConstruct', name, ... };
+}
+```
+
+### Error Message Pattern
+```typescript
+throw new ParseError(
+  `Expected ${expected} but found ${actual}`,
+  { line: token.line, column: token.column }
+);
+```
+
+## Planned Constructs (from TODO.md)
+- `is` type checking: `assume .items is array`
+- Parallel execution: `run Step1, Step2 then Step3`
+- Conditional actions: `run Step1 then Step2 if condition`
+- Variables/let bindings: `let x = expression`

package/.claude/skills/property-testing.md

@@ -0,0 +1,143 @@
+# Property-Based Testing Skill
+
+Use this skill when implementing property-based tests, fuzzing the parser, or generating mock API responses.
+
+## Capabilities
+
+### Generating Fuzzing Inputs for the Parser
+- Generate random valid Reqon programs
+- Generate edge-case inputs (empty strings, unicode, deeply nested)
+- Generate semi-valid inputs to test error recovery
+- Mutate valid programs to find parser bugs
+
+### Writing Vitest Property-Based Tests
+- Use fast-check or similar libraries
+- Define properties that should hold for all inputs
+- Shrink failing cases to minimal reproductions
+- Combine with traditional example-based tests
+
+### Creating Mock API Responses
+- Generate responses matching Reqon schemas
+- Create varied test data (edge cases, nulls, arrays)
+- Simulate pagination responses
+- Mock error responses (4xx, 5xx)
+
+## Context Files
+When using this skill, read:
+- `src/parser/parser.test.ts` - Existing parser tests
+- `src/integration.test.ts` - Integration test patterns
+- `src/ast/nodes.ts` - AST structure for generation
+- `src/lexer/tokens.ts` - Valid tokens for fuzzing
+
+## Implementation Patterns
+
+### Property-Based Test Setup
+```typescript
+import { describe, it, expect } from 'vitest';
+import * as fc from 'fast-check';
+import { parse } from './parser';
+import { lex } from '../lexer';
+
+describe('Parser Properties', () => {
+  it('should parse any valid mission without throwing', () => {
+    fc.assert(
+      fc.property(validMissionArb, (mission) => {
+        expect(() => parse(lex(mission))).not.toThrow();
+      })
+    );
+  });
+});
+```
+
+### Arbitrary Generators for Reqon
+
+```typescript
+// Generate valid identifiers
+const identifierArb = fc.stringOf(
+  fc.constantFrom(...'abcdefghijklmnopqrstuvwxyz_'),
+  { minLength: 1, maxLength: 20 }
+).filter(s => /^[a-z_][a-z0-9_]*$/i.test(s));
+
+// Generate valid string literals
+const stringLiteralArb = fc.string().map(s => `"${s.replace(/"/g, '\\"')}"`);
+
+// Generate valid fetch paths
+const pathArb = fc.array(identifierArb, { minLength: 1, maxLength: 5 })
+  .map(parts => '/' + parts.join('/'));
+
+// Generate valid actions
+const actionArb = fc.record({
+  name: identifierArb,
+  path: pathArb,
+}).map(({ name, path }) => `action ${name} { fetch ${path} }`);
+
+// Generate valid missions
+const validMissionArb = fc.record({
+  name: identifierArb,
+  url: fc.webUrl(),
+  actions: fc.array(actionArb, { minLength: 1, maxLength: 5 }),
+}).map(({ name, url, actions }) => `
+  mission ${name} {
+    source api { url: "${url}" }
+    ${actions.join('\n')}
+  }
+`);
+```
+
+### Mock Response Generation
+```typescript
+// Generate mock response matching schema
+const mockResponseArb = (schema: SchemaNode): fc.Arbitrary<unknown> => {
+  switch (schema.type) {
+    case 'string': return fc.string();
+    case 'number': return fc.double();
+    case 'boolean': return fc.boolean();
+    case 'array': return fc.array(mockResponseArb(schema.items));
+    case 'object': return fc.record(
+      Object.fromEntries(
+        Object.entries(schema.properties).map(
+          ([k, v]) => [k, mockResponseArb(v)]
+        )
+      )
+    );
+  }
+};
+
+// Generate paginated response
+const paginatedResponseArb = <T>(itemArb: fc.Arbitrary<T>) =>
+  fc.record({
+    items: fc.array(itemArb, { minLength: 0, maxLength: 100 }),
+    next_cursor: fc.option(fc.hexaString({ minLength: 16, maxLength: 16 })),
+    total: fc.nat(),
+  });
+```
+
+### Parser Fuzzing Strategies
+```typescript
+// Mutation-based fuzzing
+const mutateProgram = (valid: string): fc.Arbitrary<string> =>
+  fc.oneof(
+    // Delete random character
+    fc.nat({ max: valid.length - 1 }).map(i =>
+      valid.slice(0, i) + valid.slice(i + 1)
+    ),
+    // Insert random character
+    fc.tuple(fc.nat({ max: valid.length }), fc.char()).map(([i, c]) =>
+      valid.slice(0, i) + c + valid.slice(i)
+    ),
+    // Replace random character
+    fc.tuple(fc.nat({ max: valid.length - 1 }), fc.char()).map(([i, c]) =>
+      valid.slice(0, i) + c + valid.slice(i + 1)
+    )
+  );
+```
+
+## Dependencies
+Add to `package.json`:
+```json
+{
+  "devDependencies": {
+    "fast-check": "^3.15.0"
+  }
+}
+```

package/.claude/skills/reqon/SKILL.md

@@ -0,0 +1,44 @@
+---
+name: reqon
+# prettier-ignore
+description: Use when writing or editing .vague files for Reqon declarative API data pipelines
+---
+
+# Reqon
+
+Declarative DSL for fetch, map, validate pipelines. File extension: `.vague`
+
+## Quick Start
+
+```
+mission SyncData {
+  source API { auth: bearer, base: "https://api.example.com" }
+  store items: memory("items")
+
+  action Fetch {
+    get "/items" { paginate: page(page, 100), until: length(response) == 0 }
+    store response -> items { key: .id }
+  }
+
+  run Fetch
+}
+```
+
+## Core Constructs
+
+- `mission` - Pipeline container (sources, stores, schemas, actions)
+- `source` - API: auth (bearer/basic/api_key/oauth2), base, headers, rateLimit
+- `store` - Storage: `memory("name")`, `file("path")`, `sql("table")`
+- `action` - Pipeline step: fetch, map, validate, store
+- `run [A, B] then C` - Parallel then sequential execution
+- `match response { Schema -> ..., _ -> skip }` - Pattern matching
+- `for item in store where .active { ... }` - Iteration with filter
+
+## Flow Control
+
+`continue`, `skip`, `abort "msg"`, `retry {...}`, `queue dlq`, `jump Action then retry`
+
+## Reference Files
+
+- [references/syntax.md](references/syntax.md) - Full DSL syntax
+- [references/examples.md](references/examples.md) - Complete examples

package/.claude/skills/reqon/references/examples.md

@@ -0,0 +1,206 @@
+# Reqon Examples
+
+## Simple API Sync
+
+```
+mission SyncUsers {
+  source API {
+    auth: bearer,
+    base: "https://api.example.com"
+  }
+
+  store users: memory("users")
+
+  action FetchUsers {
+    get "/users" {
+      paginate: page(page, 50),
+      until: length(response) == 0
+    }
+    store response -> users { key: .id }
+  }
+
+  run FetchUsers
+}
+```
+
+## Multi-File Mission Structure
+
+```
+project/
+├── mission.vague        # Main: sources, stores, schemas, pipeline
+├── fetch-data.vague     # Action: FetchData
+├── transform.vague      # Action: Transform
+└── validate.vague       # Action: Validate
+```
+
+Actions in separate files are automatically merged into the mission.
+
+## Error Handling with Match
+
+```
+action FetchWithErrorHandling {
+  get "/data"
+
+  match response {
+    [DataSchema] -> {
+      store response -> data { key: .id }
+    },
+
+    RateLimitError -> retry {
+      maxAttempts: 5,
+      backoff: exponential,
+      initialDelay: 60000
+    },
+
+    AuthError -> jump RefreshToken then retry,
+
+    NotFoundError -> abort "Resource not found",
+
+    _ -> {
+      store {
+        error: "Unexpected response",
+        response: response,
+        timestamp: now()
+      } -> errors { key: now() }
+    }
+  }
+}
+```
+
+## Data Transformation Pipeline
+
+```
+action TransformData {
+  for item in raw_items {
+    map item -> NormalizedItem {
+      id: "item_" + .id,
+      title: .name,
+      description: .body,
+      status: match .state {
+        "open" => "active",
+        "closed" => "completed",
+        _ => "unknown"
+      },
+      author: .user.login,
+      created_at: .created_at,
+      synced_at: now()
+    }
+
+    validate response {
+      assume length(.title) > 0
+      assume .id != null
+    }
+
+    store response -> normalized_items {
+      key: .id,
+      upsert: true
+    }
+  }
+}
+```
+
+## Parallel Execution
+
+```
+mission ParallelSync {
+  source GitHubAPI { auth: bearer, base: "https://api.github.com" }
+
+  store issues: memory("issues")
+  store prs: memory("prs")
+  store work_items: memory("work_items")
+
+  action FetchIssues {
+    get "/repos/{owner}/{repo}/issues"
+    store response -> issues { key: .id }
+  }
+
+  action FetchPRs {
+    get "/repos/{owner}/{repo}/pulls"
+    store response -> prs { key: .id }
+  }
+
+  action Normalize {
+    // Process both issues and PRs after parallel fetch
+    for issue in issues { ... }
+    for pr in prs { ... }
+  }
+
+  // FetchIssues and FetchPRs run in parallel, then Normalize
+  run [FetchIssues, FetchPRs] then Normalize
+}
+```
+
+## Incremental Sync
+
+```
+action IncrementalFetch {
+  get "/items" {
+    body: { "updated_after": lastSync },
+    since: lastSync
+  }
+  store response -> items { key: .id, upsert: true }
+}
+```
+
+## Dead Letter Queue Pattern
+
+```
+action ProcessWithDLQ {
+  for item in pending {
+    post "/process/{item.id}"
+
+    match response {
+      SuccessSchema -> {
+        store response -> completed { key: .id }
+      },
+
+      TransientError -> retry {
+        maxAttempts: 3,
+        backoff: exponential,
+        initialDelay: 1000
+      },
+
+      // After retries exhausted or permanent error, queue for later
+      _ -> queue dead_letter_queue
+    }
+  }
+}
+```
+
+## Conditional Processing
+
+```
+action ProcessConditionally {
+  for payment in pending_payments
+    where not exists(fraud_queue[payment.id]) {
+
+    post "/payments/{payment.id}/capture"
+
+    match response {
+      PaymentSuccess -> {
+        store response -> completed { key: .id }
+      },
+      _ -> skip
+    }
+  }
+}
+```
+
+## OpenAPI Integration
+
+```
+mission OpenAPIExample {
+  source API from "./openapi.yaml" {
+    auth: bearer,
+    base: "https://api.example.com"
+  }
+
+  action FetchUsers {
+    // Use operation ID from OpenAPI spec
+    call API.listUsers { query: { limit: 100 } }
+    store response -> users { key: .id }
+  }
+
+  run FetchUsers
+}
+```