reqon-dsl 0.2.0

package/src/sync/store.ts

@@ -0,0 +1,139 @@
+import { join } from 'node:path';
+import type { SyncCheckpoint } from './state.js';
+import { EPOCH } from './state.js';
+import {
+  ensureParentDirectory,
+  writeJsonFile,
+  readJsonFile,
+  restoreDates,
+} from '../utils/file.js';
+
+/**
+ * Sync store interface - persists sync checkpoints
+ */
+export interface SyncStore {
+  /** Get last sync timestamp for a key */
+  getLastSync(key: string): Promise<Date>;
+
+  /** Get checkpoint details for a key */
+  getCheckpoint(key: string): Promise<SyncCheckpoint | null>;
+
+  /** Record a successful sync */
+  recordSync(checkpoint: SyncCheckpoint): Promise<void>;
+
+  /** List all checkpoints */
+  list(): Promise<SyncCheckpoint[]>;
+
+  /** Clear a specific checkpoint */
+  clear(key: string): Promise<void>;
+
+  /** Clear all checkpoints */
+  clearAll(): Promise<void>;
+}
+
+/**
+ * File-based sync store
+ * Stores sync state in .reqon-data/sync/{mission}.json
+ */
+export class FileSyncStore implements SyncStore {
+  private filePath: string;
+  private checkpoints: Map<string, SyncCheckpoint> = new Map();
+  private initialized: Promise<void>;
+
+  constructor(mission: string, baseDir = '.reqon-data/sync') {
+    this.filePath = join(baseDir, `${mission}.json`);
+    this.initialized = this.init();
+  }
+
+  private async init(): Promise<void> {
+    await ensureParentDirectory(this.filePath);
+    await this.load();
+  }
+
+  private async load(): Promise<void> {
+    const data = await readJsonFile<Record<string, SyncCheckpoint>>(this.filePath);
+    if (data) {
+      for (const [key, checkpoint] of Object.entries(data)) {
+        // Restore Date objects
+        restoreDates(checkpoint as unknown as Record<string, unknown>, ['syncedAt']);
+        this.checkpoints.set(key, checkpoint);
+      }
+    } else {
+      this.checkpoints = new Map();
+    }
+  }
+
+  private async persist(): Promise<void> {
+    const data: Record<string, SyncCheckpoint> = {};
+    for (const [key, checkpoint] of this.checkpoints) {
+      data[key] = checkpoint;
+    }
+    await writeJsonFile(this.filePath, data);
+  }
+
+  async getLastSync(key: string): Promise<Date> {
+    await this.initialized;
+    const checkpoint = this.checkpoints.get(key);
+    return checkpoint?.syncedAt ?? EPOCH;
+  }
+
+  async getCheckpoint(key: string): Promise<SyncCheckpoint | null> {
+    await this.initialized;
+    return this.checkpoints.get(key) ?? null;
+  }
+
+  async recordSync(checkpoint: SyncCheckpoint): Promise<void> {
+    await this.initialized;
+    this.checkpoints.set(checkpoint.key, checkpoint);
+    await this.persist();
+  }
+
+  async list(): Promise<SyncCheckpoint[]> {
+    await this.initialized;
+    return Array.from(this.checkpoints.values());
+  }
+
+  async clear(key: string): Promise<void> {
+    await this.initialized;
+    this.checkpoints.delete(key);
+    await this.persist();
+  }
+
+  async clearAll(): Promise<void> {
+    await this.initialized;
+    this.checkpoints.clear();
+    await this.persist();
+  }
+}
+
+/**
+ * In-memory sync store (for testing)
+ */
+export class MemorySyncStore implements SyncStore {
+  private checkpoints: Map<string, SyncCheckpoint> = new Map();
+
+  async getLastSync(key: string): Promise<Date> {
+    const checkpoint = this.checkpoints.get(key);
+    return checkpoint?.syncedAt ?? EPOCH;
+  }
+
+  async getCheckpoint(key: string): Promise<SyncCheckpoint | null> {
+    return this.checkpoints.get(key) ?? null;
+  }
+
+  async recordSync(checkpoint: SyncCheckpoint): Promise<void> {
+    this.checkpoints.set(checkpoint.key, { ...checkpoint });
+  }
+
+  async list(): Promise<SyncCheckpoint[]> {
+    return Array.from(this.checkpoints.values());
+  }
+
+  async clear(key: string): Promise<void> {
+    this.checkpoints.delete(key);
+  }
+
+  async clearAll(): Promise<void> {
+    this.checkpoints.clear();
+  }
+}

package/src/sync/sync.test.ts

@@ -0,0 +1,271 @@
+import { describe, it, expect, beforeEach, afterEach } from 'vitest';
+import { rmSync, existsSync } from 'node:fs';
+import {
+  generateCheckpointKey,
+  formatSinceDate,
+  parseSinceDate,
+  EPOCH,
+} from './state.js';
+import { FileSyncStore, MemorySyncStore } from './store.js';
+import { ReqonLexer } from '../lexer/index.js';
+import { ReqonParser } from '../parser/parser.js';
+import type { FetchStep, MissionDefinition } from '../ast/nodes.js';
+
+const TEST_DIR = '.reqon-test-sync';
+
+describe('Sync State Utilities', () => {
+  describe('generateCheckpointKey', () => {
+    it('generates key from source and operationId', () => {
+      const key = generateCheckpointKey('Xero', 'getInvoices');
+      expect(key).toBe('Xero:getInvoices');
+    });
+
+    it('generates key from source and endpoint', () => {
+      const key = generateCheckpointKey('API', undefined, '/invoices');
+      expect(key).toBe('API:/invoices');
+    });
+
+    it('normalizes endpoints (removes trailing slashes)', () => {
+      const key = generateCheckpointKey('API', undefined, '/invoices/');
+      expect(key).toBe('API:/invoices');
+    });
+
+    it('falls back to source only', () => {
+      const key = generateCheckpointKey('API');
+      expect(key).toBe('API');
+    });
+  });
+
+  describe('formatSinceDate', () => {
+    const date = new Date('2024-01-15T10:30:00.000Z');
+
+    it('formats as ISO string', () => {
+      expect(formatSinceDate(date, 'iso')).toBe('2024-01-15T10:30:00.000Z');
+    });
+
+    it('formats as Unix timestamp (seconds)', () => {
+      expect(formatSinceDate(date, 'unix')).toBe('1705314600');
+    });
+
+    it('formats as Unix timestamp (milliseconds)', () => {
+      expect(formatSinceDate(date, 'unix-ms')).toBe('1705314600000');
+    });
+
+    it('formats as date only', () => {
+      expect(formatSinceDate(date, 'date-only')).toBe('2024-01-15');
+    });
+  });
+
+  describe('parseSinceDate', () => {
+    it('parses ISO string', () => {
+      const result = parseSinceDate('2024-01-15T10:30:00.000Z');
+      expect(result).toBeInstanceOf(Date);
+      expect(result?.toISOString()).toBe('2024-01-15T10:30:00.000Z');
+    });
+
+    it('parses Unix timestamp (seconds)', () => {
+      const result = parseSinceDate(1705314600);
+      expect(result).toBeInstanceOf(Date);
+      expect(result?.getTime()).toBe(1705314600000);
+    });
+
+    it('parses Unix timestamp (milliseconds)', () => {
+      const result = parseSinceDate(1705314600000);
+      expect(result).toBeInstanceOf(Date);
+      expect(result?.getTime()).toBe(1705314600000);
+    });
+
+    it('returns null for invalid input', () => {
+      expect(parseSinceDate('not a date')).toBeNull();
+      expect(parseSinceDate(null)).toBeNull();
+      expect(parseSinceDate(undefined)).toBeNull();
+    });
+
+    it('passes through Date objects', () => {
+      const date = new Date();
+      expect(parseSinceDate(date)).toBe(date);
+    });
+  });
+});
+
+describe('MemorySyncStore', () => {
+  let store: MemorySyncStore;
+
+  beforeEach(() => {
+    store = new MemorySyncStore();
+  });
+
+  it('returns EPOCH for unknown keys', async () => {
+    const lastSync = await store.getLastSync('unknown');
+    expect(lastSync.getTime()).toBe(EPOCH.getTime());
+  });
+
+  it('records and retrieves sync checkpoints', async () => {
+    const syncedAt = new Date();
+    await store.recordSync({
+      key: 'Xero:getInvoices',
+      syncedAt,
+      recordCount: 100,
+    });
+
+    const lastSync = await store.getLastSync('Xero:getInvoices');
+    expect(lastSync.getTime()).toBe(syncedAt.getTime());
+
+    const checkpoint = await store.getCheckpoint('Xero:getInvoices');
+    expect(checkpoint?.recordCount).toBe(100);
+  });
+
+  it('lists all checkpoints', async () => {
+    await store.recordSync({ key: 'A', syncedAt: new Date() });
+    await store.recordSync({ key: 'B', syncedAt: new Date() });
+
+    const all = await store.list();
+    expect(all).toHaveLength(2);
+  });
+
+  it('clears specific checkpoint', async () => {
+    await store.recordSync({ key: 'A', syncedAt: new Date() });
+    await store.recordSync({ key: 'B', syncedAt: new Date() });
+    await store.clear('A');
+
+    expect(await store.getCheckpoint('A')).toBeNull();
+    expect(await store.getCheckpoint('B')).not.toBeNull();
+  });
+
+  it('clears all checkpoints', async () => {
+    await store.recordSync({ key: 'A', syncedAt: new Date() });
+    await store.recordSync({ key: 'B', syncedAt: new Date() });
+    await store.clearAll();
+
+    const all = await store.list();
+    expect(all).toHaveLength(0);
+  });
+});
+
+describe('FileSyncStore', () => {
+  let store: FileSyncStore;
+
+  beforeEach(() => {
+    if (existsSync(TEST_DIR)) {
+      rmSync(TEST_DIR, { recursive: true });
+    }
+    store = new FileSyncStore('test-mission', TEST_DIR);
+  });
+
+  afterEach(() => {
+    if (existsSync(TEST_DIR)) {
+      rmSync(TEST_DIR, { recursive: true });
+    }
+  });
+
+  it('creates sync directory', async () => {
+    // Trigger initialization by calling any async method
+    await store.list();
+    expect(existsSync(TEST_DIR)).toBe(true);
+  });
+
+  it('persists checkpoints to disk', async () => {
+    const syncedAt = new Date();
+    await store.recordSync({
+      key: 'test-key',
+      syncedAt,
+      recordCount: 50,
+    });
+
+    // Create new store instance
+    const newStore = new FileSyncStore('test-mission', TEST_DIR);
+    const checkpoint = await newStore.getCheckpoint('test-key');
+
+    expect(checkpoint).not.toBeNull();
+    expect(checkpoint?.syncedAt.getTime()).toBe(syncedAt.getTime());
+    expect(checkpoint?.recordCount).toBe(50);
+  });
+});
+
+describe('Parser: since config', () => {
+  function parse(source: string) {
+    const lexer = new ReqonLexer(source);
+    const tokens = lexer.tokenize();
+    const parser = new ReqonParser(tokens);
+    return parser.parse();
+  }
+
+  it('parses since: lastSync', () => {
+    const source = `
+      mission Test {
+        source API { auth: bearer, base: "https://api.example.com" }
+        store items: memory("items")
+
+        action Fetch {
+          get "/items" {
+            since: lastSync
+          }
+          store response -> items { key: .id }
+        }
+
+        run Fetch
+      }
+    `;
+
+    const program = parse(source);
+    const mission = program.statements[0] as MissionDefinition;
+    const fetch = mission.actions[0].steps[0] as FetchStep;
+
+    expect(fetch.since).toBeDefined();
+    expect(fetch.since?.type).toBe('lastSync');
+  });
+
+  it('parses since: lastSync("custom-key")', () => {
+    const source = `
+      mission Test {
+        source API { auth: bearer, base: "https://api.example.com" }
+        store items: memory("items")
+
+        action Fetch {
+          get "/items" {
+            since: lastSync("my-custom-key")
+          }
+          store response -> items { key: .id }
+        }
+
+        run Fetch
+      }
+    `;
+
+    const program = parse(source);
+    const mission = program.statements[0] as MissionDefinition;
+    const fetch = mission.actions[0].steps[0] as FetchStep;
+
+    expect(fetch.since?.type).toBe('lastSync');
+    expect(fetch.since?.key).toBe('my-custom-key');
+  });
+
+  it('parses since: lastSync with options', () => {
+    const source = `
+      mission Test {
+        source API { auth: bearer, base: "https://api.example.com" }
+        store items: memory("items")
+
+        action Fetch {
+          get "/items" {
+            since: lastSync {
+              param: "modified_since",
+              format: unix
+            }
+          }
+          store response -> items { key: .id }
+        }
+
+        run Fetch
+      }
+    `;
+
+    const program = parse(source);
+    const mission = program.statements[0] as MissionDefinition;
+    const fetch = mission.actions[0].steps[0] as FetchStep;
+
+    expect(fetch.since?.type).toBe('lastSync');
+    expect(fetch.since?.param).toBe('modified_since');
+    expect(fetch.since?.format).toBe('unix');
+  });
+});

package/src/utils/async.ts

@@ -0,0 +1,10 @@
+/**
+ * Utility functions for async operations
+ */
+
+/**
+ * Sleep for a specified number of milliseconds
+ */
+export function sleep(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}

package/src/utils/file.ts

@@ -0,0 +1,106 @@
+import { mkdir, readFile, writeFile, access, readdir, unlink } from 'node:fs/promises';
+import { dirname, join } from 'node:path';
+
+/**
+ * Ensure a directory exists, creating it if necessary
+ */
+export async function ensureDirectory(path: string): Promise<void> {
+  try {
+    await access(path);
+  } catch {
+    await mkdir(path, { recursive: true });
+  }
+}
+
+/**
+ * Ensure the parent directory of a file exists
+ */
+export async function ensureParentDirectory(filePath: string): Promise<void> {
+  await ensureDirectory(dirname(filePath));
+}
+
+/**
+ * Serialize data to JSON with proper formatting
+ */
+export function serialize(data: unknown, pretty = true): string {
+  return pretty ? JSON.stringify(data, null, 2) : JSON.stringify(data);
+}
+
+/**
+ * Write JSON data to a file
+ */
+export async function writeJsonFile(filePath: string, data: unknown, pretty = true): Promise<void> {
+  await writeFile(filePath, serialize(data, pretty), 'utf-8');
+}
+
+/**
+ * Read JSON data from a file
+ * Returns null if file doesn't exist or is corrupted
+ */
+export async function readJsonFile<T>(filePath: string): Promise<T | null> {
+  try {
+    const content = await readFile(filePath, 'utf-8');
+    return JSON.parse(content) as T;
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * List files in a directory with optional extension filter
+ */
+export async function listFiles(
+  dir: string,
+  extension?: string
+): Promise<string[]> {
+  try {
+    const entries = await readdir(dir);
+    const filtered = extension
+      ? entries.filter((f) => f.endsWith(extension))
+      : entries;
+    return filtered.map((f) => join(dir, f));
+  } catch {
+    return [];
+  }
+}
+
+/**
+ * Delete a file (silently ignores if file doesn't exist)
+ */
+export async function deleteFile(filePath: string): Promise<void> {
+  try {
+    await unlink(filePath);
+  } catch {
+    // File doesn't exist, nothing to delete
+  }
+}
+
+/**
+ * Restore Date objects in a parsed JSON object
+ * Mutates the object in place and returns it
+ */
+export function restoreDates<T extends Record<string, unknown>>(
+  obj: T,
+  dateFields: string[]
+): T {
+  const mutableObj = obj as Record<string, unknown>;
+  for (const field of dateFields) {
+    if (field in mutableObj && mutableObj[field] !== null && mutableObj[field] !== undefined) {
+      mutableObj[field] = new Date(mutableObj[field] as string | number);
+    }
+  }
+  return obj;
+}
+
+/**
+ * Restore Date objects in nested arrays
+ */
+export function restoreDatesInArray<T extends Record<string, unknown>>(
+  arr: T[],
+  dateFields: string[]
+): T[] {
+  for (const item of arr) {
+    restoreDates(item, dateFields);
+  }
+  return arr;
+}

package/src/utils/index.ts

@@ -0,0 +1,14 @@
+export { sleep } from './async.js';
+export { extractNestedValue, traversePath } from './path.js';
+export { type Logger, ConsoleLogger, SilentLogger, createLogger } from './logger.js';
+export {
+  ensureDirectory,
+  ensureParentDirectory,
+  serialize,
+  writeJsonFile,
+  readJsonFile,
+  listFiles,
+  deleteFile,
+  restoreDates,
+  restoreDatesInArray,
+} from './file.js';

package/src/utils/logger.ts

@@ -0,0 +1,53 @@
+/**
+ * Logger interface for injectable logging
+ */
+export interface Logger {
+  debug(message: string, ...args: unknown[]): void;
+  info(message: string, ...args: unknown[]): void;
+  warn(message: string, ...args: unknown[]): void;
+  error(message: string, ...args: unknown[]): void;
+}
+
+/**
+ * Console-based logger implementation
+ */
+export class ConsoleLogger implements Logger {
+  constructor(private prefix?: string) {}
+
+  private format(message: string): string {
+    return this.prefix ? `[${this.prefix}] ${message}` : message;
+  }
+
+  debug(message: string, ...args: unknown[]): void {
+    console.debug(this.format(message), ...args);
+  }
+
+  info(message: string, ...args: unknown[]): void {
+    console.log(this.format(message), ...args);
+  }
+
+  warn(message: string, ...args: unknown[]): void {
+    console.warn(this.format(message), ...args);
+  }
+
+  error(message: string, ...args: unknown[]): void {
+    console.error(this.format(message), ...args);
+  }
+}
+
+/**
+ * Silent logger that discards all messages
+ */
+export class SilentLogger implements Logger {
+  debug(): void {}
+  info(): void {}
+  warn(): void {}
+  error(): void {}
+}
+
+/**
+ * Create a logger with optional prefix
+ */
+export function createLogger(prefix?: string, silent = false): Logger {
+  return silent ? new SilentLogger() : new ConsoleLogger(prefix);
+}

package/src/utils/path.ts

@@ -0,0 +1,47 @@
+/**
+ * Utility functions for path traversal and value extraction
+ */
+
+/**
+ * Extract a nested value from an object using dot notation path
+ * @example extractNestedValue({ a: { b: 1 } }, 'a.b') // => 1
+ */
+export function extractNestedValue(data: Record<string, unknown>, path: string): unknown {
+  const parts = path.split('.');
+  let value: unknown = data;
+
+  for (const part of parts) {
+    if (value && typeof value === 'object') {
+      value = (value as Record<string, unknown>)[part];
+    } else {
+      return undefined;
+    }
+  }
+
+  return value;
+}
+
+/**
+ * Traverse an object path and return the value, with fallback lookup
+ */
+export function traversePath(
+  parts: string[],
+  current: unknown,
+  fallbackLookup?: (key: string) => unknown
+): unknown {
+  let value = current;
+
+  for (let i = 0; i < parts.length; i++) {
+    if (value && typeof value === 'object' && value !== null) {
+      value = (value as Record<string, unknown>)[parts[i]];
+    } else if (i === 0 && fallbackLookup) {
+      // Try fallback for first part
+      value = fallbackLookup(parts[0]);
+      if (value === undefined) return undefined;
+    } else {
+      return undefined;
+    }
+  }
+
+  return value;
+}

package/src/webhook/index.ts

@@ -0,0 +1,15 @@
+/**
+ * Webhook Module
+ *
+ * Provides webhook support for waiting on external callbacks.
+ */
+
+export { WebhookServer } from './server.js';
+export { MemoryWebhookStore, FileWebhookStore, type WebhookStore } from './store.js';
+export type {
+  WebhookServerConfig,
+  WebhookServerCallbacks,
+  WebhookRegistration,
+  WebhookEvent,
+  WaitResult,
+} from './types.js';