reqon-dsl 0.2.0

package/src/benchmark/utils.ts

@@ -0,0 +1,230 @@
+/**
+ * Benchmarking utilities for Reqon
+ */
+
+export interface BenchmarkResult {
+  name: string;
+  iterations: number;
+  totalMs: number;
+  avgMs: number;
+  minMs: number;
+  maxMs: number;
+  opsPerSec: number;
+  samples: number[];
+}
+
+export interface BenchmarkOptions {
+  iterations?: number;
+  warmupIterations?: number;
+  name?: string;
+}
+
+const DEFAULT_OPTIONS: Required<BenchmarkOptions> = {
+  iterations: 1000,
+  warmupIterations: 100,
+  name: 'benchmark',
+};
+
+/**
+ * High-resolution timer using process.hrtime.bigint()
+ */
+export function hrTimeMs(): number {
+  return Number(process.hrtime.bigint()) / 1_000_000;
+}
+
+/**
+ * Run a synchronous benchmark
+ */
+export function benchmarkSync<T>(
+  fn: () => T,
+  options: BenchmarkOptions = {}
+): BenchmarkResult {
+  const opts = { ...DEFAULT_OPTIONS, ...options };
+  const samples: number[] = [];
+
+  // Warmup phase
+  for (let i = 0; i < opts.warmupIterations; i++) {
+    fn();
+  }
+
+  // Measurement phase
+  for (let i = 0; i < opts.iterations; i++) {
+    const start = hrTimeMs();
+    fn();
+    const end = hrTimeMs();
+    samples.push(end - start);
+  }
+
+  return calculateStats(opts.name, samples);
+}
+
+/**
+ * Run an asynchronous benchmark
+ */
+export async function benchmarkAsync<T>(
+  fn: () => Promise<T>,
+  options: BenchmarkOptions = {}
+): Promise<BenchmarkResult> {
+  const opts = { ...DEFAULT_OPTIONS, ...options };
+  const samples: number[] = [];
+
+  // Warmup phase
+  for (let i = 0; i < opts.warmupIterations; i++) {
+    await fn();
+  }
+
+  // Measurement phase
+  for (let i = 0; i < opts.iterations; i++) {
+    const start = hrTimeMs();
+    await fn();
+    const end = hrTimeMs();
+    samples.push(end - start);
+  }
+
+  return calculateStats(opts.name, samples);
+}
+
+/**
+ * Calculate statistics from samples
+ */
+function calculateStats(name: string, samples: number[]): BenchmarkResult {
+  const sorted = [...samples].sort((a, b) => a - b);
+  const totalMs = samples.reduce((sum, s) => sum + s, 0);
+  const avgMs = totalMs / samples.length;
+  const minMs = sorted[0];
+  const maxMs = sorted[sorted.length - 1];
+  const opsPerSec = 1000 / avgMs;
+
+  return {
+    name,
+    iterations: samples.length,
+    totalMs,
+    avgMs,
+    minMs,
+    maxMs,
+    opsPerSec,
+    samples,
+  };
+}
+
+/**
+ * Format a benchmark result for display
+ */
+export function formatResult(result: BenchmarkResult): string {
+  const lines = [
+    `┌─ ${result.name}`,
+    `│  Iterations: ${result.iterations.toLocaleString()}`,
+    `│  Total time: ${result.totalMs.toFixed(2)}ms`,
+    `│  Avg time:   ${result.avgMs.toFixed(4)}ms`,
+    `│  Min time:   ${result.minMs.toFixed(4)}ms`,
+    `│  Max time:   ${result.maxMs.toFixed(4)}ms`,
+    `│  Ops/sec:    ${result.opsPerSec.toFixed(2)}`,
+    `└──────────────────────────────────`,
+  ];
+  return lines.join('\n');
+}
+
+/**
+ * Format multiple benchmark results as a comparison table
+ */
+export function formatResultsTable(results: BenchmarkResult[]): string {
+  const nameWidth = Math.max(...results.map((r) => r.name.length), 10);
+  const header = `| ${'Name'.padEnd(nameWidth)} | Ops/sec      | Avg (ms)   | Min (ms)   | Max (ms)   |`;
+  const separator = `|${'-'.repeat(nameWidth + 2)}|--------------|------------|------------|------------|`;
+
+  const rows = results.map((r) => {
+    return `| ${r.name.padEnd(nameWidth)} | ${r.opsPerSec.toFixed(2).padStart(12)} | ${r.avgMs.toFixed(4).padStart(10)} | ${r.minMs.toFixed(4).padStart(10)} | ${r.maxMs.toFixed(4).padStart(10)} |`;
+  });
+
+  return [separator, header, separator, ...rows, separator].join('\n');
+}
+
+/**
+ * Suite for grouping related benchmarks
+ */
+export class BenchmarkSuite {
+  private results: BenchmarkResult[] = [];
+  private name: string;
+
+  constructor(name: string) {
+    this.name = name;
+  }
+
+  addSync<T>(name: string, fn: () => T, options?: BenchmarkOptions): this {
+    const result = benchmarkSync(fn, { ...options, name });
+    this.results.push(result);
+    return this;
+  }
+
+  async addAsync<T>(
+    name: string,
+    fn: () => Promise<T>,
+    options?: BenchmarkOptions
+  ): Promise<this> {
+    const result = await benchmarkAsync(fn, { ...options, name });
+    this.results.push(result);
+    return this;
+  }
+
+  getResults(): BenchmarkResult[] {
+    return [...this.results];
+  }
+
+  print(): void {
+    console.log(`\n${'='.repeat(60)}`);
+    console.log(`Benchmark Suite: ${this.name}`);
+    console.log(`${'='.repeat(60)}\n`);
+    console.log(formatResultsTable(this.results));
+    console.log('');
+  }
+
+  toJSON(): object {
+    return {
+      name: this.name,
+      timestamp: new Date().toISOString(),
+      results: this.results.map((r) => ({
+        name: r.name,
+        iterations: r.iterations,
+        totalMs: r.totalMs,
+        avgMs: r.avgMs,
+        minMs: r.minMs,
+        maxMs: r.maxMs,
+        opsPerSec: r.opsPerSec,
+      })),
+    };
+  }
+}
+
+/**
+ * Generate a string of specified size for testing
+ */
+export function generateString(sizeKb: number): string {
+  const chars = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789 \n';
+  const targetLength = sizeKb * 1024;
+  let result = '';
+  while (result.length < targetLength) {
+    result += chars[Math.floor(Math.random() * chars.length)];
+  }
+  return result;
+}
+
+/**
+ * Generate random data for store benchmarks
+ */
+export function generateRecords(count: number): Record<string, unknown>[] {
+  const records: Record<string, unknown>[] = [];
+  for (let i = 0; i < count; i++) {
+    records.push({
+      id: `record-${i}`,
+      name: `Test Record ${i}`,
+      value: Math.random() * 1000,
+      active: Math.random() > 0.5,
+      tags: ['tag1', 'tag2', 'tag3'].slice(0, Math.floor(Math.random() * 3) + 1),
+      nested: {
+        field1: `nested-${i}`,
+        field2: i * 2,
+      },
+    });
+  }
+  return records;
+}

package/src/cli.ts

@@ -0,0 +1,313 @@
+#!/usr/bin/env node
+
+import { readFile, writeFile, mkdir } from 'node:fs/promises';
+import { resolve, dirname } from 'node:path';
+import { fromPath, parse, Scheduler, loadMission } from './index.js';
+import type { ScheduleEvent } from './scheduler/index.js';
+import { ReqonError } from './errors/index.js';
+import { loadEnv, loadCredentials } from './auth/credentials.js';
+import { WebhookServer } from './webhook/index.js';
+
+async function main() {
+  const args = process.argv.slice(2);
+
+  if (args.length === 0 || args.includes('--help') || args.includes('-h')) {
+    console.log(`
+Reqon - A DSL for fetch, map, validate pipelines
+
+Usage:
+  reqon <file.reqon|folder> [options]
+
+Options:
+  --dry-run            Run without making actual HTTP requests
+  --verbose            Enable verbose logging
+  --auth <file>        JSON file with auth credentials (supports env var interpolation)
+  --env <file>         Path to .env file (default: .env in current directory)
+  --output <path>      Export stores to JSON (file or directory)
+  --daemon             Run as daemon, executing scheduled missions
+  --once               Run scheduled missions once immediately, then exit
+  --webhook            Enable webhook server for 'wait' steps
+  --webhook-port <n>   Port for webhook server (default: 3000)
+  --webhook-url <url>  Base URL for webhook endpoints (default: http://localhost:3000)
+  --help, -h           Show this help message
+
+Environment Variables:
+  Credentials in --auth files support env var interpolation:
+    $VAR_NAME, \${VAR_NAME}, \${VAR_NAME:-default}
+
+  Auto-discovery from env vars (no --auth file needed):
+    REQON_{SOURCE}_TOKEN      Bearer token for source
+    REQON_{SOURCE}_TYPE       Auth type (bearer, oauth2, api_key, basic)
+    REQON_{SOURCE}_API_KEY    API key for source
+
+Examples:
+  reqon sync-invoices.reqon --verbose
+  reqon ./sync-invoices/ --verbose        # folder with mission.reqon + action files
+  reqon sync-invoices.reqon --auth ./credentials.json
+  reqon sync-invoices.reqon --env .env.production --auth ./credentials.json
+  reqon sync-invoices.reqon --output ./output.json
+  reqon sync-invoices.reqon --daemon --verbose
+  reqon sync-invoices.reqon --webhook --webhook-port 8080 --verbose
+`);
+    process.exit(0);
+  }
+
+  const filePath = args[0];
+  const dryRun = args.includes('--dry-run');
+  const verbose = args.includes('--verbose');
+  const daemon = args.includes('--daemon');
+  const once = args.includes('--once');
+  const webhookEnabled = args.includes('--webhook');
+
+  // Parse webhook options
+  let webhookPort = 3000;
+  const webhookPortIndex = args.indexOf('--webhook-port');
+  if (webhookPortIndex !== -1 && args[webhookPortIndex + 1]) {
+    webhookPort = parseInt(args[webhookPortIndex + 1], 10);
+  }
+
+  let webhookUrl: string | undefined;
+  const webhookUrlIndex = args.indexOf('--webhook-url');
+  if (webhookUrlIndex !== -1 && args[webhookUrlIndex + 1]) {
+    webhookUrl = args[webhookUrlIndex + 1];
+  }
+
+  // Load .env file(s)
+  let envFile: string | undefined;
+  const envIndex = args.indexOf('--env');
+  if (envIndex !== -1 && args[envIndex + 1]) {
+    envFile = args[envIndex + 1];
+  }
+
+  const envResult = loadEnv({ envFile });
+  if (verbose && envResult.loaded) {
+    console.log(`Loaded ${envResult.count} env vars from: ${envResult.files.join(', ')}`);
+  }
+
+  // Load and resolve auth credentials
+  let auth: Record<string, unknown> | undefined;
+  const authIndex = args.indexOf('--auth');
+  if (authIndex !== -1 && args[authIndex + 1]) {
+    const authPath = resolve(args[authIndex + 1]);
+    const authContent = await readFile(authPath, 'utf-8');
+    const rawAuth = JSON.parse(authContent);
+    // Resolve env var references in the auth config
+    auth = loadCredentials(rawAuth);
+  }
+
+  let outputPath: string | undefined;
+  const outputIndex = args.indexOf('--output');
+  if (outputIndex !== -1 && args[outputIndex + 1]) {
+    outputPath = resolve(args[outputIndex + 1]);
+  }
+
+  // Daemon mode: run scheduled missions
+  if (daemon || once) {
+    await runDaemon(filePath, {
+      verbose,
+      dryRun,
+      auth: auth as Record<string, { type: 'bearer' | 'oauth2'; token?: string; accessToken?: string }>,
+      once,
+    });
+    return;
+  }
+
+  // Single run mode (default)
+  console.log(`Running: ${filePath}`);
+
+  // Start webhook server if enabled
+  let webhookServer: WebhookServer | undefined;
+  if (webhookEnabled) {
+    webhookServer = new WebhookServer({
+      port: webhookPort,
+      baseUrl: webhookUrl ?? `http://localhost:${webhookPort}`,
+      verbose,
+    });
+    await webhookServer.start();
+    console.log(`Webhook server started on port ${webhookPort}`);
+  }
+
+  try {
+    const result = await fromPath(filePath, {
+      dryRun,
+      verbose,
+      auth: auth as Record<string, { type: 'bearer' | 'oauth2'; token?: string; accessToken?: string }>,
+      webhookServer,
+    });
+
+    if (result.success) {
+      console.log(`\n✓ Mission completed successfully`);
+      console.log(`  Duration: ${result.duration}ms`);
+      console.log(`  Actions run: ${result.actionsRun.join(' → ')}`);
+
+      // Print store stats and optionally export
+      const storeData: Record<string, unknown[]> = {};
+      for (const [name, store] of result.stores) {
+        const items = await store.list();
+        console.log(`  Store "${name}": ${items.length} items`);
+        storeData[name] = items;
+      }
+
+      // Export to JSON if requested
+      if (outputPath) {
+        await mkdir(dirname(outputPath), { recursive: true });
+        await writeFile(outputPath, JSON.stringify(storeData, null, 2));
+        console.log(`  Output written to: ${outputPath}`);
+      }
+    } else {
+      console.log(`\n✗ Mission failed`);
+      for (const error of result.errors) {
+        console.error(`  [${error.action}/${error.step}] ${error.message}`);
+      }
+      process.exit(1);
+    }
+  } catch (error) {
+    if (error instanceof ReqonError) {
+      console.error(error.format());
+    } else {
+      console.error(`Error: ${(error as Error).message}`);
+    }
+    process.exit(1);
+  } finally {
+    // Stop webhook server if it was started
+    if (webhookServer) {
+      await webhookServer.stop();
+    }
+  }
+}
+
+interface DaemonOptions {
+  verbose: boolean;
+  dryRun: boolean;
+  auth?: Record<string, { type: 'bearer' | 'oauth2'; token?: string; accessToken?: string }>;
+  once: boolean;
+}
+
+async function runDaemon(filePath: string, options: DaemonOptions): Promise<void> {
+  const absolutePath = resolve(filePath);
+
+  let program;
+  let baseDir: string;
+  try {
+    const result = await loadMission(absolutePath);
+    program = result.program;
+    baseDir = result.baseDir;
+  } catch (error) {
+    if (error instanceof ReqonError) {
+      console.error(error.format());
+    } else {
+      console.error(`Error loading mission: ${(error as Error).message}`);
+    }
+    process.exit(1);
+  }
+
+  // Find scheduled missions
+  const scheduledMissions = program.statements.filter(
+    (s) => s.type === 'MissionDefinition' && s.schedule
+  );
+
+  if (scheduledMissions.length === 0) {
+    console.error('No scheduled missions found in the file');
+    console.error('Add a schedule to your mission, e.g.:');
+    console.error('  schedule: every 6 hours');
+    console.error('  schedule: cron "0 */6 * * *"');
+    process.exit(1);
+  }
+
+  console.log(`Found ${scheduledMissions.length} scheduled mission(s)`);
+
+  // Create scheduler with callbacks
+  const scheduler = new Scheduler(
+    {
+      verbose: options.verbose,
+      callbacks: {
+        onJobStarted: (event: ScheduleEvent) => {
+          console.log(`[${formatTime(event.timestamp)}] Starting: ${event.missionName}`);
+        },
+        onJobCompleted: (event: ScheduleEvent) => {
+          console.log(
+            `[${formatTime(event.timestamp)}] Completed: ${event.missionName} (${event.duration}ms)`
+          );
+        },
+        onJobFailed: (event: ScheduleEvent) => {
+          console.error(
+            `[${formatTime(event.timestamp)}] Failed: ${event.missionName} - ${event.error}`
+          );
+        },
+        onJobSkipped: (event: ScheduleEvent) => {
+          console.log(
+            `[${formatTime(event.timestamp)}] Skipped: ${event.missionName} - ${event.reason}`
+          );
+        },
+      },
+    },
+    {
+      dryRun: options.dryRun,
+      verbose: options.verbose,
+      auth: options.auth,
+    }
+  );
+
+  // Register scheduled missions
+  scheduler.registerProgram(program, absolutePath);
+
+  // Print job info
+  const jobs = scheduler.getJobs();
+  console.log('\nScheduled jobs:');
+  for (const job of jobs) {
+    const schedule = job.schedule;
+    let scheduleStr: string;
+    if (schedule.scheduleType === 'interval') {
+      scheduleStr = `every ${schedule.interval!.value} ${schedule.interval!.unit}`;
+    } else if (schedule.scheduleType === 'cron') {
+      scheduleStr = `cron "${schedule.cronExpression}"`;
+    } else {
+      scheduleStr = `at "${schedule.runAt}"`;
+    }
+    console.log(`  - ${job.missionName}: ${scheduleStr}`);
+    if (job.nextRun) {
+      console.log(`    Next run: ${job.nextRun.toISOString()}`);
+    }
+  }
+  console.log('');
+
+  // Handle --once mode
+  if (options.once) {
+    console.log('Running all scheduled missions once...\n');
+    for (const job of jobs) {
+      await scheduler.trigger(job.missionName);
+    }
+    console.log('\nAll missions completed.');
+    return;
+  }
+
+  // Start daemon mode
+  console.log('Starting scheduler daemon (Ctrl+C to stop)...\n');
+
+  // Handle graceful shutdown
+  let shuttingDown = false;
+  const shutdown = async () => {
+    if (shuttingDown) return;
+    shuttingDown = true;
+    console.log('\nShutting down scheduler...');
+    await scheduler.stop();
+    process.exit(0);
+  };
+
+  process.on('SIGINT', shutdown);
+  process.on('SIGTERM', shutdown);
+
+  // Start the scheduler
+  await scheduler.start();
+
+  // Keep the process running
+  await new Promise(() => {
+    // This promise never resolves - we wait for SIGINT/SIGTERM
+  });
+}
+
+function formatTime(date: Date): string {
+  return date.toISOString().replace('T', ' ').substring(0, 19);
+}
+
+main();