serverless-ircd 0.5.0 → 0.7.0

package/tools/load-test/src/harness.ts

@@ -0,0 +1,116 @@
+/**
+ * Load-test orchestrator (the "client farm") — PLAN §6.
+ *
+ * {@link runLoadTest} spawns `config.connections` synthetic clients under
+ * a concurrency cap (`config.connectConcurrency`) and ramps them over
+ * `config.rampMs`, then aggregates their {@link ClientResult}s into the
+ * {@link LoadTestRun} the report renders.
+ *
+ * The default client factory builds real {@link IrcLoadClient}s; tests
+ * inject a fake factory so the concurrency cap, ramp pacing, and
+ * aggregation math can be pinned without touching real sockets. The
+ * end-to-end smoke (real server, real sockets) lives in
+ * `tests/smoke.test.ts` and is the ticket's acceptance criterion.
+ */
+
+import { IrcLoadClient, type IrcLoadClientOptions } from './client.js';
+import type { LoadTestConfig } from './config.js';
+import { type ClientResult, type LoadTestSummary, summarizeResults } from './metrics.js';
+import type { LoadTestRun } from './report.js';
+
+/** Minimal client surface the harness drives (so tests can inject fakes). */
+export interface SyntheticClient {
+  /** Runs the full lifecycle; never throws — failures become `ok: false`. */
+  run(): Promise<ClientResult>;
+}
+
+/** Builds a synthetic client for one connection. */
+export type ClientFactory = (opts: IrcLoadClientOptions) => SyntheticClient;
+
+/** Optional harness dependencies (all have production defaults). */
+export interface LoadTestDeps {
+  /**
+   * Overrides the default {@link IrcLoadClient} factory. Tests inject a
+   * fake so the orchestration math can be pinned without real sockets.
+   */
+  clientFactory?: ClientFactory;
+  /** Progress callback fired as each client completes. */
+  onProgress?: (done: number, total: number) => void;
+  /** Injectable sleep (tests can substitute for deterministic ramp pacing). */
+  sleep?: (ms: number) => Promise<void>;
+}
+
+/** Default client factory: real {@link IrcLoadClient}s. */
+const defaultFactory: ClientFactory = (opts) => new IrcLoadClient(opts);
+
+/** Default sleep helper (real timers). */
+function defaultSleep(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+/**
+ * Runs the configured load test and returns the aggregate.
+ *
+ * Concurrency: at most `config.connectConcurrency` clients are in flight
+ * at once. Ramp: when `config.rampMs > 0`, each client's launch is
+ * staggered by its share of the ramp window
+ * (`floor(rampMs / connections * launchIndex)`), smoothing the
+ * connection-rate onto the target.
+ */
+export async function runLoadTest(
+  config: LoadTestConfig,
+  deps?: LoadTestDeps,
+): Promise<LoadTestRun> {
+  const factory = deps?.clientFactory ?? defaultFactory;
+  const sleep = deps?.sleep ?? defaultSleep;
+  const onProgress = deps?.onProgress;
+
+  const total = config.connections;
+  // Pre-allocated so out-of-order completion lands at the right index.
+  const results: ClientResult[] = new Array(total);
+  let nextId = 0;
+  let launched = 0;
+  let completed = 0;
+
+  const startedAt = new Date();
+  const startTime = Date.now();
+
+  // Cap workers at the requested concurrency, but never above the number
+  // of connections (so a `connections: 1, concurrency: 100` run starts
+  // exactly one worker rather than 100 idle ones).
+  const workerCount = Math.max(1, Math.min(config.connectConcurrency, total));
+
+  const worker = async (): Promise<void> => {
+    for (;;) {
+      const id = nextId++;
+      if (id >= total) return;
+
+      // Ramp: stagger this launch by its share of the window. `launched`
+      // is a monotonic launch counter (distinct from `id`) so the stagger
+      // reflects actual launch order, not the worker that picked it up.
+      if (config.rampMs > 0 && total > 1) {
+        const launchIndex = launched++;
+        const stagger = Math.floor((config.rampMs / total) * launchIndex);
+        if (stagger > 0) await sleep(stagger);
+      }
+
+      const client = factory({ id, config });
+      const result = await client.run();
+      results[id] = result;
+      completed++;
+      if (onProgress !== undefined) onProgress(completed, total);
+    }
+  };
+
+  const workers = Array.from({ length: workerCount }, () => worker());
+  await Promise.all(workers);
+
+  const summary: LoadTestSummary = summarizeResults(results);
+  return {
+    config,
+    summary,
+    startedAt,
+    finishedAt: new Date(),
+    durationMs: Date.now() - startTime,
+  };
+}

package/tools/load-test/src/main.ts

@@ -0,0 +1,120 @@
+#!/usr/bin/env node
+/**
+ * `tools/load-test` entry point — the synthetic WebSocket IRC client farm
+ * (PLAN §6).
+ *
+ * Boots a load-test run against a `ws://` / `wss://` target: opens N
+ * connections, registers (NICK/USER), joins a channel, and optionally
+ * chats. Captures p50/p95/p99 latency per phase and the drop rate, then
+ * prints a markdown report to stdout (and writes it to `--report <path>`
+ * when given).
+ *
+ * Usage:
+ *   node --import tsx tools/load-test/src/main.ts \
+ *     --target wss://irc.staging.example.com/ \
+ *     --connections 10000 --concurrency 200 --ramp-ms 60000 \
+ *     --channel '#loadtest' --messages 3 --platform cf-staging \
+ *     --report reports/cf-staging-2026-08-03.md
+ *
+ * Or, once built:
+ *   pnpm --filter load-test start -- --target wss://irc.staging.example.com/ ...
+ */
+
+import { writeFileSync } from 'node:fs';
+import { HELP_TEXT, parseArgs } from './config.js';
+import { runLoadTest } from './harness.js';
+import { generateReport } from './report.js';
+
+async function main(): Promise<void> {
+  // Handle -h/--help before parseArgs so the help text is printed and the
+  // process exits cleanly without parseArgs having to call process.exit
+  // (which would make it untestable).
+  if (process.argv.slice(2).some((a) => a === '-h' || a === '--help')) {
+    process.stdout.write(`${HELP_TEXT}\n`);
+    process.exit(0);
+  }
+
+  const config = parseArgs(process.argv.slice(2));
+
+  process.stderr.write(
+    `${JSON.stringify({
+      ts: new Date().toISOString(),
+      level: 'info',
+      msg: 'load-test starting',
+      target: config.target,
+      platform: config.platform,
+      connections: config.connections,
+      concurrency: config.connectConcurrency,
+      rampMs: config.rampMs,
+      channel: config.channel,
+      messagesPerClient: config.messagesPerClient,
+      echoMessage: config.echoMessage,
+    })}\n`,
+  );
+
+  const lastReport = { value: Date.now() };
+  const run = await runLoadTest(config, {
+    onProgress: (done, total) => {
+      // Throttle progress to one line per second so a 10k run doesn't
+      // flood stderr while still showing live progress.
+      const now = Date.now();
+      if (now - lastReport.value >= 1000 || done === total) {
+        lastReport.value = now;
+        process.stderr.write(
+          `${JSON.stringify({
+            ts: new Date().toISOString(),
+            level: 'info',
+            msg: 'load-test progress',
+            done,
+            total,
+            pct: Number(((done / total) * 100).toFixed(1)),
+          })}\n`,
+        );
+      }
+    },
+  });
+
+  const report = generateReport(run);
+  process.stdout.write(`${report}\n`);
+
+  if (config.reportPath !== undefined) {
+    writeFileSync(config.reportPath, report, 'utf8');
+    process.stderr.write(
+      `${JSON.stringify({
+        ts: new Date().toISOString(),
+        level: 'info',
+        msg: 'load-test report written',
+        path: config.reportPath,
+      })}\n`,
+    );
+  }
+
+  process.stderr.write(
+    `${JSON.stringify({
+      ts: new Date().toISOString(),
+      level: 'info',
+      msg: 'load-test complete',
+      durationMs: run.durationMs,
+      succeeded: run.summary.succeeded,
+      failed: run.summary.failed,
+      dropRate: run.summary.dropRate,
+    })}\n`,
+  );
+
+  // A non-zero drop rate is a signal, not a hard failure: the run still
+  // "completes without crashing" (the ticket's acceptance). Operators
+  // inspect the report. Exit 0 unless a catastrophic error threw above.
+  process.exit(0);
+}
+
+main().catch((err) => {
+  process.stderr.write(
+    `${JSON.stringify({
+      ts: new Date().toISOString(),
+      level: 'error',
+      msg: 'load-test failed',
+      error: err instanceof Error ? err.message : String(err),
+    })}\n`,
+  );
+  process.exit(1);
+});

package/tools/load-test/src/metrics.ts

@@ -0,0 +1,168 @@
+/**
+ * Latency statistics for the load-test harness.
+ *
+ * This module is intentionally pure: it turns raw per-client latency
+ * samples into the p50/p95/p99 summary the report renders, with no socket
+ * or timer dependencies. Keeping it pure means the math can be pinned
+ * independently of any real connection behaviour (PLAN §6).
+ *
+ * Percentile convention: **nearest-rank**. For `n` sorted samples and a
+ * percentile `p ∈ [0,100]`, the rank index is `ceil(p/100 * n) - 1`,
+ * clamped to `[0, n-1]`. This is the same method `hd`/`ping`/k6 report;
+ * it is conservative for small `n` (p95 of 3 samples returns the max
+ * rather than interpolating) which suits "worst-case tail" reporting for
+ * an IRC control plane.
+ */
+
+/** The lifecycle phases a synthetic client measures. */
+export type PhaseName = 'connect' | 'register' | 'join' | 'privmsg';
+
+/** All phases, in lifecycle order (used for deterministic report rows). */
+export const PHASE_NAMES: readonly PhaseName[] = [
+  'connect',
+  'register',
+  'join',
+  'privmsg',
+] as const;
+
+/**
+ * Aggregated latency for one phase across every client that reached it.
+ * All fields are milliseconds. A phase a client never reached contributes
+ * no samples (so a client that died at `register` does not inflate the
+ * `join` bucket).
+ */
+export interface PhaseSummary {
+  /** Number of samples recorded for this phase. */
+  count: number;
+  /** Minimum latency observed (ms). */
+  min: number;
+  /** Maximum latency observed (ms). */
+  max: number;
+  /** 50th percentile latency (ms). */
+  p50: number;
+  /** 95th percentile latency (ms). */
+  p95: number;
+  /** 99th percentile latency (ms). */
+  p99: number;
+}
+
+/**
+ * The result one synthetic client reports back to the harness. The harness
+ * flattens an array of these into a {@link LoadTestSummary}.
+ */
+export interface ClientResult {
+  /** 0-based client index. */
+  id: number;
+  /** `true` iff the client reached the end of every measured phase. */
+  ok: boolean;
+  /** Human-readable failure reason when `ok === false`. */
+  error?: string;
+  /** The phase at which the client failed (drop attribution). */
+  failedPhase?: PhaseName;
+  /** Latency samples per phase in ms (may be empty for unreached phases). */
+  samples: Record<PhaseName, number[]>;
+}
+
+/** Top-level aggregate the report renders. */
+export interface LoadTestSummary {
+  /** Total clients attempted (connections the harness tried to open). */
+  totalClients: number;
+  /** Clients that completed every measured phase. */
+  succeeded: number;
+  /** Clients that failed at some phase. */
+  failed: number;
+  /** Fraction of clients that failed: `failed / total` (0 when total === 0). */
+  dropRate: number;
+  /** Per-phase latency summary. Phases no client reached are zeroed. */
+  phases: Record<PhaseName, PhaseSummary>;
+}
+
+/**
+ * Returns the `p`-th percentile of `samples` using the nearest-rank method.
+ *
+ * - `p` must be in `[0, 100]`; outside that range throws (programming error).
+ * - An empty sample set returns `0` (so a phase nobody reached renders as
+ *   `0ms` rather than `NaN`).
+ *
+ * The input is not assumed to be sorted; a defensive copy is sorted
+ * in place on that copy so callers' arrays are left untouched.
+ */
+export function percentile(samples: readonly number[], p: number): number {
+  if (p < 0 || p > 100) {
+    throw new RangeError(`percentile must be in [0, 100] (got ${p})`);
+  }
+  if (samples.length === 0) return 0;
+  const sorted = [...samples].sort((a, b) => a - b);
+  // nearest-rank: rank = ceil(p/100 * n), index = rank - 1, clamped.
+  const rank = Math.ceil((p / 100) * sorted.length);
+  const index = Math.max(0, Math.min(sorted.length - 1, rank - 1));
+  return sorted[index] ?? 0;
+}
+
+/**
+ * Summarises one phase's sample set into a {@link PhaseSummary}. An empty
+ * sample set yields a zeroed summary (count 0, every percentile 0) so the
+ * report renders `—`/`0` cleanly instead of `NaN`.
+ */
+export function summarizePhase(samples: readonly number[]): PhaseSummary {
+  if (samples.length === 0) {
+    return { count: 0, min: 0, max: 0, p50: 0, p95: 0, p99: 0 };
+  }
+  let min = Number.POSITIVE_INFINITY;
+  let max = Number.NEGATIVE_INFINITY;
+  for (const v of samples) {
+    if (v < min) min = v;
+    if (v > max) max = v;
+  }
+  return {
+    count: samples.length,
+    min,
+    max,
+    p50: percentile(samples, 50),
+    p95: percentile(samples, 95),
+    p99: percentile(samples, 99),
+  };
+}
+
+/**
+ * Flattens per-client results into a {@link LoadTestSummary}.
+ *
+ * - `totalClients` / `succeeded` / `failed` come from the `ok` flag.
+ * - `dropRate = failed / total` (0 when no clients were attempted).
+ * - Each phase aggregates every sample across all clients that recorded
+ *   one. A client that failed before a phase contributes no sample to
+ *   that phase (drop attribution: a `register` timeout does not pollute
+ *   the `join` bucket).
+ */
+export function summarizeResults(results: readonly ClientResult[]): LoadTestSummary {
+  const totalClients = results.length;
+  let succeeded = 0;
+  const buckets: Record<PhaseName, number[]> = {
+    connect: [],
+    register: [],
+    join: [],
+    privmsg: [],
+  };
+  for (const r of results) {
+    if (r.ok) succeeded++;
+    for (const phase of PHASE_NAMES) {
+      const samples = r.samples[phase];
+      if (samples !== undefined && samples.length > 0) {
+        buckets[phase].push(...samples);
+      }
+    }
+  }
+  const failed = totalClients - succeeded;
+  return {
+    totalClients,
+    succeeded,
+    failed,
+    dropRate: totalClients === 0 ? 0 : failed / totalClients,
+    phases: {
+      connect: summarizePhase(buckets.connect),
+      register: summarizePhase(buckets.register),
+      join: summarizePhase(buckets.join),
+      privmsg: summarizePhase(buckets.privmsg),
+    },
+  };
+}

package/tools/load-test/src/report.ts

@@ -0,0 +1,106 @@
+/**
+ * Markdown report rendering for a load-test run (PLAN §6).
+ *
+ * Pure function over a {@link LoadTestRun}: no I/O, no sockets. The CLI
+ * (`main.ts`) writes the returned string to stdout (and optionally to
+ * `--report <path>`). Keeping it pure lets the structural pieces a human
+ * scans for (platform, target, drop rate, the per-phase latency table) be
+ * pinned independently of any real connection behaviour.
+ */
+
+import type { LoadTestConfig } from './config.js';
+import { type LoadTestSummary, PHASE_NAMES, type PhaseName, type PhaseSummary } from './metrics.js';
+
+/** Everything needed to render a report: config + aggregate + timing. */
+export interface LoadTestRun {
+  config: LoadTestConfig;
+  summary: LoadTestSummary;
+  /** When the run started (UTC). */
+  startedAt: Date;
+  /** When the run finished (UTC). */
+  finishedAt: Date;
+  /** Wall-clock duration of the run (ms). */
+  durationMs: number;
+}
+
+/**
+ * Formats a millisecond duration into a compact human-readable string:
+ * `<1s → "X.XXms"`, `<60s → "X.XXs"`, otherwise `"XmYY.YYs"`.
+ */
+export function formatMs(ms: number): string {
+  if (ms < 1000) return `${ms.toFixed(2)}ms`;
+  if (ms < 60_000) return `${(ms / 1000).toFixed(2)}s`;
+  const totalSec = ms / 1000;
+  const minutes = Math.floor(totalSec / 60);
+  const seconds = totalSec - minutes * 60;
+  return `${minutes}m${seconds.toFixed(2)}s`;
+}
+
+/** Formats a `[0,1]` fraction as a percentage with two decimals. */
+export function formatPercent(fraction: number): string {
+  return `${(fraction * 100).toFixed(2)}%`;
+}
+
+/**
+ * Renders the run as a markdown document. Layout:
+ *
+ *   # ServerlessIRCd Load Test Report
+ *   - header (platform, target, date, duration)
+ *   - ## Configuration table
+ *   - ## Results table (attempted / succeeded / failed / drop rate)
+ *   - ## Latency (ms) table — one row per phase
+ *   - ## Notes — cost modeling is platform-specific
+ */
+export function generateReport(run: LoadTestRun): string {
+  const { config, summary, startedAt, durationMs } = run;
+  const lines: string[] = [];
+  lines.push('# ServerlessIRCd Load Test Report', '');
+  lines.push(`- **Platform:** ${config.platform}`);
+  lines.push(`- **Target:** ${config.target}`);
+  lines.push(`- **Date:** ${startedAt.toISOString()}`);
+  lines.push(`- **Duration:** ${formatMs(durationMs)}`, '');
+
+  lines.push('## Configuration', '');
+  lines.push('| Setting | Value |');
+  lines.push('|---|---|');
+  lines.push(`| Connections | ${config.connections} |`);
+  lines.push(`| Concurrency | ${config.connectConcurrency} |`);
+  lines.push(`| Ramp | ${formatMs(config.rampMs)} |`);
+  lines.push(`| Channel | ${config.channel} |`);
+  lines.push(`| Messages/client | ${config.messagesPerClient} |`);
+  lines.push(`| Echo-message | ${config.echoMessage ? 'on' : 'off'} |`);
+  lines.push('');
+
+  lines.push('## Results', '');
+  lines.push('| Metric | Value |');
+  lines.push('|---|---|');
+  lines.push(`| Attempted | ${summary.totalClients} |`);
+  lines.push(`| Succeeded | ${summary.succeeded} |`);
+  lines.push(`| Failed | ${summary.failed} |`);
+  lines.push(`| Drop rate | ${formatPercent(summary.dropRate)} |`);
+  lines.push('');
+
+  lines.push('## Latency (ms)', '');
+  lines.push('| Phase | Count | Min | p50 | p95 | p99 | Max |');
+  lines.push('|---|---:|---:|---:|---:|---:|---:|');
+  for (const phase of PHASE_NAMES) {
+    lines.push(renderPhaseRow(phase, summary.phases[phase]));
+  }
+  lines.push('');
+
+  lines.push('## Notes', '');
+  lines.push(
+    'Cost modeling is platform-specific (Cloudflare Workers vs AWS API Gateway ' +
+      'WebSocket pricing) and is not computed here; operators should multiply the ' +
+      "connection-minutes and message counts above by the deployed platform's unit " +
+      'price and record the result alongside this report.',
+  );
+  lines.push('');
+  return lines.join('\n');
+}
+
+/** Renders one latency-table row, right-aligning the numeric columns. */
+function renderPhaseRow(phase: PhaseName, s: PhaseSummary): string {
+  const f = (n: number): string => n.toFixed(2);
+  return `| ${phase} | ${s.count} | ${f(s.min)} | ${f(s.p50)} | ${f(s.p95)} | ${f(s.p99)} | ${f(s.max)} |`;
+}