serverless-ircd 0.5.0 → 0.6.0

package/tools/load-test/tests/metrics.test.ts

@@ -0,0 +1,174 @@
+import { describe, expect, it } from 'vitest';
+import {
+  type ClientResult,
+  PHASE_NAMES,
+  type PhaseName,
+  percentile,
+  summarizePhase,
+  summarizeResults,
+} from '../src/metrics';
+
+/**
+ * The metrics module is the statistical core of the load tester. It is the
+ * piece that turns raw per-client latency samples into the p50/p95/p99
+ * summary the report renders, so its math must be pinned independently of
+ * any real socket behaviour.
+ */
+describe('percentile (nearest-rank)', () => {
+  it('returns the upper-median for p50 of an even-sized sample set', () => {
+    // nearest-rank: index = ceil(p/100 * n) - 1
+    // n=4, p=50 -> ceil(2)-1 = 1 -> sorted[1] = 20
+    expect(percentile([10, 20, 30, 40], 50)).toBe(20);
+  });
+
+  it('returns the exact middle for p50 of an odd-sized sample set', () => {
+    expect(percentile([10, 20, 30], 50)).toBe(20);
+  });
+
+  it('returns the max for p100', () => {
+    expect(percentile([5, 9, 3, 12, 7], 100)).toBe(12);
+  });
+
+  it('returns the min for p0', () => {
+    expect(percentile([5, 9, 3, 12, 7], 0)).toBe(3);
+  });
+
+  it('clamps the rank index to the last element for p99 on a small set', () => {
+    // n=3, p=99 -> ceil(2.97)-1 = 3-1 = 2 -> sorted[2] = 90
+    expect(percentile([10, 50, 90], 99)).toBe(90);
+  });
+
+  it('throws for a percentile below 0 (programming error)', () => {
+    expect(() => percentile([10, 20, 30], -5)).toThrow(/percentile/i);
+  });
+
+  it('works on an unsorted input (sorts internally)', () => {
+    expect(percentile([100, 1, 50, 25, 75], 50)).toBe(50);
+    expect(percentile([100, 1, 50, 25, 75], 100)).toBe(100);
+  });
+
+  it('returns 0 for an empty sample set', () => {
+    expect(percentile([], 50)).toBe(0);
+  });
+
+  it('returns the sole sample for a single-element set at any percentile', () => {
+    expect(percentile([42], 50)).toBe(42);
+    expect(percentile([42], 99)).toBe(42);
+  });
+
+  it('throws for a percentile above 100 (programming error)', () => {
+    expect(() => percentile([1, 2, 3], 101)).toThrow(/percentile/i);
+  });
+});
+
+describe('summarizePhase', () => {
+  it('reports count and the percentile triple for a populated phase', () => {
+    const s = summarizePhase([10, 20, 30, 40, 50, 60, 70, 80, 90, 100]);
+    expect(s.count).toBe(10);
+    expect(s.min).toBe(10);
+    expect(s.max).toBe(100);
+    expect(s.p50).toBe(50);
+    expect(s.p95).toBe(100);
+    expect(s.p99).toBe(100);
+  });
+
+  it('returns a zeroed summary for an empty sample set', () => {
+    const s = summarizePhase([]);
+    expect(s).toEqual({ count: 0, min: 0, max: 0, p50: 0, p95: 0, p99: 0 });
+  });
+
+  it('handles a single sample', () => {
+    expect(summarizePhase([77])).toEqual({
+      count: 1,
+      min: 77,
+      max: 77,
+      p50: 77,
+      p95: 77,
+      p99: 77,
+    });
+  });
+});
+
+describe('summarizeResults', () => {
+  const emptySamples = (): Record<PhaseName, number[]> => ({
+    connect: [],
+    register: [],
+    join: [],
+    privmsg: [],
+  });
+
+  it('computes drop rate as failed / total', () => {
+    const results: ClientResult[] = [
+      { id: 0, ok: true, samples: emptySamples() },
+      { id: 1, ok: true, samples: emptySamples() },
+      {
+        id: 2,
+        ok: false,
+        error: 'connect timeout',
+        failedPhase: 'connect',
+        samples: emptySamples(),
+      },
+    ];
+    const summary = summarizeResults(results);
+    expect(summary.totalClients).toBe(3);
+    expect(summary.succeeded).toBe(2);
+    expect(summary.failed).toBe(1);
+    expect(summary.dropRate).toBeCloseTo(1 / 3, 5);
+  });
+
+  it('flatten-aggregates samples per phase across all clients (including privmsg repeats)', () => {
+    const results: ClientResult[] = [
+      {
+        id: 0,
+        ok: true,
+        samples: {
+          connect: [12],
+          register: [40],
+          join: [8],
+          privmsg: [3, 4],
+        },
+      },
+      {
+        id: 1,
+        ok: true,
+        samples: {
+          connect: [20],
+          register: [60],
+          join: [10],
+          privmsg: [5],
+        },
+      },
+    ];
+    const summary = summarizeResults(results);
+    expect(summary.phases.connect).toEqual(summarizePhase([12, 20]));
+    expect(summary.phases.privmsg).toEqual(summarizePhase([3, 4, 5]));
+    expect(summary.phases.privmsg.count).toBe(3);
+  });
+
+  it('only counts samples up to the failed phase (drop attribution)', () => {
+    // A client that failed at `register` contributes a connect sample but
+    // no register/join/privmsg sample.
+    const results: ClientResult[] = [
+      {
+        id: 0,
+        ok: false,
+        error: 'register timeout',
+        failedPhase: 'register',
+        samples: { connect: [12], register: [], join: [], privmsg: [] },
+      },
+    ];
+    const summary = summarizeResults(results);
+    expect(summary.phases.connect.count).toBe(1);
+    expect(summary.phases.register.count).toBe(0);
+    expect(summary.phases.join.count).toBe(0);
+  });
+
+  it('returns a zeroed summary for no clients', () => {
+    const summary = summarizeResults([]);
+    expect(summary.totalClients).toBe(0);
+    expect(summary.dropRate).toBe(0);
+    for (const phase of PHASE_NAMES) {
+      expect(summary.phases[phase].count).toBe(0);
+    }
+  });
+});

package/tools/load-test/tests/report.test.ts

@@ -0,0 +1,161 @@
+import { describe, expect, it } from 'vitest';
+import { type LoadTestConfig, resolveConfig } from '../src/config';
+import {
+  type ClientResult,
+  type LoadTestSummary,
+  PHASE_NAMES,
+  summarizeResults,
+} from '../src/metrics';
+import { type LoadTestRun, formatMs, formatPercent, generateReport } from '../src/report';
+
+/**
+ * The report module renders the run's aggregate into a markdown string.
+ * Tests pin the structural pieces a human scans for (platform, target,
+ * drop rate, the per-phase latency table) without coupling to exact
+ * whitespace — the layout can evolve without churn.
+ */
+
+function baseConfig(overrides: Partial<LoadTestConfig> = {}): LoadTestConfig {
+  return resolveConfig({
+    target: 'wss://irc.staging.example.com/',
+    platform: 'cf-staging',
+    connections: 5,
+    channel: '#loadtest',
+    ...overrides,
+  });
+}
+
+function emptySummary(): LoadTestSummary {
+  return summarizeResults([]);
+}
+
+function makeRun(
+  summary: LoadTestSummary = emptySummary(),
+  config: LoadTestConfig = baseConfig(),
+  durationMs = 1000,
+): LoadTestRun {
+  const startedAt = new Date('2026-08-03T14:00:00.000Z');
+  return {
+    config,
+    summary,
+    startedAt,
+    finishedAt: new Date(startedAt.getTime() + durationMs),
+    durationMs,
+  };
+}
+
+function okClient(
+  id: number,
+  latencies: Partial<Record<(typeof PHASE_NAMES)[number], number[]>>,
+): ClientResult {
+  const samples: ClientResult['samples'] = { connect: [], register: [], join: [], privmsg: [] };
+  for (const [phase, vals] of Object.entries(latencies) as [keyof typeof samples, number[]][]) {
+    samples[phase] = vals;
+  }
+  return { id, ok: true, samples };
+}
+
+describe('formatMs', () => {
+  it('formats sub-second durations in ms', () => {
+    expect(formatMs(12)).toBe('12.00ms');
+  });
+
+  it('formats seconds with two decimals', () => {
+    expect(formatMs(42_300)).toBe('42.30s');
+  });
+
+  it('formats minutes with seconds', () => {
+    expect(formatMs(90_000)).toBe('1m30.00s');
+  });
+
+  it('formats zero as 0.00ms', () => {
+    expect(formatMs(0)).toBe('0.00ms');
+  });
+});
+
+describe('formatPercent', () => {
+  it('formats a fraction as a percentage with two decimals', () => {
+    expect(formatPercent(0.0002)).toBe('0.02%');
+  });
+
+  it('formats 1 as 100.00%', () => {
+    expect(formatPercent(1)).toBe('100.00%');
+  });
+
+  it('formats 0 as 0.00%', () => {
+    expect(formatPercent(0)).toBe('0.00%');
+  });
+});
+
+describe('generateReport', () => {
+  it('includes the platform label and target URL in the header', () => {
+    const md = generateReport(makeRun());
+    expect(md).toContain('cf-staging');
+    expect(md).toContain('wss://irc.staging.example.com/');
+  });
+
+  it('includes the ISO date and the human-readable duration', () => {
+    const md = generateReport(makeRun(emptySummary(), baseConfig(), 42_300));
+    expect(md).toContain('2026-08-03T14:00:00.000Z');
+    expect(md).toContain('42.30s');
+  });
+
+  it('reports the configured connection count, channel, and messages/client', () => {
+    const cfg = baseConfig({ connections: 10_000, channel: '#perf', messagesPerClient: 3 });
+    const md = generateReport(makeRun(emptySummary(), cfg));
+    expect(md).toContain('10000');
+    expect(md).toContain('#perf');
+    expect(md).toContain('3');
+  });
+
+  it('renders the drop rate from the summary', () => {
+    const results: ClientResult[] = [
+      okClient(0, { connect: [10] }),
+      okClient(1, { connect: [12] }),
+      {
+        id: 2,
+        ok: false,
+        error: 'timeout',
+        failedPhase: 'connect',
+        samples: { connect: [], register: [], join: [], privmsg: [] },
+      },
+    ];
+    const run = makeRun(summarizeResults(results));
+    const md = generateReport(run);
+    expect(md).toContain('Attempted');
+    expect(md).toContain('3');
+    expect(md).toContain('2'); // succeeded
+    expect(md).toContain('1'); // failed
+    expect(md).toContain('33.33%'); // drop rate
+  });
+
+  it('renders a latency table row per phase with p50/p95/p99 columns', () => {
+    const results: ClientResult[] = [
+      okClient(0, { connect: [12, 20], register: [40], join: [8], privmsg: [3, 4] }),
+      okClient(1, { connect: [30], register: [60], join: [10], privmsg: [5] }),
+    ];
+    const md = generateReport(makeRun(summarizeResults(results)));
+    // Column headers present.
+    expect(md).toContain('p50');
+    expect(md).toContain('p95');
+    expect(md).toContain('p99');
+    // Each phase name appears as a row.
+    for (const phase of PHASE_NAMES) {
+      expect(md).toContain(`| ${phase} `);
+    }
+    // Sample counts reach the table.
+    expect(md).toContain('| connect |'); // row marker
+  });
+
+  it('renders a zeroed latency table when no clients succeeded', () => {
+    const md = generateReport(makeRun(emptySummary()));
+    for (const phase of PHASE_NAMES) {
+      expect(md).toContain(`| ${phase} `);
+    }
+  });
+
+  it('documents that cost modeling is platform-specific (operator-filled)', () => {
+    const md = generateReport(makeRun());
+    expect(md.toLowerCase()).toContain('cost');
+  });
+});

package/tools/load-test/tests/smoke.test.ts

@@ -0,0 +1,67 @@
+import { type LocalServer, startLocalServer } from '@serverless-ircd/local-cli/dist/server.js';
+import { afterAll, beforeAll, describe, expect, it } from 'vitest';
+import { resolveConfig } from '../src/config';
+import { runLoadTest } from '../src/harness';
+import { generateReport } from '../src/report';
+
+/**
+ * Smoke acceptance: "load test script completes without crashing."
+ *
+ * This is the end-to-end smoke: the real `IrcLoadClient` factory, a real
+ * in-process IRC server (`startLocalServer`, the project's e2e fixture
+ * target), and a small fanout of real WebSocket connections through the
+ * full lifecycle. Sizes are kept tiny so the suite stays fast; the same
+ * machinery scales to 10k by changing `--connections` on the CLI.
+ */
+describe('load-test smoke (real IRC server)', () => {
+  let irc: LocalServer;
+  let target: string;
+
+  beforeAll(async () => {
+    irc = await startLocalServer({
+      port: 0,
+      hostname: '127.0.0.1',
+      serverName: 'irc.load-test.local',
+      networkName: 'LoadTestNet',
+    });
+    target = `ws://127.0.0.1:${irc.port}/`;
+  });
+
+  afterAll(async () => {
+    await irc.close();
+  });
+
+  it('completes a small fanout with no crashes and sane metrics', async () => {
+    const channel = `#smoke-${Math.random().toString(36).slice(2, 8)}`;
+    const config = resolveConfig({
+      target,
+      connections: 6,
+      connectConcurrency: 3,
+      rampMs: 0,
+      channel,
+      nickPrefix: 'smoke',
+      messagesPerClient: 2,
+      messageIntervalMs: 0,
+      platform: 'local-cli-smoke',
+      connectTimeoutMs: 3000,
+      registerTimeoutMs: 3000,
+    });
+
+    const run = await runLoadTest(config);
+
+    // Every client should succeed against the healthy in-process server.
+    expect(run.summary.totalClients).toBe(6);
+    expect(run.summary.succeeded).toBe(6);
+    expect(run.summary.failed).toBe(0);
+    expect(run.summary.dropRate).toBe(0);
+    // Each phase that was reached has samples.
+    expect(run.summary.phases.connect.count).toBe(6);
+    expect(run.summary.phases.register.count).toBe(6);
+    expect(run.summary.phases.join.count).toBe(6);
+    expect(run.summary.phases.privmsg.count).toBe(6 * 2);
+    // The report renders without throwing and mentions the platform.
+    const md = generateReport(run);
+    expect(md).toContain('local-cli-smoke');
+    expect(md).toContain('Latency');
+  }, 20_000);
+});

package/tools/load-test/tsconfig.build.json

@@ -0,0 +1,12 @@
+{
+  "extends": "../../tsconfig.base.json",
+  "compilerOptions": {
+    "composite": true,
+    "rootDir": "./src",
+    "outDir": "./dist",
+    "tsBuildInfoFile": "./.tsbuildinfo",
+    "types": ["node"]
+  },
+  "include": ["src/**/*.ts"],
+  "exclude": ["dist", "tests", "**/*.test.ts"]
+}

package/tools/load-test/tsconfig.test.json

@@ -0,0 +1,10 @@
+{
+  "extends": "../../tsconfig.base.json",
+  "compilerOptions": {
+    "noEmit": true,
+    "types": ["node"],
+    "composite": false
+  },
+  "include": ["src/**/*.ts", "tests/**/*.ts"],
+  "exclude": ["dist"]
+}

package/tools/load-test/vitest.config.ts

@@ -0,0 +1,29 @@
+import { defineConfig } from 'vitest/config';
+
+export default defineConfig({
+  test: {
+    environment: 'node',
+    include: ['tests/**/*.test.ts'],
+    // The harness smoke test spins up a real in-process WS IRC server
+    // (via @serverless-ircd/local-cli) and ramps a small fanout of real
+    // WebSocket connections against it; give it headroom while staying
+    // fast enough for watch mode.
+    testTimeout: 15_000,
+    hookTimeout: 15_000,
+    coverage: {
+      provider: 'v8',
+      all: false,
+      include: ['src/**/*.ts'],
+      // `main.ts` is CLI glue (argv parsing, signal handlers); exercised
+      // manually by `pnpm start`. Mirrors the apps/local-cli policy.
+      exclude: ['src/main.ts'],
+      reporter: ['text', 'html', 'json-summary'],
+      thresholds: {
+        lines: 90,
+        functions: 90,
+        branches: 80,
+        statements: 90,
+      },
+    },
+  },
+});

package/tools/tcp-ws-forwarder/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@serverless-ircd/tcp-ws-forwarder",
-  "version": "0.5.0",
+  "version": "0.6.0",
   "private": true,
   "description": "Local TCP to ws/wss forwarder so standard IRC clients can reach the WebSocket-only ServerlessIRCd transport",
   "license": "BSD-3-Clause",