serverless-ircd 0.9.0 → 0.11.0

package/packages/aws-adapter/src/handlers/default.ts

@@ -12,10 +12,10 @@ import type { ApiGatewayManagementApi } from '@aws-sdk/client-apigatewaymanageme
 import type { DynamoDBDocumentClient } from '@aws-sdk/lib-dynamodb';
 import { GetCommand, UpdateCommand } from '@aws-sdk/lib-dynamodb';
 import {
-  type AccountStore,
   type ChannelState,
   type Clock,
   type ConnectionState,
+  FRAME_FLOOD_ERROR_LINE,
   type IdFactory,
   type MessageStore,
   type MotdProvider,
@@ -26,6 +26,7 @@ import {
   SystemClock,
   UuidIdFactory,
   type WsFrameMode,
+  admitFrameWindow,
   createConnection,
   frameToLines,
 } from '@serverless-ircd/irc-core';
@@ -63,19 +64,14 @@ export interface DefaultParams {
    * behaviour of callers that have not opted in.
    */
   messages?: MessageStore;
-  /**
-   * SASL account verification source. When omitted the actor runs with
-   * `ctx.accounts` undefined so `AUTHENTICATE PLAIN` fails with `904`
-   * — preserves the behaviour of callers that have not opted in.
-   */
-  accounts?: AccountStore;
   /**
    * Persistent services store (NickServ / ChanServ / HostServ / MemoServ /
-   * OperServ + read-marker). When omitted the actor runs with
-   * `ctx.services` undefined so `PRIVMSG NickServ` / ChanServ routing is
-   * disabled — preserves the behaviour of callers that have not opted in.
-   * On success the handler flushes pending write-behind ops after the
-   * actor run.
+   * OperServ + read-marker + the unified SASL/PASS credential home). When
+   * omitted the actor runs with `ctx.services` undefined so
+   * `PRIVMSG NickServ` / ChanServ routing is disabled and every SASL/PASS
+   * credential verify fails — preserves the behaviour of callers that
+   * have not opted in. On success the handler flushes pending
+   * write-behind ops after the actor run.
    */
   services?: DynamoServicesStore;
   /**
@@ -128,113 +124,194 @@ export interface DefaultParams {
 export async function handleDefault(params: DefaultParams): Promise<{ statusCode: number }> {
   const clock = params.clock ?? SystemClock;
   const ids = params.ids ?? new UuidIdFactory();
-  // Load persisted state. Strongly consistent: $connect writes this row and
-  // $default reads it on the very next frame — an eventually-consistent Get
-  // can miss the just-written row (DynamoDB replication lag) and silently
-  // drop the client's first frame as a 410. Strong consistency closes that
-  // read-after-write window.
-  const result = await params.dynamo.send(
-    new GetCommand({
-      TableName: params.tables.Connections,
-      Key: { connectionId: params.connectionId },
-      ConsistentRead: true,
-    }),
-  );
-  if (result.Item === undefined) {
-    // Previously this branch returned silently, making a missing-row
-    // $default invisible to operators (client sees nothing, logs show
-    // nothing). Warn so the 410 path is diagnosable.
-    console.warn(
-      `[irc-handler] $default connId=${params.connectionId} wsMode=? row=missing -> 410`,
-    );
-    return { statusCode: 410 };
-  }
-  const persisted = result.Item as unknown as MarshalledConnection;
-  const state = unmarshalConnection(persisted);
 
-  // Recover the IRCv3 WebSocket frame mode persisted at `$connect`.
-  // Absent = legacy (no subprotocol negotiated). Drives both the inbound
-  // line-framing transport (spec: one IRC message per frame, never split)
-  // and the outbound delivery shape (spec: one postToConnection per line).
-  const wsMode: WsFrameMode = persisted.wsMode ?? 'legacy';
-
-  // Mark this frame's activity.
-  state.lastSeen = clock.now();
-
-  // Build the runtime + handlers.
+  // Optimistic-concurrency retry loop. API Gateway dispatches a client's
+  // rapid frames (CAP END, NICK, USER; or two PRIVMSG) as CONCURRENT
+  // Lambda invocations that each read-modify-write the same Connections
+  // row. Each attempt:
+  //   1. Strongly-consistent read (closes the $connect→$default window).
+  //   2. Run the actor over the frame against the fresh state.
+  //   3. Conditional write requiring `revision = :expected`; on conflict
+  //      (another invocation wrote first) re-read and re-run.
+  // Membership writes (TransactWriteItems + ADD/DELETE on a String Set)
+  // are idempotent by design, so a retried frame never doubles roster
+  // state. NOTE: peer `postToConnection` calls issued inside the actor
+  // run MAY double-deliver on the rare retry; registration (the deadlock
+  // this fixes) has no peer sends, and a duplicated PRIVMSG on a rare
+  // retry is strictly better than the total state-loss this loop fixes.
+  // Full peer-send buffering is a documented follow-up.
+  const MAX_RETRIES = 8;
   const outbound: string[] = [];
-  const handlers: AwsRuntimeHandlers = {
-    send: (lines) => {
-      // Self-send: APIGW echoes back to the caller's open socket via the
-      // management API in production; in tests we collect for assertions.
-      for (const l of lines) outbound.push(l.text);
-    },
-    disconnect: () => {
-      // No-op: a Lambda `$default` invocation cannot close its own APIGW
-      // socket. The canonical teardown is the `$disconnect` route (which
-      // fires when APIGW observes the close) plus the sweeper/ping-checker
-      // for connections that vanish without one. The actor's state
-      // mutations (nick release, roster deltas) are persisted below before
-      // this invocation returns, so the eventual `$disconnect` completes
-      // the fanout. See docs/AWS-Deployment.md §14.1.
-    },
-    snapshot: () => state,
-  };
-  const runtime = new AwsRuntime({
-    dynamo: params.dynamo,
-    tables: params.tables,
-    connId: params.connectionId,
-    handlers,
-    managementApi: params.managementApi,
-    clock,
-    ...(params.configLoader !== undefined ? { configLoader: params.configLoader } : {}),
-  });
+  // Captured from the committed attempt so the post-loop outbound flush
+  // knows the connection's frame mode (spec vs legacy line framing).
+  let wsModeHolder: WsFrameMode = 'legacy';
 
-  const channelAccess = new LambdaChannelAccess(runtime, clock.now());
-
-  const actor = new ConnectionActor({
-    state,
-    runtime,
-    channels: channelAccess,
-    serverConfig: params.serverConfig,
-    configSource: 'Secrets Manager',
-    clock,
-    ids,
-    motd: params.motd,
-    transport: { feed: (chunk) => frameToLines(chunk, wsMode) },
-    // AwsStats Scans Connections + ChannelMeta for LUSERS/STATS.
-    // `uptimeStartedAt` is captured at the Lambda's cold start so `STATS u`
-    // reports per-isolate uptime; a deployment-wide uptime would require a
-    // persistent record (future work).
-    stats: new AwsStats({
+  /** Builds a fresh actor for `state` and runs this invocation's frame. */
+  const runFrame = async (state: ConnectionState, wsMode: WsFrameMode): Promise<void> => {
+    const handlers: AwsRuntimeHandlers = {
+      send: (lines) => {
+        // Self-send: APIGW echoes back to the caller's open socket via the
+        // management API in production; in tests we collect for assertions.
+        for (const l of lines) outbound.push(l.text);
+      },
+      disconnect: () => {
+        // No-op: a Lambda `$default` invocation cannot close its own APIGW
+        // socket. The canonical teardown is the `$disconnect` route (which
+        // fires when APIGW observes the close) plus the sweeper/ping-checker
+        // for connections that vanish without one. The actor's state
+        // mutations (nick release, roster deltas) are persisted below before
+        // this invocation returns, so the eventual `$disconnect` completes
+        // the fanout. See docs/AWS-Deployment.md §14.1.
+      },
+      snapshot: () => state,
+    };
+    const runtime = new AwsRuntime({
       dynamo: params.dynamo,
       tables: params.tables,
-      uptimeStartedAt: LAMBDA_STARTUP_AT,
-    }),
-    ...(params.messages !== undefined ? { messages: params.messages } : {}),
-    ...(params.accounts !== undefined ? { accounts: params.accounts } : {}),
-    ...(params.services !== undefined ? { services: params.services } : {}),
-    ...(params.mtlsIdentity !== undefined ? { mtlsIdentity: params.mtlsIdentity } : {}),
-    ...(params.history !== undefined ? { history: params.history } : {}),
-    // API Gateway terminates TLS before the Lambda is invoked, so every
-    // WebSocket connection is secure → user mode `S`.
-    secure: true,
-  });
-
-  try {
+      connId: params.connectionId,
+      handlers,
+      managementApi: params.managementApi,
+      clock,
+      ...(params.configLoader !== undefined ? { configLoader: params.configLoader } : {}),
+    });
+    const channelAccess = new LambdaChannelAccess(runtime, clock.now());
+    const actor = new ConnectionActor({
+      state,
+      runtime,
+      channels: channelAccess,
+      serverConfig: params.serverConfig,
+      configSource: 'Secrets Manager',
+      clock,
+      ids,
+      motd: params.motd,
+      transport: { feed: (chunk) => frameToLines(chunk, wsMode) },
+      // AwsStats Scans Connections + ChannelMeta for LUSERS/STATS.
+      // `uptimeStartedAt` is captured at the Lambda's cold start so `STATS u`
+      // reports per-isolate uptime; a deployment-wide uptime would require a
+      // persistent record (future work).
+      stats: new AwsStats({
+        dynamo: params.dynamo,
+        tables: params.tables,
+        uptimeStartedAt: LAMBDA_STARTUP_AT,
+      }),
+      ...(params.messages !== undefined ? { messages: params.messages } : {}),
+      ...(params.services !== undefined ? { services: params.services } : {}),
+      ...(params.mtlsIdentity !== undefined ? { mtlsIdentity: params.mtlsIdentity } : {}),
+      ...(params.history !== undefined ? { history: params.history } : {}),
+      // API Gateway terminates TLS before the Lambda is invoked, so every
+      // WebSocket connection is secure → user mode `S`.
+      secure: true,
+    });
     await actor.receiveTextFrame(params.body);
-  } catch (err: unknown) {
-    console.error('[irc-handler] actor.receiveTextFrame failed', err);
-    // Still persist the state — partial mutations are valuable for debug.
-    await persistState(params.dynamo, params.tables, params.connectionId, state, clock.now());
-    await params.services?.flush();
-    return { statusCode: 500 };
+  };
+
+  for (let attempt = 0; ; attempt++) {
+    // 1. Strongly-consistent read.
+    const result = await params.dynamo.send(
+      new GetCommand({
+        TableName: params.tables.Connections,
+        Key: { connectionId: params.connectionId },
+        ConsistentRead: true,
+      }),
+    );
+    if (result.Item === undefined) {
+      // Row gone (client reconnected elsewhere, swept, or stale connId).
+      // Not retriable: surface the 410 so the client reconnects.
+      console.warn(
+        `[irc-handler] $default connId=${params.connectionId} wsMode=? row=missing -> 410`,
+      );
+      return { statusCode: 410 };
+    }
+    const persisted = result.Item as unknown as MarshalledConnection & { frameTimes?: unknown };
+    const state = unmarshalConnection(persisted);
+    const expectedRevision = persisted.revision ?? 0;
+    const wsMode: WsFrameMode = persisted.wsMode ?? 'legacy';
+    wsModeHolder = wsMode;
+
+    // 1a. Adapter-boundary frame-rate gate. Each frame is a billed read +
+    // write (and the actor run); a hostile client must not be able to
+    // drive that cost unboundedly with low-cost commands. The sliding
+    // window persists on the row (admitted frames persist it atomically
+    // with the state write below), so the limit holds across
+    // invocations. An over-limit frame never reaches the actor: the
+    // canonical ERROR line is delivered, the socket is closed via
+    // DeleteConnection (which fires the $disconnect cleanup), and the
+    // invocation returns without a write.
+    const frameDecision = admitFrameWindow(
+      readFrameTimes(persisted),
+      clock.now(),
+      params.serverConfig.adapter.maxFramesPerWindow,
+      params.serverConfig.adapter.frameWindowSeconds * 1000,
+    );
+    if (!frameDecision.admitted) {
+      console.warn(
+        `[irc-handler] $default connId=${params.connectionId} frame limit exceeded ` +
+          `(${params.serverConfig.adapter.maxFramesPerWindow}/${params.serverConfig.adapter.frameWindowSeconds}s) -> closing`,
+      );
+      if (params.managementApi !== null) {
+        try {
+          await postOutbound(
+            params.managementApi,
+            params.connectionId,
+            [FRAME_FLOOD_ERROR_LINE],
+            wsMode,
+          );
+          await closeConnectionIfSupported(params.managementApi, params.connectionId);
+        } catch (err) {
+          // The socket may already be gone (close raced a concurrent
+          // over-limit frame) — surface for observability, nothing to retry.
+          console.error('[irc-handler] flood close post failed', err);
+        }
+      }
+      return { statusCode: 200 };
+    }
+
+    // 2. Mark activity + run the actor. Self-sends reset per attempt.
+    state.lastSeen = clock.now();
+    outbound.length = 0;
+    try {
+      await runFrame(state, wsMode);
+    } catch (err: unknown) {
+      console.error('[irc-handler] actor.receiveTextFrame failed', err);
+      // Best-effort persist of partial state (swallow a conflict here —
+      // the actor already failed, no point retrying).
+      await persistState(
+        params.dynamo,
+        params.tables,
+        state,
+        clock.now(),
+        expectedRevision,
+        frameDecision.times,
+      ).catch(() => {});
+      await params.services?.flush();
+      return { statusCode: 500 };
+    }
+
+    // 3. Conditional persist.
+    try {
+      await persistState(
+        params.dynamo,
+        params.tables,
+        state,
+        clock.now(),
+        expectedRevision,
+        frameDecision.times,
+      );
+      break; // committed — exit the retry loop.
+    } catch (err: unknown) {
+      if (isConditionalCheckFailed(err) && attempt < MAX_RETRIES) {
+        // Another invocation wrote first; re-read + re-run against fresh state.
+        continue;
+      }
+      // Exhausted retries or an unrelated DynamoDB error.
+      console.error('[irc-handler] persist failed (retries exhausted or unrecoverable)', err);
+      await params.services?.flush();
+      return { statusCode: 500 };
+    }
   }
 
-  // Persist the mutated state and flush outbound bytes.
-  await persistState(params.dynamo, params.tables, params.connectionId, state, clock.now());
-  // Drain any services write-behind ops (NickServ/ChanServ/...) so a
-  // Lambda freeze / evict does not lose state.
+  // Persist committed. Drain any services write-behind ops (NickServ/ChanServ/...)
+  // so a Lambda freeze / evict does not lose state.
   await params.services?.flush();
 
   // In production, deliver outbound bytes back to the caller via APIGW.
@@ -243,7 +320,7 @@ export async function handleDefault(params: DefaultParams): Promise<{ statusCode
   // receive every line CRLF-joined in a single message.
   if (params.managementApi !== null && outbound.length > 0) {
     try {
-      await postOutbound(params.managementApi, params.connectionId, outbound, wsMode);
+      await postOutbound(params.managementApi, params.connectionId, outbound, wsModeHolder);
     } catch (err) {
       // Surface the failure (gone connection, stale MANAGEMENT_URL, …)
       // so it shows in CloudWatch instead of vanishing. The handler still
@@ -255,6 +332,35 @@ export async function handleDefault(params: DefaultParams): Promise<{ statusCode
   return { statusCode: 200 };
 }
 
+/**
+ * Reads the persisted inbound frame-window timestamps off a Connections
+ * row. Rows written before the frame gate shipped (or by paths that do
+ * not track frames, e.g. NLB chunks) carry no attribute — an empty
+ * window starts the connection fresh.
+ */
+function readFrameTimes(row: MarshalledConnection & { frameTimes?: unknown }): number[] {
+  const raw = row.frameTimes;
+  if (!Array.isArray(raw)) return [];
+  return raw as number[];
+}
+
+/**
+ * Closes the client's APIGW WebSocket via the management API
+ * (`DeleteConnection` — API Gateway observes the close and fires the
+ * `$disconnect` route, which runs the canonical QUIT cleanup). The
+ * `PostToConnection` test-double arm has no delete method; the close is
+ * skipped there (the ERROR post alone still informs the client).
+ */
+async function closeConnectionIfSupported(
+  api: ApiGatewayManagementApi | PostToConnection,
+  connectionId: string,
+): Promise<void> {
+  const deleter = api as Partial<Pick<ApiGatewayManagementApi, 'deleteConnection'>>;
+  if (typeof deleter.deleteConnection === 'function') {
+    await deleter.deleteConnection({ ConnectionId: connectionId });
+  }
+}
+
 /**
  * Delivers `lines` to `connectionId` via APIGW according to `mode`:
  *   - `legacy` — all lines joined with `\r\n` in a single `postToConnection`.
@@ -280,28 +386,35 @@ async function postOutbound(
 /**
  * Writes the actor's mutated `state` back to the Connections row.
  *
- * Uses an `UpdateCommand` that SETs every actor-owned field EXCEPT
- * `joinedChannels`. The membership transaction (see `AwsRuntime`)
- * owns `joinedChannels` atomically; if this frame's full-state write
- * touched it, a concurrent `$default` for the same connection would
- * lose the other frame's JOIN/PART. Optional fields (nick, user, …)
- * are SET when present and REMOVE'd when absent so a cleared value
- * does not linger from a previous frame.
+ * Uses a conditional `UpdateCommand` that SETs every actor-owned field
+ * EXCEPT `joinedChannels`, bumps the `revision` counter, and requires
+ * the row's `revision` to still equal `expectedRevision` (the value read
+ * at the start of this invocation). A `ConditionalCheckFailedException`
+ * means another concurrent `$default` for the same connection wrote in
+ * between; {@link handleDefault} catches it and retries the whole
+ * read → run → persist cycle against the fresh row. The membership
+ * transaction (see `AwsRuntime`) owns `joinedChannels` atomically and
+ * is idempotent (`ADD`/`DELETE` on a String Set), so a retried frame
+ * never doubles roster state. Optional fields (nick, user, …) are SET
+ * when present and REMOVE'd when absent so a cleared value does not
+ * linger from a previous frame.
  */
 async function persistState(
   dynamo: DynamoDBDocumentClient,
   tables: TablesConfig,
-  _connId: string,
   state: ConnectionState,
   now: number,
+  expectedRevision: number,
+  frameTimes: readonly number[],
 ): Promise<void> {
-  const expr = buildPersistUpdate(state, now);
+  const expr = buildPersistUpdate(state, now, expectedRevision, frameTimes);
   const hasNames = Object.keys(expr.expressionAttributeNames).length > 0;
   await dynamo.send(
     new UpdateCommand({
       TableName: tables.Connections,
       Key: { connectionId: state.id },
       UpdateExpression: expr.updateExpression,
+      ConditionExpression: expr.conditionExpression,
       ExpressionAttributeValues: expr.expressionAttributeValues,
       ...(hasNames ? { ExpressionAttributeNames: expr.expressionAttributeNames } : {}),
     }),
@@ -326,12 +439,28 @@ const OPTIONAL_FIELDS = [
  * the required fields; SETs optional fields that are present and
  * REMOVEs those that are absent. `joinedChannels` is intentionally
  * excluded — owned by the membership transaction.
+ *
+ * **Optimistic concurrency:** the expression also `ADD`s `1` to the
+ * `revision` counter and attaches a `ConditionExpression` requiring
+ * `revision = :expectedRev` (or `attribute_not_exists(#rev)` for rows
+ * written before the field shipped). {@link handleDefault} captures the
+ * row's revision at read time and passes it as `expectedRevision`; a
+ * `ConditionalCheckFailedException` means another concurrent invocation
+ * for the same connection wrote in between → the caller re-reads and
+ * re-runs the frame (see {@link handleDefault}'s retry loop). Without
+ * this guard, two concurrent `$default` invocations (API Gateway
+ * dispatches a client's rapid frames as overlapping Lambdas) clobber
+ * each other's state — the registration deadlock where `CAP END`'s
+ * `capNegotiating=false` was overwritten by `NICK`/`USER`'s stale read.
  */
-function buildPersistUpdate(
+export function buildPersistUpdate(
   state: ConnectionState,
   now: number,
+  expectedRevision: number,
+  frameTimes: readonly number[],
 ): {
   updateExpression: string;
+  conditionExpression: string;
   expressionAttributeNames: Record<string, string>;
   expressionAttributeValues: Record<string, unknown>;
 } {
@@ -343,6 +472,15 @@ function buildPersistUpdate(
     ':ls': state.lastSeen,
     ':is': now,
     ':v': CONNECTION_VERSION,
+    // Sliding-window timestamps for the adapter-boundary frame limit.
+    // Committed atomically with the state write (same revision
+    // condition), so each frame is counted exactly once even when
+    // concurrent invocations race through the retry loop.
+    ':ft': [...frameTimes],
+    // Optimistic-concurrency: increment the revision counter, gated by
+    // a condition that the row is still at the revision we read.
+    ':revInc': 1,
+    ':expectedRev': expectedRevision,
   };
   const setNames = [
     'registration = :reg',
@@ -352,9 +490,10 @@ function buildPersistUpdate(
     'lastSeen = :ls',
     'idleSince = :is',
     'version = :v',
+    'frameTimes = :ft',
   ];
   const removeNames: string[] = [];
-  const nameMap: Record<string, string> = {};
+  const nameMap: Record<string, string> = { '#rev': 'revision' };
 
   for (const field of OPTIONAL_FIELDS) {
     const placeholder = `:${field}`;
@@ -373,13 +512,30 @@ function buildPersistUpdate(
   if (removeNames.length > 0) {
     parts.push(`REMOVE ${removeNames.join(', ')}`);
   }
+  // Bump the optimistic-concurrency counter atomically with the state write.
+  parts.push('ADD #rev :revInc');
   return {
     updateExpression: parts.join(' '),
+    // `attribute_not_exists(#rev)` lets the first write after deploy land on
+    // pre-existing rows that predate the column (treats them as revision 0).
+    conditionExpression: 'attribute_not_exists(#rev) OR #rev = :expectedRev',
     expressionAttributeNames: nameMap,
     expressionAttributeValues: values,
   };
 }
 
+/**
+ * Returns true when a DynamoDB error is a conditional-check failure
+ * (the row's `revision` no longer matches the value the invocation
+ * read). Such failures are retried by {@link handleDefault}'s
+ * optimistic-concurrency loop after re-reading and re-running the frame.
+ */
+export function isConditionalCheckFailed(err: unknown): boolean {
+  if (err === null || typeof err !== 'object') return false;
+  const name = (err as { name?: string }).name;
+  return name === 'ConditionalCheckFailedException';
+}
+
 /**
  * `ActorChannelAccess` backed by the per-invocation {@link AwsRuntime}.
  *