serverless-ircd 0.3.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/ci.yml +2 -2
- package/.github/workflows/deploy-aws.yml +1 -3
- package/.github/workflows/deploy-cf-tcp.yml +87 -0
- package/.github/workflows/deploy-cf.yml +1 -1
- package/.node-version +1 -0
- package/.nvmrc +1 -0
- package/CHANGELOG.md +349 -18
- package/README.md +132 -30
- package/apps/aws-stack/README.md +6 -5
- package/apps/aws-stack/bin/aws.ts +7 -0
- package/apps/aws-stack/package.json +4 -4
- package/apps/aws-stack/src/aws-stack.ts +150 -10
- package/apps/aws-stack/tests/stack.test.ts +145 -4
- package/apps/cf-tcp-container/Dockerfile +69 -0
- package/apps/cf-tcp-container/package.json +34 -0
- package/apps/cf-tcp-container/src/config-loader.ts +145 -0
- package/apps/cf-tcp-container/src/container-do.ts +38 -0
- package/apps/cf-tcp-container/src/container-server.ts +363 -0
- package/apps/cf-tcp-container/src/main.ts +77 -0
- package/apps/cf-tcp-container/src/persistence.ts +144 -0
- package/apps/cf-tcp-container/src/worker.ts +41 -0
- package/apps/cf-tcp-container/terraform/provider.tf +24 -0
- package/apps/cf-tcp-container/terraform/spectrum.tf +81 -0
- package/apps/cf-tcp-container/tests/config-loader.test.ts +217 -0
- package/apps/cf-tcp-container/tests/container-server.test.ts +465 -0
- package/apps/cf-tcp-container/tests/persistence.test.ts +227 -0
- package/apps/cf-tcp-container/tests/tls-e2e.test.ts +275 -0
- package/apps/cf-tcp-container/tsconfig.build.json +17 -0
- package/apps/cf-tcp-container/tsconfig.test.json +15 -0
- package/apps/cf-tcp-container/vitest.config.ts +26 -0
- package/apps/cf-tcp-container/wrangler.toml +63 -0
- package/apps/cf-worker/package.json +1 -1
- package/apps/cf-worker/wrangler.test.toml +6 -0
- package/apps/cf-worker/wrangler.toml +28 -2
- package/apps/local-cli/package.json +1 -1
- package/apps/local-cli/src/config-loader.ts +10 -0
- package/apps/local-cli/src/server.ts +149 -24
- package/apps/local-cli/tests/e2e.test.ts +1 -1
- package/apps/local-cli/tests/ws-subprotocol.test.ts +257 -0
- package/package.json +14 -10
- package/packages/aws-adapter/package.json +3 -3
- package/packages/aws-adapter/src/account-store.ts +1 -1
- package/packages/aws-adapter/src/admission.ts +74 -0
- package/packages/aws-adapter/src/aws-runtime.ts +75 -4
- package/packages/aws-adapter/src/config-loader.ts +32 -0
- package/packages/aws-adapter/src/dynamo-account-store.ts +35 -96
- package/packages/aws-adapter/src/handlers/connect.ts +96 -9
- package/packages/aws-adapter/src/handlers/default.ts +98 -7
- package/packages/aws-adapter/src/handlers/index.ts +109 -4
- package/packages/aws-adapter/src/handlers/nlb-stream.ts +490 -0
- package/packages/aws-adapter/src/handlers/ping-checker.ts +1 -1
- package/packages/aws-adapter/src/index.ts +8 -0
- package/packages/aws-adapter/src/serialize.ts +11 -1
- package/packages/aws-adapter/src/stats.ts +80 -0
- package/packages/aws-adapter/tests/account-store-dynamo.test.ts +45 -34
- package/packages/aws-adapter/tests/account-store.test.ts +19 -20
- package/packages/aws-adapter/tests/admission.test.ts +70 -0
- package/packages/aws-adapter/tests/aws-harness.ts +13 -1
- package/packages/aws-adapter/tests/aws-integration.test.ts +1 -1
- package/packages/aws-adapter/tests/aws-runtime.test.ts +61 -0
- package/packages/aws-adapter/tests/config-loader.test.ts +20 -0
- package/packages/aws-adapter/tests/connect.test.ts +174 -0
- package/packages/aws-adapter/tests/disconnect-fanout.test.ts +47 -40
- package/packages/aws-adapter/tests/gone-exception.test.ts +31 -26
- package/packages/aws-adapter/tests/handlers.test.ts +302 -53
- package/packages/aws-adapter/tests/nlb-stream.test.ts +480 -0
- package/packages/aws-adapter/tests/ping-checker.test.ts +34 -29
- package/packages/aws-adapter/tests/stats.test.ts +317 -0
- package/packages/aws-adapter/tests/sweeper.test.ts +25 -18
- package/packages/aws-adapter/tests/transactions.test.ts +25 -20
- package/packages/cf-adapter/package.json +5 -1
- package/packages/cf-adapter/src/cf-runtime.ts +68 -5
- package/packages/cf-adapter/src/channel-do.ts +2 -2
- package/packages/cf-adapter/src/config-loader.ts +33 -0
- package/packages/cf-adapter/src/connection-do.ts +278 -85
- package/packages/cf-adapter/src/d1-account-store.ts +198 -0
- package/packages/cf-adapter/src/env.ts +54 -11
- package/packages/cf-adapter/src/index.ts +11 -8
- package/packages/cf-adapter/src/registry-do.ts +22 -3
- package/packages/cf-adapter/src/sharding.ts +1 -2
- package/packages/cf-adapter/src/stats.ts +65 -0
- package/packages/cf-adapter/tests/cf-harness.ts +12 -2
- package/packages/cf-adapter/tests/cf-integration.test.ts +6 -5
- package/packages/cf-adapter/tests/cf-runtime.test.ts +38 -2
- package/packages/cf-adapter/tests/channel-do.test.ts +2 -2
- package/packages/cf-adapter/tests/config-loader.test.ts +22 -0
- package/packages/cf-adapter/tests/connection-do-channel-registration.test.ts +37 -0
- package/packages/cf-adapter/tests/connection-do-no-batching-reservation.test.ts +52 -0
- package/packages/cf-adapter/tests/connection-do-sasl-d1.test.ts +166 -0
- package/packages/cf-adapter/tests/connection-do-ws-spec-contract.test.ts +289 -0
- package/packages/cf-adapter/tests/connection-do-ws-subprotocol.test.ts +184 -0
- package/packages/cf-adapter/tests/connection-do.test.ts +27 -2
- package/packages/cf-adapter/tests/d1-account-store.test.ts +226 -0
- package/packages/cf-adapter/tests/raw-modules.d.ts +11 -0
- package/packages/cf-adapter/tests/registry-do.test.ts +4 -4
- package/packages/cf-adapter/tests/sharding.test.ts +1 -1
- package/packages/cf-adapter/tests/stats.test.ts +120 -0
- package/packages/cf-adapter/tests/worker/main.ts +15 -8
- package/packages/cf-adapter/tests/worker/stubs/channel-stub.ts +2 -2
- package/packages/cf-adapter/tests/worker/stubs/registry-stub.ts +8 -2
- package/packages/cf-adapter/wrangler.test.toml +15 -0
- package/packages/in-memory-runtime/package.json +1 -1
- package/packages/in-memory-runtime/src/in-memory-runtime.ts +39 -0
- package/packages/in-memory-runtime/tests/in-memory-runtime.test.ts +259 -0
- package/packages/irc-core/package.json +6 -1
- package/packages/irc-core/scripts/generate-build-info.mjs +31 -0
- package/packages/irc-core/src/admission.ts +16 -15
- package/packages/irc-core/src/caps/capabilities.ts +24 -3
- package/packages/irc-core/src/cloak.ts +1 -1
- package/packages/irc-core/src/commands/cap.ts +8 -1
- package/packages/irc-core/src/commands/index.ts +17 -0
- package/packages/irc-core/src/commands/invite.ts +2 -4
- package/packages/irc-core/src/commands/ison.ts +61 -0
- package/packages/irc-core/src/commands/isupport.ts +6 -2
- package/packages/irc-core/src/commands/kick.ts +2 -4
- package/packages/irc-core/src/commands/kill.ts +127 -0
- package/packages/irc-core/src/commands/list.ts +1 -1
- package/packages/irc-core/src/commands/lusers.ts +204 -0
- package/packages/irc-core/src/commands/mode.ts +4 -8
- package/packages/irc-core/src/commands/names.ts +3 -5
- package/packages/irc-core/src/commands/part.ts +2 -4
- package/packages/irc-core/src/commands/quit.ts +12 -0
- package/packages/irc-core/src/commands/registration.ts +18 -12
- package/packages/irc-core/src/commands/rehash.ts +119 -0
- package/packages/irc-core/src/commands/sasl.ts +72 -9
- package/packages/irc-core/src/commands/server-info.ts +129 -0
- package/packages/irc-core/src/commands/setname.ts +109 -0
- package/packages/irc-core/src/commands/stats.ts +152 -0
- package/packages/irc-core/src/commands/topic.ts +2 -4
- package/packages/irc-core/src/commands/trace.ts +137 -0
- package/packages/irc-core/src/commands/userhost.ts +84 -0
- package/packages/irc-core/src/commands/wallops.ts +118 -0
- package/packages/irc-core/src/commands/whowas.ts +113 -0
- package/packages/irc-core/src/config.ts +65 -0
- package/packages/irc-core/src/credential-hashing.ts +124 -0
- package/packages/irc-core/src/effects.ts +33 -30
- package/packages/irc-core/src/index.ts +3 -0
- package/packages/irc-core/src/ports.ts +360 -12
- package/packages/irc-core/src/protocol/numerics.ts +48 -11
- package/packages/irc-core/src/protocol/outbound.ts +20 -3
- package/packages/irc-core/src/types.ts +46 -2
- package/packages/irc-core/src/ws-framing.ts +132 -0
- package/packages/irc-core/src/ws-subprotocol.ts +66 -0
- package/packages/irc-core/tests/account-store.test.ts +45 -2
- package/packages/irc-core/tests/admission.test.ts +18 -0
- package/packages/irc-core/tests/caps/capabilities.test.ts +4 -3
- package/packages/irc-core/tests/commands/cap.test.ts +33 -1
- package/packages/irc-core/tests/commands/ison.test.ts +166 -0
- package/packages/irc-core/tests/commands/kill.test.ts +243 -0
- package/packages/irc-core/tests/commands/lusers.test.ts +368 -0
- package/packages/irc-core/tests/commands/mode.test.ts +57 -0
- package/packages/irc-core/tests/commands/quit.test.ts +69 -2
- package/packages/irc-core/tests/commands/registration.test.ts +151 -6
- package/packages/irc-core/tests/commands/rehash.test.ts +171 -0
- package/packages/irc-core/tests/commands/sasl.test.ts +118 -10
- package/packages/irc-core/tests/commands/server-info.test.ts +274 -0
- package/packages/irc-core/tests/commands/setname.test.ts +225 -0
- package/packages/irc-core/tests/commands/stats.test.ts +294 -0
- package/packages/irc-core/tests/commands/tagmsg.test.ts +9 -35
- package/packages/irc-core/tests/commands/trace.test.ts +282 -0
- package/packages/irc-core/tests/commands/userhost.test.ts +264 -0
- package/packages/irc-core/tests/commands/wallops.test.ts +231 -0
- package/packages/irc-core/tests/commands/whowas.test.ts +312 -0
- package/packages/irc-core/tests/config.test.ts +95 -1
- package/packages/irc-core/tests/credential-hashing.test.ts +170 -0
- package/packages/irc-core/tests/dropped-s2s-and-obsolete-verbs.test.ts +90 -0
- package/packages/irc-core/tests/effects.test.ts +14 -27
- package/packages/irc-core/tests/nick-history-store.test.ts +162 -0
- package/packages/irc-core/tests/numerics.test.ts +102 -0
- package/packages/irc-core/tests/outbound.test.ts +51 -0
- package/packages/irc-core/tests/ports.test.ts +22 -0
- package/packages/irc-core/tests/raw-modules.d.ts +11 -0
- package/packages/irc-core/tests/stats-store.test.ts +222 -0
- package/packages/irc-core/tests/types.test.ts +35 -1
- package/packages/irc-core/tests/ws-framing.test.ts +213 -0
- package/packages/irc-core/tests/ws-subprotocol.test.ts +111 -0
- package/packages/irc-core/tsconfig.build.json +1 -1
- package/packages/irc-core/tsconfig.test.json +1 -1
- package/packages/irc-server/package.json +1 -1
- package/packages/irc-server/src/actor.ts +393 -16
- package/packages/irc-server/src/dispatch.ts +1 -3
- package/packages/irc-server/src/index.ts +10 -2
- package/packages/irc-server/src/routing.ts +15 -0
- package/packages/irc-server/src/runtime.ts +31 -0
- package/packages/irc-server/src/transport.ts +104 -0
- package/packages/irc-server/tests/actor.test.ts +1489 -4
- package/packages/irc-server/tests/dispatch.test.ts +37 -17
- package/packages/irc-server/tests/raw-modules.d.ts +11 -0
- package/packages/irc-server/tests/routing.test.ts +5 -0
- package/packages/irc-server/tests/runtime.test.ts +7 -0
- package/packages/irc-server/tests/transport.test.ts +230 -0
- package/packages/irc-test-support/package.json +1 -1
- package/packages/irc-test-support/src/harness.ts +44 -9
- package/packages/irc-test-support/src/in-memory-harness.ts +73 -9
- package/packages/irc-test-support/src/index.ts +3 -0
- package/packages/irc-test-support/src/scenarios.ts +141 -3
- package/packages/irc-test-support/tests/in-memory-harness.test.ts +2 -1
- package/packages/irc-test-support/tests/in-memory-scenarios.test.ts +23 -9
- package/pnpm-workspace.yaml +10 -1
- package/tools/ci-hardening/package.json +1 -1
- package/tools/package.json +9 -0
- package/tools/seed-aws-accounts.ts +5 -6
- package/tools/seed-cf-accounts.ts +107 -0
- package/tools/tcp-ws-forwarder/package.json +1 -1
- package/tools/tcp-ws-forwarder/src/forwarder.ts +57 -9
- package/tools/tcp-ws-forwarder/tests/forwarder.test.ts +34 -1
- package/tools/tcp-ws-forwarder/tests/framing.test.ts +65 -1
- package/docs/ADR-001-pure-reducers-and-effect-system.md +0 -74
- package/docs/ADR-002-location-of-authority.md +0 -82
- package/docs/ADR-003-durable-object-sharding.md +0 -93
- package/docs/ADR-004-dynamodb-schema.md +0 -96
- package/docs/ADR-005-wss-only-transport-v1.md +0 -83
- package/docs/ADR-006-sasl-mechanism-scope.md +0 -86
- package/docs/ADR-007-deterministic-ports.md +0 -82
- package/docs/ADR-008-monorepo-tooling.md +0 -60
- package/docs/AWS-Adapter-Architecture.md +0 -496
- package/docs/AWS-Deployment.md +0 -1186
- package/docs/Cloudflare-Deployment-Guide.md +0 -660
- package/docs/Home.md +0 -11
- package/docs/Observability.md +0 -87
- package/docs/PlanIRCv3Websocket.md +0 -489
- package/docs/PlanWebClient.md +0 -451
- package/docs/Release-Process.md +0 -443
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Pure admission policy for the AWS `$connect` handler.
|
|
3
|
+
*
|
|
4
|
+
* The I/O (counting live rows in DynamoDB) lives in `handlers/connect.ts`;
|
|
5
|
+
* the *decision* — "does this count fit under the caps?" — is a pure
|
|
6
|
+
* function extracted here so it is trivial to unit-test every branch
|
|
7
|
+
* without standing up DynamoDB Local.
|
|
8
|
+
*
|
|
9
|
+
* This is deliberately separate from irc-core's `decideAdmission` /
|
|
10
|
+
* `AdmissionStats`: that helper is an in-memory counter design that
|
|
11
|
+
* assumes a single long-lived process. AWS Lambda invocations are
|
|
12
|
+
* independent processes with no shared memory, so the authoritative
|
|
13
|
+
* counts MUST come from DynamoDB (the shared `Connections` table) and
|
|
14
|
+
* are fed into this pure policy.
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* Live connection counts gathered from DynamoDB.
|
|
19
|
+
*
|
|
20
|
+
* - `total` is always supplied (a paginated `Scan COUNT` over
|
|
21
|
+
* `Connections`).
|
|
22
|
+
* - `perIp` is optional: counting connections per source IP needs a
|
|
23
|
+
* GSI on `sourceIp` (see `handlers/connect.ts` for the deferred
|
|
24
|
+
* per-IP follow-up). Until that lands, callers omit `perIp` and the
|
|
25
|
+
* per-IP branch of {@link decideConnectAdmission} is skipped.
|
|
26
|
+
*/
|
|
27
|
+
export interface ConnectionCounts {
|
|
28
|
+
readonly total: number;
|
|
29
|
+
readonly perIp?: number;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* Effective caps sourced from {@link ParsedServerConfig}. `perIp` is
|
|
34
|
+
* optional because per-IP enforcement is not yet wired on the AWS path.
|
|
35
|
+
*/
|
|
36
|
+
export interface AdmissionLimits {
|
|
37
|
+
readonly maxClients: number;
|
|
38
|
+
readonly maxConnectionsPerIp?: number;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* Outcome of an admission decision. Mirrors the {@link ConnectOutcome}
|
|
43
|
+
* shape returned by `handleConnect` so the handler can return the
|
|
44
|
+
* decision verbatim.
|
|
45
|
+
*/
|
|
46
|
+
export type AdmissionOutcome =
|
|
47
|
+
| { readonly admitted: true }
|
|
48
|
+
| { readonly admitted: false; readonly statusCode: number; readonly reason: string };
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* Decides whether a new connection should be admitted given the live
|
|
52
|
+
* counts and the configured caps. Pure: same inputs ⇒ same output, no
|
|
53
|
+
* ambient state.
|
|
54
|
+
*
|
|
55
|
+
* Precedence: the global `maxClients` cap is checked first (it is the
|
|
56
|
+
* cheap, always-available gate), then the optional per-IP cap when both
|
|
57
|
+
* the limit and the `perIp` count are present.
|
|
58
|
+
*/
|
|
59
|
+
export function decideConnectAdmission(
|
|
60
|
+
counts: ConnectionCounts,
|
|
61
|
+
limits: AdmissionLimits,
|
|
62
|
+
): AdmissionOutcome {
|
|
63
|
+
if (counts.total >= limits.maxClients) {
|
|
64
|
+
return { admitted: false, statusCode: 429, reason: 'server full' };
|
|
65
|
+
}
|
|
66
|
+
if (
|
|
67
|
+
limits.maxConnectionsPerIp !== undefined &&
|
|
68
|
+
counts.perIp !== undefined &&
|
|
69
|
+
counts.perIp >= limits.maxConnectionsPerIp
|
|
70
|
+
) {
|
|
71
|
+
return { admitted: false, statusCode: 429, reason: 'too many connections from this IP' };
|
|
72
|
+
}
|
|
73
|
+
return { admitted: true };
|
|
74
|
+
}
|
|
@@ -51,6 +51,7 @@ import {
|
|
|
51
51
|
type Nick,
|
|
52
52
|
type RawLine,
|
|
53
53
|
type RosterEntry,
|
|
54
|
+
type ServerConfig,
|
|
54
55
|
SystemClock,
|
|
55
56
|
emptyModes,
|
|
56
57
|
toSnapshot as toConnSnapshot,
|
|
@@ -80,7 +81,8 @@ import type { TablesConfig } from './tables.js';
|
|
|
80
81
|
* is the in-memory `received` array.
|
|
81
82
|
* - `disconnect(reason?)` closes the bound transport. In Lambda this
|
|
82
83
|
* is a no-op — the canonical disconnect path is the APIGW `$disconnect`
|
|
83
|
-
* route, which fires after the socket is already gone.
|
|
84
|
+
* route, which fires after the socket is already gone. See
|
|
85
|
+
* `docs/AWS-Deployment.md` §14.1 for the full platform-limit write-up.
|
|
84
86
|
* - `snapshot()` returns the in-memory ConnectionState the actor is
|
|
85
87
|
* mutating this frame. Critical so `getConnection(boundId)` returns
|
|
86
88
|
* the live view (and dispatch's reducer-observable effects see
|
|
@@ -118,6 +120,14 @@ export interface AwsRuntimeOptions {
|
|
|
118
120
|
*/
|
|
119
121
|
managementApi: ApiGatewayManagementApi | PostToConnection | null;
|
|
120
122
|
clock?: Clock;
|
|
123
|
+
/**
|
|
124
|
+
* Config-reload source for `REHASH`. The Lambda handler binds this to a
|
|
125
|
+
* closure that re-invokes {@link loadServerConfigFromLambdaEnv} against the
|
|
126
|
+
* invocation env (Secrets Manager / SSM / env vars). When omitted
|
|
127
|
+
* `reloadConfig` rejects so the actor applies a graceful error-suffixed
|
|
128
|
+
* `382` and retains the prior config.
|
|
129
|
+
*/
|
|
130
|
+
readonly configLoader?: () => Promise<ServerConfig>;
|
|
121
131
|
}
|
|
122
132
|
|
|
123
133
|
/**
|
|
@@ -132,6 +142,7 @@ export class AwsRuntime implements IrcRuntime {
|
|
|
132
142
|
private readonly handlers: AwsRuntimeHandlers;
|
|
133
143
|
private readonly managementApi: ApiGatewayManagementApi | PostToConnection | null;
|
|
134
144
|
private readonly clock: Clock;
|
|
145
|
+
private readonly configLoader: (() => Promise<ServerConfig>) | undefined;
|
|
135
146
|
|
|
136
147
|
constructor(opts: AwsRuntimeOptions) {
|
|
137
148
|
this.dynamo = opts.dynamo;
|
|
@@ -140,6 +151,7 @@ export class AwsRuntime implements IrcRuntime {
|
|
|
140
151
|
this.handlers = opts.handlers;
|
|
141
152
|
this.managementApi = opts.managementApi;
|
|
142
153
|
this.clock = opts.clock ?? SystemClock;
|
|
154
|
+
this.configLoader = opts.configLoader;
|
|
143
155
|
}
|
|
144
156
|
|
|
145
157
|
// -------------------------------------------------------------------------
|
|
@@ -185,11 +197,12 @@ export class AwsRuntime implements IrcRuntime {
|
|
|
185
197
|
this.handlers.disconnect(reason);
|
|
186
198
|
return;
|
|
187
199
|
}
|
|
188
|
-
// Cross-connection disconnect is a no-op
|
|
189
|
-
//
|
|
200
|
+
// Cross-connection disconnect is a no-op: the Lambda runtime cannot
|
|
201
|
+
// directly close another connection's WebSocket. The canonical
|
|
190
202
|
// disconnect path is the APIGW `$disconnect` route, which fires when
|
|
191
203
|
// the socket is already gone. KICK/PART remove membership rows but
|
|
192
|
-
// do not need to force-close the socket.
|
|
204
|
+
// do not need to force-close the socket. See docs/AWS-Deployment.md
|
|
205
|
+
// §14.1 for the full platform-limit write-up and the mitigations.
|
|
193
206
|
}
|
|
194
207
|
|
|
195
208
|
async sendToNick(
|
|
@@ -208,6 +221,51 @@ export class AwsRuntime implements IrcRuntime {
|
|
|
208
221
|
}
|
|
209
222
|
}
|
|
210
223
|
|
|
224
|
+
/**
|
|
225
|
+
* Global cross-connection fan-out for `WALLOPS`. `Scan`s the `Connections`
|
|
226
|
+
* table filtered to rows whose `userModes.wallops` is `true`, then
|
|
227
|
+
* `PostToConnection`s each one (skipping `except`). The bound connection's
|
|
228
|
+
* own delivery goes through the in-process handlers (no APIGW round-trip),
|
|
229
|
+
* mirroring `send`.
|
|
230
|
+
*
|
|
231
|
+
* Cost: a DynamoDB `Scan` reads the ENTIRE table (chargeable per row read,
|
|
232
|
+
* not per match), so this is expensive at high connection counts. A fleet
|
|
233
|
+
* of 10k connections costs one full scan per WALLOPS plus one
|
|
234
|
+
* `PostToConnection` per `+w` recipient. Deployments should bound the
|
|
235
|
+
* oper's WALLOPS rate; a projected GSI on `userModes.wallops` would lift
|
|
236
|
+
* the scan to a targeted query (documented follow-up). The scan uses
|
|
237
|
+
* `FilterExpression` so only `+w` rows survive to the `PostToConnection`
|
|
238
|
+
* phase — the APIGW call count is bounded by the genuine recipient set.
|
|
239
|
+
*/
|
|
240
|
+
async broadcastWallops(lines: RawLine[], except?: ConnId): Promise<void> {
|
|
241
|
+
let startKey: Record<string, NativeAttributeValue> | undefined;
|
|
242
|
+
// Paginate the scan: DynamoDB caps a single Scan at 1 MB. Loop until
|
|
243
|
+
// no `LastEvaluatedKey` remains so every +w connection is reached.
|
|
244
|
+
do {
|
|
245
|
+
const result = await this.dynamo.send(
|
|
246
|
+
new ScanCommand({
|
|
247
|
+
TableName: this.tables.Connections,
|
|
248
|
+
FilterExpression: 'userModes.#m = :true',
|
|
249
|
+
ExpressionAttributeNames: { '#m': 'wallops' },
|
|
250
|
+
ExpressionAttributeValues: { ':true': true },
|
|
251
|
+
...(startKey !== undefined ? { ExclusiveStartKey: startKey } : {}),
|
|
252
|
+
}),
|
|
253
|
+
);
|
|
254
|
+
const items = (result.Items ?? []) as unknown as MarshalledConnection[];
|
|
255
|
+
for (const row of items) {
|
|
256
|
+
const connId = row.connectionId;
|
|
257
|
+
if (connId === except) continue;
|
|
258
|
+
// `send` short-cuts the bound connection through the in-process
|
|
259
|
+
// handlers and catches GoneException so vanished sockets are
|
|
260
|
+
// cleaned up rather than aborting the fan-out mid-loop.
|
|
261
|
+
if (row.userModes.wallops) {
|
|
262
|
+
await this.send(connId, lines);
|
|
263
|
+
}
|
|
264
|
+
}
|
|
265
|
+
startKey = result.LastEvaluatedKey as Record<string, NativeAttributeValue> | undefined;
|
|
266
|
+
} while (startKey !== undefined);
|
|
267
|
+
}
|
|
268
|
+
|
|
211
269
|
// -------------------------------------------------------------------------
|
|
212
270
|
// Nick registry
|
|
213
271
|
// -------------------------------------------------------------------------
|
|
@@ -363,6 +421,19 @@ export class AwsRuntime implements IrcRuntime {
|
|
|
363
421
|
return out;
|
|
364
422
|
}
|
|
365
423
|
|
|
424
|
+
/**
|
|
425
|
+
* Re-invokes the bound config loader (Secrets Manager / SSM / env) and
|
|
426
|
+
* returns the fresh {@link ServerConfig}. The Lambda handler binds the
|
|
427
|
+
* loader at construction; when no loader was supplied this rejects so the
|
|
428
|
+
* actor's graceful failure path applies.
|
|
429
|
+
*/
|
|
430
|
+
async reloadConfig(): Promise<ServerConfig> {
|
|
431
|
+
if (this.configLoader === undefined) {
|
|
432
|
+
throw new Error('no config loader bound to AwsRuntime');
|
|
433
|
+
}
|
|
434
|
+
return this.configLoader();
|
|
435
|
+
}
|
|
436
|
+
|
|
366
437
|
// -------------------------------------------------------------------------
|
|
367
438
|
// Public helpers (re-exported surface)
|
|
368
439
|
// -------------------------------------------------------------------------
|
|
@@ -19,6 +19,17 @@ import { type ParsedServerConfig, parseServerConfig } from '@serverless-ircd/irc
|
|
|
19
19
|
export interface LambdaConfigEnv {
|
|
20
20
|
SERVER_NAME?: string;
|
|
21
21
|
NETWORK_NAME?: string;
|
|
22
|
+
/**
|
|
23
|
+
* Server version surfaced in `002`/`004`/`351`/`371`. When unset the
|
|
24
|
+
* schema default ({@link DEFAULT_SERVER_VERSION}) applies.
|
|
25
|
+
*/
|
|
26
|
+
SERVER_VERSION?: string;
|
|
27
|
+
/**
|
|
28
|
+
* "Created" text for `003 RPL_CREATED`. A numeric string is parsed into
|
|
29
|
+
* an epoch-ms number so the reducer formats it as a UTC timestamp; any
|
|
30
|
+
* other value is passed through verbatim.
|
|
31
|
+
*/
|
|
32
|
+
CREATED_AT?: string;
|
|
22
33
|
MOTD?: string;
|
|
23
34
|
MAX_CLIENTS?: string;
|
|
24
35
|
CHANNEL_PREFIXES?: string;
|
|
@@ -53,6 +64,12 @@ export function buildLambdaConfigInput(env: LambdaConfigEnv): Record<string, unk
|
|
|
53
64
|
serverName: env.SERVER_NAME,
|
|
54
65
|
networkName: env.NETWORK_NAME,
|
|
55
66
|
};
|
|
67
|
+
if (env.SERVER_VERSION !== undefined) {
|
|
68
|
+
input.serverVersion = env.SERVER_VERSION;
|
|
69
|
+
}
|
|
70
|
+
if (env.CREATED_AT !== undefined) {
|
|
71
|
+
input.createdAt = parseCreatedAt(env.CREATED_AT);
|
|
72
|
+
}
|
|
56
73
|
if (env.MOTD !== undefined && env.MOTD.length > 0) {
|
|
57
74
|
input.motdLines = env.MOTD.split('\n');
|
|
58
75
|
}
|
|
@@ -97,6 +114,21 @@ export function buildLambdaConfigInput(env: LambdaConfigEnv): Record<string, unk
|
|
|
97
114
|
return input;
|
|
98
115
|
}
|
|
99
116
|
|
|
117
|
+
/**
|
|
118
|
+
* Parses the `CREATED_AT` env var into the schema's `createdAt` field.
|
|
119
|
+
*
|
|
120
|
+
* A purely numeric string is treated as epoch-ms (returned as a number so
|
|
121
|
+
* the reducer formats it as a UTC timestamp); any other value is returned
|
|
122
|
+
* verbatim as a human-readable string.
|
|
123
|
+
*/
|
|
124
|
+
function parseCreatedAt(raw: string): string | number {
|
|
125
|
+
if (/^\d+$/.test(raw.trim())) {
|
|
126
|
+
const n = Number(raw.trim());
|
|
127
|
+
if (Number.isFinite(n)) return n;
|
|
128
|
+
}
|
|
129
|
+
return raw;
|
|
130
|
+
}
|
|
131
|
+
|
|
100
132
|
/**
|
|
101
133
|
* Parses a newline-delimited `username:password` string into the config's
|
|
102
134
|
* `saslAccounts` shape. Malformed entries (missing colon, empty fields)
|
|
@@ -16,108 +16,37 @@
|
|
|
16
16
|
* `irc-core/src/ports.ts` and used by `MotdProvider` / `MessageStore`:
|
|
17
17
|
* adapters that need an async backend pre-load the data at boot into a
|
|
18
18
|
* synchronously-readable store.
|
|
19
|
+
*
|
|
20
|
+
* The scrypt hashing helpers (`hashAccountCredential` /
|
|
21
|
+
* `verifyHashedPassword` + the `HashedAccountCredential` type) live in
|
|
22
|
+
* `irc-core` and are shared with the CF adapter's `D1AccountStore`. They
|
|
23
|
+
* are re-exported below so existing call sites importing from this module
|
|
24
|
+
* keep working.
|
|
19
25
|
*/
|
|
20
26
|
|
|
21
|
-
import { randomBytes, scryptSync, timingSafeEqual } from 'node:crypto';
|
|
22
27
|
import type { ScanCommandOutput } from '@aws-sdk/client-dynamodb';
|
|
23
28
|
import { ScanCommand } from '@aws-sdk/lib-dynamodb';
|
|
24
29
|
import type { DynamoDBDocumentClient } from '@aws-sdk/lib-dynamodb';
|
|
25
|
-
import
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
30
|
+
import {
|
|
31
|
+
type HashedAccountCredential,
|
|
32
|
+
HashedAccountStore,
|
|
33
|
+
hashAccountCredential,
|
|
34
|
+
verifyHashedPassword,
|
|
35
|
+
} from '@serverless-ircd/irc-core';
|
|
29
36
|
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
* Every field is a DynamoDB attribute — this object IS the table row
|
|
34
|
-
* (plus the PK `account`). `algorithm` is retained so future migrations
|
|
35
|
-
* to Argon2/bcrypt can be detected and handled gracefully.
|
|
36
|
-
*/
|
|
37
|
-
export interface HashedAccountCredential {
|
|
38
|
-
readonly account: string;
|
|
39
|
-
readonly algorithm: 'scrypt';
|
|
40
|
-
readonly salt: string;
|
|
41
|
-
readonly hash: string;
|
|
42
|
-
}
|
|
43
|
-
|
|
44
|
-
/**
|
|
45
|
-
* Hashes a plaintext password into a {@link HashedAccountCredential}
|
|
46
|
-
* suitable for writing to the `Accounts` table via `putAccountCredential`.
|
|
47
|
-
*
|
|
48
|
-
* Uses `scryptSync` (memory-hard, GPU-resistant) with a random salt.
|
|
49
|
-
* Pass `{ salt }` for deterministic test scenarios; pass `{ keyLen }` to
|
|
50
|
-
* override the default 64-byte derived-key length.
|
|
51
|
-
*/
|
|
52
|
-
export function hashAccountCredential(
|
|
53
|
-
username: string,
|
|
54
|
-
password: string,
|
|
55
|
-
opts?: { salt?: Buffer; keyLen?: number },
|
|
56
|
-
): HashedAccountCredential {
|
|
57
|
-
const salt = opts?.salt ?? randomBytes(SALT_LEN);
|
|
58
|
-
const keyLen = opts?.keyLen ?? SCRYPT_KEY_LEN;
|
|
59
|
-
const hash = scryptSync(password, salt, keyLen);
|
|
60
|
-
return {
|
|
61
|
-
account: username,
|
|
62
|
-
algorithm: 'scrypt',
|
|
63
|
-
salt: salt.toString('base64'),
|
|
64
|
-
hash: hash.toString('base64'),
|
|
65
|
-
};
|
|
66
|
-
}
|
|
37
|
+
// Re-export so callers importing these from `@serverless-ircd/aws-adapter`
|
|
38
|
+
// (the historical home of the hashing helpers) keep compiling.
|
|
39
|
+
export { type HashedAccountCredential, hashAccountCredential, verifyHashedPassword };
|
|
67
40
|
|
|
68
41
|
/**
|
|
69
|
-
*
|
|
70
|
-
* using `timingSafeEqual` (no short-circuit on first mismatched byte).
|
|
42
|
+
* Synchronous {@link AccountStore} backed by pre-loaded hashed credentials.
|
|
71
43
|
*
|
|
72
|
-
*
|
|
73
|
-
*
|
|
44
|
+
* Now a re-export of the shared `HashedAccountStore` from `irc-core` —
|
|
45
|
+
* both adapters use one store implementation. The AWS-specific surface
|
|
46
|
+
* (`loadDynamoAccountStore`) lives below; it diverges from CF only at the
|
|
47
|
+
* scan boundary.
|
|
74
48
|
*/
|
|
75
|
-
export
|
|
76
|
-
if (entry.algorithm !== 'scrypt') return false;
|
|
77
|
-
let salt: Buffer;
|
|
78
|
-
let expected: Buffer;
|
|
79
|
-
try {
|
|
80
|
-
salt = Buffer.from(entry.salt, 'base64');
|
|
81
|
-
expected = Buffer.from(entry.hash, 'base64');
|
|
82
|
-
} catch {
|
|
83
|
-
return false;
|
|
84
|
-
}
|
|
85
|
-
if (expected.length === 0) return false;
|
|
86
|
-
const computed = scryptSync(password, salt, expected.length);
|
|
87
|
-
return computed.length === expected.length && timingSafeEqual(computed, expected);
|
|
88
|
-
}
|
|
89
|
-
|
|
90
|
-
/**
|
|
91
|
-
* Synchronous {@link AccountStore} backed by pre-loaded hashed
|
|
92
|
-
* credentials. Construct with entries from {@link loadDynamoAccountStore}
|
|
93
|
-
* (or {@link hashAccountCredential} for unit tests).
|
|
94
|
-
*/
|
|
95
|
-
export class DynamoAccountStore implements AccountStore {
|
|
96
|
-
private readonly entries: ReadonlyMap<string, HashedAccountCredential>;
|
|
97
|
-
|
|
98
|
-
constructor(entries: ReadonlyArray<HashedAccountCredential>) {
|
|
99
|
-
this.entries = new Map(entries.map((e) => [e.account, e]));
|
|
100
|
-
}
|
|
101
|
-
|
|
102
|
-
verify(mech: string, payload: SaslPayload): SaslResult {
|
|
103
|
-
if (mech.toUpperCase() !== 'PLAIN' || payload.kind !== 'PLAIN') {
|
|
104
|
-
return { ok: false, reason: `unsupported mechanism: ${mech}` };
|
|
105
|
-
}
|
|
106
|
-
const entry = this.entries.get(payload.username);
|
|
107
|
-
if (entry === undefined) {
|
|
108
|
-
return { ok: false, reason: 'invalid credentials' };
|
|
109
|
-
}
|
|
110
|
-
if (!verifyHashedPassword(payload.password, entry)) {
|
|
111
|
-
return { ok: false, reason: 'invalid credentials' };
|
|
112
|
-
}
|
|
113
|
-
return { ok: true, account: payload.username };
|
|
114
|
-
}
|
|
115
|
-
|
|
116
|
-
/** Number of loaded accounts (diagnostics / logging). */
|
|
117
|
-
get size(): number {
|
|
118
|
-
return this.entries.size;
|
|
119
|
-
}
|
|
120
|
-
}
|
|
49
|
+
export { HashedAccountStore as DynamoAccountStore };
|
|
121
50
|
|
|
122
51
|
/**
|
|
123
52
|
* Scans the `Accounts` table and returns a fully-populated
|
|
@@ -130,7 +59,7 @@ export class DynamoAccountStore implements AccountStore {
|
|
|
130
59
|
export async function loadDynamoAccountStore(
|
|
131
60
|
docClient: DynamoDBDocumentClient,
|
|
132
61
|
tableName: string,
|
|
133
|
-
): Promise<
|
|
62
|
+
): Promise<HashedAccountStore | undefined> {
|
|
134
63
|
const entries: HashedAccountCredential[] = [];
|
|
135
64
|
let exclusiveStartKey: Record<string, unknown> | undefined = undefined;
|
|
136
65
|
do {
|
|
@@ -145,12 +74,22 @@ export async function loadDynamoAccountStore(
|
|
|
145
74
|
const algorithm = typeof item.algorithm === 'string' ? item.algorithm : undefined;
|
|
146
75
|
const salt = typeof item.salt === 'string' ? item.salt : undefined;
|
|
147
76
|
const hash = typeof item.hash === 'string' ? item.hash : undefined;
|
|
148
|
-
if (
|
|
149
|
-
|
|
77
|
+
if (
|
|
78
|
+
account !== undefined &&
|
|
79
|
+
algorithm !== undefined &&
|
|
80
|
+
salt !== undefined &&
|
|
81
|
+
hash !== undefined
|
|
82
|
+
) {
|
|
83
|
+
entries.push({
|
|
84
|
+
account,
|
|
85
|
+
algorithm: algorithm as HashedAccountCredential['algorithm'],
|
|
86
|
+
salt,
|
|
87
|
+
hash,
|
|
88
|
+
});
|
|
150
89
|
}
|
|
151
90
|
}
|
|
152
91
|
exclusiveStartKey = result.LastEvaluatedKey;
|
|
153
92
|
} while (exclusiveStartKey !== undefined);
|
|
154
93
|
if (entries.length === 0) return undefined;
|
|
155
|
-
return new
|
|
94
|
+
return new HashedAccountStore(entries);
|
|
156
95
|
}
|
|
@@ -4,11 +4,24 @@
|
|
|
4
4
|
* Persists a fresh `Connections` row keyed by the APIGW connection id.
|
|
5
5
|
* Subsequent `$default` invocations load this row, run the actor, and
|
|
6
6
|
* persist the mutated state back.
|
|
7
|
+
*
|
|
8
|
+
* Admission gate: before writing the row, the handler counts the live
|
|
9
|
+
* `Connections` rows and rejects the upgrade with `429 Too Many Requests`
|
|
10
|
+
* once `serverConfig.maxClients` is reached. A `429` (not `403`) is used
|
|
11
|
+
* because the rejection is a capacity / back-pressure signal — the client
|
|
12
|
+
* is welcome to retry — rather than an authorisation failure.
|
|
7
13
|
*/
|
|
8
14
|
|
|
9
15
|
import type { DynamoDBDocumentClient } from '@aws-sdk/lib-dynamodb';
|
|
10
|
-
import { PutCommand } from '@aws-sdk/lib-dynamodb';
|
|
11
|
-
import
|
|
16
|
+
import { PutCommand, ScanCommand } from '@aws-sdk/lib-dynamodb';
|
|
17
|
+
import {
|
|
18
|
+
type ParsedServerConfig,
|
|
19
|
+
type WsSubprotocol,
|
|
20
|
+
parseSecWsProtocolOffers,
|
|
21
|
+
selectSubprotocol,
|
|
22
|
+
wsFrameModeFor,
|
|
23
|
+
} from '@serverless-ircd/irc-core';
|
|
24
|
+
import { type AdmissionOutcome, decideConnectAdmission } from '../admission.js';
|
|
12
25
|
import { marshalConnection } from '../serialize.js';
|
|
13
26
|
import type { TablesConfig } from '../tables.js';
|
|
14
27
|
import { createInitialConnectionState } from './state.js';
|
|
@@ -20,25 +33,99 @@ export interface ConnectParams {
|
|
|
20
33
|
connectionId: string;
|
|
21
34
|
/** Wall clock used to seed `connectedSince` / `idleSince` (default: now). */
|
|
22
35
|
now?: number;
|
|
23
|
-
/**
|
|
24
|
-
serverConfig
|
|
36
|
+
/** Server config — consulted for the `maxClients` admission cap. */
|
|
37
|
+
serverConfig: ParsedServerConfig;
|
|
38
|
+
/**
|
|
39
|
+
* Raw `Sec-WebSocket-Protocol` header offered by the client on the
|
|
40
|
+
* `$connect` upgrade, if present. Parsed into an ordered offer list and
|
|
41
|
+
* the first supported IRCv3 subprotocol is agreed (legacy fallback when
|
|
42
|
+
* none match or the header is absent).
|
|
43
|
+
*/
|
|
44
|
+
secWebSocketProtocol?: string | null;
|
|
25
45
|
}
|
|
26
46
|
|
|
27
47
|
/**
|
|
28
|
-
*
|
|
48
|
+
* Outcome of a `$connect`. Mirrors {@link AdmissionOutcome} on the rejection
|
|
49
|
+
* branch and carries the negotiated IRCv3 {@link WsSubprotocol} on success so
|
|
50
|
+
* the dispatcher can echo it back to the client in the integration response.
|
|
51
|
+
* `subprotocol === null` signals a legacy (no-subprotocol) connection.
|
|
52
|
+
*/
|
|
53
|
+
export type ConnectOutcome =
|
|
54
|
+
| { readonly admitted: true; readonly subprotocol: WsSubprotocol | null }
|
|
55
|
+
| { readonly admitted: false; readonly statusCode: number; readonly reason: string };
|
|
56
|
+
|
|
57
|
+
/**
|
|
58
|
+
* Enforces the `maxClients` admission cap, then inserts the Connections
|
|
59
|
+
* row for a fresh WebSocket upgrade.
|
|
60
|
+
*
|
|
61
|
+
* Idempotent on success: APIGW retries `$connect` on transient failures,
|
|
62
|
+
* so the write uses `PutCommand` (overwrites) rather than a conditional
|
|
63
|
+
* insert.
|
|
29
64
|
*
|
|
30
|
-
*
|
|
31
|
-
*
|
|
65
|
+
* TOCTOU note: DynamoDB cannot conditionally write based on a row count,
|
|
66
|
+
* so the count-then-put sequence has a race window. This is an accepted
|
|
67
|
+
* trade-off for an admission cap — it bounds growth under steady load
|
|
68
|
+
* rather than enforcing a hard invariant. Two concurrent upgrades that
|
|
69
|
+
* both observe `count == cap - 1` will both be admitted.
|
|
70
|
+
*
|
|
71
|
+
* Per-IP admission (`maxConnectionsPerIp`) is intentionally NOT enforced
|
|
72
|
+
* here. irc-core's in-memory `AdmissionStats` is unsuitable for Lambda
|
|
73
|
+
* (each invocation is a fresh process with no shared counters), and a
|
|
74
|
+
* filtered full-table `Scan` per `$connect` would double the DynamoDB
|
|
75
|
+
* cost on every connect (and `maxConnectionsPerIp` is on by default).
|
|
76
|
+
* The correct fix is to persist `sourceIp` on the Connections row and
|
|
77
|
+
* add a DynamoDB GSI keyed by `sourceIp`, then feed the GSI's count
|
|
78
|
+
* into {@link decideConnectAdmission}'s `perIp` branch. That schema
|
|
79
|
+
* change is tracked as a follow-up; the pure policy already supports it.
|
|
32
80
|
*/
|
|
33
|
-
export async function handleConnect(params: ConnectParams): Promise<
|
|
81
|
+
export async function handleConnect(params: ConnectParams): Promise<ConnectOutcome> {
|
|
34
82
|
const now = params.now ?? Date.now();
|
|
83
|
+
const total = await countConnections(params.dynamo, params.tables.Connections);
|
|
84
|
+
const outcome = decideConnectAdmission({ total }, { maxClients: params.serverConfig.maxClients });
|
|
85
|
+
if (!outcome.admitted) return outcome;
|
|
86
|
+
// IRCv3 WebSocket subprotocol negotiation: parse the offered
|
|
87
|
+
// `Sec-WebSocket-Protocol` list, select the first supported entry, and
|
|
88
|
+
// persist the resulting frame mode so every subsequent `$default`
|
|
89
|
+
// invocation recovers it (a Lambda `$connect` has no live socket to
|
|
90
|
+
// tag, unlike the CF adapter's hibernation tags). `null` means no
|
|
91
|
+
// supported subprotocol was offered → legacy framing, no mode column.
|
|
92
|
+
const chosen = selectSubprotocol(parseSecWsProtocolOffers(params.secWebSocketProtocol));
|
|
93
|
+
const mode = wsFrameModeFor(chosen ?? undefined);
|
|
35
94
|
const state = createInitialConnectionState(params.connectionId, now);
|
|
36
|
-
void params.serverConfig; // reserved for max-clients gating (later ticket)
|
|
37
95
|
const row = marshalConnection(state, now);
|
|
96
|
+
if (mode !== 'legacy') {
|
|
97
|
+
row.wsMode = mode;
|
|
98
|
+
}
|
|
38
99
|
await params.dynamo.send(
|
|
39
100
|
new PutCommand({
|
|
40
101
|
TableName: params.tables.Connections,
|
|
41
102
|
Item: row,
|
|
42
103
|
}),
|
|
43
104
|
);
|
|
105
|
+
return { admitted: true, subprotocol: chosen };
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
/**
|
|
109
|
+
* Paginated `Scan` with `Select: COUNT` over the `Connections` table.
|
|
110
|
+
* A single Scan page caps at 1MB, so the loop follows
|
|
111
|
+
* `LastEvaluatedKey` until every page has been summed.
|
|
112
|
+
*/
|
|
113
|
+
async function countConnections(
|
|
114
|
+
dynamo: DynamoDBDocumentClient,
|
|
115
|
+
tableName: string,
|
|
116
|
+
): Promise<number> {
|
|
117
|
+
let total = 0;
|
|
118
|
+
let lastKey: unknown = undefined;
|
|
119
|
+
do {
|
|
120
|
+
const res = await dynamo.send(
|
|
121
|
+
new ScanCommand({
|
|
122
|
+
TableName: tableName,
|
|
123
|
+
Select: 'COUNT',
|
|
124
|
+
...(lastKey !== undefined ? { ExclusiveStartKey: lastKey as never } : {}),
|
|
125
|
+
}),
|
|
126
|
+
);
|
|
127
|
+
total += res.Count ?? 0;
|
|
128
|
+
lastKey = res.LastEvaluatedKey;
|
|
129
|
+
} while (lastKey !== undefined);
|
|
130
|
+
return total;
|
|
44
131
|
}
|