tickcli-js 0.1.4 → 0.1.6

package/src/hub.ts

@@ -474,6 +474,8 @@ export interface LiveStats {
   /** over the data cap or a late loop now; rooms slowed by their own budget */
   pressure: boolean;
   slowedRooms: number;
+  /** namespaces slowed because they are over their own outBytes cap (ScopeCaps) */
+  slowedScopes: number;
   /** the 5 rooms sending the most */
   top: { players: number; bytesOut: number; sharedBytes: number }[];
 }
@@ -487,6 +489,15 @@ export interface SnapRoom {
 }
 export interface RoomsSnapshot { format: number; at: number; rooms: SnapRoom[] }
 
+/** One namespace's own share of the server (a host app's per-customer caps, tickcli-js/remote). */
+export interface ScopeCaps {
+  /** data going out to its rooms, bytes a second (moving average over all its rooms): over it, all its rooms send half as often (up to 8 times
+   *  less) until the faster rate would be under 80% of it again. Other namespaces keep their full rate. */
+  outBytes?: number;
+  /** joins a minute across the namespace (reconnects count), on top of the per-address limit; over it: 1013 busy, the page tries again */
+  joinsPerMinute?: number;
+}
+
 export interface HubStats { rooms: number; players: number; online: number; connections: number; pending: number; refused: Record<string, number>; live: LiveStats }
 
 export class RoomHub {
@@ -500,6 +511,10 @@ export class RoomHub {
   private perAddr = new Map<string, number>();
   private addrHits = new Map<string, { at: number; n: number }>();
   private joins = new Map<string, number[]>();
+  /** per namespace with caps: its slow-down factor and moving average out (bytes a second) */
+  private scopeSlow = new Map<string, { f: number; avg: number }>();
+  /** per namespace with a joins cap: joins in this minute and the last one (a sliding estimate, no list per join) */
+  private scopeJoins = new Map<string, { at: number; n: number; prev: number }>();
   private refused: Record<string, number> = {};
   private timer: ReturnType<typeof setInterval> | null = null;
   private sub = 0;
@@ -531,6 +546,8 @@ export class RoomHub {
   pressure = false;
   /** world save leases: "scope|slot|who" -> the room key that may write it */
   private leases = new Map<string, string>();
+  /** namespaces being handed to another process now (exportScope): joins to them are refused with 1012 */
+  private moving = new Set<string>();
 
   /** shutdown() hands the live rooms to this (a restart writes them to disk; importRooms() in the next process takes them back) */
   private readonly onSnapshot: ((snap: RoomsSnapshot) => Promise<void> | void) | null;
@@ -541,9 +558,12 @@ export class RoomHub {
 
   /** rooms allowed in one namespace when it differs from limits.maxRoomsPerScope (a host app's per-customer cap, tickcli-js/remote) */
   private readonly roomsPerScope: ((scope: string) => number | undefined) | null;
+  /** a namespace's own data and joins caps (tickcli-js/remote: a deployment's caps); undefined = only the server-wide limits */
+  private readonly scopeCaps: ((scope: string) => ScopeCaps | undefined) | null;
 
-  constructor(opts: { secret: string; now?: () => number; limits?: Partial<Limits>; saves?: SavePort; onSnapshot?: (snap: RoomsSnapshot) => Promise<void> | void; listRooms?: boolean; roomsPerScope?: (scope: string) => number | undefined }) {
+  constructor(opts: { secret: string; now?: () => number; limits?: Partial<Limits>; saves?: SavePort; onSnapshot?: (snap: RoomsSnapshot) => Promise<void> | void; listRooms?: boolean; roomsPerScope?: (scope: string) => number | undefined; scopeCaps?: (scope: string) => ScopeCaps | undefined }) {
     this.roomsPerScope = opts.roomsPerScope ?? null;
+    this.scopeCaps = opts.scopeCaps ?? null;
     this.saves = opts.saves ?? null;
     this.listRooms = opts.listRooms !== false;
     this.onSnapshot = opts.onSnapshot ?? null;
@@ -675,12 +695,14 @@ export class RoomHub {
   private join(s: Session, m: Record<string, unknown>) {
     const L = this.limits, now = this.now();
     if (s.room) return this.bad(s, "already in a room");
+    if (this.moving.has(s.scope)) return s.end(CLOSE.restart, "moving to another server");
     const spec = parseSpec(m.spec, L.maxPlayers);
     if (!spec) return this.bad(s, "bad multiplayer option");
     const hits = (this.joins.get(s.addr) || []).filter((x) => now - x < 60_000);
     if (hits.length >= L.joinsPerMinute) { this.refuse("joins"); return s.end(CLOSE.tooFast, "too many joins, wait a minute"); }
     hits.push(now); this.joins.set(s.addr, hits);
     if (this.joins.size > 10_000) for (const [k, v] of this.joins) if (!v.length || now - v[v.length - 1] > 60_000) this.joins.delete(k);
+    if (!this.scopeJoinAllowed(s.scope, now)) { this.refuse("scope_joins"); return s.end(CLOSE.busy, "this game is busy right now, trying again"); }
     const who = s.who!;
     if (who.guest && typeof m.name === "string") { const n = cleanName(m.name); if (n) who.name = n; }
     const specKey = stable(spec);
@@ -1313,7 +1335,7 @@ export class RoomHub {
         if (!on) continue;
         const h = r.host?.session;
         rooms.push({ _r: r, code: r.code, host: r.host?.name ?? null, players: on, away, max: r.spec.max, started: r.started, match: r.match,
-          ping: h?.rtt ?? 0, slowed: r.slowF > 1 || (this.pressure && r.seats.size > 4), full: taken >= r.spec.max, spec: r.specKey, created: r.created });
+          ping: h?.rtt ?? 0, slowed: r.slowF > 1 || (this.pressure && r.seats.size > 4) || this.scopeF(r.scope) > 1, full: taken >= r.spec.max, spec: r.specKey, created: r.created });
       }
       rooms.sort((a, b) => Number(a.full) - Number(b.full) || Number(a.started) - Number(b.started) || (b.players as number) - (a.players as number) || (b.created as number) - (a.created as number));
       if (this.lists.size > 1000) this.lists.clear();
@@ -1345,9 +1367,46 @@ export class RoomHub {
       if (!r.slowTold && r.host?.session) { r.slowTold = true; this.err(r.host.session, "too_much_data"); }
     } else if (r.slowF > 1 && r.outAvg * 2 < L.roomOutBytes * 0.8) { r.slowF /= 2; r.outAvg *= 2; }
   }
-  /** How many ticks apart a room sends its state: 1 (50 ms), 2 in rooms over 8 players, times its own budget factor, times 2 under server pressure
-   *  for rooms over 4 players. */
-  private every(r: Room) { return (r.seats.size > 8 ? 2 : 1) * r.slowF * (this.pressure && r.seats.size > 4 ? 2 : 1); }
+  /** How many ticks apart a room sends its state: 1 (50 ms), 2 in rooms over 8 players, times its own budget factor, times its namespace's factor
+   *  (over its outBytes cap), times 2 under server pressure for rooms over 4 players. */
+  private every(r: Room) { return (r.seats.size > 8 ? 2 : 1) * r.slowF * this.scopeF(r.scope) * (this.pressure && r.seats.size > 4 ? 2 : 1); }
+  private scopeF(scope: string) { return this.scopeSlow.get(scope)?.f ?? 1; }
+
+  /** Once a second: each namespace with an outBytes cap over its share slows all its rooms (twice as slow per second over, up to 8 times), and
+   *  speeds up one step when the faster rate would be under 80% of the cap. A busy customer then costs its own players smoothness, not everyone's. */
+  private scopeBudgets() {
+    if (!this.scopeCaps) return;
+    const out = new Map<string, number>();
+    for (const r of this.rooms.values()) out.set(r.scope, (out.get(r.scope) || 0) + r.outRate);
+    for (const k of this.scopeSlow.keys()) if (!out.has(k)) this.scopeSlow.delete(k);
+    for (const [scope, rate] of out) {
+      let cap: number | undefined;
+      try { cap = this.scopeCaps(scope)?.outBytes; } catch { cap = undefined; }
+      if (!cap || cap <= 0) { this.scopeSlow.delete(scope); continue; }
+      let x = this.scopeSlow.get(scope);
+      if (!x) { if (rate <= cap) continue; x = { f: 1, avg: 0 }; this.scopeSlow.set(scope, x); }
+      x.avg = x.avg ? x.avg * 0.7 + rate * 0.3 : rate;
+      if (x.avg > cap && x.f < 8) { x.f *= 2; x.avg /= 2; this.refuse("scope_budget"); }
+      else if (x.f > 1 && x.avg * 2 < cap * 0.8) { x.f /= 2; x.avg *= 2; }
+      if (x.f === 1 && x.avg <= cap) this.scopeSlow.delete(scope);
+    }
+  }
+
+  /** One more join in a namespace with a joinsPerMinute cap, counted over a sliding minute (this minute plus the share of the last one still in it). */
+  private scopeJoinAllowed(scope: string, now: number) {
+    if (!this.scopeCaps) return true;
+    let cap: number | undefined;
+    try { cap = this.scopeCaps(scope)?.joinsPerMinute; } catch { cap = undefined; }
+    if (!cap || cap <= 0) return true;
+    let w = this.scopeJoins.get(scope);
+    if (!w || now - w.at >= 120_000) { w = { at: now, n: 0, prev: 0 }; this.scopeJoins.set(scope, w); }
+    else if (now - w.at >= 60_000) { w.prev = w.n; w.n = 0; w.at += 60_000; }
+    if (this.scopeJoins.size > 10_000) for (const [k, v] of this.scopeJoins) if (now - v.at >= 120_000) this.scopeJoins.delete(k);
+    const used = w.n + w.prev * Math.max(0, 1 - (now - w.at) / 60_000);
+    if (used >= cap) return false;
+    w.n++;
+    return true;
+  }
 
   /** Once a second: the rates over the last second (server and per room). */
   private meters(now: number) {
@@ -1356,6 +1415,7 @@ export class RoomHub {
     for (const k of ["framesOut", "bytesOut", "msgsIn"] as const) { this.rates[k] = Math.round((this.meter[k] - this.mark[k]) / sec); this.mark[k] = this.meter[k]; }
     this.mark.at = now;
     for (const r of this.rooms.values()) { r.outRate = Math.round((r.outBytes - r.outMark) / sec); r.outMark = r.outBytes; this.budget(r); }
+    this.scopeBudgets();
     // the whole server: over 80% of its data cap, or the event loop running late (p99 over 30 ms for 10 s, from the host process), bigger rooms
     // send at half rate and new rooms wait; back to normal under 60% and a loop on time
     const mbit = (this.rates.bytesOut * 8) / 1e6, cap = this.limits.maxOutMbit;
@@ -1478,11 +1538,50 @@ export class RoomHub {
     if (snap) await this.onSnapshot!(snap);
   }
 
-  /** The live rooms as plain JSON: everything a new process needs to carry on (sockets, timers and open actions are not kept). */
-  exportRooms(): RoomsSnapshot {
+  /** Blue/green hand-over of one namespace to another process: its pending changes go out, open actions fail ("server restarting"), saves are
+   *  written, its rooms are exported, its sockets hear 1012 (a router in front re-joins them on the other process over the same page socket; a
+   *  page on its own reconnects) and its rooms close here, giving up their save leases. Joins to it are refused with 1012 meanwhile. Resolves
+   *  once every save is in the store, so the other process may take the snapshot (importRooms) and write the same saves. */
+  async exportScope(scope: string): Promise<RoomsSnapshot> {
+    this.moving.add(scope);
+    try {
+      // a profile save waits for its profile to load: let those land (2 s at most) so the write below carries them
+      const loads: Promise<unknown>[] = [];
+      for (const r of this.rooms.values()) if (r.scope === scope) for (const x of r.seats.values()) if (x.profLoad) loads.push(x.profLoad.catch(() => null));
+      if (loads.length) await Promise.race([Promise.all(loads), new Promise((r) => setTimeout(r, 2000).unref?.())]);
+      const mine = [...this.rooms.values()].filter((r) => r.scope === scope);
+      for (const room of mine) {
+        this.flush(room);
+        for (const [aid, a] of room.actions) { a.from.session?.send({ t: "result", id: a.ref, ok: false, error: "server restarting" }); room.actions.delete(aid); }
+        this.saveWorld(room, true);
+        for (const x of room.seats.values()) this.flushProfile(room, x, true);
+      }
+      const snap = this.exportRooms(scope);
+      for (const s of [...this.sessions]) if (s.scope === scope) s.end(CLOSE.restart, "moving to another server");
+      for (const room of mine) this.closeRoom(room);
+      await this.settle();
+      return snap;
+    } finally { this.moving.delete(scope); }
+  }
+
+  /** Namespaces with live rooms here and their size (a router's hand-over list, a dashboard). */
+  scopeList(): { scope: string; rooms: number; players: number; online: number }[] {
+    const by = new Map<string, { scope: string; rooms: number; players: number; online: number }>();
+    for (const r of this.rooms.values()) {
+      let x = by.get(r.scope);
+      if (!x) { x = { scope: r.scope, rooms: 0, players: 0, online: 0 }; by.set(r.scope, x); }
+      x.rooms++;
+      for (const z of r.seats.values()) { x.players++; if (z.session) x.online++; }
+    }
+    return [...by.values()];
+  }
+
+  /** The live rooms (of one namespace when given) as plain JSON: everything a new process needs to carry on (sockets, timers and open actions
+   *  are not kept). */
+  exportRooms(scope?: string): RoomsSnapshot {
     const rooms: SnapRoom[] = [];
     for (const r of this.rooms.values()) {
-      if (r.seats.size === 0) continue;
+      if (r.seats.size === 0 || (scope !== undefined && r.scope !== scope)) continue;
       rooms.push({
         key: r.key, scope: r.scope, game: r.game, spec: r.spec, code: r.code, priv: r.priv, slot: r.slot, owner: r.owner, world: r.world, lease: r.lease, saveBusy: r.saveBusy,
         worldState: r.worldState === "ready" ? "ready" : "loading", saved: r.saved, wrev: r.wrev, savedWrev: r.savedWrev, noSave: r.noSave, resumed: r.resumed, sid: r.sid,
@@ -1548,14 +1647,15 @@ export class RoomHub {
   }
 
   /** One namespace's use right now: rooms, seats, connected players and bytes out a second (over the last second). For per-customer caps. */
-  scopeStats(scope: string): { rooms: number; players: number; online: number; bytesOut: number } {
+  /** `slowed`: how many times less often its rooms send because the namespace is over its own outBytes cap (1 = full rate). */
+  scopeStats(scope: string): { rooms: number; players: number; online: number; bytesOut: number; slowed: number } {
     let rooms = 0, players = 0, online = 0, bytesOut = 0;
     for (const r of this.rooms.values()) {
       if (r.scope !== scope) continue;
       rooms++; bytesOut += r.outRate;
       for (const x of r.seats.values()) { players++; if (x.session) online++; }
     }
-    return { rooms, players, online, bytesOut };
+    return { rooms, players, online, bytesOut, slowed: this.scopeF(scope) };
   }
 
   stats(): HubStats {
@@ -1570,6 +1670,7 @@ export class RoomHub {
       ...this.rates, tickMs: { p50: pct(0.5), p99: pct(0.99), max: n ? Math.round(sorted[n - 1] * 100) / 100 : 0 }, slow, started,
       savesOk: this.counts.saves_ok || 0, savesFailed: this.refused.save_failed || 0, loadsFailed: this.refused.load_failed || 0, worldWritesPerMin: this.worldWrites.length,
       loop: this.loop, top, pressure: this.pressure, slowedRooms: [...this.rooms.values()].filter((r) => r.slowF > 1).length,
+      slowedScopes: [...this.scopeSlow.values()].filter((x) => x.f > 1).length,
     };
     return { rooms: this.rooms.size, players, online, connections: this.sessions.size, pending: this.pending, refused: { ...this.refused }, live };
   }

package/src/remote.ts

@@ -19,7 +19,7 @@ import crypto from "node:crypto";
 import type { Duplex } from "node:stream";
 import { fileURLToPath } from "node:url";
 import { WebSocketServer, type WebSocket } from "ws";
-import { CLOSE, RoomHub, addressKey, type Limits, type SavePort } from "./hub.ts";
+import { CLOSE, RoomHub, addressKey, type Limits, type SavePort, type ScopeCaps } from "./hub.ts";
 import { GuestPasses, Heartbeat, WS_OPTIONS, attach, clientAddress, fileSaves, gameHash, hardenServer } from "./node.ts";
 
 export interface Deployment {
@@ -36,8 +36,12 @@ export interface Deployment {
     rooms?: number;
     /** connected sockets at once */
     players?: number;
-    /** data going out to its players, KB a second (over the last second); over it, new sockets wait (busy) and open rooms carry on */
+    /** data going out to its players, KB a second: over it, new sockets wait (busy) and all its open rooms send less often (half, down to an
+     *  eighth) until it is back under 80% of it, so one busy deployment cannot take the server's whole uplink */
     kbPerSec?: number;
+    /** joins a minute across the deployment, reconnects included (on top of the hub's per-address joinsPerMinute); over it: 1013 busy, the page
+     *  tries again. Leave room for a restart: every connected player joins again within a few seconds. */
+    joinsPerMinute?: number;
   };
 }
 
@@ -65,6 +69,8 @@ export interface Remote {
   scope(id: string): string;
   /** The rooms cap the hub should use for a namespace (pass to new RoomHub({ roomsPerScope })). */
   roomsPerScope(scope: string): number | undefined;
+  /** The data and joins caps the hub should use for a namespace (pass to new RoomHub({ scopeCaps })). */
+  scopeCaps(scope: string): ScopeCaps | undefined;
   /** connected sockets per deployment id */
   sockets(id: string): number;
 }
@@ -175,6 +181,10 @@ export function createRemote(o: RemoteOptions): Remote {
     scope,
     sockets: (id) => live.get(id) || 0,
     roomsPerScope: (sc) => (sc.startsWith("dep:") ? cache.get(sc.slice(4))?.caps?.rooms : undefined),
+    scopeCaps: (sc) => {
+      const c = sc.startsWith("dep:") ? cache.get(sc.slice(4))?.caps : undefined;
+      return c && (c.kbPerSec !== undefined || c.joinsPerMinute !== undefined) ? { outBytes: c.kbPerSec !== undefined ? c.kbPerSec * 1024 : undefined, joinsPerMinute: c.joinsPerMinute } : undefined;
+    },
     revoke(id) { cache.delete(id); seen.delete(id); o.hub.closeScope(scope(id), "this game was turned off by its owner"); },
     forget(id) { cache.delete(id); seen.delete(id); },
 
@@ -241,13 +251,15 @@ export interface RemoteServerOptions {
   address?: RemoteOptions["address"];
   /** behind a proxy on a private network: players' addresses from X-Forwarded-For (see clientAddress in tickcli-js/node) */
   trustProxy?: boolean;
+  /** close() hands the live rooms to this (write them somewhere; the next process passes them to hub.importRooms): a restart keeps matches */
+  onSnapshot?: ConstructorParameters<typeof RoomHub>[0]["onSnapshot"];
 }
 
 /** A complete remote room server: node:http + the hub + createRemote. Add your own routes on `server` before listen(). */
 export function createRemoteServer(o: RemoteServerOptions) {
   let remote: Remote | null = null;
   const saves = typeof o.saves === "string" ? fileSaves(o.saves) : o.saves;
-  const hub = new RoomHub({ secret: o.secret ?? crypto.randomBytes(32).toString("base64url"), limits: o.limits, saves, roomsPerScope: (sc) => remote?.roomsPerScope(sc) });
+  const hub = new RoomHub({ secret: o.secret ?? crypto.randomBytes(32).toString("base64url"), limits: o.limits, saves, onSnapshot: o.onSnapshot, roomsPerScope: (sc) => remote?.roomsPerScope(sc), scopeCaps: (sc) => remote?.scopeCaps(sc) });
   hub.start();
   const heartbeat = new Heartbeat();
   remote = createRemote({ hub, deployment: o.deployment, heartbeat, address: o.address ?? (o.trustProxy ? (req) => clientAddress(req, true) : undefined) });