rockcord 4.0.0-dev.4 → 4.0.0

package/src/client/voice/dispatcher/VideoDispatcher.js

@@ -1,6 +1,8 @@
 'use strict';
 
+const { setTimeout } = require('node:timers');
 const BaseDispatcher = require('./BaseDispatcher');
+const CongestionControl = require('../networking/CongestionControl');
 
 /**
  * The class that sends video packet data to the voice connection.
@@ -14,13 +16,13 @@ const BaseDispatcher = require('./BaseDispatcher');
  * @extends {BaseDispatcher}
  */
 class VideoDispatcher extends BaseDispatcher {
-  constructor(player, highWaterMark = 12, streams, fps, payloadType) {
+  constructor(player, highWaterMark = 12, streams, fps = 30, payloadType) {
     super(player, highWaterMark, payloadType, true, streams);
     /**
      * Video FPS
      * @type {number}
      */
-    this.fps = fps;
+    this.setFPSSource(fps);
 
     this.mtu = 1200;
   }
@@ -57,12 +59,94 @@ class VideoDispatcher extends BaseDispatcher {
    * @param {number} value fps
    */
   setFPSSource(value) {
+    if (!Number.isFinite(value) || value <= 0) throw new RangeError('Video FPS must be a positive finite number');
     this.fps = value;
   }
 
   _codecCallback() {
     throw new Error('The _codecCallback method must be implemented');
   }
+
+  configureCongestion(options = {}) {
+    if (options.congestionControl === false) return;
+    const config = typeof options.congestionControl === 'object' ? options.congestionControl : {};
+    const bitrate = config.bitrate ?? (typeof options.bitrate === 'number' ? options.bitrate : 2000);
+    this.congestionControl = new CongestionControl({
+      bitrate,
+      minBitrate: Math.min(128, bitrate),
+      ...config,
+      onChange: state => this.emit('congestion', state),
+    });
+    this.player.voiceConnection._congestionControl = this.congestionControl;
+  }
+
+  _write(chunk, encoding, done) {
+    if (this.congestionControl) {
+      this._pacedPackets = [];
+      this._pacedBytes = 0;
+      this._pacingKey = this.player.voiceConnection.authentication.secret_key;
+    }
+    super._write(chunk, encoding, done);
+  }
+
+  _sendPacket(packet) {
+    if (!this._pacedPackets) return super._sendPacket(packet);
+    this._pacedBytes += packet.length;
+    if (this._pacedBytes > 4 * 1024 * 1024 || this._pacedPackets.length >= 4096) {
+      throw new RangeError('Video frame exceeds the pacing queue limit');
+    }
+    this._pacedPackets.push(packet);
+    return undefined;
+  }
+
+  _step(done) {
+    if (!this._pacedPackets) return super._step(done);
+    const packets = this._pacedPackets;
+    this._pacedPackets = null;
+    let index = 0;
+    const sendNext = () => {
+      if (this.destroyed) return;
+      if (this.paused) {
+        this._pacingTimer = setTimeout(sendNext, 20).unref();
+        return;
+      }
+      if (index === packets.length) {
+        super._step(done);
+        return;
+      }
+      const packet = packets[index++];
+      const delay = this.congestionControl.delay(packet.length);
+      this._pacingTimer = setTimeout(
+        () => {
+          if (this.destroyed) return;
+          if (this._pacingKey !== this.player.voiceConnection.authentication.secret_key) {
+            this.destroy(new Error('Video transport key changed during paced playback'));
+            return;
+          }
+          if (this.paused) {
+            index--;
+            sendNext();
+            return;
+          }
+          this.congestionControl.packetSent();
+          super._sendPacket(packet);
+          sendNext();
+        },
+        Math.max(0, delay),
+      ).unref();
+    };
+    sendNext();
+    return undefined;
+  }
+
+  _cleanup() {
+    clearTimeout(this._pacingTimer);
+    this._pacedPackets = null;
+    if (this.player.voiceConnection._congestionControl === this.congestionControl) {
+      this.player.voiceConnection._congestionControl = null;
+    }
+    super._cleanup();
+  }
 }
 
 module.exports = VideoDispatcher;

package/src/client/voice/networking/CongestionControl.js

@@ -0,0 +1,84 @@
+'use strict';
+
+/** Loss-based AIMD rate control and bounded video packet pacing. @private */
+class CongestionControl {
+  constructor({ bitrate = 2000, minBitrate = 128, onChange = () => {} } = {}) {
+    if (!Number.isFinite(bitrate) || !Number.isFinite(minBitrate) || minBitrate <= 0 || bitrate < minBitrate) {
+      throw new RangeError('Congestion bitrates must be finite, positive and ordered');
+    }
+    this.maximum = bitrate * 1000;
+    this.minimum = minBitrate * 1000;
+    this.targetBitrate = this.maximum;
+    this.onChange = onChange;
+    this.loss = 0;
+    this.reason = 'initial';
+    this.nextSendAt = 0;
+    this.windowStart = performance.now();
+    this.lastUpdate = -Infinity;
+    this.lastReport = -Infinity;
+    this.sent = 0;
+    this.totalSent = 0;
+    this.missing = new Set();
+    this.reports = new Map();
+  }
+
+  get state() {
+    return { targetBitrate: this.targetBitrate, loss: this.loss, packetsSent: this.totalSent, reason: this.reason };
+  }
+
+  report(loss, highest, reporter, now = performance.now()) {
+    if (!Number.isFinite(loss) || loss < 0 || loss > 1) return;
+    const previous = this.reports.get(reporter);
+    const distance = previous ? (highest - previous.highest) >>> 0 : 1;
+    if (distance >= 0x80000000 || !distance) return;
+    for (const [id, report] of this.reports) if (now - report.time > 3000) this.reports.delete(id);
+    if (!this.reports.has(reporter) && this.reports.size >= 32) return;
+    this.reports.set(reporter, { highest, loss, time: now });
+    this.lastReport = now;
+    this._update(Math.max(...[...this.reports.values()].map(report => report.loss)), now, true);
+  }
+
+  nack(sequence) {
+    if (this.missing.size < 512) this.missing.add(sequence);
+  }
+
+  packetSent(now = performance.now()) {
+    if (now - this.windowStart >= 1000) {
+      if (now - this.lastReport > 2000 && this.sent >= 20 && this.missing.size) {
+        this._update(Math.min(1, this.missing.size / this.sent), now, false);
+      }
+      this.windowStart = now;
+      this.sent = 0;
+      this.missing.clear();
+    }
+    this.sent++;
+    this.totalSent++;
+  }
+
+  _update(loss, now, canIncrease) {
+    if (now - this.lastUpdate < 1000) return;
+    this.lastUpdate = now;
+    this.loss = loss;
+    const before = this.targetBitrate;
+    if (loss >= 0.1) {
+      this.targetBitrate *= 0.75;
+      this.reason = 'high-loss';
+    } else if (loss >= 0.02) {
+      this.targetBitrate *= 0.9;
+      this.reason = 'loss';
+    } else if (canIncrease && loss < 0.02) {
+      this.targetBitrate += Math.max(8000, this.targetBitrate * 0.05);
+      this.reason = 'recovery';
+    }
+    this.targetBitrate = Math.round(Math.max(this.minimum, Math.min(this.maximum, this.targetBitrate)));
+    if (before !== this.targetBitrate) this.onChange(this.state);
+  }
+
+  delay(bytes, now = performance.now()) {
+    const due = Math.max(now, this.nextSendAt);
+    this.nextSendAt = due + (bytes * 8000) / this.targetBitrate;
+    return due - now;
+  }
+}
+
+module.exports = CongestionControl;

package/src/client/voice/networking/DAVE.js

@@ -51,6 +51,8 @@ class DAVE {
   }
 
   reinit(version = this.targetVersion) {
+    this._mediaStarted = false;
+    this._bootstrapSender = null;
     this.validateVersion(version);
     this.blocked = true;
     if (!version) return;
@@ -81,6 +83,8 @@ class DAVE {
     this.version = this.targetVersion = this.pending.get(id);
     this.pending.delete(id);
     this.lastTransition = id;
+    this._mediaStarted = false;
+    this._bootstrapSender = null;
     this.recovering = false;
     this.blocked = this.version > 0 && !this.session?.ready;
     // When upgrading, expire the explicitly permitted plaintext grace period after ten seconds.
@@ -137,7 +141,11 @@ class DAVE {
   encrypt(frame, codec = 'OPUS') {
     if (this.closed || !this.negotiated) return null;
     // The protocol's Opus silence marker is exempt from frame encryption, including while joining.
-    if (codec === 'OPUS' && SILENCE_FRAME.equals(frame)) return frame;
+    if (codec === 'OPUS' && SILENCE_FRAME.equals(frame)) {
+      // Exempt silence must not unlock nonce-zero startup ordering.
+      this._bootstrapSender = null;
+      return frame;
+    }
     if (this.blocked) return null;
     if (this.version === 0) return frame;
     if (!this.session?.ready) return null;
@@ -146,6 +154,23 @@ class DAVE {
     return this.session.encrypt(codec === 'OPUS' ? api.MediaType.AUDIO : api.MediaType.VIDEO, api.Codec[codec], frame);
   }
 
+  beginMedia(sender) {
+    if (this._mediaStarted || !this.version || this.blocked || !this.negotiated || !this.session?.ready) return true;
+    if (this._bootstrapSender && this._bootstrapSender !== sender) return false;
+    this._bootstrapSender = sender;
+    return true;
+  }
+
+  sentMedia(sender) {
+    if (this._bootstrapSender !== sender) return;
+    this._mediaStarted = true;
+    this._bootstrapSender = null;
+  }
+
+  releaseMedia(sender) {
+    if (this._bootstrapSender === sender) this._bootstrapSender = null;
+  }
+
   decrypt(frame, userId, video = false) {
     if (this.closed || !this.negotiated) return null;
     if (!video && SILENCE_FRAME.equals(frame)) return frame;

package/src/client/voice/networking/MediaRecovery.js

@@ -0,0 +1,300 @@
+'use strict';
+
+const { Buffer } = require('node:buffer');
+const { setInterval, setTimeout } = require('node:timers');
+const Util = require('../../../util/Util');
+const { encrypt, decrypt } = require('../util/TransportCrypto');
+
+/** Bounded RFC 4585 NACK feedback and RFC 4588 video RTX handling. @private */
+class MediaRecovery {
+  constructor(connection) {
+    this.connection = connection;
+    this.cache = new Map();
+    this.sources = new Map();
+    this.pendingRtx = new Set();
+    this.rtxSequence = 0;
+    this.stats = { requested: 0, retransmitted: 0, recovered: 0, expired: 0 };
+  }
+
+  remember(header, payload) {
+    const type = Util.getAllPayloadType().find(
+      codec => codec.rtx_payload_type && codec.payload_type === (header[1] & 127),
+    );
+    if (!type || payload.length > 4096) return;
+    const key = Buffer.from(this.connection.authentication.secret_key);
+    if (!this.key?.equals(key)) {
+      this.reset();
+      this.key = key;
+    }
+    const now = performance.now();
+    for (const [id, item] of this.cache) if (now - item.time > 2000) this.cache.delete(id);
+    if (this.cache.size >= 512) this.cache.delete(this.cache.keys().next().value);
+    const extension = header[0] & 0x10 ? header.readUInt16BE(header.length - 2) * 4 : 0;
+    this.cache.set(`${header.readUInt32BE(8)}:${header.readUInt16BE(2)}`, {
+      header: Buffer.from(header.subarray(0, 12)),
+      payload: Buffer.from(payload.subarray(extension)),
+      type: type.rtx_payload_type,
+      time: now,
+      retries: 0,
+      last: -Infinity,
+    });
+  }
+
+  receive(packet, retransmission = false) {
+    const { ssrc, sequenceNumber: seq } = packet.header;
+    let source = this.sources.get(ssrc);
+    if (retransmission) {
+      if (source?.missing.delete(seq)) this.stats.recovered++;
+      return;
+    }
+    if (!source) {
+      if (this.sources.size >= 32) return;
+      this.sources.set(ssrc, {
+        highest: seq,
+        highestExtended: seq,
+        first: seq,
+        received: 1,
+        seen: new Set([seq]),
+        missing: new Map(),
+        reportedExpected: 0,
+        reportedReceived: 0,
+      });
+      if (!this.reportTimer) this.reportTimer = setInterval(() => this._report(), 1000).unref();
+      return;
+    }
+    if (!source.seen.has(seq)) {
+      const age = (source.highest - seq) & 65535;
+      if (age < 128 || age >= 32768) source.received++;
+      source.seen.add(seq);
+      if (source.seen.size > 512) source.seen.delete(source.seen.values().next().value);
+    }
+    if (source.missing.delete(seq)) this.stats.recovered++;
+    const distance = (seq - source.highest) & 65535;
+    if (!distance || distance >= 32768) return;
+    if (distance > 128) {
+      this.stats.expired += source.missing.size;
+      source.missing.clear();
+    } else {
+      const time = performance.now();
+      for (let i = 1; i < distance; i++) {
+        if (source.missing.size >= 128) break;
+        source.missing.set((source.highest + i) & 65535, { time, last: -Infinity, retries: 0 });
+      }
+    }
+    source.highest = seq;
+    source.highestExtended += distance;
+    if (source.missing.size && !this.timer) this.timer = setInterval(() => this._tick(), 20).unref();
+  }
+
+  _tick() {
+    const now = performance.now();
+    let pending = false;
+    for (const [ssrc, source] of this.sources) {
+      const sequences = [];
+      for (const [seq, item] of source.missing) {
+        if (now - item.time >= 500) {
+          source.missing.delete(seq);
+          this.stats.expired++;
+        } else if (now - item.time >= 20 && now - item.last >= 60 && item.retries < 3) {
+          item.last = now;
+          item.retries++;
+          sequences.push(seq);
+        }
+      }
+      if (sequences.length) this._nack(ssrc, sequences);
+      pending ||= source.missing.size > 0;
+    }
+    if (!pending) {
+      clearInterval(this.timer);
+      this.timer = null;
+    }
+  }
+
+  _nack(ssrc, sequences) {
+    const entries = [];
+    for (const seq of sequences) {
+      const entry = entries.at(-1);
+      const delta = entry ? (seq - entry.seq) & 65535 : 0;
+      if (entry && delta > 0 && delta <= 16) entry.mask |= 1 << (delta - 1);
+      else entries.push({ seq, mask: 0 });
+    }
+    const packet = Buffer.alloc(12 + entries.length * 4);
+    packet[0] = 0x81;
+    packet[1] = 205;
+    packet.writeUInt16BE(packet.length / 4 - 1, 2);
+    packet.writeUInt32BE(this.connection.authentication.ssrc, 4);
+    packet.writeUInt32BE(ssrc, 8);
+    entries.forEach((entry, index) => {
+      packet.writeUInt16BE(entry.seq, 12 + index * 4);
+      packet.writeUInt16BE(entry.mask, 14 + index * 4);
+    });
+    this.stats.requested += sequences.length;
+    try {
+      const header = packet.subarray(0, 8);
+      this._send(Buffer.concat([header, ...encrypt(this.connection, packet.subarray(8), header)]));
+    } catch (error) {
+      this.connection.receiver?.emit('debug', error);
+    }
+  }
+
+  handle(buffer) {
+    let packet;
+    try {
+      packet = decrypt(this.connection, buffer, 8);
+    } catch {
+      return;
+    }
+    const messages = [];
+    let offset = 0;
+    while (offset + 8 <= packet.length) {
+      const size = (packet.readUInt16BE(offset + 2) + 1) * 4;
+      if (packet[offset] >> 6 !== 2 || size < 8 || offset + size > packet.length) return;
+      if (packet[offset + 1] === 205 && (packet[offset] & 31) === 1 && size >= 16 && size % 4 === 0) {
+        messages.push(packet.subarray(offset, offset + size));
+      } else if (packet[offset + 1] === 201 || packet[offset + 1] === 200) {
+        const base = packet[offset + 1] === 201 ? 8 : 28;
+        if (size < base + (packet[offset] & 31) * 24) return;
+        messages.push(packet.subarray(offset, offset + size));
+      }
+      offset += size;
+    }
+    if (offset !== packet.length) return;
+    for (const message of messages) {
+      if (message[1] === 205) this._handleNack(message);
+      else this._handleReport(message);
+    }
+  }
+
+  _handleReport(packet) {
+    const control = this.connection._congestionControl;
+    if (!control) return;
+    const reporter = packet.readUInt32BE(4);
+    const videoSsrc = this.connection.authentication.ssrc + 1;
+    const base = packet[1] === 201 ? 8 : 28;
+    for (let index = 0; index < (packet[0] & 31); index++) {
+      const offset = base + index * 24;
+      if (packet.readUInt32BE(offset) === videoSsrc) {
+        control.report(packet[offset + 4] / 256, packet.readUInt32BE(offset + 8), reporter);
+      }
+    }
+  }
+
+  _report() {
+    for (const [ssrc, source] of this.sources) {
+      const expected = source.highestExtended - source.first + 1;
+      const intervalExpected = expected - source.reportedExpected;
+      const intervalReceived = source.received - source.reportedReceived;
+      source.reportedExpected = expected;
+      source.reportedReceived = source.received;
+      if (!intervalExpected) continue;
+      const packet = Buffer.alloc(32);
+      packet[0] = 0x81;
+      packet[1] = 201;
+      packet.writeUInt16BE(7, 2);
+      packet.writeUInt32BE(this.connection.authentication.ssrc, 4);
+      packet.writeUInt32BE(ssrc, 8);
+      packet[12] = Math.max(
+        0,
+        Math.min(255, Math.floor(((intervalExpected - intervalReceived) * 256) / intervalExpected)),
+      );
+      packet.writeUIntBE(Math.max(0, Math.min(0x7fffff, expected - source.received)), 13, 3);
+      packet.writeUInt32BE(source.highestExtended >>> 0, 16);
+      try {
+        const header = packet.subarray(0, 8);
+        this._send(Buffer.concat([header, ...encrypt(this.connection, packet.subarray(8), header)]));
+      } catch (error) {
+        this.connection.receiver?.emit('debug', error);
+      }
+    }
+  }
+
+  _handleNack(packet) {
+    const now = performance.now();
+    const ssrc = packet.readUInt32BE(8);
+    for (let offset = 12; offset < packet.length; offset += 4) {
+      const seq = packet.readUInt16BE(offset),
+        mask = packet.readUInt16BE(offset + 2);
+      this._retransmit(ssrc, seq, now);
+      for (let bit = 0; bit < 16; bit++) {
+        if (mask & (1 << bit)) this._retransmit(ssrc, (seq + bit + 1) & 65535, now);
+      }
+    }
+  }
+
+  _retransmit(ssrc, sequence, now) {
+    const item = this.cache.get(`${ssrc}:${sequence}`);
+    const auth = this.connection.authentication;
+    if (
+      !item ||
+      !auth.secret_key ||
+      !this.key?.equals(Buffer.from(auth.secret_key)) ||
+      now - item.time > 2000 ||
+      item.retries >= 3 ||
+      now - item.last < 20
+    ) {
+      return;
+    }
+    const stream = auth.streams?.find(value => value.ssrc === ssrc);
+    const rtx = stream?.rtx_ssrc ?? (ssrc === auth.ssrc + 1 ? auth.ssrc + 2 : null);
+    if (!rtx) return;
+    item.retries++;
+    item.last = now;
+    if (ssrc === auth.ssrc + 1) this.connection._congestionControl?.nack(sequence);
+    const header = Buffer.from(item.header);
+    header[0] = 0x80;
+    header[1] = (header[1] & 0x80) | item.type;
+    header.writeUInt16BE(this.rtxSequence++ & 65535, 2);
+    header.writeUInt32BE(rtx, 8);
+    const original = Buffer.alloc(2);
+    original.writeUInt16BE(sequence);
+    const send = () => {
+      const current = this.connection.authentication;
+      if (
+        !current.secret_key ||
+        !this.key?.equals(Buffer.from(current.secret_key)) ||
+        performance.now() - item.time > 2000
+      )
+        return;
+      try {
+        this._send(
+          Buffer.concat([header, ...encrypt(this.connection, Buffer.concat([original, item.payload]), header)]),
+        );
+        this.stats.retransmitted++;
+      } catch (error) {
+        this.connection.receiver?.emit('debug', error);
+      }
+    };
+    const control = this.connection._congestionControl;
+    if (!control) {
+      send();
+      return;
+    }
+    if (this.pendingRtx.size >= 128) return;
+    if (control.nextSendAt - performance.now() > 500) return;
+    const delay = control.delay(header.length + original.length + item.payload.length + 20);
+    if (delay > 500) return;
+    const timer = setTimeout(() => {
+      this.pendingRtx.delete(timer);
+      send();
+    }, delay).unref();
+    this.pendingRtx.add(timer);
+  }
+
+  _send(packet) {
+    this.connection.sockets?.udp?.send(packet).catch(error => this.connection.receiver?.emit('debug', error));
+  }
+
+  reset() {
+    clearInterval(this.timer);
+    clearInterval(this.reportTimer);
+    this.reportTimer = null;
+    this.timer = null;
+    this.cache.clear();
+    this.sources.clear();
+    for (const timer of this.pendingRtx) clearTimeout(timer);
+    this.pendingRtx.clear();
+    this.rtxSequence = 0;
+  }
+}
+
+module.exports = MediaRecovery;

package/src/client/voice/networking/VoiceUDPClient.js

@@ -137,7 +137,16 @@ class VoiceConnectionUDPClient extends EventEmitter {
           op: VoiceOpcodes.SELECT_PROTOCOL,
           d: {
             protocol: 'udp',
-            codecs: Util.getAllPayloadType(),
+            codecs: Util.getAllPayloadType().map(codec =>
+              codec.type === 'video'
+                ? {
+                    ...codec,
+                    priority: codec.name === this.voiceConnection.videoCodec ? 1 : codec.priority,
+                    encode: codec.name === this.voiceConnection.videoCodec,
+                    decode: ['H264', 'H265', 'VP8'].includes(codec.name),
+                  }
+                : codec,
+            ),
             data: {
               address: packet.address,
               port: packet.port,

package/src/client/voice/networking/VoiceWebSocket.js

@@ -263,7 +263,10 @@ class VoiceWebSocket extends EventEmitter {
       case VoiceOpcodes.CLIENT_DISCONNECT:
         this.dave.clients.delete(packet.d.user_id);
         for (const [ssrc, info] of this.connection.ssrcMap) {
-          if (info.userId === packet.d.user_id) this.connection.ssrcMap.delete(ssrc);
+          if (info.userId === packet.d.user_id) {
+            this.connection.ssrcMap.delete(ssrc);
+            this.connection.receiver?.packets.forgetSource(ssrc);
+          }
         }
         const streamInfo = this.connection.receiver && this.connection.receiver.packets.streams.get(packet.d.user_id);
         if (streamInfo) {