rockcord 4.0.0-dev.4 → 4.0.0

package/src/client/voice/player/MediaPlayer.js

@@ -22,7 +22,7 @@ const { Readable: ReadableStream } = require('stream');
 const prism = require('prism-media');
 const { H264NalSplitter, H265NalSplitter } = require('./processing/AnnexBNalSplitter');
 const { IvfTransformer } = require('./processing/IvfSplitter');
-const { H264Dispatcher } = require('../dispatcher/AnnexBDispatcher');
+const { H264Dispatcher, H265Dispatcher } = require('../dispatcher/AnnexBDispatcher');
 const AudioDispatcher = require('../dispatcher/AudioDispatcher');
 const { VP8Dispatcher } = require('../dispatcher/VPxDispatcher');
 
@@ -38,7 +38,20 @@ const FFMPEG_INPUT_PREFIX = [
   '4294',
 ];
 const FFMPEG_PCM_ARGUMENTS = ['-f', 's16le', '-ar', '48000', '-ac', '2'];
-const FFMPEG_VP8_ARGUMENTS = ['-f', 'ivf', '-deadline', 'realtime', '-c:v', 'libvpx'];
+const FFMPEG_VP8_ARGUMENTS = options => [
+  '-f',
+  'ivf',
+  '-deadline',
+  'realtime',
+  '-c:v',
+  'libvpx',
+  '-g',
+  String(Math.max(1, Math.round(options?.fps || 30))),
+  '-lag-in-frames',
+  '0',
+  '-auto-alt-ref',
+  '0',
+];
 const FFMPEG_H264_ARGUMENTS = options => [
   '-c:v',
   'libx264',
@@ -52,10 +65,8 @@ const FFMPEG_H264_ARGUMENTS = options => [
   options?.presetH26x || 'faster',
   '-profile:v',
   'baseline',
-  // '-g',
-  // `${options?.fps}`,
-  // '-x264-params',
-  // `keyint=${options?.fps}:min-keyint=${options?.fps}`,
+  '-g',
+  String(Math.max(1, Math.round(options?.fps || 30))),
   '-bf',
   '0',
   '-bsf:v',
@@ -68,9 +79,15 @@ const FFMPEG_H265_ARGUMENTS = options => [
   '-f',
   'hevc',
   '-preset',
-  options?.presetH265 || 'faster',
+  options?.presetH26x || 'faster',
+  '-tune',
+  'zerolatency',
   '-profile:v',
   'main',
+  '-g',
+  String(Math.max(1, Math.round(options?.fps || 30))),
+  '-x265-params',
+  'repeat-headers=1:aud=1',
   '-bf',
   '0',
 ];
@@ -173,7 +190,10 @@ class MediaPlayer extends EventEmitter {
   playUnknownVideo(input, options = {}) {
     this.destroyVideoDispatcher();
     const isStream = input instanceof ReadableStream;
-    if (!options?.fps) options.fps = 30;
+    if (options.fps == null) options.fps = 30;
+    if (!Number.isFinite(options.fps) || options.fps <= 0) {
+      throw new RangeError('Video FPS must be a positive finite number');
+    }
 
     const args = [
       '-i',
@@ -202,7 +222,7 @@ class MediaPlayer extends EventEmitter {
 
     // Get stream type
     if (this.voiceConnection.videoCodec === 'VP8') {
-      args.push(...FFMPEG_VP8_ARGUMENTS);
+      args.push(...FFMPEG_VP8_ARGUMENTS(options));
       // Remove  '-speed', '5' bc bad quality
     }
 
@@ -245,8 +265,11 @@ class MediaPlayer extends EventEmitter {
       case 'H264': {
         return this.playAnnexBVideo(ffmpeg, options, streams, 'H264');
       }
+      case 'H265': {
+        return this.playAnnexBVideo(ffmpeg, options, streams, 'H265');
+      }
       default: {
-        throw new Error('Invalid codec (Supported: VP8, H264)');
+        throw new Error('Invalid codec (Supported: VP8, H264, H265)');
       }
     }
   }
@@ -261,7 +284,6 @@ class MediaPlayer extends EventEmitter {
     return dispatcher;
   }
 
-  // eslint-disable-next-line no-unused-vars
   playAnnexBVideo(stream, options, streams, type) {
     this.destroyVideoDispatcher();
     let videoStream;
@@ -300,6 +322,7 @@ class MediaPlayer extends EventEmitter {
           streams,
           options?.fps,
         ));
+        dispatcher.configureCongestion(options);
         return dispatcher;
       }
       case 'H264': {
@@ -309,6 +332,17 @@ class MediaPlayer extends EventEmitter {
           streams,
           options?.fps,
         ));
+        dispatcher.configureCongestion(options);
+        return dispatcher;
+      }
+      case 'H265': {
+        const dispatcher = (this.videoDispatcher = new H265Dispatcher(
+          this,
+          options?.highWaterMark,
+          streams,
+          options?.fps,
+        ));
+        dispatcher.configureCongestion(options);
         return dispatcher;
       }
       default: {

package/src/client/voice/player/processing/AnnexBNalSplitter.js

@@ -147,7 +147,6 @@ class AnnexBNalSplitter extends Transform {
     const newData = Buffer.allocUnsafe(data.length);
     let newLength = 0;
 
-    // eslint-disable-next-line no-constant-condition
     while (true) {
       const epbsPos = data.indexOf(epbPrefix);
       if (epbsPos === -1) {

package/src/client/voice/receiver/AudioJitter.js

@@ -0,0 +1,101 @@
+'use strict';
+
+const { Buffer } = require('node:buffer');
+const { setTimeout } = require('node:timers');
+const { RtpHeader, RtpPacket } = require('werift-rtp');
+const { SILENCE_FRAME } = require('../util/Silence');
+
+// Opus TOC frame durations from RFC 6716, section 3.1.
+function duration(payload) {
+  if (!payload.length) return 20;
+  const config = payload[0] >> 3;
+  const frame =
+    config >= 16 ? 2.5 * 2 ** (config & 3) : config >= 12 ? 10 * 2 ** (config & 1) : [10, 20, 40, 60][config & 3];
+  const code = payload[0] & 3;
+  const frames = code === 0 ? 1 : code === 3 ? (payload[1] || 0) & 63 : 2;
+  const milliseconds = frame * frames;
+  return milliseconds > 0 && milliseconds <= 120 ? milliseconds : 20;
+}
+
+/** Forty-millisecond playout window with bounded reordering and loss padding. @private */
+class AudioJitter {
+  constructor(deliver) {
+    this.deliver = deliver;
+    this.packets = new Map();
+    this.expected = null;
+    this.lastArrival = -Infinity;
+  }
+
+  push(packet) {
+    const now = performance.now();
+    const seq = packet.header.sequenceNumber;
+    if (packet.payload.length > 4096) return;
+    if (
+      this.expected === null ||
+      now - this.lastArrival > 500 ||
+      (((seq - this.expected) & 65535) > 1024 && ((seq - this.expected) & 65535) < 32768)
+    ) {
+      this.reset();
+      this.expected = seq;
+      this.next = now + 40;
+    }
+    const ahead = (seq - this.expected) & 65535;
+    if (ahead >= 32768 || this.packets.has(seq)) return;
+    if (this.packets.size >= 64) return;
+    this.lastArrival = now;
+    this.packets.set(seq, packet);
+    this._schedule();
+  }
+
+  _schedule() {
+    if (this.timer || !this.packets.size) return;
+    this.timer = setTimeout(
+      () => {
+        this.timer = null;
+        this._play();
+      },
+      Math.max(0, this.next - performance.now()),
+    ).unref();
+  }
+
+  _play() {
+    let packet = this.packets.get(this.expected);
+    if (!packet) {
+      const sequence = [...this.packets.keys()].sort(
+        (a, b) => ((a - this.expected) & 65535) - ((b - this.expected) & 65535),
+      )[0];
+      if (sequence === undefined) return;
+      const gap = (sequence - this.expected) & 65535;
+      if (gap > 10 || !this.last) {
+        this.expected = sequence;
+        packet = this.packets.get(sequence);
+      } else {
+        packet = new RtpPacket(
+          new RtpHeader({
+            ...this.last.header,
+            sequenceNumber: this.expected,
+            timestamp: (this.last.header.timestamp + duration(this.last.payload) * 48) >>> 0,
+          }),
+          Buffer.from(SILENCE_FRAME),
+        );
+        packet.concealed = true;
+      }
+    }
+    this.packets.delete(this.expected);
+    this.expected = (this.expected + 1) & 65535;
+    this.last = packet;
+    this.next += duration(packet.payload);
+    this.deliver(packet);
+    this._schedule();
+  }
+
+  reset() {
+    clearTimeout(this.timer);
+    this.timer = null;
+    this.packets.clear();
+    this.expected = null;
+    this.last = null;
+  }
+}
+
+module.exports = { AudioJitter, duration };

package/src/client/voice/receiver/PacketHandler.js

@@ -5,9 +5,11 @@ const { Buffer } = require('node:buffer');
 const crypto = require('node:crypto');
 const { setTimeout } = require('node:timers');
 const { RtpPacket, RtpHeader } = require('werift-rtp');
+const { AudioJitter } = require('./AudioJitter');
 const Recorder = require('./Recorder');
 const Speaking = require('../../../util/Speaking');
 const Util = require('../../../util/Util');
+const MediaRecovery = require('../networking/MediaRecovery');
 const secretbox = require('../util/Secretbox');
 const { SILENCE_FRAME } = require('../util/Silence');
 const { VideoFrames, packetize } = require('../util/VideoFrames');
@@ -21,7 +23,7 @@ const UNPADDED_NONCE_LENGTH = 4;
 const AUTH_TAG_LENGTH = 16;
 
 class Readable extends require('stream').Readable {
-  _read() {} // eslint-disable-line no-empty-function
+  _read() {}
 }
 
 /**
@@ -38,6 +40,9 @@ class PacketHandler extends EventEmitter {
     this.speakingTimeouts = new Map();
     this.videoFrames = new VideoFrames();
     this.videoSequences = new Map();
+    this.videoPictures = new Map();
+    this.audioJitters = new Map();
+    this.recovery = this.connection._mediaRecovery = new MediaRecovery(this.connection);
   }
 
   getNonceBuffer() {
@@ -70,9 +75,10 @@ class PacketHandler extends EventEmitter {
     return stream;
   }
 
-  makeVideoStream(user, output) {
+  makeVideoStream(user, output, options = {}) {
     if (this.videoStreams.has(user)) return this.videoStreams.get(user);
     const stream = new Recorder(this, {
+      ...options,
       userId: user,
       output,
     });
@@ -164,7 +170,7 @@ class PacketHandler extends EventEmitter {
       if (!opusPacket.payload) {
         return;
       }
-      if (SILENCE_FRAME.equals(opusPacket.payload)) {
+      if (SILENCE_FRAME.equals(opusPacket.payload) && !opusPacket.concealed) {
         // If this is a silence frame, pretend we never received it
         return;
       }
@@ -239,13 +245,17 @@ class PacketHandler extends EventEmitter {
       if (this.connection.dave) {
         const codec = ['H264', 'H265', 'VP8'].find(name => Util.getPayloadType(name) === packet.header.payloadType);
         if (!codec) return;
-        const frame = this.videoFrames.push(packet, codec);
+        let plain;
+        const frame = this.videoFrames.push(packet, codec, assembled => {
+          plain = this.connection.dave.decrypt(assembled, userStat.userId, true);
+          return Boolean(plain);
+        });
         if (!frame) return;
-        const plain = this.connection.dave.decrypt(frame, userStat.userId, true);
-        if (!plain) return;
         this.receiver.emit('videoFrame', userStat, plain, codec);
         if (streamInfo) {
-          const payloads = packetize(plain, codec);
+          const picture = this.videoPictures.get(ssrc) ?? 0;
+          const payloads = packetize(plain, codec, 1200, picture);
+          this.videoPictures.set(ssrc, (picture + 1) & 0x7fff);
           let sequence = this.videoSequences.get(ssrc) ?? packet.header.sequenceNumber;
           for (let i = 0; i < payloads.length; i++) {
             streamInfo.feed(
@@ -270,6 +280,16 @@ class PacketHandler extends EventEmitter {
   }
 
   push(buffer) {
+    if (
+      Buffer.isBuffer(buffer) &&
+      buffer.length >= 28 &&
+      buffer[0] >> 6 === 2 &&
+      buffer[1] >= 192 &&
+      buffer[1] <= 223
+    ) {
+      this.recovery.handle(buffer);
+      return;
+    }
     if (!Buffer.isBuffer(buffer) || buffer.length < 32 || buffer[0] >> 6 !== 2) return;
     const ssrc = buffer.readUInt32BE(8);
     let userStat, packet;
@@ -280,6 +300,8 @@ class PacketHandler extends EventEmitter {
       return;
     }
     if (packet instanceof Error) return;
+    const type = packet.header.payloadType;
+    if (!Util.getAllPayloadType().some(codec => codec.payload_type === type || codec.rtx_payload_type === type)) return;
     if (this.connection.ssrcMap.has(ssrc)) {
       userStat = this.connection.ssrcMap.get(ssrc); // Audio_ssrc
       if (userStat.kind === 'video-rtx') {
@@ -289,17 +311,35 @@ class PacketHandler extends EventEmitter {
         packet.header.sequenceNumber = packet.payload.readUInt16BE();
         packet.header.payloadType = codec.payload_type;
         packet.payload = packet.payload.subarray(2);
+        this.recovery.receive(packet, true);
         this.videoReceiver(userStat.primarySsrc, userStat, packet);
       } else if (userStat.kind === 'video') {
+        this.recovery.receive(packet);
         this.videoReceiver(ssrc, userStat, packet);
       } else {
+        if (type !== Util.getPayloadType('opus')) return;
         if (this.connection.dave) {
           const plain = this.connection.dave.decrypt(packet.payload, userStat.userId);
           if (!plain) return;
           packet.payload = plain;
         }
-        this.audioReceiver(ssrc, userStat, packet);
-        this.audioReceiverForStream(ssrc, userStat, packet);
+        let jitter = this.audioJitters.get(ssrc);
+        if (jitter && jitter.userId !== userStat.userId) {
+          jitter.reset();
+          this.audioJitters.delete(ssrc);
+          jitter = null;
+        }
+        if (!jitter) {
+          if (this.audioJitters.size >= 32) return;
+          jitter = new AudioJitter(ordered => {
+            this.audioReceiver(ssrc, jitter.userStat, ordered);
+            this.audioReceiverForStream(ssrc, jitter.userStat, ordered);
+          });
+          jitter.userId = userStat.userId;
+          this.audioJitters.set(ssrc, jitter);
+        }
+        jitter.userStat = userStat;
+        jitter.push(packet);
       }
     } else if (this.connection.ssrcMap.has(ssrc - 1)) {
       userStat = this.connection.ssrcMap.get(ssrc - 1); // Video_ssrc
@@ -309,12 +349,29 @@ class PacketHandler extends EventEmitter {
   }
 
   // When udp connection is closed (STREAM_DELETE), destroy all streams (Memory leak)
+  forgetSource(ssrc) {
+    this.audioJitters.get(ssrc)?.reset();
+    this.audioJitters.delete(ssrc);
+    this.recovery.sources.delete(ssrc);
+    this.videoSequences.delete(ssrc);
+    this.videoPictures.delete(ssrc);
+    clearTimeout(this.speakingTimeouts.get(ssrc));
+    this.speakingTimeouts.delete(ssrc);
+    for (const key of this.videoFrames.frames.keys()) {
+      if (key.startsWith(`${ssrc}:`)) this.videoFrames.frames.delete(key);
+    }
+  }
+
   destroyAllStream() {
+    this.recovery.reset();
+    for (const jitter of this.audioJitters.values()) jitter.reset();
+    this.audioJitters.clear();
     for (const timeout of this.speakingTimeouts.values()) clearTimeout(timeout);
     this.speakingTimeouts.clear();
     this.videoFrames.frames.clear();
     this.videoFrames.completed.clear();
     this.videoSequences.clear();
+    this.videoPictures.clear();
     for (const stream of this.streams.values()) {
       stream.stream.destroy();
     }

package/src/client/voice/receiver/Receiver.js

@@ -34,7 +34,9 @@ class VoiceReceiver extends EventEmitter {
   }
 
   _updateVideoSubscriptions(frames = this.listenerCount('videoFrame') > 0) {
-    const wants = { any: 0 };
+    // Keep default media delivery enabled: any: 0 also suppresses audio when
+    // switching on explicit video subscriptions.
+    const wants = { any: 100 };
     for (const [ssrc, info] of this.connection.ssrcMap) {
       if (info.kind === 'video') wants[ssrc] = frames || this.packets.videoStreams.has(info.userId) ? 100 : 0;
     }
@@ -90,15 +92,18 @@ class VoiceReceiver extends EventEmitter {
   /**
    * Creates a new video receiving stream. If a stream already exists for a user, then that stream will be returned
    * rather than generating a new one.
-   * <info>Proof of concept - Requires a very good internet connection</info>
+   * Records received H264, H265 or VP8 video and Opus audio using FFmpeg.
+   * Wait for `ready` before sending media and await `stop()` to finalize the output.
    * @param {UserResolvable} user The user to start listening to.
    * @param {WritableStream|string} output Output stream or file path to write the video stream to.
+   * @param {Object} [options] Recording options
+   * @param {VideoCodec} [options.codec='H264'] The codec sent by the recorded participant
    * @returns {Recorder} The video stream for the specified user.
    */
-  createVideoStream(user, output) {
+  createVideoStream(user, output, options = {}) {
     user = this.connection.client.users.resolve(user);
     if (!user) throw new Error('VOICE_USER_MISSING');
-    const stream = this.packets.makeVideoStream(user.id, output);
+    const stream = this.packets.makeVideoStream(user.id, output, options);
     this._updateVideoSubscriptions();
     stream.once('closed', () => this._updateVideoSubscriptions());
     return stream;