rockcord 4.0.0-dev.4

package/src/client/voice/receiver/PacketHandler.js

@@ -0,0 +1,329 @@
+'use strict';
+
+const EventEmitter = require('events');
+const { Buffer } = require('node:buffer');
+const crypto = require('node:crypto');
+const { setTimeout } = require('node:timers');
+const { RtpPacket, RtpHeader } = require('werift-rtp');
+const Recorder = require('./Recorder');
+const Speaking = require('../../../util/Speaking');
+const Util = require('../../../util/Util');
+const secretbox = require('../util/Secretbox');
+const { SILENCE_FRAME } = require('../util/Silence');
+const { VideoFrames, packetize } = require('../util/VideoFrames');
+
+// The delay between packets when a user is considered to have stopped speaking
+// https://github.com/discordjs/discord.js/issues/3524#issuecomment-540373200
+const DISCORD_SPEAKING_DELAY = 250;
+
+// Unused: const HEADER_EXTENSION_BYTE = Buffer.from([0xbe, 0xde]);
+const UNPADDED_NONCE_LENGTH = 4;
+const AUTH_TAG_LENGTH = 16;
+
+class Readable extends require('stream').Readable {
+  _read() {} // eslint-disable-line no-empty-function
+}
+
+/**
+ * Internal receive packet and media-frame handler.
+ * @extends {EventEmitter}
+ * @private
+ */
+class PacketHandler extends EventEmitter {
+  constructor(receiver) {
+    super();
+    this.receiver = receiver;
+    this.streams = new Map();
+    this.videoStreams = new Map();
+    this.speakingTimeouts = new Map();
+    this.videoFrames = new VideoFrames();
+    this.videoSequences = new Map();
+  }
+
+  getNonceBuffer() {
+    return this.receiver.connection.authentication.mode === 'aead_aes256_gcm_rtpsize'
+      ? Buffer.alloc(12)
+      : Buffer.alloc(24);
+  }
+
+  get connection() {
+    return this.receiver.connection;
+  }
+
+  _stoppedSpeaking(userId) {
+    const streamInfo = this.streams.get(userId);
+    if (streamInfo && streamInfo.end === 'silence') {
+      this.streams.delete(userId);
+      streamInfo.stream.push(null);
+    }
+  }
+
+  makeStream(user, end) {
+    if (this.streams.has(user)) return this.streams.get(user).stream;
+    const stream = new Readable();
+    const cleanup = () => {
+      if (this.streams.get(user)?.stream === stream) this.streams.delete(user);
+    };
+    stream.once('end', cleanup);
+    stream.once('close', cleanup);
+    this.streams.set(user, { stream, end });
+    return stream;
+  }
+
+  makeVideoStream(user, output) {
+    if (this.videoStreams.has(user)) return this.videoStreams.get(user);
+    const stream = new Recorder(this, {
+      userId: user,
+      output,
+    });
+    this.videoStreams.set(user, stream);
+    stream.once('closed', () => {
+      if (this.videoStreams.get(user) === stream) this.videoStreams.delete(user);
+    });
+    return stream;
+  }
+
+  parseBuffer(buffer) {
+    const { secret_key, mode } = this.receiver.connection.authentication;
+    // Open packet
+    if (!secret_key) return new Error('secret_key cannot be null or undefined');
+    const nonce = this.getNonceBuffer();
+    // Copy the last 4 bytes of unpadded nonce to the padding of (12 - 4) or (24 - 4) bytes
+    buffer.copy(nonce, 0, buffer.length - UNPADDED_NONCE_LENGTH);
+
+    let headerSize = 12 + (buffer[0] & 15) * 4;
+    const first = buffer.readUint8();
+    if ((first >> 4) & 0x01) headerSize += 4;
+    if (headerSize > buffer.length - AUTH_TAG_LENGTH - UNPADDED_NONCE_LENGTH) {
+      throw new RangeError('Truncated RTP header');
+    }
+
+    // The unencrypted RTP header contains 12 bytes, HEADER_EXTENSION and the extension size
+    const header = buffer.slice(0, headerSize);
+
+    // Encrypted contains the extension, if any, the opus packet, and the auth tag
+    const encrypted = buffer.slice(headerSize, buffer.length - AUTH_TAG_LENGTH - UNPADDED_NONCE_LENGTH);
+    const authTag = buffer.slice(
+      buffer.length - AUTH_TAG_LENGTH - UNPADDED_NONCE_LENGTH,
+      buffer.length - UNPADDED_NONCE_LENGTH,
+    );
+
+    let packet;
+    switch (mode) {
+      case 'aead_aes256_gcm_rtpsize': {
+        const decipheriv = crypto.createDecipheriv('aes-256-gcm', secret_key, nonce);
+        decipheriv.setAAD(header);
+        decipheriv.setAuthTag(authTag);
+
+        packet = Buffer.concat([decipheriv.update(encrypted), decipheriv.final()]);
+        break;
+      }
+      case 'aead_xchacha20_poly1305_rtpsize': {
+        // Combined mode expects authtag in the encrypted message
+        packet = secretbox.methods.crypto_aead_xchacha20poly1305_ietf_decrypt(
+          Buffer.concat([encrypted, authTag]),
+          header,
+          nonce,
+          secret_key,
+        );
+
+        packet = Buffer.from(packet);
+        break;
+      }
+      default: {
+        return new RangeError(`Unsupported decryption method: ${mode}`);
+      }
+    }
+
+    /*
+    // Strip decrypted RTP Header Extension if present
+    if (buffer.slice(12, 14).compare(HEADER_EXTENSION_BYTE) === 0) {
+      const headerExtensionLength = buffer.slice(14).readUInt16BE();
+      packet = packet.subarray(4 * headerExtensionLength);
+    }
+    */
+
+    return RtpPacket.deSerialize(Buffer.concat([header, packet]));
+  }
+
+  audioReceiver(ssrc, userStat, opusPacket) {
+    const streamInfo = this.streams.get(userStat.userId);
+    // If the user is in video, we need to check if the packet is just silence
+    if (userStat.hasVideo) {
+      if (opusPacket instanceof Error) {
+        // Only emit an error if we were actively receiving packets from this user
+        if (streamInfo) {
+          this.emit('error', opusPacket);
+        }
+        return;
+      }
+      // Check payload type
+      if (opusPacket.header.payloadType !== Util.getPayloadType('opus')) {
+        return;
+      }
+      if (!opusPacket.payload) {
+        return;
+      }
+      if (SILENCE_FRAME.equals(opusPacket.payload)) {
+        // If this is a silence frame, pretend we never received it
+        return;
+      }
+    }
+
+    let speakingTimeout = this.speakingTimeouts.get(ssrc);
+    if (typeof speakingTimeout === 'undefined') {
+      // Ensure at least the speaking bit is set.
+      // As the object is by reference, it's only needed once per client re-connect.
+      if (userStat.speaking === 0) {
+        userStat.speaking = Speaking.FLAGS.SPEAKING;
+      }
+      this.connection.onSpeaking({ user_id: userStat.userId, ssrc: ssrc, speaking: userStat.speaking });
+      speakingTimeout = setTimeout(() => {
+        try {
+          this.connection.onSpeaking({ user_id: userStat.userId, ssrc: ssrc, speaking: 0 });
+          clearTimeout(speakingTimeout);
+          this.speakingTimeouts.delete(ssrc);
+        } catch {
+          // Connection already closed, ignore
+        }
+      }, DISCORD_SPEAKING_DELAY).unref();
+      this.speakingTimeouts.set(ssrc, speakingTimeout);
+    } else {
+      speakingTimeout.refresh();
+    }
+
+    if (streamInfo) {
+      const { stream } = streamInfo;
+      if (opusPacket instanceof Error) {
+        this.emit('error', opusPacket);
+        return;
+      }
+      if (opusPacket.header.payloadType !== Util.getPayloadType('opus')) {
+        return;
+      }
+      stream.push(opusPacket.payload);
+    }
+  }
+
+  audioReceiverForStream(ssrc, userStat, packet) {
+    const streamInfo = this.videoStreams.get(userStat.userId);
+    if (!streamInfo) return;
+    if (packet instanceof Error) {
+      return;
+    }
+    if (packet.header.payloadType !== Util.getPayloadType('opus')) {
+      return;
+    }
+    streamInfo.feed(packet);
+  }
+
+  /**
+   * Test
+   * @param {number} ssrc ssrc
+   * @param {Object} userStat { userId, hasVideo }
+   * @param {RtpPacket} packet RtpPacket
+   * @returns {void}
+   */
+  videoReceiver(ssrc, userStat, packet) {
+    const streamInfo = this.videoStreams.get(userStat.userId);
+    // If the user is in video, we need to check if the packet is just silence
+    if (userStat.hasVideo) {
+      if (packet instanceof Error) {
+        return;
+      }
+
+      if (packet.header.payloadType === Util.getPayloadType('opus')) {
+        return;
+      }
+
+      if (this.connection.dave) {
+        const codec = ['H264', 'H265', 'VP8'].find(name => Util.getPayloadType(name) === packet.header.payloadType);
+        if (!codec) return;
+        const frame = this.videoFrames.push(packet, codec);
+        if (!frame) return;
+        const plain = this.connection.dave.decrypt(frame, userStat.userId, true);
+        if (!plain) return;
+        this.receiver.emit('videoFrame', userStat, plain, codec);
+        if (streamInfo) {
+          const payloads = packetize(plain, codec);
+          let sequence = this.videoSequences.get(ssrc) ?? packet.header.sequenceNumber;
+          for (let i = 0; i < payloads.length; i++) {
+            streamInfo.feed(
+              new RtpPacket(
+                new RtpHeader({
+                  ssrc,
+                  timestamp: packet.header.timestamp,
+                  payloadType: packet.header.payloadType,
+                  sequenceNumber: sequence++ & 65535,
+                  marker: i === payloads.length - 1,
+                }),
+                payloads[i],
+              ),
+            );
+          }
+          this.videoSequences.set(ssrc, sequence & 65535);
+        }
+      } else if (streamInfo) {
+        streamInfo.feed(packet);
+      }
+    }
+  }
+
+  push(buffer) {
+    if (!Buffer.isBuffer(buffer) || buffer.length < 32 || buffer[0] >> 6 !== 2) return;
+    const ssrc = buffer.readUInt32BE(8);
+    let userStat, packet;
+    try {
+      packet = this.parseBuffer(buffer);
+    } catch {
+      // Malformed or unauthenticated datagrams must not escape the UDP callback.
+      return;
+    }
+    if (packet instanceof Error) return;
+    if (this.connection.ssrcMap.has(ssrc)) {
+      userStat = this.connection.ssrcMap.get(ssrc); // Audio_ssrc
+      if (userStat.kind === 'video-rtx') {
+        const codec = Util.getAllPayloadType().find(value => value.rtx_payload_type === packet.header.payloadType);
+        if (!codec || packet.payload.length < 3) return;
+        packet.header.ssrc = userStat.primarySsrc;
+        packet.header.sequenceNumber = packet.payload.readUInt16BE();
+        packet.header.payloadType = codec.payload_type;
+        packet.payload = packet.payload.subarray(2);
+        this.videoReceiver(userStat.primarySsrc, userStat, packet);
+      } else if (userStat.kind === 'video') {
+        this.videoReceiver(ssrc, userStat, packet);
+      } else {
+        if (this.connection.dave) {
+          const plain = this.connection.dave.decrypt(packet.payload, userStat.userId);
+          if (!plain) return;
+          packet.payload = plain;
+        }
+        this.audioReceiver(ssrc, userStat, packet);
+        this.audioReceiverForStream(ssrc, userStat, packet);
+      }
+    } else if (this.connection.ssrcMap.has(ssrc - 1)) {
+      userStat = this.connection.ssrcMap.get(ssrc - 1); // Video_ssrc
+      this.videoReceiver(ssrc, userStat, packet);
+    }
+    if (userStat && !(packet instanceof Error)) this.receiver.emit('receiverData', userStat, packet);
+  }
+
+  // When udp connection is closed (STREAM_DELETE), destroy all streams (Memory leak)
+  destroyAllStream() {
+    for (const timeout of this.speakingTimeouts.values()) clearTimeout(timeout);
+    this.speakingTimeouts.clear();
+    this.videoFrames.frames.clear();
+    this.videoFrames.completed.clear();
+    this.videoSequences.clear();
+    for (const stream of this.streams.values()) {
+      stream.stream.destroy();
+    }
+    this.streams.clear();
+    for (const stream of this.videoStreams.values()) {
+      stream.destroy();
+    }
+    this.videoStreams.clear();
+  }
+}
+
+module.exports = PacketHandler;

package/src/client/voice/receiver/Receiver.js

@@ -0,0 +1,115 @@
+'use strict';
+
+const EventEmitter = require('events');
+const prism = require('prism-media');
+const PacketHandler = require('./PacketHandler');
+const { Error } = require('../../../errors');
+const { VoiceOpcodes } = require('../../../util/Constants');
+const { PCMInsertSilence } = require('../player/processing/PCMInsertSilence');
+
+/**
+ * Receives audio packets from a voice connection.
+ * @example
+ * const receiver = connection.createReceiver();
+ * // opusStream is a ReadableStream - that means you could play it back to a voice channel if you wanted to!
+ * const opusStream = receiver.createStream(user);
+ */
+class VoiceReceiver extends EventEmitter {
+  constructor(connection) {
+    super();
+    this.connection = connection;
+    this.packets = new PacketHandler(this);
+    /**
+     * Emitted whenever there is a warning
+     * @event VoiceReceiver#debug
+     * @param {Error|string} error The error or message to debug
+     */
+    this.packets.on('error', err => this.emit('debug', err));
+    this.on('newListener', event => {
+      if (event === 'videoFrame') this._updateVideoSubscriptions(true);
+    });
+    this.on('removeListener', event => {
+      if (event === 'videoFrame') this._updateVideoSubscriptions();
+    });
+  }
+
+  _updateVideoSubscriptions(frames = this.listenerCount('videoFrame') > 0) {
+    const wants = { any: 0 };
+    for (const [ssrc, info] of this.connection.ssrcMap) {
+      if (info.kind === 'video') wants[ssrc] = frames || this.packets.videoStreams.has(info.userId) ? 100 : 0;
+    }
+    this.connection.sockets.ws
+      ?.sendPacket({ op: VoiceOpcodes.MEDIA_SINK_WANTS, d: wants })
+      .catch(error => this.emit('debug', error));
+  }
+
+  /**
+   * Options passed to `VoiceReceiver#createStream`.
+   * @typedef {Object} ReceiveStreamOptions
+   * @property {string} [mode='opus'] The mode for audio output. This defaults to opus, meaning discord.js won't decode
+   * the packets for you. You can set this to 'pcm' so that the stream's output will be 16-bit little-endian stereo
+   * audio
+   * @property {string} [end='silence'] When the stream should be destroyed. If `silence`, this will be when the user
+   * stops talking. Otherwise, if `manual`, this should be handled by you.
+   * @property {boolean} [paddingSilence=false] Whether to add silence padding
+   * If 'end' is set to 'silence', this property automatically defaults to `false`
+   */
+
+  /**
+   * Creates a new audio receiving stream. If a stream already exists for a user, then that stream will be returned
+   * rather than generating a new one.
+   * @param {UserResolvable} user The user to start listening to.
+   * @param {ReceiveStreamOptions} options Options.
+   * @returns {ReadableStream}
+   */
+  createStream(user, { mode = 'opus', end = 'silence', paddingSilence = false } = {}) {
+    user = this.connection.client.users.resolve(user);
+    if (end === 'silence') paddingSilence = false;
+    if (!user) throw new Error('VOICE_USER_MISSING');
+    const stream = this.packets.makeStream(user.id, end); // Opus stream
+    if (paddingSilence) {
+      const decoder = new prism.opus.Decoder({ channels: 2, rate: 48000, frameSize: 960 });
+      const pcmTransformer = new PCMInsertSilence();
+      stream.pipe(decoder).pipe(pcmTransformer);
+      if (mode === 'opus') {
+        const encoder = new prism.opus.Encoder({ channels: 2, rate: 48000, frameSize: 960 });
+        pcmTransformer.pipe(encoder);
+        return encoder;
+      }
+      return pcmTransformer;
+    } else {
+      if (mode === 'pcm') {
+        const decoder = new prism.opus.Decoder({ channels: 2, rate: 48000, frameSize: 960 });
+        stream.pipe(decoder);
+        return decoder;
+      }
+      return stream;
+    }
+  }
+
+  /**
+   * Creates a new video receiving stream. If a stream already exists for a user, then that stream will be returned
+   * rather than generating a new one.
+   * <info>Proof of concept - Requires a very good internet connection</info>
+   * @param {UserResolvable} user The user to start listening to.
+   * @param {WritableStream|string} output Output stream or file path to write the video stream to.
+   * @returns {Recorder} The video stream for the specified user.
+   */
+  createVideoStream(user, output) {
+    user = this.connection.client.users.resolve(user);
+    if (!user) throw new Error('VOICE_USER_MISSING');
+    const stream = this.packets.makeVideoStream(user.id, output);
+    this._updateVideoSubscriptions();
+    stream.once('closed', () => this._updateVideoSubscriptions());
+    return stream;
+  }
+
+  /**
+   * Emitted whenever there is a data packet received.
+   * @event VoiceReceiver#receiverData
+   * @param {{ userId: Snowflake, hasVideo: boolean }} ssrcData SSRC Data
+   * @param {RtpPacket} header RTP Packet
+   */
+}
+
+module.exports = VoiceReceiver;

package/src/client/voice/receiver/Recorder.js

@@ -0,0 +1,197 @@
+'use strict';
+
+const { spawn } = require('child_process');
+const { createSocket } = require('dgram');
+const { EventEmitter } = require('events');
+const { Buffer } = require('node:buffer');
+const { Writable } = require('stream');
+const kill = require('tree-kill');
+const { RtpPacket } = require('werift-rtp');
+const Util = require('../../../util/Util');
+const { randomPort } = require('../util/Function');
+const { StreamOutput } = require('../util/Socket');
+
+/**
+ * Represents a FFmpeg handler
+ * @extends {EventEmitter}
+ */
+class Recorder extends EventEmitter {
+  constructor(receiver, { userId, portUdpH264, portUdpOpus, output } = {}) {
+    super();
+
+    Object.defineProperty(this, 'receiver', { value: receiver });
+
+    /**
+     * The user ID
+     * @type {Snowflake}
+     */
+    this.userId = userId;
+
+    this.portUdpH264 = portUdpH264 || null;
+    this.portUdpH265 = null;
+    this.portUdpOpus = portUdpOpus || null;
+
+    this.promise = null;
+
+    if (!portUdpH264 || !portUdpOpus) {
+      this.promise = (async () => {
+        const selected = new Set([this.portUdpH264, this.portUdpOpus].filter(Boolean));
+        const missing = Number(!this.portUdpH264) + Number(!this.portUdpOpus);
+        const ports = [];
+        for (let attempt = 0; ports.length < missing && attempt < 64; attempt++) {
+          const port = await randomPort('udp4');
+          if (port % 2 === 0 && !selected.has(port)) {
+            selected.add(port);
+            ports.push(port);
+          }
+        }
+        if (ports.length < missing) throw new Error('Could not allocate recording ports');
+        this.portUdpH264 ??= ports.shift();
+        this.portUdpOpus ??= ports.shift();
+      })();
+    }
+
+    /**
+     * The output of the stream
+     * @type {string|Readable}
+     */
+    this.output = output;
+
+    /**
+     * The FFmpeg process is ready or not
+     * @type {boolean}
+     */
+    this.ready = false;
+    this.destroyed = false;
+
+    this.socket = createSocket('udp4');
+
+    this.socket.on('error', error => this.fail(error));
+    this.init(output).catch(error => this.fail(error));
+  }
+
+  fail(error) {
+    if (this.listenerCount('error')) this.emit('error', error);
+    else this.emit('debug', error.message);
+    this.destroy();
+  }
+  async init(output) {
+    await this.promise;
+    if (this.destroyed) return;
+    const sdpData = Util.getSDPCodecName(this.portUdpH264, this.portUdpH265, this.portUdpOpus);
+    const isStream = output instanceof Writable;
+    if (isStream) {
+      this.outputStream = StreamOutput(output);
+    }
+    const stream = spawn('ffmpeg', [
+      '-reorder_queue_size',
+      '500',
+      '-thread_queue_size',
+      '500',
+      '-err_detect',
+      'ignore_err',
+      '-flags2',
+      '+export_mvs',
+      '-fflags',
+      '+genpts+discardcorrupt',
+      '-use_wallclock_as_timestamps',
+      '1',
+      '-f',
+      'sdp',
+      '-analyzeduration',
+      '1M',
+      '-probesize',
+      '1M',
+      '-protocol_whitelist',
+      'file,udp,rtp,pipe,fd',
+      '-i',
+      '-', // Read from stdin
+      '-buffer_size',
+      '4M',
+      '-max_delay',
+      '500000', // 500ms
+      '-rtbufsize',
+      '4M',
+      '-c',
+      'copy',
+      '-y',
+      '-f',
+      'matroska',
+      isStream ? this.outputStream.url : output,
+    ]);
+
+    /**
+     * The FFmpeg process
+     * @type {ChildProcessWithoutNullStreams}
+     */
+    this.stream = stream;
+    stream.on('error', error => this.fail(error));
+    stream.stdin.on('error', error => this.fail(error));
+    stream.once('exit', () => this.destroy());
+    this.stream.stdin.write(sdpData);
+    this.stream.stdin.end();
+    this.stream.stderr.once('data', data => {
+      this.emit('debug', `stderr: ${data}`);
+      this.ready = true;
+      this.emit('ready');
+    });
+  }
+  /**
+   * Send a payload to FFmpeg via UDP
+   * @param {RtpPacket|string|Buffer} payload The payload
+   * @param {*} callback Callback
+   */
+  feed(
+    payload,
+    callback = e => {
+      if (e) {
+        console.error('Error sending packet:', e);
+      }
+    },
+  ) {
+    if (this.destroyed) return;
+    if (!(payload instanceof RtpPacket)) {
+      payload = RtpPacket.deSerialize(Buffer.isBuffer(payload) ? payload : Buffer.from(payload));
+    }
+    const message = payload.serialize();
+    // Get port from payloadType
+    let port;
+    if (payload.header.payloadType === Util.getPayloadType('opus')) {
+      port = this.portUdpOpus;
+    } else if (payload.header.payloadType === Util.getPayloadType('H264')) {
+      port = this.portUdpH264;
+    } else if (payload.header.payloadType === Util.getPayloadType('H265')) {
+      port = this.portUdpH265;
+    } else {
+      return;
+    }
+    this.socket.send(message, 0, message.length, port, '127.0.0.1', callback);
+  }
+
+  destroy() {
+    if (this.destroyed) return;
+    this.destroyed = true;
+    this.ready = false;
+    try {
+      this.socket.close();
+    } catch {
+      /* Socket may not have bound yet. */
+    }
+    this.outputStream?.destroy();
+    if (this.stream?.pid) kill(this.stream.pid, () => {});
+    if (this.receiver?.videoStreams?.get(this.userId) === this) this.receiver.videoStreams.delete(this.userId);
+    this.emit('closed');
+  }
+
+  /**
+   * Emitted when the Recorder becomes ready to start working.
+   * @event Recorder#ready
+   */
+
+  /**
+   * Emitted when the Recorder is closed.
+   * @event Recorder#closed
+   */
+}
+
+module.exports = Recorder;