piixishstream 0.0.0-stage → 1.0.2

package/src/client/voice/MediaUdp.ts.bak

@@ -0,0 +1,171 @@
+import udpCon from 'node:dgram';
+import { isIP } from 'node:net';
+import { AudioPacketizer } from '../packet/AudioPacketizer.js';
+import {
+    VideoPacketizerH264,
+    VideoPacketizerH265
+} from '../packet/VideoPacketizerAnnexB.js';
+import { VideoPacketizerVP8 } from '../packet/VideoPacketizerVP8.js';
+import { normalizeVideoCodec } from '../../utils.js';
+import type { BaseMediaPacketizer } from '../packet/BaseMediaPacketizer.js';
+import type { BaseMediaConnection } from './BaseMediaConnection.js';
+
+// credit to discord.js
+function parseLocalPacket(message: Buffer) {
+    const packet = Buffer.from(message);
+
+	const ip = packet.subarray(8, packet.indexOf(0, 8)).toString('utf8');
+
+	if (!isIP(ip)) {
+		throw new Error('Malformed IP address');
+	}
+
+	const port = packet.readUInt16BE(packet.length - 2);
+
+	return { ip, port };
+}
+
+export class MediaUdp {
+    private _mediaConnection: BaseMediaConnection;
+    private _socket: udpCon.Socket | null = null;
+    private _ready = false;
+    private _audioPacketizer?: BaseMediaPacketizer;
+    private _videoPacketizer?: BaseMediaPacketizer;
+    private _ip?: string;
+    private _port?: number;
+
+    constructor(voiceConnection: BaseMediaConnection) {
+        this._mediaConnection = voiceConnection;
+    }
+
+    public get audioPacketizer(): BaseMediaPacketizer | undefined {
+        return this._audioPacketizer;
+    }
+
+    public get videoPacketizer(): BaseMediaPacketizer | undefined {
+        // This will never be undefined anyway, so it's safe
+        return this._videoPacketizer;
+    }
+
+    public get mediaConnection(): BaseMediaConnection {
+        return this._mediaConnection;
+    }
+
+    public get ip()
+    {
+        return this._ip;
+    }
+
+    public get port()
+    {
+        return this._port;
+    }
+
+    public async sendAudioFrame(frame: Buffer, frametime: number): Promise<void> {
+        if(!this.ready) return;
+        await this.audioPacketizer?.sendFrame(frame, frametime);
+    }
+
+    public async sendVideoFrame(frame: Buffer, frametime: number): Promise<void> {
+        if(!this.ready) return;
+        await this.videoPacketizer?.sendFrame(frame, frametime);
+    }
+
+    public setPacketizer(videoCodec: string): void {
+        if (!this.mediaConnection.webRtcParams)
+            throw new Error("WebRTC connection not ready");
+        const { audioSsrc, videoSsrc } = this.mediaConnection.webRtcParams;
+        this._audioPacketizer = new AudioPacketizer(this, audioSsrc);
+        switch (normalizeVideoCodec(videoCodec))
+        {
+            case "H264":
+                this._videoPacketizer = new VideoPacketizerH264(this, videoSsrc);
+                break;
+            case "H265":
+                this._videoPacketizer = new VideoPacketizerH265(this, videoSsrc);
+                break;
+            case "VP8":
+                this._videoPacketizer = new VideoPacketizerVP8(this, videoSsrc);
+                break;
+            default:
+                throw new Error(`Packetizer not implemented for ${videoCodec}`)
+        }
+    }
+
+    public sendPacket(packet: Buffer): Promise<void> {
+        if (!this.mediaConnection.webRtcParams)
+            throw new Error("WebRTC connection not ready");
+        const { address, port } = this.mediaConnection.webRtcParams;
+        return new Promise<void>((resolve, reject) => {
+            try {
+            this._socket?.send(packet, 0, packet.length, port, address, (error, bytes) => {
+                if (error) {
+                    console.log("ERROR", error);
+                    reject(error);
+                }
+                resolve();
+            });
+        } catch(e) {reject(e)}
+        });
+    }
+
+    handleIncoming(buf: unknown): void {
+        //console.log("RECEIVED PACKET", buf);
+    }
+
+    public get ready(): boolean {
+        return this._ready;
+    }
+
+    public set ready(val: boolean) {
+        this._ready = val;
+    }
+
+    public stop(): void {
+        try {
+            this.ready = false;
+            this._socket?.disconnect();
+        }catch(e) {}
+    }
+
+    public createUdp(): Promise<void> {
+        if (!this.mediaConnection.webRtcParams)
+            throw new Error("WebRTC connection not ready");
+        const { audioSsrc, address, port } = this.mediaConnection.webRtcParams;
+        return new Promise<void>((resolve, reject) => {
+            this._socket = udpCon.createSocket('udp4');
+
+            this._socket.on('error', (error: Error) => {
+                console.error("Error connecting to media udp server", error);
+                reject(error);
+            });
+
+            this._socket.once('message', (message) => {
+                if (message.readUInt16BE(0) !== 2) {
+                    reject('wrong handshake packet for udp')
+                }
+                try {
+                    const packet = parseLocalPacket(message);
+                    this._ip = packet.ip;
+                    this._port = packet.port;
+                    this._ready = true;
+                } catch(e) { reject(e) }
+                
+                resolve();
+                this._socket?.on('message', this.handleIncoming);
+            });
+
+            const blank = Buffer.alloc(74);
+            
+            blank.writeUInt16BE(1, 0);
+			blank.writeUInt16BE(70, 2);
+            blank.writeUInt32BE(audioSsrc, 4);
+
+            this._socket.send(blank, 0, blank.length, port, address, (error, bytes) => {
+                if (error) {
+                    reject(error)
+                }
+            });
+        });
+    }
+}

package/src/client/voice/StreamConnection.ts

@@ -0,0 +1,38 @@
+import { VoiceOpCodes } from "../voice/VoiceOpCodes.js";
+import { BaseMediaConnection } from "./BaseMediaConnection.js";
+
+export class StreamConnection extends BaseMediaConnection {
+  private _streamKey: string | null = null;
+  private _serverId: string | null = null;
+
+  public override setSpeaking(speaking: boolean): void {
+    if (!this.webRtcParams) throw new Error("WebRTC connection not ready");
+    this.sendOpcode(VoiceOpCodes.SPEAKING, {
+      delay: 0,
+      speaking: speaking ? 2 : 0,
+      ssrc: this.webRtcParams.audioSsrc,
+    });
+  }
+
+  public override get daveChannelId() {
+    if (this._serverId === null)
+      throw new Error("Server ID not set (this shouldn't happen)");
+    const channelId = BigInt(this._serverId) - 1n;
+    return channelId.toString();
+  }
+
+  public override get serverId(): string | null {
+    return this._serverId;
+  }
+
+  public set serverId(id: string) {
+    this._serverId = id;
+  }
+
+  public get streamKey(): string | null {
+    return this._streamKey;
+  }
+  public set streamKey(value: string) {
+    this._streamKey = value;
+  }
+}

package/src/client/voice/VoiceConnection.ts

@@ -0,0 +1,19 @@
+import { BaseMediaConnection } from "./BaseMediaConnection.js";
+import type { StreamConnection } from "./StreamConnection.js";
+
+export class VoiceConnection extends BaseMediaConnection {
+  public streamConnection?: StreamConnection;
+
+  public override get daveChannelId() {
+    return this.channelId;
+  }
+
+  public override get serverId(): string {
+    return this.guildId ?? this.channelId; // for guild vc it is the guild id, for dm voice it is the channel id
+  }
+
+  public override stop(): void {
+    super.stop();
+    this.streamConnection?.stop();
+  }
+}

package/src/client/voice/VoiceMessageTypes.ts

@@ -0,0 +1,263 @@
+import type { VoiceOpCodes } from "./VoiceOpCodes.js";
+import type { SupportedEncryptionModes } from "../../utils.js";
+
+type StreamInfo = {
+  active: boolean;
+  quality: number;
+  rid: string;
+  ssrc: number;
+  rtx_ssrc: number;
+  /**
+   * always "video" from what I observed
+   */
+  type: string;
+};
+
+type SimulcastInfo = {
+  type: string;
+  rid: string;
+  quality: number;
+};
+
+type CodecPayloadType =
+  | {
+      name: string;
+      type: "audio";
+      priority: number;
+      payload_type: number;
+    }
+  | {
+      name: string;
+      type: "video";
+      priority: number;
+      payload_type: number;
+      rtx_payload_type: number;
+      encode: boolean;
+      decode: boolean;
+    };
+
+export namespace Message {
+  // Request messages
+  export type Identify = {
+    server_id: string;
+    user_id: string;
+    session_id: string;
+    token: string;
+    video: boolean;
+    streams: SimulcastInfo[];
+    max_dave_protocol_version?: number;
+  };
+
+  export type Resume = {
+    server_id: string;
+    session_id: string;
+    token: string;
+    seq_ack: number;
+  };
+
+  export type Heartbeat = {
+    t: number;
+    seq_ack?: number;
+  };
+
+  export type SelectProtocol =
+    | {
+        protocol: "udp";
+        codecs: CodecPayloadType[];
+        data: {
+          address: string;
+          port: number;
+          mode: SupportedEncryptionModes;
+        };
+      }
+    | {
+        protocol: "webrtc";
+        codecs: CodecPayloadType[];
+        data: string;
+        sdp: string;
+        rtc_connection_id: string;
+      };
+
+  export type Video = {
+    audio_ssrc: number;
+    video_ssrc: number;
+    rtx_ssrc: number;
+    streams: {
+      type: "video";
+      rid: string;
+      ssrc: number;
+      active: boolean;
+      quality: number;
+      rtx_ssrc: number;
+      max_bitrate: number;
+      max_framerate: number;
+      max_resolution: {
+        type: "fixed";
+        width: number;
+        height: number;
+      };
+    }[];
+  };
+
+  // Response messages
+  export type Hello = {
+    heartbeat_interval: number;
+  };
+
+  export type Ready = {
+    ssrc: number;
+    ip: string;
+    port: number;
+    modes: SupportedEncryptionModes[];
+    experiments: string[];
+    streams: StreamInfo[];
+  };
+
+  export type Speaking = {
+    speaking: 0 | 1 | 2;
+    delay: number;
+    ssrc: number;
+  };
+
+  export type SelectProtocolAck = {
+    audio_codec: string;
+    video_codec: string;
+    dave_protocol_version: number;
+  } & (
+    | {
+        secret_key: number[];
+        mode: string;
+      }
+    | {
+        media_session_id: number;
+        sdp: string;
+      }
+  );
+
+  export type HeartbeatAck = {
+    t: number;
+  };
+
+  export type ClientsConnect = {
+    user_ids: string[];
+  };
+
+  export type ClientDisconnect = {
+    user_id: string;
+  };
+
+  export type DavePrepareTransition = {
+    transition_id: number;
+    protocol_version: number;
+  };
+
+  export type DaveExecuteTransition = {
+    transition_id: number;
+  };
+
+  export type DaveTransitionReady = {
+    transition_id: number;
+  };
+
+  export type DavePrepareEpoch = {
+    epoch: number;
+    protocol_version: number;
+  };
+
+  export type MlsInvalidCommitWelcome = {
+    transition_id: number;
+  };
+}
+
+export namespace GatewayResponse {
+  type Generic<
+    Op extends VoiceOpCodes,
+    T extends Record<string, unknown> | null,
+  > = {
+    op: Op;
+    d: T;
+    seq?: number;
+  };
+  export type Hello = Generic<VoiceOpCodes.HELLO, Message.Hello>;
+  export type Ready = Generic<VoiceOpCodes.READY, Message.Ready>;
+  export type Resumed = Generic<VoiceOpCodes.RESUMED, null>;
+  export type Speaking = Generic<VoiceOpCodes.SPEAKING, Message.Speaking>;
+  export type SelectProtocolAck = Generic<
+    VoiceOpCodes.SELECT_PROTOCOL_ACK,
+    Message.SelectProtocolAck
+  >;
+  export type HeartbeatAck = Generic<
+    VoiceOpCodes.HEARTBEAT_ACK,
+    Message.HeartbeatAck
+  >;
+  export type ClientsConnect = Generic<
+    VoiceOpCodes.CLIENTS_CONNECT,
+    Message.ClientsConnect
+  >;
+  export type ClientDisconnect = Generic<
+    VoiceOpCodes.CLIENT_DISCONNECT,
+    Message.ClientDisconnect
+  >;
+  export type DavePrepareTransition = Generic<
+    VoiceOpCodes.DAVE_PREPARE_TRANSITION,
+    Message.DavePrepareTransition
+  >;
+  export type DaveExecuteTransition = Generic<
+    VoiceOpCodes.DAVE_EXECUTE_TRANSITION,
+    Message.DaveExecuteTransition
+  >;
+  export type DavePrepareEpoch = Generic<
+    VoiceOpCodes.DAVE_PREPARE_EPOCH,
+    Message.DavePrepareEpoch
+  >;
+}
+
+export type GatewayResponse =
+  | GatewayResponse.Hello
+  | GatewayResponse.Ready
+  | GatewayResponse.Resumed
+  | GatewayResponse.Speaking
+  | GatewayResponse.SelectProtocolAck
+  | GatewayResponse.HeartbeatAck
+  | GatewayResponse.ClientsConnect
+  | GatewayResponse.ClientDisconnect
+  | GatewayResponse.DavePrepareTransition
+  | GatewayResponse.DaveExecuteTransition
+  | GatewayResponse.DavePrepareEpoch;
+
+export namespace GatewayRequest {
+  type Generic<
+    Op extends VoiceOpCodes,
+    T extends Record<string, unknown> | null,
+  > = {
+    op: Op;
+    d: T;
+  };
+  export type Identify = Generic<VoiceOpCodes.IDENTIFY, Message.Identify>;
+  export type Resume = Generic<VoiceOpCodes.RESUME, Message.Resume>;
+  export type Heartbeat = Generic<VoiceOpCodes.HEARTBEAT, Message.Heartbeat>;
+  export type SelectProtocol = Generic<
+    VoiceOpCodes.SELECT_PROTOCOL,
+    Message.SelectProtocol
+  >;
+  export type Video = Generic<VoiceOpCodes.VIDEO, Message.Video>;
+  export type Speaking = Generic<VoiceOpCodes.SPEAKING, Message.Speaking>;
+  export type DaveTransitionReady = Generic<
+    VoiceOpCodes.DAVE_TRANSITION_READY,
+    Message.DaveTransitionReady
+  >;
+  export type MlsInvalidCommitWelcome = Generic<
+    VoiceOpCodes.MLS_INVALID_COMMIT_WELCOME,
+    Message.MlsInvalidCommitWelcome
+  >;
+}
+
+export type GatewayRequest =
+  | GatewayRequest.Identify
+  | GatewayRequest.Resume
+  | GatewayRequest.Heartbeat
+  | GatewayRequest.SelectProtocol
+  | GatewayRequest.Video
+  | GatewayRequest.Speaking
+  | GatewayRequest.DaveTransitionReady
+  | GatewayRequest.MlsInvalidCommitWelcome;

package/src/client/voice/VoiceOpCodes.ts

@@ -0,0 +1,36 @@
+export enum VoiceOpCodes {
+  IDENTIFY = 0,
+  SELECT_PROTOCOL = 1,
+  READY = 2,
+  HEARTBEAT = 3,
+  SELECT_PROTOCOL_ACK = 4,
+  SPEAKING = 5,
+  HEARTBEAT_ACK = 6,
+  RESUME = 7,
+  HELLO = 8,
+  RESUMED = 9,
+  CLIENTS_CONNECT = 11,
+  VIDEO = 12,
+  CLIENT_DISCONNECT = 13,
+  SESSION_UPDATE = 14,
+  MEDIA_SINK_WANTS = 15,
+  VOICE_BACKEND_VERSION = 16,
+  CHANNEL_OPTIONS_UPDATE = 17,
+  FLAGS = 18,
+  SPEED_TEST = 19,
+  PLATFORM = 20,
+  DAVE_PREPARE_TRANSITION = 21,
+  DAVE_EXECUTE_TRANSITION = 22,
+  DAVE_TRANSITION_READY = 23,
+  DAVE_PREPARE_EPOCH = 24,
+  MLS_INVALID_COMMIT_WELCOME = 31,
+}
+
+export enum VoiceOpCodesBinary {
+  MLS_EXTERNAL_SENDER = 25,
+  MLS_KEY_PACKAGE = 26,
+  MLS_PROPOSALS = 27,
+  MLS_COMMIT_WELCOME = 28,
+  MLS_ANNOUNCE_COMMIT_TRANSITION = 29,
+  MLS_WELCOME = 30,
+}