piixishstream 0.0.0-stage → 1.0.2

package/src/media/LibavDemuxer.ts

@@ -0,0 +1,293 @@
+import pDebounce from "p-debounce";
+import {
+  BitStreamFilterAPI,
+  Demuxer,
+  avGetCodecName,
+  type Stream,
+} from "node-av";
+import { Log } from "debug-level";
+import { randomUUID } from "node:crypto";
+import { AVCodecID } from "./LibavCodecId.js";
+import { once } from "node:events";
+import { PassThrough } from "node:stream";
+import { finished } from "node:stream/promises";
+import type { CodecParameters, Packet } from "node-av";
+import type { Readable } from "node:stream";
+
+type MediaStreamInfoCommon = {
+  index: number;
+  codec: AVCodecID;
+  codecpar: CodecParameters;
+  avStream: Stream;
+};
+
+export type VideoStreamInfo = MediaStreamInfoCommon & {
+  width: number;
+  height: number;
+  framerate_num: number;
+  framerate_den: number;
+};
+export type AudioStreamInfo = MediaStreamInfoCommon & {
+  sample_rate: number;
+};
+
+const allowedVideoCodec = new Set([
+  AVCodecID.AV_CODEC_ID_H264,
+  AVCodecID.AV_CODEC_ID_H265,
+  AVCodecID.AV_CODEC_ID_VP8,
+  AVCodecID.AV_CODEC_ID_VP9,
+  AVCodecID.AV_CODEC_ID_AV1,
+]);
+
+const allowedAudioCodec = new Set([AVCodecID.AV_CODEC_ID_OPUS]);
+
+function parseOpusPacketDuration(frame: Uint8Array) {
+  // https://datatracker.ietf.org/doc/html/rfc6716#section-3.1
+  const frameSizes = [
+    // SILK only, narrow band
+    10, 20, 40, 60,
+
+    // SILK only, medium band
+    10, 20, 40, 60,
+
+    // SILK only, wide band
+    10, 20, 40, 60,
+
+    // Hybrid, super wide band
+    10, 20,
+
+    // Hybrid, full band
+    10, 20,
+
+    // CELT only, narrow band
+    2.5, 5, 10, 20,
+
+    // CELT only, wide band
+    2.5, 5, 10, 20,
+
+    // CELT only, super wide band
+    2.5, 5, 10, 20,
+
+    // CELT only, full band
+    2.5, 5, 10, 20,
+  ];
+
+  const frameSize = (48000 / 1000) * frameSizes[frame[0] >> 3];
+
+  let frameCount = 0;
+  const c = frame[0] & 0b11;
+  switch (c) {
+    case 0:
+      frameCount = 1;
+      break;
+
+    case 1:
+    case 2:
+      frameCount = 2;
+      break;
+
+    case 3:
+      frameCount = frame[1] & 0b111111;
+      break;
+  }
+
+  return frameSize * frameCount;
+}
+
+type DemuxerOptions = {
+  format: "matroska" | "nut";
+};
+
+export async function demux(input: Readable, { format }: DemuxerOptions) {
+  const loggerFormat = new Log("demux:format");
+  const loggerFrameCommon = new Log("demux:frame:common");
+  const loggerFrameVideo = new Log("demux:frame:video");
+  const loggerFrameAudio = new Log("demux:frame:audio");
+
+  const filename = randomUUID();
+  const demuxer = await Demuxer.open(input, {
+    options: {
+      fflags: "nobuffer",
+    },
+    format,
+    bufferSize: 8192,
+  });
+
+  const cleanup = () => {
+    input.destroy();
+    demuxer.close();
+    vPipe.off("drain", readFrame);
+    aPipe.off("drain", readFrame);
+    vPipe.end();
+    aPipe.end();
+    vbsf.forEach((e) => {
+      e.close();
+    });
+  };
+
+  const vStream = demuxer.video();
+  const aStream = demuxer.audio();
+
+  let vInfo: VideoStreamInfo | undefined;
+  let aInfo: AudioStreamInfo | undefined;
+  const vPipe = new PassThrough({
+    objectMode: true,
+    writableHighWaterMark: 128,
+  });
+  const aPipe = new PassThrough({
+    objectMode: true,
+    writableHighWaterMark: 128,
+  });
+
+  const vbsf: BitStreamFilterAPI[] = [];
+  if (vStream) {
+    const codecId = vStream.codecpar.codecId;
+    if (!allowedVideoCodec.has(codecId)) {
+      const codecName = avGetCodecName(codecId);
+      cleanup();
+      throw new Error(`Video codec ${codecName} is not allowed`);
+    }
+    try {
+      switch (codecId) {
+        case AVCodecID.AV_CODEC_ID_H264:
+          vbsf.push(BitStreamFilterAPI.create("h264_mp4toannexb", vStream));
+          vbsf.push(
+            BitStreamFilterAPI.create("h264_metadata", vStream, {
+              options: {
+                aud: "remove",
+              },
+            }),
+          );
+          vbsf.push(BitStreamFilterAPI.create("dump_extra", vStream));
+          break;
+        case AVCodecID.AV_CODEC_ID_HEVC:
+          vbsf.push(BitStreamFilterAPI.create("hevc_mp4toannexb", vStream));
+          vbsf.push(
+            BitStreamFilterAPI.create("hevc_metadata", vStream, {
+              options: {
+                aud: "remove",
+              },
+            }),
+          );
+          vbsf.push(BitStreamFilterAPI.create("dump_extra", vStream));
+          break;
+        default:
+          vbsf.push(BitStreamFilterAPI.create("null", vStream));
+          break;
+      }
+    } catch (e) {
+      cleanup();
+      throw new Error(`Failed to construct bitstream filterchain`, {
+        cause: (e as Error).cause,
+      });
+    }
+
+    const codecpar = vbsf.at(-1)?.outputCodecParameters ?? vStream.codecpar;
+    vInfo = {
+      index: vStream.index,
+      codec: codecId,
+      codecpar,
+      width: codecpar.width ?? 0,
+      height: codecpar.height ?? 0,
+      framerate_num: codecpar.frameRate.num,
+      framerate_den: codecpar.frameRate.den,
+      avStream: vStream,
+    };
+    loggerFormat.info(
+      {
+        info: vInfo,
+      },
+      `Found video stream in input ${filename}`,
+    );
+  }
+  if (aStream) {
+    const codecId = aStream.codecpar.codecId;
+    if (!allowedAudioCodec.has(codecId)) {
+      const codecName = avGetCodecName(codecId);
+      cleanup();
+      throw new Error(`Audio codec ${codecName} is not allowed`);
+    }
+    aInfo = {
+      index: aStream.index,
+      codec: codecId,
+      codecpar: aStream.codecpar,
+      sample_rate: aStream.codecpar.sampleRate || 0,
+      avStream: aStream,
+    };
+    loggerFormat.info(
+      {
+        info: aInfo,
+      },
+      `Found audio stream in input ${filename}`,
+    );
+  }
+
+  const packetIterator = demuxer.packets();
+  const applyBitStreamFilters = async (
+    input: Packet | null,
+    filters: BitStreamFilterAPI[],
+  ) => {
+    let packets = [input];
+    for (const filter of filters) {
+      let newPackets: (Packet | null)[] = [];
+      for (const packet of packets) {
+        newPackets = [...newPackets, ...(await filter.filterAll(packet))];
+        packet?.free();
+      }
+      if (!input) newPackets.push(null);
+      packets = newPackets;
+    }
+    return packets;
+  };
+  const readFrame = pDebounce.promise(async () => {
+    let resume = true;
+    while (resume) {
+      try {
+        const { value: inPacket, done } = await packetIterator.next();
+        if (done) {
+          loggerFrameCommon.info("Reached end of stream. Stopping");
+          const packets = await applyBitStreamFilters(null, vbsf);
+          for (const packet of packets) {
+            if (packet) vPipe.write(packet);
+          }
+          cleanup();
+          return;
+        } else if (inPacket) {
+          const streamIndex = inPacket.streamIndex;
+          if (vInfo && vInfo.index === streamIndex) {
+            loggerFrameVideo.trace("Received a video packet");
+            const packets = await applyBitStreamFilters(inPacket.clone(), vbsf);
+            for (const packet of packets) {
+              if (packet) resume &&= vPipe.write(packet);
+            }
+          } else if (aInfo && aInfo.index === streamIndex) {
+            const packet = inPacket.clone()!;
+            packet.duration ||= BigInt(parseOpusPacketDuration(packet.data!));
+            resume &&= aPipe.write(packet);
+          }
+          inPacket.free();
+        }
+      } catch (e) {
+        loggerFrameCommon.info(
+          { error: e },
+          "Received an error during frame extraction. Stopping",
+        );
+        cleanup();
+        return;
+      }
+    }
+  });
+  vPipe.on("drain", () => {
+    loggerFrameVideo.trace("Video pipe drained");
+    readFrame();
+  });
+  aPipe.on("drain", () => {
+    loggerFrameAudio.trace("Audio pipe drained");
+    readFrame();
+  });
+  readFrame();
+  return {
+    video: vInfo ? { ...vInfo, stream: vPipe as Readable } : undefined,
+    audio: aInfo ? { ...aInfo, stream: aPipe as Readable } : undefined,
+  };
+}

package/src/media/VideoStream.ts

@@ -0,0 +1,17 @@
+import { BaseMediaStream } from "./BaseMediaStream.js";
+import type { WebRtcConnWrapper } from "../client/voice/WebRtcWrapper.js";
+
+export class VideoStream extends BaseMediaStream {
+  private _conn: WebRtcConnWrapper;
+  constructor(conn: WebRtcConnWrapper, noSleep = false) {
+    super("video", noSleep);
+    this._conn = conn;
+  }
+
+  protected override async _sendFrame(
+    frame: Buffer,
+    frametime: number,
+  ): Promise<void> {
+    this._conn.sendVideoFrame(frame, frametime);
+  }
+}

package/src/media/encoders/index.ts

@@ -0,0 +1,27 @@
+import type { SupportedVideoCodec } from "../../utils.js";
+
+export type EncoderSettings = {
+  name: string;
+  options: string[];
+  globalOptions?: string[];
+  outFilters?: string[];
+};
+
+export type EncoderSettingsGetter = (
+  bitrate: number,
+  bitrateMax: number,
+) => Partial<Record<SupportedVideoCodec, EncoderSettings>>;
+
+import { software } from "./software.js";
+import { nvenc } from "./nvenc.js";
+import { vaapi } from "./vaapi.js";
+import { merge } from "./merge.js";
+
+const Encoders = {
+  software,
+  nvenc,
+  vaapi,
+  merge,
+};
+
+export { Encoders };

package/src/media/encoders/merge.ts

@@ -0,0 +1,14 @@
+import type { EncoderSettingsGetter } from "./index.js";
+import type { SupportedVideoCodec } from "../../utils.js";
+
+export const merge = (
+  encoder: Partial<Record<SupportedVideoCodec, EncoderSettingsGetter>>,
+) => {
+  return ((bitrate, bitrateMax) => ({
+    H264: encoder.H264?.(bitrate, bitrateMax),
+    H265: encoder.H265?.(bitrate, bitrateMax),
+    VP8: encoder.VP8?.(bitrate, bitrateMax),
+    VP9: encoder.VP9?.(bitrate, bitrateMax),
+    AV1: encoder.AV1?.(bitrate, bitrateMax),
+  })) as EncoderSettingsGetter;
+};

package/src/media/encoders/nvenc.ts

@@ -0,0 +1,38 @@
+import type { EncoderSettingsGetter } from "./index.js";
+
+type NvencPreset = "p1" | "p2" | "p3" | "p4" | "p5" | "p6" | "p7";
+
+type NvencSettings = {
+  preset: NvencPreset;
+  spatialAq: boolean;
+  temporalAq: boolean;
+  gpu: number;
+};
+
+export function nvenc({
+  preset = "p4",
+  spatialAq = false,
+  temporalAq = false,
+  gpu,
+}: Partial<NvencSettings> = {}) {
+  const options = [
+    `-preset ${preset}`,
+    `-spatial-aq ${spatialAq}`,
+    `-temporal-aq ${temporalAq}`,
+    ...(gpu !== undefined ? [`-gpu ${gpu}`] : []),
+  ];
+  return (() => ({
+    H264: {
+      name: "h264_nvenc",
+      options,
+    },
+    H265: {
+      name: "hevc_nvenc",
+      options,
+    },
+    AV1: {
+      name: "av1_nvenc",
+      options,
+    },
+  })) as EncoderSettingsGetter;
+}

package/src/media/encoders/software.ts

@@ -0,0 +1,76 @@
+import type { EncoderSettingsGetter } from "./index.js";
+
+type DeepPartial<T> = T extends unknown[]
+  ? T
+  : { [P in keyof T]?: DeepPartial<T[P]> };
+type x26xPreset =
+  | "ultrafast"
+  | "superfast"
+  | "veryfast"
+  | "faster"
+  | "fast"
+  | "medium"
+  | "slow"
+  | "slower"
+  | "veryslow"
+  | "placebo";
+
+export type SoftwareEncoderSettings = {
+  x264: {
+    preset: x26xPreset;
+    tune:
+      | "film"
+      | "animation"
+      | "grain"
+      | "stillimage"
+      | "fastdecode"
+      | "zerolatency"
+      | "psnr"
+      | "ssim";
+  };
+  x265: {
+    preset: x26xPreset;
+    tune:
+      | "psnr"
+      | "ssim"
+      | "grain"
+      | "fastdecode"
+      | "zerolatency"
+      | "animation";
+  };
+};
+
+export const software = ({
+  x264,
+  x265,
+}: DeepPartial<SoftwareEncoderSettings> = {}) => {
+  const { preset: x264Preset = "superfast", tune: x264Tune = "film" } =
+    x264 ?? {};
+  const { preset: x265Preset = "superfast", tune: x265Tune } = x265 ?? {};
+  return (() => ({
+    H264: {
+      name: "libx264",
+      options: ["-forced-idr 1", `-tune ${x264Tune}`, `-preset ${x264Preset}`],
+    },
+    H265: {
+      name: "libx265",
+      options: [
+        "-forced-idr 1",
+        ...(x265Tune ? [`-tune ${x265Tune}`] : []),
+        `-preset ${x265Preset}`,
+      ],
+    },
+    VP8: {
+      name: "libvpx",
+      options: ["-deadline 20000"],
+    },
+    VP9: {
+      name: "libvpx-vp9",
+      options: ["-deadline 20000"],
+    },
+    AV1: {
+      name: "libsvtav1",
+      options: [],
+    },
+  })) as EncoderSettingsGetter;
+};

package/src/media/encoders/vaapi.ts

@@ -0,0 +1,29 @@
+import type { EncoderSettingsGetter } from "./index.js";
+
+type VaapiSettings = {
+  device?: string;
+};
+
+export function vaapi({
+  device = "/dev/dri/renderD128",
+}: Partial<VaapiSettings> = {}) {
+  const props = {
+    options: [],
+    globalOptions: ["-vaapi_device", device],
+    outFilters: ["format=nv12|vaapi", "hwupload"],
+  };
+  return (() => ({
+    H264: {
+      name: "h264_vaapi",
+      ...props,
+    },
+    H265: {
+      name: "hevc_vaapi",
+      ...props,
+    },
+    AV1: {
+      name: "av1_vaapi",
+      ...props,
+    },
+  })) as EncoderSettingsGetter;
+}

package/src/media/index.ts

@@ -0,0 +1,4 @@
+export * from "./LibavDemuxer.js";
+export * from "./newApi.js";
+export * as NewApi from "./newApi.js";
+export * from "./encoders/index.js";