werift 0.12.8 → 0.13.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "werift",
-  "version": "0.12.8",
+  "version": "0.13.2",
   "description": "WebRTC Implementation for TypeScript (Node.js)",
   "keywords": [
     "WebRTC",
@@ -36,6 +36,7 @@
     "@fidm/x509": "^1.2.1",
     "@peculiar/webcrypto": "^1.1.6",
     "@peculiar/x509": "^1.2.2",
+    "@shinyoshiaki/ebml-builder": "^0.0.1",
     "aes-js": "^3.1.2",
     "big-integer": "^1.6.48",
     "binary-data": "^0.6.0",
@@ -58,12 +59,12 @@
     "@types/aes-js": "^3.1.1",
     "@types/big-integer": "^0.0.31",
     "@types/buffer-crc32": "^0.2.0",
-    "@types/debug": "^4.1.6",
+    "@types/debug": "^4.1.7",
     "@types/elliptic": "^6.4.12",
     "@types/ip": "^1.1.0",
-    "@types/lodash": "^4.14.170",
+    "@types/lodash": "^4.14.172",
     "@types/uuid": "^8.3.0",
-    "typedoc": "^0.21.4"
+    "typedoc": "^0.21.5"
   },
   "engines": {
     "node": ">=15"

package/src/const.ts

@@ -1,3 +1,7 @@
+import {
+  ProtectionProfileAeadAes128Gcm,
+  ProtectionProfileAes128CmHmacSha1_80,
+} from "../../dtls/src/context/srtp";
 import { DtlsRole } from "./transport/dtls";
 
 // data channel export constants
@@ -47,9 +51,10 @@ export const FMTP_INT_PARAMETERS = [
 
 export const SSRC_INFO_ATTRS = ["cname", "msid", "mslabel", "label"];
 
-export enum SRTP_PROFILE {
-  SRTP_AES128_CM_HMAC_SHA1_80 = 1,
-}
+export const SRTP_PROFILE = {
+  SRTP_AES128_CM_HMAC_SHA1_80: ProtectionProfileAes128CmHmacSha1_80,
+  SRTP_AEAD_AES_128_GCM: ProtectionProfileAeadAes128Gcm,
+} as const;
 
 export const SenderDirections = ["sendonly", "sendrecv"];
 export const NotSenderDirections = ["inactive", "recvonly"];

package/src/helper.ts

@@ -10,21 +10,21 @@ export function divide(from: string, split: string): [string, string] {
 }
 
 export class PromiseQueue {
-  queue: { promise: () => Promise<any>; call: () => void }[] = [];
+  queue: { promise: () => Promise<any>; done: () => void }[] = [];
   running = false;
 
   push = (promise: () => Promise<any>) =>
     new Promise<void>((r) => {
-      this.queue.push({ promise, call: r });
+      this.queue.push({ promise, done: r });
       if (!this.running) this.run();
     });
 
-  async run() {
+  private async run() {
     const task = this.queue.shift();
     if (task) {
       this.running = true;
       await task.promise();
-      task.call();
+      task.done();
 
       this.run();
     } else {

package/src/index.ts

@@ -1,12 +1,16 @@
 export * from "../../ice/src";
 export * from "../../rtp/src";
 export * from "./dataChannel";
-export * from "./extension/rtcpFeedback";
-export * from "./extension/rtpExtension";
 export * from "./helper";
+export * from "./media/extension/rtcpFeedback";
+export * from "./media/extension/rtpExtension";
 export * from "./media/parameters";
 export * from "./media/rtpTransceiver";
 export * from "./media/track";
+export * from "./nonstandard/lipsync";
+export * from "./nonstandard/recorder";
+export * from "./nonstandard/sampleBuilder";
+export * from "./nonstandard/userMedia";
 export * from "./peerConnection";
 export * from "./sdp";
 export * from "./transport/dtls";

package/src/{extension → media/extension}/rtcpFeedback.ts

@@ -1,4 +1,4 @@
-import { RTCPFB } from "../media/parameters";
+import { RTCPFB } from "../parameters";
 
 export const useFIR = (): RTCPFB => ({ type: "ccm", parameter: "fir" });
 

package/src/{extension → media/extension}/rtpExtension.ts

@@ -1,4 +1,4 @@
-import { RTCRtpHeaderExtensionParameters } from "../media/parameters";
+import { RTCRtpHeaderExtensionParameters } from "../parameters";
 
 export const RTP_EXTENSION_URI = {
   sdesMid: "urn:ietf:params:rtp-hdrext:sdes:mid",

package/src/media/{nack.ts → receiver/nack.ts}

@@ -1,13 +1,13 @@
 import { range } from "lodash";
 import Event from "rx.mini";
 
-import { uint16Add } from "../../../common/src";
+import { uint16Add } from "../../../../common/src";
 import {
   GenericNack,
   RtcpTransportLayerFeedback,
   RtpPacket,
-} from "../../../rtp/src";
-import { RTCRtpReceiver } from "./rtpReceiver";
+} from "../../../../rtp/src";
+import { RTCRtpReceiver } from "../rtpReceiver";
 
 const LOST_SIZE = 30 * 5;
 

package/src/media/{statistics.ts → receiver/statistics.ts}

@@ -1,5 +1,5 @@
-import { int, uint16Gt } from "../../../common/src";
-import { RtpPacket } from "../../../rtp/src";
+import { int, uint16Gt } from "../../../../common/src";
+import { RtpPacket } from "../../../../rtp/src";
 
 // from aiortc
 

package/src/media/router.ts

@@ -12,7 +12,7 @@ import {
   RtcpTransportLayerFeedback,
   RtpPacket,
 } from "../../../rtp/src";
-import { RTP_EXTENSION_URI } from "../extension/rtpExtension";
+import { RTP_EXTENSION_URI } from "./extension/rtpExtension";
 import {
   RTCRtpReceiveParameters,
   RTCRtpSimulcastParameters,

package/src/media/rtpReceiver.ts

@@ -1,5 +1,6 @@
 import { debug } from "debug";
 import { jspack } from "jspack";
+import Event from "rx.mini";
 import { setTimeout } from "timers/promises";
 import { v4 as uuid } from "uuid";
 
@@ -14,14 +15,15 @@ import {
   RtpHeader,
   RtpPacket,
 } from "../../../rtp/src";
-import { RTP_EXTENSION_URI } from "../extension/rtpExtension";
 import { RTCDtlsTransport } from "../transport/dtls";
 import { Kind } from "../types/domain";
-import { Nack } from "./nack";
+import { compactNtp } from "../utils";
+import { RTP_EXTENSION_URI } from "./extension/rtpExtension";
 import { RTCRtpCodecParameters, RTCRtpReceiveParameters } from "./parameters";
+import { Nack } from "./receiver/nack";
 import { ReceiverTWCC } from "./receiver/receiverTwcc";
+import { StreamStatistics } from "./receiver/statistics";
 import { Extensions } from "./router";
-import { StreamStatistics } from "./statistics";
 import { MediaStreamTrack } from "./track";
 
 const log = debug("werift:packages/webrtc/src/media/rtpReceiver.ts");
@@ -37,9 +39,10 @@ export class RTCRtpReceiver {
   readonly trackBySSRC: { [ssrc: string]: MediaStreamTrack } = {};
   readonly trackByRID: { [rid: string]: MediaStreamTrack } = {};
   // last senderReport
-  readonly lsr: { [ssrc: number]: BigInt } = {};
+  readonly lsr: { [ssrc: number]: number } = {};
   readonly lsrTime: { [ssrc: number]: number } = {};
   readonly onPacketLost = this.nack.onPacketLost;
+  readonly onRtcp = new Event<[RtcpPacket]>();
 
   sdesMid?: string;
   latestRid?: string;
@@ -128,7 +131,7 @@ export class RTCRtpReceiver {
 
         const reports = Object.entries(this.remoteStreams).map(
           ([ssrc, stream]) => {
-            let lsr = 0n,
+            let lsr = 0,
               dlsr = 0;
             if (this.lsr[ssrc]) {
               lsr = this.lsr[ssrc];
@@ -144,7 +147,7 @@ export class RTCRtpReceiver {
               packetsLost: stream.packets_lost,
               highestSequence: stream.max_seq,
               jitter: stream.jitter,
-              lsr: Number(lsr),
+              lsr,
               dlsr,
             });
           }
@@ -184,11 +187,12 @@ export class RTCRtpReceiver {
       case RtcpSrPacket.type:
         {
           const sr = packet as RtcpSrPacket;
-          this.lsr[sr.ssrc] = (sr.senderInfo.ntpTimestamp >> 16n) & 0xffffffffn;
+          this.lsr[sr.ssrc] = compactNtp(sr.senderInfo.ntpTimestamp);
           this.lsrTime[sr.ssrc] = Date.now() / 1000;
         }
         break;
     }
+    this.onRtcp.execute(packet);
   }
 
   handleRtpBySsrc = (packet: RtpPacket, extensions: Extensions) => {

package/src/media/rtpSender.ts

@@ -29,16 +29,16 @@ import {
   SourceDescriptionItem,
   TransportWideCC,
 } from "../../../rtp/src";
-import { RTP_EXTENSION_URI } from "../extension/rtpExtension";
 import { RTCDtlsTransport } from "../transport/dtls";
 import { Kind } from "../types/domain";
-import { milliTime, ntpTime } from "../utils";
+import { compactNtp, milliTime, ntpTime } from "../utils";
+import { RTP_EXTENSION_URI } from "./extension/rtpExtension";
 import {
   RTCRtpCodecParameters,
   RTCRtpHeaderExtensionParameters,
   RTCRtpSendParameters,
 } from "./parameters";
-import { SenderBandwidthEstimator, SentInfo } from "./senderBWE/senderBWE";
+import { SenderBandwidthEstimator, SentInfo } from "./sender/senderBWE";
 import { MediaStreamTrack } from "./track";
 
 const log = debug("werift:packages/webrtc/src/media/rtpSender.ts");
@@ -54,7 +54,7 @@ export class RTCRtpSender {
       : this.trackOrKind.kind;
   readonly ssrc = jspack.Unpack("!L", randomBytes(4))[0];
   readonly rtxSsrc = jspack.Unpack("!L", randomBytes(4))[0];
-  readonly streamId = uuid.v4();
+  streamId = uuid.v4();
   readonly trackId = uuid.v4();
   readonly onReady = new Event();
   readonly onRtcp = new Event<[RtcpPacket]>();
@@ -72,7 +72,7 @@ export class RTCRtpSender {
   private disposeTrack?: () => void;
 
   // # stats
-  private lsr?: bigint;
+  private lsr?: number;
   private lsrTime: number = Date.now() / 1000;
   private ntpTimestamp = 0n;
   private rtpTimestamp = 0;
@@ -104,6 +104,9 @@ export class RTCRtpSender {
       }
     });
     if (trackOrKind instanceof MediaStreamTrack) {
+      if (trackOrKind.streamId) {
+        this.streamId = trackOrKind.streamId;
+      }
       this.registerTrack(trackOrKind);
     }
   }
@@ -149,6 +152,10 @@ export class RTCRtpSender {
     if (this.codec) {
       track.codec = this.codec;
     }
+
+    track.onSourceChanged.subscribe((header) => {
+      this.replaceRTP(header);
+    });
   }
 
   async replaceTrack(track: MediaStreamTrack | null) {
@@ -167,7 +174,7 @@ export class RTCRtpSender {
     }
 
     this.registerTrack(track);
-    log("replaceTrack", track.ssrc, track.rid);
+    log("replaceTrack", "ssrc", track.ssrc, "rid", track.rid);
   }
 
   stop() {
@@ -199,7 +206,7 @@ export class RTCRtpSender {
             }),
           }),
         ];
-        this.lsr = (this.ntpTimestamp >> 16n) & 0xffffffffn;
+        this.lsr = compactNtp(this.ntpTimestamp);
         this.lsrTime = Date.now() / 1000;
 
         if (this.cname) {
@@ -227,7 +234,10 @@ export class RTCRtpSender {
     } catch (error) {}
   }
 
-  private replaceRTP({ sequenceNumber, timestamp }: RtpHeader) {
+  replaceRTP({
+    sequenceNumber,
+    timestamp,
+  }: Pick<RtpHeader, "sequenceNumber" | "timestamp">) {
     if (this.sequenceNumber != undefined) {
       this.seqOffset = uint16Add(this.sequenceNumber, -sequenceNumber);
     }
@@ -330,7 +340,7 @@ export class RTCRtpSender {
           packet.reports
             .filter((report) => report.ssrc === this.ssrc)
             .forEach((report) => {
-              if (this.lsr === BigInt(report.lsr) && report.dlsr) {
+              if (this.lsr === report.lsr && report.dlsr) {
                 const rtt =
                   Date.now() / 1000 - this.lsrTime - report.dlsr / 65536;
                 if (this.rtt === undefined) {

package/src/media/track.ts

@@ -8,6 +8,8 @@ import { RTCRtpCodecParameters } from "./parameters";
 
 export class MediaStreamTrack extends EventTarget {
   readonly uuid = v4();
+  /**MediaStream ID*/
+  streamId?: string;
   remote = false;
   label: string;
   kind!: Kind;
@@ -21,6 +23,9 @@ export class MediaStreamTrack extends EventTarget {
   enabled = true;
 
   readonly onReceiveRtp = new Event<[RtpPacket]>();
+  readonly onSourceChanged = new Event<
+    [Pick<RtpHeader, "sequenceNumber" | "timestamp">]
+  >();
 
   stopped = false;
   muted = true;
@@ -64,6 +69,7 @@ export class MediaStream {
   }
 
   addTrack(track: MediaStreamTrack) {
+    track.streamId = this.id;
     this.tracks.push(track);
   }
 

package/src/nonstandard/jitterBuffer.ts

@@ -0,0 +1,47 @@
+import { debug } from "debug";
+
+import { uint16Add } from "../../../common/src";
+import { RtpPacket } from "../../../rtp/src";
+
+const log = debug("werift:packages/webrtc/src/nonstandard/jitterBuffer.ts");
+
+export class JitterBuffer {
+  static MaxRetry = 100;
+  retry = 0;
+  head?: number;
+  buffer: { [sequenceNumber: number]: RtpPacket } = {};
+
+  constructor(public maxRetry = JitterBuffer.MaxRetry) {}
+
+  push(p: RtpPacket) {
+    this.buffer[p.header.sequenceNumber] = p;
+
+    if (this.head == undefined) {
+      this.head = p.header.sequenceNumber;
+    } else if (p.header.sequenceNumber != uint16Add(this.head, 1)) {
+      if (this.retry++ >= this.maxRetry) {
+        log("give up packet lost");
+        this.head = uint16Add(this.head, 2);
+      } else {
+        return [];
+      }
+    } else {
+      this.head = uint16Add(this.head, 1);
+    }
+
+    const packets: RtpPacket[] = [];
+    let tail = this.head;
+    for (; ; tail = uint16Add(tail, 1)) {
+      const p = this.buffer[tail];
+      if (p) {
+        packets.push(p);
+        delete this.buffer[tail];
+      } else {
+        break;
+      }
+    }
+    this.head = uint16Add(tail, -1);
+
+    return packets;
+  }
+}

package/src/nonstandard/lipsync.ts

@@ -0,0 +1,55 @@
+import { bufferReader, bufferWriter } from "../../../common/src";
+import { RtcpSrPacket, RtpPacket } from "../../../rtp/src";
+
+export class LipSync {
+  baseNtpTimestamp?: bigint;
+  baseRtpTimestamp?: number;
+
+  constructor(public clockRate: number) {}
+
+  srReceived(sr: RtcpSrPacket) {
+    const { ntpTimestamp, rtpTimestamp } = sr.senderInfo;
+    this.baseNtpTimestamp = ntpTimestamp;
+    this.baseRtpTimestamp = rtpTimestamp;
+  }
+
+  calcNtpTime(rtpTimestamp: number) {
+    if (!this.baseRtpTimestamp || !this.baseNtpTimestamp) {
+      return 0;
+    }
+
+    // base rtpTimestamp is rollover
+    if (rtpTimestamp - this.baseRtpTimestamp > Max32bit - this.clockRate * 60) {
+      this.baseRtpTimestamp += Max32bit;
+    }
+    // target rtpTimestamp is rollover
+    else if (
+      rtpTimestamp + (Max32bit - this.clockRate * 60) - this.baseRtpTimestamp <
+      0
+    ) {
+      rtpTimestamp += Max32bit;
+    }
+
+    const elapsed = (rtpTimestamp - this.baseRtpTimestamp) / this.clockRate;
+
+    return ntpTime2Time(this.baseNtpTimestamp) + elapsed;
+  }
+}
+
+export const ntpTime2Time = (ntp: bigint) => {
+  const [ntpSec, ntpMsec] = bufferReader(bufferWriter([8], [ntp]), [4, 4]);
+
+  return Number(`${ntpSec}.${ntpMsec}`);
+};
+
+/**4294967295 */
+export const Max32bit = Number((0x01n << 32n) - 1n);
+
+export interface BufferResolve {
+  packets: {
+    packet: RtpPacket;
+    offset: number;
+  }[];
+  /**NTP seconds */
+  startAtNtpTime: number;
+}

package/src/nonstandard/recorder.ts

@@ -0,0 +1,26 @@
+import { MediaStreamTrack } from "../media/track";
+import { WebmFactory } from "./webm";
+
+export class MediaRecorder {
+  webm?: WebmFactory;
+
+  constructor(
+    public tracks: MediaStreamTrack[],
+    public path: string,
+    public options: Partial<{ width: number; height: number }> = {}
+  ) {}
+
+  addTrack(track: MediaStreamTrack) {
+    this.tracks.push(track);
+  }
+
+  async start() {
+    this.webm = new WebmFactory(this.tracks, this.path, this.options);
+    await this.webm.start();
+  }
+
+  async stop() {
+    if (!this.webm) throw new Error();
+    await this.webm.stop();
+  }
+}

package/src/nonstandard/sampleBuilder.ts

@@ -0,0 +1,71 @@
+import { debug } from "debug";
+
+import { int } from "../../../common/src";
+import { DePacketizerBase, RtpPacket } from "../../../rtp/src";
+import { enumerate } from "../helper";
+import { JitterBuffer } from "./jitterBuffer";
+
+const log = debug("werift:packages/webrtc/src/nonstandard/sampleBuilder.ts");
+
+export class SampleBuilder {
+  private readonly jitterBuffer = new JitterBuffer();
+  private buffer: RtpPacket[] = [];
+  private baseTimestamp?: number;
+  relativeTimestamp = 0;
+
+  constructor(
+    readonly DePacketizer: typeof DePacketizerBase,
+    public clockRate: number
+  ) {}
+
+  push(p: RtpPacket) {
+    const buf = this.jitterBuffer.push(p);
+    this.buffer = [...this.buffer, ...buf];
+  }
+
+  resetTimestamp() {
+    this.baseTimestamp = undefined;
+    this.relativeTimestamp = 0;
+  }
+
+  build() {
+    let tail: number | undefined;
+    for (const [i, p] of enumerate(this.buffer)) {
+      if (this.DePacketizer.isDetectedFinalPacketInSequence(p.header)) {
+        tail = i;
+        break;
+      }
+    }
+
+    if (tail == undefined) return;
+    if (this.baseTimestamp == undefined) {
+      this.baseTimestamp = this.buffer[tail].header.timestamp;
+    }
+
+    const tailTimestamp = this.buffer[tail].header.timestamp;
+    const rotate =
+      Math.abs(tailTimestamp - this.baseTimestamp) > (Max32Uint / 4) * 3;
+    if (rotate) log({ rotate }, tailTimestamp, this.baseTimestamp);
+
+    const elapsed = rotate
+      ? tailTimestamp + Max32Uint - this.baseTimestamp
+      : tailTimestamp - this.baseTimestamp;
+
+    const relativeTimestamp = int((elapsed / this.clockRate) * 1000);
+    this.relativeTimestamp = relativeTimestamp;
+
+    const frames = this.buffer.slice(0, tail + 1).map((p) => {
+      const frame = this.DePacketizer.deSerialize(p.payload);
+      return frame;
+    });
+    const isKeyframe = !!frames.find((f) => f.isKeyframe);
+    const data = Buffer.concat(frames.map((f) => f.payload));
+
+    this.buffer = this.buffer.slice(tail + 1);
+
+    return { data, relativeTimestamp, isKeyframe };
+  }
+}
+
+/**4294967295 */
+const Max32Uint = Number(0x01n << 32n) - 1;

package/src/nonstandard/userMedia.ts

@@ -0,0 +1,74 @@
+import { exec } from "child_process";
+import { createSocket } from "dgram";
+import { setImmediate } from "timers/promises";
+import { v4 } from "uuid";
+
+import { randomPort } from "../../../ice/src";
+import { RtpPacket } from "../../../rtp/src";
+import { MediaStreamTrack } from "../media/track";
+
+export const getUserMp4 = async (path: string, loop?: boolean) => {
+  const audioPort = await randomPort();
+  const videoPort = await randomPort();
+
+  return new MediaMp4(audioPort, videoPort, path, loop);
+};
+
+class MediaMp4 {
+  private streamId = v4();
+  audio = new MediaStreamTrack({ kind: "audio", streamId: this.streamId });
+  video = new MediaStreamTrack({ kind: "video", streamId: this.streamId });
+
+  constructor(
+    private videoPort: number,
+    private audioPort: number,
+    private path: string,
+    private loop?: boolean
+  ) {
+    this.setupTrack(audioPort, this.audio);
+    this.setupTrack(videoPort, this.video);
+  }
+
+  private setupTrack = (port: number, track: MediaStreamTrack) => {
+    let payloadType = 0;
+
+    const socket = createSocket("udp4");
+    socket.bind(port);
+    socket.on("message", async (buf) => {
+      const rtp = RtpPacket.deSerialize(buf);
+      if (!payloadType) {
+        payloadType = rtp.header.payloadType;
+      }
+
+      // detect gStreamer restarted
+      if (payloadType !== rtp.header.payloadType) {
+        payloadType = rtp.header.payloadType;
+        track.onSourceChanged.execute(rtp.header);
+      }
+
+      track.writeRtp(buf);
+    });
+  };
+
+  async start() {
+    let payloadType = 96;
+    const run = async () => {
+      if (payloadType > 100) payloadType = 96;
+
+      const cmd = `gst-launch-1.0 filesrc location= ${this.path} ! \
+qtdemux name=d ! \
+queue ! h264parse ! rtph264pay config-interval=10 pt=${payloadType++} ! \
+udpsink host=127.0.0.1 port=${this.videoPort} d. ! \
+queue ! aacparse ! avdec_aac ! audioresample ! audioconvert ! opusenc ! rtpopuspay pt=${payloadType++} ! \
+udpsink host=127.0.0.1 port=${this.audioPort}`;
+      const process = exec(cmd);
+
+      if (this.loop) {
+        await new Promise((r) => process.on("close", r));
+        run();
+      }
+    };
+    await setImmediate();
+    run();
+  }
+}