werift 0.15.10 → 0.16.0

package/src/media/receiver/nack.ts

@@ -1,3 +1,4 @@
+import debug from "debug";
 import range from "lodash/range";
 import Event from "rx.mini";
 
@@ -9,20 +10,27 @@ import {
 } from "../../../../rtp/src";
 import { RTCRtpReceiver } from "../rtpReceiver";
 
+const log = debug("werift:packages/webrtc/src/media/receiver/nack.ts");
+
 const LOST_SIZE = 30 * 5;
 
-export class Nack {
+export class NackHandler {
   private newEstSeqNum = 0;
-  private _lost: { [seqNum: number]: number } = {};
-  private nackLoop = setInterval(() => this.packetLost(), 20);
+  lost: { [seqNum: number]: number } = {};
+  private nackLoop = setInterval(() => this.sendNack(), 20);
 
   readonly onPacketLost = new Event<[GenericNack]>();
   mediaSourceSsrc?: number;
+  retryCount = 10;
 
   constructor(private receiver: RTCRtpReceiver) {}
 
-  get lost() {
-    return Object.keys(this._lost).map(Number);
+  get lostNumber() {
+    return Object.keys(this.lost).map(Number);
+  }
+
+  removeLost(sequenceNumber: number) {
+    delete this.lost[sequenceNumber];
   }
 
   addPacket(packet: RtpPacket) {
@@ -34,8 +42,8 @@ export class Nack {
       return;
     }
 
-    if (this._lost[sequenceNumber]) {
-      delete this._lost[sequenceNumber];
+    if (this.lost[sequenceNumber]) {
+      this.removeLost(sequenceNumber);
       return;
     }
 
@@ -44,48 +52,59 @@ export class Nack {
     } else if (sequenceNumber > uint16Add(this.newEstSeqNum, 1)) {
       // packet lost detected
       range(uint16Add(this.newEstSeqNum, 1), sequenceNumber).forEach((seq) => {
-        this._lost[seq] = 1;
+        this.lost[seq] = 1;
       });
       this.receiver.sendRtcpPLI(this.mediaSourceSsrc);
 
       this.newEstSeqNum = sequenceNumber;
+      this.pruneLost();
+    }
+  }
 
-      if (Object.keys(this._lost).length > LOST_SIZE) {
-        this._lost = Object.entries(this._lost)
-          .slice(-LOST_SIZE)
-          .reduce((acc, [key, v]) => {
-            acc[key] = v;
-            return acc;
-          }, {} as { [seqNum: number]: number });
-      }
+  private pruneLost() {
+    if (Object.keys(this.lost).length > LOST_SIZE) {
+      this.lost = Object.entries(this.lost)
+        .slice(-LOST_SIZE)
+        .reduce((acc, [key, v]) => {
+          acc[key] = v;
+          return acc;
+        }, {} as { [seqNum: number]: number });
     }
   }
 
   close() {
     clearInterval(this.nackLoop);
+    this.lost = {};
   }
 
-  private increment() {
-    Object.keys(this._lost).forEach((seq) => {
-      if (++this._lost[seq] > 10) {
-        delete this._lost[seq];
-      }
-    });
+  private updateRetryCount() {
+    const res = Object.keys(this.lost)
+      .map((seq) => {
+        const count = this.lost[seq]++;
+        if (count > this.retryCount) {
+          delete this.lost[seq];
+          return seq;
+        }
+      })
+      .filter((v) => v != undefined);
+    if (res.length > 0) {
+      // log("failed to retransmit", res);
+    }
   }
 
-  private packetLost() {
-    if (this.lost.length > 0 && this.mediaSourceSsrc) {
+  private sendNack() {
+    if (this.lostNumber.length > 0 && this.mediaSourceSsrc) {
       const nack = new GenericNack({
         senderSsrc: this.receiver.rtcpSsrc,
         mediaSourceSsrc: this.mediaSourceSsrc,
-        lost: this.lost,
+        lost: this.lostNumber,
       });
       const rtcp = new RtcpTransportLayerFeedback({
         feedback: nack,
       });
       this.receiver.dtlsTransport.sendRtcp([rtcp]);
 
-      this.increment();
+      this.updateRetryCount();
       this.onPacketLost.execute(nack);
     }
   }

package/src/media/receiver/receiverTwcc.ts

@@ -13,7 +13,7 @@ import {
 import { RTCDtlsTransport } from "../../transport/dtls";
 import { microTime } from "../../utils";
 
-const log = debug("werift/webrtc/media/receiver/receiverTwcc");
+const log = debug("werift:packages/webrtc/media/receiver/receiverTwcc");
 
 type ExtensionInfo = { tsn: number; timestamp: number };
 

package/src/media/receiver/red.ts

@@ -1,6 +1,17 @@
 import { Red, RtpHeader, RtpPacket, uint16Add, uint32Add } from "../..";
 
-export class RedHandler {
+// 0                   1                    2                   3
+// 0 1 2 3 4 5 6 7 8 9 0 1 2 3  4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+// |F|   block PT  |  timestamp offset         |   block length    |
+// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+
+// 0 1 2 3 4 5 6 7
+// +-+-+-+-+-+-+-+-+
+// |0|   Block PT  |
+// +-+-+-+-+-+-+-+-+
+
+export class AudioRedHandler {
   private readonly size = 150;
   private sequenceNumbers: number[] = [];
 
@@ -42,9 +53,11 @@ export class RedHandler {
     });
 
     const filtered = packets.filter((p) => {
+      // duplicate
       if (this.sequenceNumbers.includes(p.header.sequenceNumber)) {
         return false;
       } else {
+        // buffer overflow
         if (this.sequenceNumbers.length > this.size) {
           this.sequenceNumbers.shift();
         }

package/src/media/router.ts

@@ -27,9 +27,9 @@ const log = debug("werift:packages/webrtc/src/media/router.ts");
 export type Extensions = { [uri: string]: number | string };
 
 export class RtpRouter {
-  private ssrcTable: { [ssrc: number]: RTCRtpReceiver | RTCRtpSender } = {};
-  private ridTable: { [rid: string]: RTCRtpReceiver | RTCRtpSender } = {};
-  private extIdUriMap: { [id: number]: string } = {};
+  ssrcTable: { [ssrc: number]: RTCRtpReceiver | RTCRtpSender } = {};
+  ridTable: { [rid: string]: RTCRtpReceiver | RTCRtpSender } = {};
+  extIdUriMap: { [id: number]: string } = {};
 
   constructor() {}
 
@@ -110,6 +110,8 @@ export class RtpRouter {
               uri,
               value: bufferReader(extension.payload, [3])[0],
             };
+          default:
+            return { uri, value: 0 };
         }
       })
       .reduce((acc: { [uri: string]: any }, cur) => {

package/src/media/rtpReceiver.ts

@@ -16,15 +16,15 @@ import {
   RtpHeader,
   RtpPacket,
 } from "../../../rtp/src";
-import { codecParametersFromString } from "..";
+import { codecParametersFromString, PeerConfig } from "..";
 import { RTCDtlsTransport } from "../transport/dtls";
 import { Kind } from "../types/domain";
-import { compactNtp } from "../utils";
+import { compactNtp, timestampSeconds } from "../utils";
 import { RTP_EXTENSION_URI } from "./extension/rtpExtension";
 import { RTCRtpCodecParameters, RTCRtpReceiveParameters } from "./parameters";
-import { Nack } from "./receiver/nack";
+import { NackHandler } from "./receiver/nack";
 import { ReceiverTWCC } from "./receiver/receiverTwcc";
-import { RedHandler } from "./receiver/red";
+import { AudioRedHandler } from "./receiver/red";
 import { StreamStatistics } from "./receiver/statistics";
 import { Extensions } from "./router";
 import { MediaStreamTrack } from "./track";
@@ -33,18 +33,26 @@ const log = debug("werift:packages/webrtc/src/media/rtpReceiver.ts");
 
 export class RTCRtpReceiver {
   private readonly codecs: { [pt: number]: RTCRtpCodecParameters } = {};
+  private get codecArray() {
+    return Object.values(this.codecs).sort(
+      (a, b) => a.payloadType - b.payloadType
+    );
+  }
   private readonly ssrcByRtx: { [rtxSsrc: number]: number } = {};
-  private readonly nack = new Nack(this);
-  private readonly redHandler = new RedHandler();
+  private readonly nack = new NackHandler(this);
+  private readonly redHandler = new AudioRedHandler();
 
   readonly type = "receiver";
   readonly uuid = uuid();
   readonly tracks: MediaStreamTrack[] = [];
   readonly trackBySSRC: { [ssrc: string]: MediaStreamTrack } = {};
   readonly trackByRID: { [rid: string]: MediaStreamTrack } = {};
-  // last senderReport
-  readonly lsr: { [ssrc: number]: number } = {};
-  readonly lsrTime: { [ssrc: number]: number } = {};
+  /**last sender Report Timestamp
+   * compactNtp
+   */
+  readonly lastSRtimestamp: { [ssrc: number]: number } = {};
+  /**seconds */
+  readonly receiveLastSRTimestamp: { [ssrc: number]: number } = {};
   readonly onPacketLost = this.nack.onPacketLost;
   readonly onRtcp = new Event<[RtcpPacket]>();
 
@@ -63,7 +71,11 @@ export class RTCRtpReceiver {
   private rtcpCancel = new AbortController();
   private remoteStreams: { [ssrc: number]: StreamStatistics } = {};
 
-  constructor(public kind: Kind, public rtcpSsrc: number) {}
+  constructor(
+    readonly config: PeerConfig,
+    public kind: Kind,
+    public rtcpSsrc: number
+  ) {}
 
   setDtlsTransport(dtls: RTCDtlsTransport) {
     this.dtlsTransport = dtls;
@@ -74,6 +86,10 @@ export class RTCRtpReceiver {
     return this.tracks[0];
   }
 
+  get nackEnabled() {
+    return this.codecArray[0].rtcpFeedback.find((f) => f.type === "nack");
+  }
+
   prepareReceive(params: RTCRtpReceiveParameters) {
     params.codecs.forEach((c) => {
       this.codecs[c.payloadType] = c;
@@ -136,13 +152,14 @@ export class RTCRtpReceiver {
 
         const reports = Object.entries(this.remoteStreams).map(
           ([ssrc, stream]) => {
-            let lsr = 0,
-              dlsr = 0;
-            if (this.lsr[ssrc]) {
-              lsr = this.lsr[ssrc];
-              const delay = Date.now() / 1000 - this.lsrTime[ssrc];
-              if (delay > 0 && delay < 65536) {
-                dlsr = int(delay * 65536);
+            let lastSRtimestamp = 0,
+              delaySinceLastSR = 0;
+            if (this.lastSRtimestamp[ssrc]) {
+              lastSRtimestamp = this.lastSRtimestamp[ssrc];
+              const delaySeconds =
+                timestampSeconds() - this.receiveLastSRTimestamp[ssrc];
+              if (delaySeconds > 0 && delaySeconds < 65536) {
+                delaySinceLastSR = int(delaySeconds * 65536);
               }
             }
 
@@ -152,8 +169,8 @@ export class RTCRtpReceiver {
               packetsLost: stream.packets_lost,
               highestSequence: stream.max_seq,
               jitter: stream.jitter,
-              lsr,
-              dlsr,
+              lsr: lastSRtimestamp,
+              dlsr: delaySinceLastSR,
             });
           }
         );
@@ -161,6 +178,9 @@ export class RTCRtpReceiver {
         const packet = new RtcpRrPacket({ ssrc: this.rtcpSsrc, reports });
 
         try {
+          if (this.config.debug.receiverReportDelay) {
+            await setTimeout(this.config.debug.receiverReportDelay);
+          }
           await this.dtlsTransport.sendRtcp([packet]);
         } catch (error) {
           log("sendRtcp failed", error);
@@ -192,8 +212,10 @@ export class RTCRtpReceiver {
       case RtcpSrPacket.type:
         {
           const sr = packet as RtcpSrPacket;
-          this.lsr[sr.ssrc] = compactNtp(sr.senderInfo.ntpTimestamp);
-          this.lsrTime[sr.ssrc] = Date.now() / 1000;
+          this.lastSRtimestamp[sr.ssrc] = compactNtp(
+            sr.senderInfo.ntpTimestamp
+          );
+          this.receiveLastSRTimestamp[sr.ssrc] = timestampSeconds();
 
           const track = this.trackBySSRC[packet.ssrc];
           if (track) {
@@ -264,18 +286,27 @@ export class RTCRtpReceiver {
     let red: Red | undefined;
     if (codec.name.toLowerCase() === "red") {
       red = Red.deSerialize(packet.payload);
+      if (
+        !Object.keys(this.codecs).includes(
+          red.header.fields[0].blockPT.toString()
+        )
+      ) {
+        return;
+      }
     }
 
-    // todo fix select use or not use nack
-    if (track?.kind === "video") {
+    if (track?.kind === "video" && this.nackEnabled) {
       this.nack.addPacket(packet);
     }
 
     if (track) {
       if (red) {
-        const payloads = this.redHandler.push(red, packet);
-        for (const packet of payloads) {
-          track.onReceiveRtp.execute(packet.clone());
+        if (track.kind === "audio") {
+          const payloads = this.redHandler.push(red, packet);
+          for (const packet of payloads) {
+            track.onReceiveRtp.execute(packet.clone());
+          }
+        } else {
         }
       } else {
         track.onReceiveRtp.execute(packet.clone());
@@ -291,11 +322,11 @@ export function unwrapRtx(rtx: RtpPacket, payloadType: number, ssrc: number) {
     new RtpHeader({
       payloadType,
       marker: rtx.header.marker,
-      sequenceNumber: jspack.Unpack("!H", rtx.payload.slice(0, 2))[0],
+      sequenceNumber: jspack.Unpack("!H", rtx.payload.subarray(0, 2))[0],
       timestamp: rtx.header.timestamp,
       ssrc,
     }),
-    rtx.payload.slice(2)
+    rtx.payload.subarray(2)
   );
   return packet;
 }

package/src/media/rtpSender.ts

@@ -1,3 +1,25 @@
+/**
+   [10 Nov 1995 11:33:25.125 UTC]       [10 Nov 1995 11:33:36.5 UTC]
+   n                 SR(n)              A=b710:8000 (46864.500 s)
+   ---------------------------------------------------------------->
+                      v                 ^
+   ntp_sec =0xb44db705 v               ^ dlsr=0x0005:4000 (    5.250s)
+   ntp_frac=0x20000000  v             ^  lsr =0xb705:2000 (46853.125s)
+     (3024992005.125 s)  v           ^
+   r                      v         ^ RR(n)
+   ---------------------------------------------------------------->
+                          |<-DLSR->|
+                           (5.250 s)
+        
+   A     0xb710:8000 (46864.500 s)
+   DLSR -0x0005:4000 (    5.250 s)
+   LSR  -0xb705:2000 (46853.125 s)
+   -------------------------------
+   delay 0x0006:2000 (    6.125 s)
+        
+Figure 2: Example for round-trip time computation
+ */
+
 import { randomBytes } from "crypto";
 import debug from "debug";
 import { jspack } from "jspack";
@@ -33,7 +55,7 @@ import {
 import { codecParametersFromString } from "..";
 import { RTCDtlsTransport } from "../transport/dtls";
 import { Kind } from "../types/domain";
-import { compactNtp, milliTime, ntpTime } from "../utils";
+import { compactNtp, milliTime, ntpTime, timestampSeconds } from "../utils";
 import { RTP_EXTENSION_URI } from "./extension/rtpExtension";
 import {
   RTCRtpCodecParameters,
@@ -77,8 +99,8 @@ export class RTCRtpSender {
   private disposeTrack?: () => void;
 
   // # stats
-  private lsr?: number;
-  private lsrTime: number = Date.now() / 1000;
+  private lastSRtimestamp?: number;
+  private lastSentSRTimestamp?: number;
   private ntpTimestamp = 0n;
   private rtpTimestamp = 0;
   private octetCount = 0;
@@ -233,8 +255,8 @@ export class RTCRtpSender {
             }),
           }),
         ];
-        this.lsr = compactNtp(this.ntpTimestamp);
-        this.lsrTime = Date.now() / 1000;
+        this.lastSRtimestamp = compactNtp(this.ntpTimestamp);
+        this.lastSentSRTimestamp = timestampSeconds();
 
         if (this.cname) {
           packets.push(
@@ -375,13 +397,17 @@ export class RTCRtpSender {
           packet.reports
             .filter((report) => report.ssrc === this.ssrc)
             .forEach((report) => {
-              if (this.lsr === report.lsr && report.dlsr) {
-                const rtt =
-                  Date.now() / 1000 - this.lsrTime - report.dlsr / 65536;
-                if (this.rtt === undefined) {
-                  this.rtt = rtt;
-                } else {
-                  this.rtt = RTT_ALPHA * this.rtt + (1 - RTT_ALPHA) * rtt;
+              if (this.lastSRtimestamp === report.lsr && report.dlsr) {
+                if (this.lastSentSRTimestamp) {
+                  const rtt =
+                    timestampSeconds() -
+                    this.lastSentSRTimestamp -
+                    report.dlsr / 65536;
+                  if (this.rtt === undefined) {
+                    this.rtt = rtt;
+                  } else {
+                    this.rtt = RTT_ALPHA * this.rtt + (1 - RTT_ALPHA) * rtt;
+                  }
                 }
               }
             });

package/src/media/rtpTransceiver.ts

@@ -14,7 +14,7 @@ import { MediaStreamTrack } from "./track";
 
 export class RTCRtpTransceiver {
   readonly uuid = uuid.v4();
-  readonly onTrack = new Event<[MediaStreamTrack]>();
+  readonly onTrack = new Event<[MediaStreamTrack, RTCRtpTransceiver]>();
   mid?: string;
   mLineIndex?: number;
   usedForSender = false;
@@ -69,7 +69,9 @@ export class RTCRtpTransceiver {
 
   addTrack(track: MediaStreamTrack) {
     const res = this.receiver.addTrack(track);
-    if (res) this.onTrack.execute(track);
+    if (res) {
+      this.onTrack.execute(track, this);
+    }
   }
 
   // todo impl
@@ -89,12 +91,12 @@ export class RTCRtpTransceiver {
   }
 }
 
-export const Directions = [
-  "inactive",
-  "sendonly",
-  "recvonly",
-  "sendrecv",
-] as const;
+export const Inactive = "inactive";
+export const Sendonly = "sendonly";
+export const Recvonly = "recvonly";
+export const Sendrecv = "sendrecv";
+
+export const Directions = [Inactive, Sendonly, Recvonly, Sendrecv] as const;
 
 export type Direction = typeof Directions[number];
 

package/src/media/track.ts

@@ -52,8 +52,12 @@ export class MediaStreamTrack extends EventTarget {
   };
 
   writeRtp = (rtp: RtpPacket | Buffer) => {
-    if (this.remote) throw new Error("this is remoteTrack");
-    if (!this.codec || this.stopped) return;
+    if (this.remote) {
+      throw new Error("this is remoteTrack");
+    }
+    if (!this.codec || this.stopped) {
+      return;
+    }
 
     const packet = Buffer.isBuffer(rtp) ? RtpPacket.deSerialize(rtp) : rtp;
     packet.header.payloadType = this.codec.payloadType;

package/src/nonstandard/recorder/index.ts

@@ -26,8 +26,8 @@ export class MediaRecorder {
     this.tracks.push(track);
   }
 
-  start() {
-    this.writer.start(this.tracks);
+  async start() {
+    await this.writer.start(this.tracks);
   }
 
   async stop() {
@@ -38,4 +38,8 @@ export class MediaRecorder {
 export interface MediaRecorderOptions {
   width: number;
   height: number;
+  jitterBufferLatency: number;
+  jitterBufferSize: number;
+  waitForKeyframe: boolean;
+  defaultDuration: number;
 }

package/src/nonstandard/recorder/writer/index.ts

@@ -7,7 +7,7 @@ export abstract class MediaWriter {
     protected options: Partial<MediaRecorderOptions>
   ) {}
 
-  start(tracks: MediaStreamTrack[]) {}
+  async start(tracks: MediaStreamTrack[]) {}
 
   async stop() {}
 }