werift 0.17.6 → 0.18.0

package/src/media/receiver/nack.ts

@@ -22,41 +22,42 @@ export class NackHandler {
   readonly onPacketLost = new Event<[GenericNack]>();
   mediaSourceSsrc?: number;
   retryCount = 10;
+  closed = false;
 
   constructor(private receiver: RTCRtpReceiver) {}
 
-  private start() {
-    if (this.nackLoop) {
-      return;
-    }
-    this.nackLoop = setInterval(async () => {
-      try {
-        await this.sendNack();
-      } catch (error) {
-        log("failed to send nack", error);
-      }
-    }, 20);
-  }
-
   get lostSeqNumbers() {
     return Object.keys(this._lost).map(Number).sort();
   }
 
-  getLost(seq: number) {
+  private getLost(seq: number) {
     return this._lost[seq];
   }
 
-  setLost(seq: number, count: number) {
+  private setLost(seq: number, count: number) {
     this._lost[seq] = count;
+
+    if (this.nackLoop || this.closed) {
+      return;
+    }
+    this.nackLoop = setInterval(async () => {
+      try {
+        await this.sendNack();
+        if (!Object.keys(this._lost).length) {
+          clearInterval(this.nackLoop);
+          this.nackLoop = undefined;
+        }
+      } catch (error) {
+        log("failed to send nack", error);
+      }
+    }, 5);
   }
 
-  removeLost(sequenceNumber: number) {
+  private removeLost(sequenceNumber: number) {
     delete this._lost[sequenceNumber];
   }
 
   addPacket(packet: RtpPacket) {
-    this.start();
-
     const { sequenceNumber, ssrc } = packet.header;
     this.mediaSourceSsrc = ssrc;
 
@@ -66,6 +67,7 @@ export class NackHandler {
     }
 
     if (this.getLost(sequenceNumber)) {
+      // log("packetLoss resolved", { sequenceNumber });
       this.removeLost(sequenceNumber);
       return;
     }
@@ -77,7 +79,7 @@ export class NackHandler {
       range(uint16Add(this.newEstSeqNum, 1), sequenceNumber).forEach((seq) => {
         this.setLost(seq, 1);
       });
-      this.receiver.sendRtcpPLI(this.mediaSourceSsrc);
+      // this.receiver.sendRtcpPLI(this.mediaSourceSsrc);
 
       this.newEstSeqNum = sequenceNumber;
       this.pruneLost();
@@ -96,6 +98,7 @@ export class NackHandler {
   }
 
   close() {
+    this.closed = true;
     clearInterval(this.nackLoop);
     this._lost = {};
   }
@@ -118,6 +121,7 @@ export class NackHandler {
           mediaSourceSsrc: this.mediaSourceSsrc,
           lost: this.lostSeqNumbers,
         });
+        // log("sendNack", nack.toJSON());
         const rtcp = new RtcpTransportLayerFeedback({
           feedback: nack,
         });

package/src/media/rtpReceiver.ts

@@ -214,6 +214,11 @@ export class RTCRtpReceiver {
       log("pli not supported", { mediaSsrc });
       return;
     }
+
+    if (this.stopped) {
+      return;
+    }
+
     log("sendRtcpPLI", { mediaSsrc });
 
     const packet = new RtcpPayloadSpecificFeedback({

package/src/media/rtpSender.ts

@@ -72,10 +72,7 @@ const RTT_ALPHA = 0.85;
 
 export class RTCRtpSender {
   readonly type = "sender";
-  readonly kind =
-    typeof this.trackOrKind === "string"
-      ? this.trackOrKind
-      : this.trackOrKind.kind;
+  readonly kind: Kind;
   readonly ssrc = jspack.Unpack("!L", randomBytes(4))[0];
   readonly rtxSsrc = jspack.Unpack("!L", randomBytes(4))[0];
   streamId = uuid.v4();
@@ -124,6 +121,10 @@ export class RTCRtpSender {
   private rtcpCancel = new AbortController();
 
   constructor(public trackOrKind: Kind | MediaStreamTrack) {
+    this.kind =
+      typeof this.trackOrKind === "string"
+        ? this.trackOrKind
+        : this.trackOrKind.kind;
     if (trackOrKind instanceof MediaStreamTrack) {
       if (trackOrKind.streamId) {
         this.streamId = trackOrKind.streamId;
@@ -315,7 +316,7 @@ export class RTCRtpSender {
 
     rtp = Buffer.isBuffer(rtp) ? RtpPacket.deSerialize(rtp) : rtp;
 
-    const header = rtp.header;
+    const { header, payload } = rtp;
     header.ssrc = this.ssrc;
     header.payloadType = this.codec.payloadType;
     header.timestamp = uint32Add(header.timestamp, this.timestampOffset);
@@ -323,6 +324,8 @@ export class RTCRtpSender {
     this.timestamp = header.timestamp;
     this.sequenceNumber = header.sequenceNumber;
 
+    const ntptime = ntpTime();
+
     header.extensions = this.headerExtensions
       .map((extension) => {
         const payload = (() => {
@@ -355,7 +358,7 @@ export class RTCRtpSender {
               );
             case RTP_EXTENSION_URI.absSendTime:
               const buf = Buffer.alloc(3);
-              const time = (ntpTime() >> 14n) & 0x00ffffffn;
+              const time = (ntptime >> 14n) & 0x00ffffffn;
               buf.writeUIntBE(Number(time), 0, 3);
               return buf;
           }
@@ -365,17 +368,14 @@ export class RTCRtpSender {
       })
       .filter((v) => v) as Extension[];
 
-    this.ntpTimestamp = ntpTime();
-    this.rtpTimestamp = rtp.header.timestamp;
-    this.octetCount += rtp.payload.length;
+    this.ntpTimestamp = ntptime;
+    this.rtpTimestamp = header.timestamp;
+    this.octetCount += payload.length;
     this.packetCount = uint32Add(this.packetCount, 1);
 
-    rtp.header = header;
-
-    this.rtpCache.push(rtp);
-    this.rtpCache = this.rtpCache.slice(-RTP_HISTORY_SIZE);
+    this.rtpCache[header.sequenceNumber % RTP_HISTORY_SIZE] = rtp;
 
-    let rtpPayload = rtp.payload;
+    let rtpPayload = payload;
 
     if (this.redRedundantPayloadType) {
       this.redEncoder.push({
@@ -390,11 +390,12 @@ export class RTCRtpSender {
     const size = await this.dtlsTransport.sendRtp(rtpPayload, header);
 
     this.runRtcp();
+    const millitime = milliTime();
     const sentInfo: SentInfo = {
       wideSeq: this.dtlsTransport.transportSequenceNumber,
       size,
-      sendingAtMs: milliTime(),
-      sentAtMs: milliTime(),
+      sendingAtMs: millitime,
+      sentAtMs: millitime,
     };
     this.senderBWE.rtpPacketSent(sentInfo);
   }
@@ -438,9 +439,11 @@ export class RTCRtpSender {
               {
                 const feedback = packet.feedback as GenericNack;
                 feedback.lost.forEach(async (seqNum) => {
-                  let packet = this.rtpCache.find(
-                    (rtp) => rtp.header.sequenceNumber === seqNum
-                  );
+                  let packet: RtpPacket | undefined =
+                    this.rtpCache[seqNum % RTP_HISTORY_SIZE];
+                  if (packet && packet.header.sequenceNumber !== seqNum) {
+                    packet = undefined;
+                  }
                   if (packet) {
                     if (this.rtxPayloadType != undefined) {
                       packet = wrapRtx(

package/src/nonstandard/recorder/index.ts

@@ -5,10 +5,12 @@ import { WebmFactory } from "./writer/webm";
 export class MediaRecorder {
   writer: MediaWriter;
   ext: string;
+  tracks: MediaStreamTrack[] = [];
+  started = false;
 
   constructor(
-    public tracks: MediaStreamTrack[],
     public path: string,
+    public numOfTracks = 1,
     public options: Partial<MediaRecorderOptions> = {}
   ) {
     this.ext = path.split(".").slice(-1)[0];
@@ -20,14 +22,19 @@ export class MediaRecorder {
           throw new Error();
       }
     })();
+    this.tracks = options.tracks ?? this.tracks;
   }
 
-  addTrack(track: MediaStreamTrack) {
+  async addTrack(track: MediaStreamTrack) {
     this.tracks.push(track);
+    await this.start();
   }
 
-  async start() {
-    await this.writer.start(this.tracks);
+  private async start() {
+    if (this.tracks.length === this.numOfTracks && this.started === false) {
+      this.started = true;
+      await this.writer.start(this.tracks);
+    }
   }
 
   async stop() {
@@ -42,4 +49,5 @@ export interface MediaRecorderOptions {
   jitterBufferSize: number;
   waitForKeyframe: boolean;
   defaultDuration: number;
+  tracks: MediaStreamTrack[];
 }

package/src/nonstandard/recorder/writer/webm.ts

@@ -1,15 +1,17 @@
-import { appendFile, open, unlink } from "fs/promises";
+import { unlink } from "fs/promises";
 import { EventDisposer } from "rx.mini";
-import { ReadableStreamDefaultReadResult } from "stream/web";
 
 import { SupportedCodec } from "../../../../../rtp/src/container/webm";
 import {
-  depacketizeTransformer,
-  jitterBufferTransformer,
+  DepacketizeCallback,
+  JitterBufferCallback,
+  LipsyncCallback,
   MediaStreamTrack,
-  RtpSourceStream,
-  WebmStream,
-  WebmStreamOutput,
+  NtpTimeCallback,
+  RtcpSourceCallback,
+  RtpSourceCallback,
+  saveToFileSystem,
+  WebmCallback,
   WeriftError,
 } from "../../..";
 import { MediaWriter } from ".";
@@ -17,7 +19,7 @@ import { MediaWriter } from ".";
 const sourcePath = "packages/webrtc/src/nonstandard/recorder/writer/webm.ts";
 
 export class WebmFactory extends MediaWriter {
-  rtpSources: RtpSourceStream[] = [];
+  rtpSources: RtpSourceCallback[] = [];
 
   unSubscribers = new EventDisposer();
 
@@ -68,67 +70,57 @@ export class WebmFactory extends MediaWriter {
       }
     });
 
-    const webm = new WebmStream(inputTracks, {
+    const webm = new WebmCallback(inputTracks, {
       duration: this.options.defaultDuration ?? 1000 * 60 * 60 * 24,
     });
+    const lipsync = new LipsyncCallback();
 
     this.rtpSources = inputTracks.map(({ track, clockRate, codec }) => {
-      const rtpSource = new RtpSourceStream();
-
+      const rtpSource = new RtpSourceCallback();
+      const rtcpSource = new RtcpSourceCallback();
       track.onReceiveRtp
         .subscribe((rtp) => {
-          rtpSource.push(rtp.clone());
+          rtpSource.input(rtp.clone());
         })
         .disposer(this.unSubscribers);
-
-      // const jitterBuffer = jitterBufferTransformer(clockRate, {
-      //   latency: this.options.jitterBufferLatency,
-      //   bufferSize: this.options.jitterBufferSize,
-      // });
+      track.onReceiveRtcp
+        .subscribe((rtcp) => {
+          rtcpSource.input(rtcp);
+        })
+        .disposer(this.unSubscribers);
+      const ntpTime = new NtpTimeCallback(clockRate);
 
       if (track.kind === "video") {
-        rtpSource.readable
-          // .pipeThrough(jitterBuffer)
-          .pipeThrough(
-            depacketizeTransformer(codec, {
-              waitForKeyframe: this.options.waitForKeyframe,
-              isFinalPacketInSequence: (h) => h.marker,
-            })
-          )
-          .pipeTo(webm.videoStream);
+        const depacketizer = new DepacketizeCallback(codec, {
+          isFinalPacketInSequence: (h) => h.marker,
+        });
+        const jitterBuffer = new JitterBufferCallback(clockRate);
+
+        rtpSource.pipe(jitterBuffer.input);
+        rtcpSource.pipe(ntpTime.input);
+
+        jitterBuffer.pipe(ntpTime.input);
+        ntpTime.pipe(depacketizer.input);
+        depacketizer.pipe(lipsync.inputVideo);
+        lipsync.pipeVideo(webm.inputVideo);
       } else {
-        rtpSource.readable
-          // .pipeThrough(jitterBuffer)
-          .pipeThrough(depacketizeTransformer(codec))
-          .pipeTo(webm.audioStream);
+        const depacketizer = new DepacketizeCallback(codec);
+
+        rtpSource.pipe(ntpTime.input);
+        rtcpSource.pipe(ntpTime.input);
+
+        ntpTime.pipe(depacketizer.input);
+        depacketizer.pipe(lipsync.inputAudio);
+        lipsync.pipeAudio(webm.inputAudio);
       }
 
       return rtpSource;
     });
-
-    const reader = webm.webmStream.getReader();
-    const readChunk = async ({
-      value,
-      done,
-    }: ReadableStreamDefaultReadResult<WebmStreamOutput>) => {
-      if (done) return;
-
-      if (value.saveToFile) {
-        await appendFile(this.path, value.saveToFile);
-      } else if (value.eol) {
-        const { durationElement } = value.eol;
-        const handler = await open(this.path, "r+");
-        await handler.write(durationElement, 0, durationElement.length, 83);
-        await handler.close();
-      }
-      reader.read().then(readChunk);
-    };
-    reader.read().then(readChunk);
+    webm.pipe(saveToFileSystem(this.path));
   }
 
   async stop() {
     await Promise.all(this.rtpSources.map((r) => r.stop()));
-
     this.unSubscribers.dispose();
   }
 }

package/src/transport/dtls.ts

@@ -49,7 +49,7 @@ export class RTCDtlsTransport {
 
   readonly onStateChange = new Event<[DtlsState]>();
 
-  localCertificate?: RTCCertificate = this.certificates[0];
+  localCertificate?: RTCCertificate;
   private remoteParameters?: RTCDtlsParameters;
 
   constructor(
@@ -58,7 +58,9 @@ export class RTCDtlsTransport {
     readonly router: RtpRouter,
     readonly certificates: RTCCertificate[],
     private readonly srtpProfiles: Profile[] = []
-  ) {}
+  ) {
+    this.localCertificate = this.certificates[0];
+  }
 
   get localParameters() {
     return new RTCDtlsParameters(
@@ -340,7 +342,9 @@ class IceTransport implements Transport {
   }
   onData?: (buf: Buffer) => void;
 
-  readonly send = this.ice.send;
+  readonly send = (data: Buffer) => {
+    return this.ice.send(data);
+  };
 
   close() {
     this.ice.close();

package/src/transport/ice.ts

@@ -9,7 +9,7 @@ const log = debug("werift:packages/webrtc/src/transport/ice.ts");
 
 export class RTCIceTransport {
   readonly id = v4();
-  connection = this.gather.connection;
+  connection: Connection;
   state: RTCIceConnectionState = "new";
 
   readonly onStateChange = new Event<[RTCIceConnectionState]>();
@@ -17,6 +17,7 @@ export class RTCIceTransport {
   private waitStart?: Event<[]>;
 
   constructor(private gather: RTCIceGatherer) {
+    this.connection = this.gather.connection;
     this.connection.stateChanged.subscribe((state) => {
       this.setState(state);
     });
@@ -116,9 +117,11 @@ export class RTCIceGatherer {
   gatheringState: IceGathererState = "new";
 
   readonly onGatheringStateChange = new Event<[IceGathererState]>();
-  readonly connection = new Connection(false, this.options);
+  readonly connection: Connection;
 
-  constructor(private options: Partial<IceOptions> = {}) {}
+  constructor(private options: Partial<IceOptions> = {}) {
+    this.connection = new Connection(false, this.options);
+  }
 
   async gather() {
     if (this.gatheringState === "new") {

package/src/transport/sctp.ts

@@ -381,6 +381,8 @@ class BridgeDtls implements Transport {
   set onData(onData: (buf: Buffer) => void) {
     this.dtls.dataReceiver = onData;
   }
-  readonly send = this.dtls.sendData;
+  readonly send = (data: Buffer) => {
+    return this.dtls.sendData(data);
+  };
   close() {}
 }

package/lib/rtp/src/processor/avBuffer.d.ts

@@ -1,36 +0,0 @@
-import { DepacketizerOutput } from "./depacketizer";
-export declare type AVBufferInput = DepacketizerOutput;
-export declare type AVBufferOutput = AVBufferInput;
-/**
- * @description [japanese]
- * audioパケットとvideoパケットを同一のタイムラインで扱い、それぞれの
- * パケットのタイムスタンプが前後しないように制御する
- */
-export declare class AVBufferBase {
-    private audioOutput;
-    private videoOutput;
-    private options;
-    bufferLength: number;
-    baseAudioTimestamp?: number;
-    baseVideoTimestamp?: number;
-    audioBuffer: (AVBufferInput & {
-        elapsed: number;
-        kind: string;
-    })[][];
-    videoBuffer: (AVBufferInput & {
-        elapsed: number;
-        kind: string;
-    })[][];
-    stopped: boolean;
-    private interval;
-    private started;
-    constructor(audioOutput: (output: AVBufferOutput) => void, videoOutput: (output: AVBufferOutput) => void, options?: Partial<AvBufferOptions>);
-    private start;
-    processAudioInput: (input: AVBufferInput) => void;
-    processVideoInput: (input: AVBufferInput) => void;
-    private calcElapsed;
-}
-export interface AvBufferOptions {
-    interval: number;
-    bufferLength: number;
-}

package/lib/rtp/src/processor/avBuffer.js

@@ -1,107 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.AVBufferBase = void 0;
-const src_1 = require("../../../common/src");
-const webm_1 = require("./webm");
-/**
- * @description [japanese]
- * audioパケットとvideoパケットを同一のタイムラインで扱い、それぞれの
- * パケットのタイムスタンプが前後しないように制御する
- */
-class AVBufferBase {
-    constructor(audioOutput, videoOutput, options = {}) {
-        this.audioOutput = audioOutput;
-        this.videoOutput = videoOutput;
-        this.options = options;
-        this.bufferLength = this.options.bufferLength ?? 50;
-        this.audioBuffer = [
-            ...new Array(this.bufferLength),
-        ].map(() => []);
-        this.videoBuffer = [
-            ...new Array(this.bufferLength),
-        ].map(() => []);
-        this.stopped = false;
-        this.interval = this.options.interval ?? 500;
-        this.started = false;
-        this.processAudioInput = (input) => {
-            if (!input.frame) {
-                this.stopped = true;
-                this.audioOutput(input);
-                return;
-            }
-            if (this.stopped) {
-                return;
-            }
-            if (this.baseAudioTimestamp == undefined) {
-                this.baseAudioTimestamp = input.frame?.timestamp;
-            }
-            const { elapsed, rotate } = this.calcElapsed(this.baseAudioTimestamp, input.frame.timestamp, 48000);
-            if (rotate) {
-                this.baseAudioTimestamp = input.frame?.timestamp;
-            }
-            const index = (0, src_1.int)(elapsed / this.interval) % this.bufferLength;
-            this.audioBuffer[index].push({ ...input, elapsed, kind: "audio" });
-            this.start();
-        };
-        this.processVideoInput = (input) => {
-            if (!input.frame) {
-                this.stopped = true;
-                this.videoOutput(input);
-                return;
-            }
-            if (this.stopped) {
-                return;
-            }
-            if (this.baseVideoTimestamp == undefined) {
-                this.baseVideoTimestamp = input.frame?.timestamp;
-            }
-            const { elapsed, rotate } = this.calcElapsed(this.baseVideoTimestamp, input.frame.timestamp, 90000);
-            if (rotate) {
-                this.baseVideoTimestamp = input.frame?.timestamp;
-            }
-            const index = (0, src_1.int)(elapsed / this.interval) % this.bufferLength;
-            this.videoBuffer[index].push({ ...input, elapsed, kind: "video" });
-            this.start();
-        };
-    }
-    start() {
-        if ([...this.audioBuffer[1], ...this.videoBuffer[1]].length === 0) {
-            return;
-        }
-        if (this.started) {
-            return;
-        }
-        this.started = true;
-        let index = 0;
-        setInterval(() => {
-            const joined = [...this.audioBuffer[index], ...this.videoBuffer[index]];
-            const sorted = joined.sort((a, b) => a.elapsed - b.elapsed);
-            this.audioBuffer[index] = [];
-            this.videoBuffer[index] = [];
-            for (const output of sorted) {
-                if (output.kind === "audio") {
-                    this.audioOutput(output);
-                }
-                else {
-                    this.videoOutput(output);
-                }
-            }
-            index++;
-            if (index === this.bufferLength) {
-                index = 0;
-            }
-        }, this.interval);
-    }
-    calcElapsed(base, timestamp, clockRate) {
-        const rotate = Math.abs(timestamp - base) > (webm_1.Max32Uint / 4) * 3;
-        if (rotate) {
-            console.log(rotate);
-        }
-        const diff = rotate ? timestamp + webm_1.Max32Uint - base : timestamp - base;
-        /**ms */
-        const elapsed = (0, src_1.int)((diff / clockRate) * 1000);
-        return { elapsed, rotate };
-    }
-}
-exports.AVBufferBase = AVBufferBase;
-//# sourceMappingURL=avBuffer.js.map

package/lib/rtp/src/processor/avBuffer.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"avBuffer.js","sourceRoot":"","sources":["../../../../../rtp/src/processor/avBuffer.ts"],"names":[],"mappings":";;;AAAA,6CAA0C;AAE1C,iCAAmC;AAMnC;;;;GAIG;AACH,MAAa,YAAY;IAcvB,YACU,WAA6C,EAC7C,WAA6C,EAC7C,UAAoC,EAAE;QAFtC,gBAAW,GAAX,WAAW,CAAkC;QAC7C,gBAAW,GAAX,WAAW,CAAkC;QAC7C,YAAO,GAAP,OAAO,CAA+B;QAhBhD,iBAAY,GAAG,IAAI,CAAC,OAAO,CAAC,YAAY,IAAI,EAAE,CAAC;QAG/C,gBAAW,GAA4D;YACrE,GAAG,IAAI,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC;SAChC,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;QAChB,gBAAW,GAA4D;YACrE,GAAG,IAAI,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC;SAChC,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;QAChB,YAAO,GAAG,KAAK,CAAC;QACR,aAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,IAAI,GAAG,CAAC;QACxC,YAAO,GAAG,KAAK,CAAC;QAwCxB,sBAAiB,GAAG,CAAC,KAAoB,EAAE,EAAE;YAC3C,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE;gBAChB,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;gBACpB,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;gBACxB,OAAO;aACR;YACD,IAAI,IAAI,CAAC,OAAO,EAAE;gBAChB,OAAO;aACR;YAED,IAAI,IAAI,CAAC,kBAAkB,IAAI,SAAS,EAAE;gBACxC,IAAI,CAAC,kBAAkB,GAAG,KAAK,CAAC,KAAK,EAAE,SAAS,CAAC;aAClD;YAED,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC,WAAW,CAC1C,IAAI,CAAC,kBAAkB,EACvB,KAAK,CAAC,KAAK,CAAC,SAAS,EACrB,KAAK,CACN,CAAC;YACF,IAAI,MAAM,EAAE;gBACV,IAAI,CAAC,kBAAkB,GAAG,KAAK,CAAC,KAAK,EAAE,SAAS,CAAC;aAClD;YAED,MAAM,KAAK,GAAG,IAAA,SAAG,EAAC,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,IAAI,CAAC,YAAY,CAAC;YAC/D,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,EAAE,GAAG,KAAK,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC;YAEnE,IAAI,CAAC,KAAK,EAAE,CAAC;QACf,CAAC,CAAC;QAEF,sBAAiB,GAAG,CAAC,KAAoB,EAAE,EAAE;YAC3C,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE;gBAChB,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;gBACpB,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;gBACxB,OAAO;aACR;YACD,IAAI,IAAI,CAAC,OAAO,EAAE;gBAChB,OAAO;aACR;YAED,IAAI,IAAI,CAAC,kBAAkB,IAAI,SAAS,EAAE;gBACxC,IAAI,CAAC,kBAAkB,GAAG,KAAK,CAAC,KAAK,EAAE,SAAS,CAAC;aAClD;YAED,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC,WAAW,CAC1C,IAAI,CAAC,kBAAkB,EACvB,KAAK,CAAC,KAAK,CAAC,SAAS,EACrB,KAAK,CACN,CAAC;YACF,IAAI,MAAM,EAAE;gBACV,IAAI,CAAC,kBAAkB,GAAG,KAAK,CAAC,KAAK,EAAE,SAAS,CAAC;aAClD;YAED,MAAM,KAAK,GAAG,IAAA,SAAG,EAAC,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,IAAI,CAAC,YAAY,CAAC;YAC/D,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,EAAE,GAAG,KAAK,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC;YAEnE,IAAI,CAAC,KAAK,EAAE,CAAC;QACf,CAAC,CAAC;IA1FC,CAAC;IAEI,KAAK;QACX,IAAI,CAAC,GAAG,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,GAAG,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,KAAK,CAAC,EAAE;YACjE,OAAO;SACR;QAED,IAAI,IAAI,CAAC,OAAO,EAAE;YAChB,OAAO;SACR;QACD,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QAEpB,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,WAAW,CAAC,GAAG,EAAE;YACf,MAAM,MAAM,GAAG,CAAC,GAAG,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,EAAE,GAAG,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC;YACxE,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC;YAC5D,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC;YAC7B,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC;YAE7B,KAAK,MAAM,MAAM,IAAI,MAAM,EAAE;gBAC3B,IAAI,MAAM,CAAC,IAAI,KAAK,OAAO,EAAE;oBAC3B,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;iBAC1B;qBAAM;oBACL,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;iBAC1B;aACF;YAED,KAAK,EAAE,CAAC;YACR,IAAI,KAAK,KAAK,IAAI,CAAC,YAAY,EAAE;gBAC/B,KAAK,GAAG,CAAC,CAAC;aACX;QACH,CAAC,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;IACpB,CAAC;IA4DO,WAAW,CAAC,IAAY,EAAE,SAAiB,EAAE,SAAiB;QACpE,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,gBAAS,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC;QAChE,IAAI,MAAM,EAAE;YACV,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;SACrB;QAED,MAAM,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,SAAS,GAAG,gBAAS,GAAG,IAAI,CAAC,CAAC,CAAC,SAAS,GAAG,IAAI,CAAC;QACtE,QAAQ;QACR,MAAM,OAAO,GAAG,IAAA,SAAG,EAAC,CAAC,IAAI,GAAG,SAAS,CAAC,GAAG,IAAI,CAAC,CAAC;QAC/C,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;IAC7B,CAAC;CACF;AAzHD,oCAyHC","sourcesContent":["import { int } from \"../../../common/src\";\nimport { DepacketizerOutput } from \"./depacketizer\";\nimport { Max32Uint } from \"./webm\";\n\nexport type AVBufferInput = DepacketizerOutput;\n\nexport type AVBufferOutput = AVBufferInput;\n\n/**\n * @description [japanese]\n * audioパケットとvideoパケットを同一のタイムラインで扱い、それぞれの\n * パケットのタイムスタンプが前後しないように制御する\n */\nexport class AVBufferBase {\n  bufferLength = this.options.bufferLength ?? 50;\n  baseAudioTimestamp?: number;\n  baseVideoTimestamp?: number;\n  audioBuffer: (AVBufferInput & { elapsed: number; kind: string })[][] = [\n    ...new Array(this.bufferLength),\n  ].map(() => []);\n  videoBuffer: (AVBufferInput & { elapsed: number; kind: string })[][] = [\n    ...new Array(this.bufferLength),\n  ].map(() => []);\n  stopped = false;\n  private interval = this.options.interval ?? 500;\n  private started = false;\n\n  constructor(\n    private audioOutput: (output: AVBufferOutput) => void,\n    private videoOutput: (output: AVBufferOutput) => void,\n    private options: Partial<AvBufferOptions> = {}\n  ) {}\n\n  private start() {\n    if ([...this.audioBuffer[1], ...this.videoBuffer[1]].length === 0) {\n      return;\n    }\n\n    if (this.started) {\n      return;\n    }\n    this.started = true;\n\n    let index = 0;\n    setInterval(() => {\n      const joined = [...this.audioBuffer[index], ...this.videoBuffer[index]];\n      const sorted = joined.sort((a, b) => a.elapsed - b.elapsed);\n      this.audioBuffer[index] = [];\n      this.videoBuffer[index] = [];\n\n      for (const output of sorted) {\n        if (output.kind === \"audio\") {\n          this.audioOutput(output);\n        } else {\n          this.videoOutput(output);\n        }\n      }\n\n      index++;\n      if (index === this.bufferLength) {\n        index = 0;\n      }\n    }, this.interval);\n  }\n\n  processAudioInput = (input: AVBufferInput) => {\n    if (!input.frame) {\n      this.stopped = true;\n      this.audioOutput(input);\n      return;\n    }\n    if (this.stopped) {\n      return;\n    }\n\n    if (this.baseAudioTimestamp == undefined) {\n      this.baseAudioTimestamp = input.frame?.timestamp;\n    }\n\n    const { elapsed, rotate } = this.calcElapsed(\n      this.baseAudioTimestamp,\n      input.frame.timestamp,\n      48000\n    );\n    if (rotate) {\n      this.baseAudioTimestamp = input.frame?.timestamp;\n    }\n\n    const index = int(elapsed / this.interval) % this.bufferLength;\n    this.audioBuffer[index].push({ ...input, elapsed, kind: \"audio\" });\n\n    this.start();\n  };\n\n  processVideoInput = (input: AVBufferInput) => {\n    if (!input.frame) {\n      this.stopped = true;\n      this.videoOutput(input);\n      return;\n    }\n    if (this.stopped) {\n      return;\n    }\n\n    if (this.baseVideoTimestamp == undefined) {\n      this.baseVideoTimestamp = input.frame?.timestamp;\n    }\n\n    const { elapsed, rotate } = this.calcElapsed(\n      this.baseVideoTimestamp,\n      input.frame.timestamp,\n      90000\n    );\n    if (rotate) {\n      this.baseVideoTimestamp = input.frame?.timestamp;\n    }\n\n    const index = int(elapsed / this.interval) % this.bufferLength;\n    this.videoBuffer[index].push({ ...input, elapsed, kind: \"video\" });\n\n    this.start();\n  };\n\n  private calcElapsed(base: number, timestamp: number, clockRate: number) {\n    const rotate = Math.abs(timestamp - base) > (Max32Uint / 4) * 3;\n    if (rotate) {\n      console.log(rotate);\n    }\n\n    const diff = rotate ? timestamp + Max32Uint - base : timestamp - base;\n    /**ms */\n    const elapsed = int((diff / clockRate) * 1000);\n    return { elapsed, rotate };\n  }\n}\n\nexport interface AvBufferOptions {\n  interval: number;\n  bufferLength: number;\n}\n"]}

package/lib/rtp/src/processor/avBufferCallback.d.ts

@@ -1,10 +0,0 @@
-import { AVBufferBase, AvBufferOptions, AVBufferOutput } from "./avBuffer";
-export declare class AvBufferCallback extends AVBufferBase {
-    private audioCb;
-    private videoCb;
-    constructor(options?: Partial<AvBufferOptions>);
-    pipeAudio: (cb: (input: AVBufferOutput) => void) => void;
-    pipeVideo: (cb: (input: AVBufferOutput) => void) => void;
-    inputAudio: (input: import("./depacketizer").DepacketizerOutput) => void;
-    inputVideo: (input: import("./depacketizer").DepacketizerOutput) => void;
-}

package/lib/rtp/src/processor/avBufferCallback.js

@@ -1,27 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.AvBufferCallback = void 0;
-const avBuffer_1 = require("./avBuffer");
-class AvBufferCallback extends avBuffer_1.AVBufferBase {
-    constructor(options = {}) {
-        super((output) => {
-            if (this.audioCb) {
-                this.audioCb(output);
-            }
-        }, (output) => {
-            if (this.videoCb) {
-                this.videoCb(output);
-            }
-        }, options);
-        this.pipeAudio = (cb) => {
-            this.audioCb = cb;
-        };
-        this.pipeVideo = (cb) => {
-            this.videoCb = cb;
-        };
-        this.inputAudio = this.processAudioInput;
-        this.inputVideo = this.processVideoInput;
-    }
-}
-exports.AvBufferCallback = AvBufferCallback;
-//# sourceMappingURL=avBufferCallback.js.map

package/lib/rtp/src/processor/avBufferCallback.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"avBufferCallback.js","sourceRoot":"","sources":["../../../../../rtp/src/processor/avBufferCallback.ts"],"names":[],"mappings":";;;AAAA,yCAA2E;AAE3E,MAAa,gBAAiB,SAAQ,uBAAY;IAGhD,YAAY,UAAoC,EAAE;QAChD,KAAK,CACH,CAAC,MAAM,EAAE,EAAE;YACT,IAAI,IAAI,CAAC,OAAO,EAAE;gBAChB,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;aACtB;QACH,CAAC,EACD,CAAC,MAAM,EAAE,EAAE;YACT,IAAI,IAAI,CAAC,OAAO,EAAE;gBAChB,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;aACtB;QACH,CAAC,EACD,OAAO,CACR,CAAC;QAGJ,cAAS,GAAG,CAAC,EAAmC,EAAE,EAAE;YAClD,IAAI,CAAC,OAAO,GAAG,EAAE,CAAC;QACpB,CAAC,CAAC;QACF,cAAS,GAAG,CAAC,EAAmC,EAAE,EAAE;YAClD,IAAI,CAAC,OAAO,GAAG,EAAE,CAAC;QACpB,CAAC,CAAC;QAEF,eAAU,GAAG,IAAI,CAAC,iBAAiB,CAAC;QACpC,eAAU,GAAG,IAAI,CAAC,iBAAiB,CAAC;IAVpC,CAAC;CAWF;AA5BD,4CA4BC","sourcesContent":["import { AVBufferBase, AvBufferOptions, AVBufferOutput } from \"./avBuffer\";\n\nexport class AvBufferCallback extends AVBufferBase {\n  private audioCb!: (input: AVBufferOutput) => void;\n  private videoCb!: (input: AVBufferOutput) => void;\n  constructor(options: Partial<AvBufferOptions> = {}) {\n    super(\n      (output) => {\n        if (this.audioCb) {\n          this.audioCb(output);\n        }\n      },\n      (output) => {\n        if (this.videoCb) {\n          this.videoCb(output);\n        }\n      },\n      options\n    );\n  }\n\n  pipeAudio = (cb: (input: AVBufferOutput) => void) => {\n    this.audioCb = cb;\n  };\n  pipeVideo = (cb: (input: AVBufferOutput) => void) => {\n    this.videoCb = cb;\n  };\n\n  inputAudio = this.processAudioInput;\n  inputVideo = this.processVideoInput;\n}\n"]}