werift 0.18.3 → 0.18.4

package/lib/rtp/src/processor/lipsync.js

@@ -1,7 +1,8 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.LipsyncBase = void 0;
-const src_1 = require("../../../common/src");
+const crypto_1 = require("crypto");
+const __1 = require("..");
 class LipsyncBase {
     constructor(audioOutput, videoOutput, options = {}) {
         Object.defineProperty(this, "audioOutput", {
@@ -22,6 +23,12 @@ class LipsyncBase {
             writable: true,
             value: options
         });
+        Object.defineProperty(this, "id", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: (0, crypto_1.randomUUID)()
+        });
         Object.defineProperty(this, "bufferLength", {
             enumerable: true,
             configurable: true,
@@ -60,11 +67,30 @@ class LipsyncBase {
             writable: true,
             value: void 0
         });
-        Object.defineProperty(this, "started", {
+        /**ms */
+        Object.defineProperty(this, "bufferDuration", {
             enumerable: true,
             configurable: true,
             writable: true,
-            value: false
+            value: void 0
+        });
+        Object.defineProperty(this, "ptime", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "index", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: 0
+        });
+        Object.defineProperty(this, "currentTimestamp", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
         });
         /**ms */
         Object.defineProperty(this, "lastCommittedTime", {
@@ -73,11 +99,11 @@ class LipsyncBase {
             writable: true,
             value: 0
         });
-        Object.defineProperty(this, "intervalId", {
+        Object.defineProperty(this, "lastExecutionTime", {
             enumerable: true,
             configurable: true,
             writable: true,
-            value: void 0
+            value: 0
         });
         Object.defineProperty(this, "internalStats", {
             enumerable: true,
@@ -85,6 +111,13 @@ class LipsyncBase {
             writable: true,
             value: {}
         });
+        /**ms */
+        Object.defineProperty(this, "lastFrameReceivedAt", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: 0
+        });
         Object.defineProperty(this, "processAudioInput", {
             enumerable: true,
             configurable: true,
@@ -96,23 +129,7 @@ class LipsyncBase {
                     this.audioOutput = undefined;
                     return;
                 }
-                if (this.stopped) {
-                    return;
-                }
-                if (this.baseTime == undefined) {
-                    this.baseTime = frame.time;
-                }
-                /**ms */
-                const elapsed = frame.time - this.baseTime;
-                if (elapsed < 0 || frame.time < this.lastCommittedTime) {
-                    return;
-                }
-                const index = (0, src_1.int)(elapsed / this.interval) % this.bufferLength;
-                this.audioBuffer[index].push({
-                    frame,
-                    kind: "audio",
-                });
-                this.startIfNeed();
+                this.processInput(frame, this.audioBuffer, "audio");
             }
         });
         Object.defineProperty(this, "processVideoInput", {
@@ -126,99 +143,130 @@ class LipsyncBase {
                     this.videoOutput = undefined;
                     return;
                 }
+                this.processInput(frame, this.videoBuffer, "video");
+            }
+        });
+        Object.defineProperty(this, "processInput", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: (frame, buffer, kind) => {
                 if (this.stopped) {
                     return;
                 }
                 if (this.baseTime == undefined) {
                     this.baseTime = frame.time;
+                    this.currentTimestamp = this.baseTime;
+                    this.lastExecutionTime = this.baseTime;
+                    this.lastCommittedTime = this.baseTime;
+                    this.lastFrameReceivedAt = Date.now();
                 }
-                /**ms */
-                const elapsed = frame.time - this.baseTime;
-                if (elapsed < 0 || frame.time < this.lastCommittedTime) {
+                // 過去のフレームを捨てる
+                if (frame.time < this.lastCommittedTime) {
                     return;
                 }
-                const index = (0, src_1.int)(elapsed / this.interval) % this.bufferLength;
-                this.videoBuffer[index].push({
+                // NTPの同期ずれが疑われるので捨てる
+                const now = Date.now();
+                const gap = 5000; // RTCP SR interval;
+                const lastCommittedElapsed = frame.time - this.lastCommittedTime;
+                const lastFrameReceivedElapsed = now - this.lastFrameReceivedAt;
+                if (gap < lastFrameReceivedElapsed && lastCommittedElapsed < gap) {
+                    this.internalStats["invalidFrameTime"] = {
+                        count: (this.internalStats["invalidFrameTime"]?.count ?? 0) + 1,
+                        at: new Date().toISOString(),
+                        lastCommittedElapsed,
+                        lastFrameReceivedElapsed,
+                    };
+                    return;
+                }
+                this.lastFrameReceivedAt = now;
+                const elapsed = frame.time - this.baseTime;
+                const index = (0, __1.int)(elapsed / this.bufferDuration) % this.bufferLength;
+                buffer[index].push({
                     frame,
-                    kind: "video",
+                    kind,
                 });
-                this.startIfNeed();
+                const diff = frame.time - this.lastExecutionTime;
+                if (diff >= this.interval) {
+                    const times = (0, __1.int)(diff / this.bufferDuration) - 1;
+                    this.lastExecutionTime = this.currentTimestamp;
+                    for (let i = 0; i < times; i++) {
+                        this.executeTask();
+                        this.lastExecutionTime += this.bufferDuration;
+                    }
+                }
+                return;
             }
         });
         this.interval = this.options.syncInterval ?? 500;
-        this.bufferLength = this.options.bufferingTimes ?? 10;
+        this.bufferDuration = this.interval / 2;
+        this.bufferLength = (this.options.bufferLength ?? 10) * 2;
         this.audioBuffer = [...new Array(this.bufferLength)].map(() => []);
         this.videoBuffer = [...new Array(this.bufferLength)].map(() => []);
+        this.ptime = this.options.ptime ?? 20;
     }
     toJSON() {
         return {
+            ...this.internalStats,
+            id: this.id,
             audioBufferLength: this.audioBuffer.flatMap((v) => v).length,
             videoBufferLength: this.videoBuffer.flatMap((v) => v).length,
             baseTime: this.baseTime,
             lastCommittedTimeSec: this.lastCommittedTime / 1000,
-            ...this.internalStats,
         };
     }
-    startIfNeed() {
-        // 2列目にカーソルが移ってから処理を始めることで1列目の処理を完了できる
-        if ([...this.audioBuffer[1], ...this.videoBuffer[1]].length === 0) {
-            return;
-        }
-        if (this.started) {
-            return;
-        }
-        this.started = true;
-        let index = 0;
-        let currentTimestamp = this.baseTime;
-        const task = () => {
-            const audioBuffer = this.audioBuffer[index].sort((a, b) => a.frame.time - b.frame.time);
-            if (this.options.fillDummyAudioPacket) {
-                const last = audioBuffer.at(-1);
-                const expect = last ? last.frame.time + 20 : currentTimestamp;
-                // パケット間の損失/muteはdtxプラグインでダミーパケットを挿入する
-                // interval中のパケットが途中から無いもしくはinterval中にパケットが無い場合はここでダミーパケットを挿入する
-                if (expect < currentTimestamp + this.interval) {
-                    for (let time = expect; time < currentTimestamp + this.interval; time += 20) {
-                        audioBuffer.push({
-                            frame: {
-                                time,
-                                data: this.options.fillDummyAudioPacket,
-                                isKeyframe: true,
-                            },
-                            kind: "audio",
-                        });
-                        this.internalStats["pushDummyPacket"] = {
-                            count: (this.internalStats["pushDummyPacket"]?.count ?? 0) + 1,
-                            at: new Date().toISOString(),
+    executeTask() {
+        const audioBuffer = this.audioBuffer[this.index].sort((a, b) => a.frame.time - b.frame.time);
+        if (this.options.fillDummyAudioPacket) {
+            const last = audioBuffer.at(-1);
+            const expect = last
+                ? last.frame.time +
+                    // offset
+                    this.ptime
+                : this.currentTimestamp;
+            // パケット間の損失/muteはdtxプラグインでダミーパケットを挿入する
+            // interval中のパケットが途中から無いもしくはinterval中にパケットが無い場合はここでダミーパケットを挿入する
+            const audioDiff = this.currentTimestamp + this.bufferDuration - expect;
+            if (audioDiff > 0) {
+                for (let time = expect; time < this.currentTimestamp + this.bufferDuration; time += this.ptime) {
+                    audioBuffer.push({
+                        frame: {
                             time,
-                        };
-                    }
+                            data: this.options.fillDummyAudioPacket,
+                            isKeyframe: true,
+                        },
+                        kind: "audio",
+                    });
+                    this.internalStats["pushDummyPacket"] = {
+                        count: (this.internalStats["pushDummyPacket"]?.count ?? 0) + 1,
+                        at: new Date().toISOString(),
+                        time,
+                    };
                 }
-                currentTimestamp += this.interval;
             }
-            const joined = [...audioBuffer, ...this.videoBuffer[index]].filter((b) => b.frame.time >= this.lastCommittedTime);
-            const sorted = joined.sort((a, b) => a.frame.time - b.frame.time);
-            this.audioBuffer[index] = [];
-            this.videoBuffer[index] = [];
-            for (const output of sorted) {
-                if (output.kind === "audio") {
-                    this.audioOutput(output);
-                }
-                else {
-                    this.videoOutput(output);
-                }
-                this.lastCommittedTime = output.frame.time;
+        }
+        this.currentTimestamp += this.bufferDuration;
+        const joined = [...audioBuffer, ...this.videoBuffer[this.index]].filter((b) => b.frame.time >= this.lastCommittedTime);
+        const sorted = joined.sort((a, b) => a.frame.time - b.frame.time);
+        this.audioBuffer[this.index] = [];
+        this.videoBuffer[this.index] = [];
+        for (const output of sorted) {
+            if (output.kind === "audio") {
+                this.audioOutput(output);
             }
-            index++;
-            if (index === this.bufferLength) {
-                index = 0;
+            else {
+                this.videoOutput(output);
             }
-        };
-        this.intervalId = setInterval(task, this.interval);
+            this.internalStats["lipsync"] = new Date().toISOString();
+            this.lastCommittedTime = output.frame.time;
+        }
+        this.index++;
+        if (this.index === this.bufferLength) {
+            this.index = 0;
+        }
     }
     stop() {
         this.stopped = true;
-        clearInterval(this.intervalId);
         this.audioBuffer = [];
         this.videoBuffer = [];
     }

package/lib/rtp/src/processor/lipsync.js.map

@@ -1 +1 @@
-{"version":3,"file":"lipsync.js","sourceRoot":"","sources":["../../../../../rtp/src/processor/lipsync.ts"],"names":[],"mappings":";;;AAAA,6CAA0C;AAc1C,MAAa,WAAW;IAetB,YACU,WAA4C,EAC5C,WAA4C,EAC5C,UAAmC,EAAE;;;;;mBAFrC;;;;;;mBACA;;;;;;mBACA;;QAjBV;;;;;WAAqB;QACrB,QAAQ;QACR;;;;;WAAkB;QAClB;;;;;WAAqD;QACrD;;;;;WAAqD;QACrD;;;;mBAAU,KAAK;WAAC;QAChB,QAAQ;QACR;;;;;WAAyB;QACzB;;;;mBAAkB,KAAK;WAAC;QACxB,QAAQ;QACR;;;;mBAAoB,CAAC;WAAC;QACtB;;;;;WAAyB;QACzB;;;;mBAAwB,EAAE;WAAC;QAqG3B;;;;mBAAoB,CAAC,EAAE,KAAK,EAAE,GAAG,EAAgB,EAAE,EAAE;gBACnD,IAAI,CAAC,KAAK,EAAE;oBACV,IAAI,CAAC,WAAW,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC;oBAC1B,IAAI,CAAC,IAAI,EAAE,CAAC;oBACZ,IAAI,CAAC,WAAW,GAAG,SAAgB,CAAC;oBACpC,OAAO;iBACR;gBACD,IAAI,IAAI,CAAC,OAAO,EAAE;oBAChB,OAAO;iBACR;gBAED,IAAI,IAAI,CAAC,QAAQ,IAAI,SAAS,EAAE;oBAC9B,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC,IAAI,CAAC;iBAC5B;gBAED,QAAQ;gBACR,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,GAAG,IAAI,CAAC,QAAS,CAAC;gBAC5C,IAAI,OAAO,GAAG,CAAC,IAAI,KAAK,CAAC,IAAI,GAAG,IAAI,CAAC,iBAAiB,EAAE;oBACtD,OAAO;iBACR;gBACD,MAAM,KAAK,GAAG,IAAA,SAAG,EAAC,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,IAAI,CAAC,YAAY,CAAC;gBAC/D,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC;oBAC3B,KAAK;oBACL,IAAI,EAAE,OAAO;iBACd,CAAC,CAAC;gBAEH,IAAI,CAAC,WAAW,EAAE,CAAC;YACrB,CAAC;WAAC;QAEF;;;;mBAAoB,CAAC,EAAE,KAAK,EAAE,GAAG,EAAgB,EAAE,EAAE;gBACnD,IAAI,CAAC,KAAK,EAAE;oBACV,IAAI,CAAC,WAAW,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC;oBAC1B,IAAI,CAAC,IAAI,EAAE,CAAC;oBACZ,IAAI,CAAC,WAAW,GAAG,SAAgB,CAAC;oBACpC,OAAO;iBACR;gBACD,IAAI,IAAI,CAAC,OAAO,EAAE;oBAChB,OAAO;iBACR;gBAED,IAAI,IAAI,CAAC,QAAQ,IAAI,SAAS,EAAE;oBAC9B,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC,IAAI,CAAC;iBAC5B;gBAED,QAAQ;gBACR,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,GAAG,IAAI,CAAC,QAAS,CAAC;gBAC5C,IAAI,OAAO,GAAG,CAAC,IAAI,KAAK,CAAC,IAAI,GAAG,IAAI,CAAC,iBAAiB,EAAE;oBACtD,OAAO;iBACR;gBACD,MAAM,KAAK,GAAG,IAAA,SAAG,EAAC,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,IAAI,CAAC,YAAY,CAAC;gBAC/D,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC;oBAC3B,KAAK;oBACL,IAAI,EAAE,OAAO;iBACd,CAAC,CAAC;gBAEH,IAAI,CAAC,WAAW,EAAE,CAAC;YACrB,CAAC;WAAC;QAtJA,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,YAAY,IAAI,GAAG,CAAC;QACjD,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,OAAO,CAAC,cAAc,IAAI,EAAE,CAAC;QACtD,IAAI,CAAC,WAAW,GAAG,CAAC,GAAG,IAAI,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;QACnE,IAAI,CAAC,WAAW,GAAG,CAAC,GAAG,IAAI,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;IACrE,CAAC;IAED,MAAM;QACJ,OAAO;YACL,iBAAiB,EAAE,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM;YAC5D,iBAAiB,EAAE,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM;YAC5D,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,oBAAoB,EAAE,IAAI,CAAC,iBAAiB,GAAG,IAAI;YACnD,GAAG,IAAI,CAAC,aAAa;SACtB,CAAC;IACJ,CAAC;IAEO,WAAW;QACjB,sCAAsC;QACtC,IAAI,CAAC,GAAG,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,GAAG,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,KAAK,CAAC,EAAE;YACjE,OAAO;SACR;QAED,IAAI,IAAI,CAAC,OAAO,EAAE;YAChB,OAAO;SACR;QACD,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QAEpB,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,IAAI,gBAAgB,GAAG,IAAI,CAAC,QAAS,CAAC;QACtC,MAAM,IAAI,GAAG,GAAG,EAAE;YAChB,MAAM,WAAW,GAAG,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,IAAI,CAC9C,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,GAAG,CAAC,CAAC,KAAK,CAAC,IAAI,CACtC,CAAC;YAEF,IAAI,IAAI,CAAC,OAAO,CAAC,oBAAoB,EAAE;gBACrC,MAAM,IAAI,GAAG,WAAW,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;gBAChC,MAAM,MAAM,GAAG,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,EAAE,CAAC,CAAC,CAAC,gBAAgB,CAAC;gBAE9D,sCAAsC;gBACtC,+DAA+D;gBAC/D,IAAI,MAAM,GAAG,gBAAgB,GAAG,IAAI,CAAC,QAAQ,EAAE;oBAC7C,KACE,IAAI,IAAI,GAAG,MAAM,EACjB,IAAI,GAAG,gBAAgB,GAAG,IAAI,CAAC,QAAQ,EACvC,IAAI,IAAI,EAAE,EACV;wBACA,WAAW,CAAC,IAAI,CAAC;4BACf,KAAK,EAAE;gCACL,IAAI;gCACJ,IAAI,EAAE,IAAI,CAAC,OAAO,CAAC,oBAAoB;gCACvC,UAAU,EAAE,IAAI;6BACjB;4BACD,IAAI,EAAE,OAAO;yBACd,CAAC,CAAC;wBACH,IAAI,CAAC,aAAa,CAAC,iBAAiB,CAAC,GAAG;4BACtC,KAAK,EAAE,CAAC,IAAI,CAAC,aAAa,CAAC,iBAAiB,CAAC,EAAE,KAAK,IAAI,CAAC,CAAC,GAAG,CAAC;4BAC9D,EAAE,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;4BAC5B,IAAI;yBACL,CAAC;qBACH;iBACF;gBACD,gBAAgB,IAAI,IAAI,CAAC,QAAQ,CAAC;aACnC;YACD,MAAM,MAAM,GAAG,CAAC,GAAG,WAAW,EAAE,GAAG,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAChE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,IAAI,IAAI,CAAC,iBAAiB,CAC9C,CAAC;YACF,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,GAAG,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YAClE,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC;YAC7B,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC;YAE7B,KAAK,MAAM,MAAM,IAAI,MAAM,EAAE;gBAC3B,IAAI,MAAM,CAAC,IAAI,KAAK,OAAO,EAAE;oBAC3B,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;iBAC1B;qBAAM;oBACL,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;iBAC1B;gBACD,IAAI,CAAC,iBAAiB,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC;aAC5C;YAED,KAAK,EAAE,CAAC;YACR,IAAI,KAAK,KAAK,IAAI,CAAC,YAAY,EAAE;gBAC/B,KAAK,GAAG,CAAC,CAAC;aACX;QACH,CAAC,CAAC;QACF,IAAI,CAAC,UAAU,GAAG,WAAW,CAAC,IAAI,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;IACrD,CAAC;IAEO,IAAI;QACV,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACpB,aAAa,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QAC/B,IAAI,CAAC,WAAW,GAAG,EAAE,CAAC;QACtB,IAAI,CAAC,WAAW,GAAG,EAAE,CAAC;IACxB,CAAC;CA2DF;AA3KD,kCA2KC","sourcesContent":["import { int } from \"../../../common/src\";\nimport { CodecFrame } from \"./depacketizer\";\nimport { AVProcessor } from \"./interface\";\n\nexport type LipsyncInput = {\n  frame?: CodecFrame;\n  eol?: boolean;\n};\n\nexport type LipsyncOutput = {\n  frame?: CodecFrame;\n  eol?: boolean;\n};\n\nexport class LipsyncBase implements AVProcessor<LipsyncInput> {\n  bufferLength: number;\n  /**ms */\n  baseTime?: number;\n  audioBuffer: { frame: CodecFrame; kind: string }[][];\n  videoBuffer: { frame: CodecFrame; kind: string }[][];\n  stopped = false;\n  /**ms */\n  private interval: number;\n  private started = false;\n  /**ms */\n  lastCommittedTime = 0;\n  private intervalId?: any;\n  private internalStats = {};\n\n  constructor(\n    private audioOutput: (output: LipsyncOutput) => void,\n    private videoOutput: (output: LipsyncOutput) => void,\n    private options: Partial<LipSyncOptions> = {}\n  ) {\n    this.interval = this.options.syncInterval ?? 500;\n    this.bufferLength = this.options.bufferingTimes ?? 10;\n    this.audioBuffer = [...new Array(this.bufferLength)].map(() => []);\n    this.videoBuffer = [...new Array(this.bufferLength)].map(() => []);\n  }\n\n  toJSON(): Record<string, any> {\n    return {\n      audioBufferLength: this.audioBuffer.flatMap((v) => v).length,\n      videoBufferLength: this.videoBuffer.flatMap((v) => v).length,\n      baseTime: this.baseTime,\n      lastCommittedTimeSec: this.lastCommittedTime / 1000,\n      ...this.internalStats,\n    };\n  }\n\n  private startIfNeed() {\n    // 2列目にカーソルが移ってから処理を始めることで1列目の処理を完了できる\n    if ([...this.audioBuffer[1], ...this.videoBuffer[1]].length === 0) {\n      return;\n    }\n\n    if (this.started) {\n      return;\n    }\n    this.started = true;\n\n    let index = 0;\n    let currentTimestamp = this.baseTime!;\n    const task = () => {\n      const audioBuffer = this.audioBuffer[index].sort(\n        (a, b) => a.frame.time - b.frame.time\n      );\n\n      if (this.options.fillDummyAudioPacket) {\n        const last = audioBuffer.at(-1);\n        const expect = last ? last.frame.time + 20 : currentTimestamp;\n\n        // パケット間の損失/muteはdtxプラグインでダミーパケットを挿入する\n        // interval中のパケットが途中から無いもしくはinterval中にパケットが無い場合はここでダミーパケットを挿入する\n        if (expect < currentTimestamp + this.interval) {\n          for (\n            let time = expect;\n            time < currentTimestamp + this.interval;\n            time += 20\n          ) {\n            audioBuffer.push({\n              frame: {\n                time,\n                data: this.options.fillDummyAudioPacket,\n                isKeyframe: true,\n              },\n              kind: \"audio\",\n            });\n            this.internalStats[\"pushDummyPacket\"] = {\n              count: (this.internalStats[\"pushDummyPacket\"]?.count ?? 0) + 1,\n              at: new Date().toISOString(),\n              time,\n            };\n          }\n        }\n        currentTimestamp += this.interval;\n      }\n      const joined = [...audioBuffer, ...this.videoBuffer[index]].filter(\n        (b) => b.frame.time >= this.lastCommittedTime\n      );\n      const sorted = joined.sort((a, b) => a.frame.time - b.frame.time);\n      this.audioBuffer[index] = [];\n      this.videoBuffer[index] = [];\n\n      for (const output of sorted) {\n        if (output.kind === \"audio\") {\n          this.audioOutput(output);\n        } else {\n          this.videoOutput(output);\n        }\n        this.lastCommittedTime = output.frame.time;\n      }\n\n      index++;\n      if (index === this.bufferLength) {\n        index = 0;\n      }\n    };\n    this.intervalId = setInterval(task, this.interval);\n  }\n\n  private stop() {\n    this.stopped = true;\n    clearInterval(this.intervalId);\n    this.audioBuffer = [];\n    this.videoBuffer = [];\n  }\n\n  processAudioInput = ({ frame, eol }: LipsyncInput) => {\n    if (!frame) {\n      this.audioOutput({ eol });\n      this.stop();\n      this.audioOutput = undefined as any;\n      return;\n    }\n    if (this.stopped) {\n      return;\n    }\n\n    if (this.baseTime == undefined) {\n      this.baseTime = frame.time;\n    }\n\n    /**ms */\n    const elapsed = frame.time - this.baseTime!;\n    if (elapsed < 0 || frame.time < this.lastCommittedTime) {\n      return;\n    }\n    const index = int(elapsed / this.interval) % this.bufferLength;\n    this.audioBuffer[index].push({\n      frame,\n      kind: \"audio\",\n    });\n\n    this.startIfNeed();\n  };\n\n  processVideoInput = ({ frame, eol }: LipsyncInput) => {\n    if (!frame) {\n      this.videoOutput({ eol });\n      this.stop();\n      this.videoOutput = undefined as any;\n      return;\n    }\n    if (this.stopped) {\n      return;\n    }\n\n    if (this.baseTime == undefined) {\n      this.baseTime = frame.time;\n    }\n\n    /**ms */\n    const elapsed = frame.time - this.baseTime!;\n    if (elapsed < 0 || frame.time < this.lastCommittedTime) {\n      return;\n    }\n    const index = int(elapsed / this.interval) % this.bufferLength;\n    this.videoBuffer[index].push({\n      frame,\n      kind: \"video\",\n    });\n\n    this.startIfNeed();\n  };\n}\n\nexport interface LipSyncOptions {\n  /**ms */\n  syncInterval: number;\n  /**\n   * int\n   * @description syncInterval * bufferingTimes=bufferTimeLength\n   * */\n  bufferingTimes: number;\n  fillDummyAudioPacket: Buffer;\n}\n"]}
+{"version":3,"file":"lipsync.js","sourceRoot":"","sources":["../../../../../rtp/src/processor/lipsync.ts"],"names":[],"mappings":";;;AAAA,mCAAoC;AAEpC,0BAAyB;AAezB,MAAa,WAAW;IA0BtB,YACU,WAA4C,EAC5C,WAA4C,EAC5C,UAAmC,EAAE;;;;;mBAFrC;;;;;;mBACA;;;;;;mBACA;;QA5BV;;;;mBAAa,IAAA,mBAAU,GAAE;WAAC;QAC1B;;;;;WAAqB;QACrB,QAAQ;QACR;;;;;WAAkB;QAClB;;;;;WAIM;QACN;;;;;WAAwD;QACxD;;;;mBAAU,KAAK;WAAC;QAChB,QAAQ;QACR;;;;;WAAyB;QACzB,QAAQ;QACR;;;;;WAA+B;QAC/B;;;;;WAAsB;QACtB;;;;mBAAgB,CAAC;WAAC;QAClB;;;;;WAAkC;QAClC,QAAQ;QACR;;;;mBAA4B,CAAC;WAAC;QAC9B;;;;mBAA4B,CAAC;WAAC;QAC9B;;;;mBAAwB,EAAE;WAAC;QAC3B,QAAQ;QACR;;;;mBAA8B,CAAC;WAAC;QA8FhC;;;;mBAAoB,CAAC,EAAE,KAAK,EAAE,GAAG,EAAgB,EAAE,EAAE;gBACnD,IAAI,CAAC,KAAK,EAAE;oBACV,IAAI,CAAC,WAAW,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC;oBAC1B,IAAI,CAAC,IAAI,EAAE,CAAC;oBACZ,IAAI,CAAC,WAAW,GAAG,SAAgB,CAAC;oBACpC,OAAO;iBACR;gBAED,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,IAAI,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;YACtD,CAAC;WAAC;QAEF;;;;mBAAoB,CAAC,EAAE,KAAK,EAAE,GAAG,EAAgB,EAAE,EAAE;gBACnD,IAAI,CAAC,KAAK,EAAE;oBACV,IAAI,CAAC,WAAW,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC;oBAC1B,IAAI,CAAC,IAAI,EAAE,CAAC;oBACZ,IAAI,CAAC,WAAW,GAAG,SAAgB,CAAC;oBACpC,OAAO;iBACR;gBAED,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,IAAI,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;YACtD,CAAC;WAAC;QAEF;;;;mBAAuB,CACrB,KAAiB,EACjB,MAGK,EACL,IAAe,EACf,EAAE;gBACF,IAAI,IAAI,CAAC,OAAO,EAAE;oBAChB,OAAO;iBACR;gBAED,IAAI,IAAI,CAAC,QAAQ,IAAI,SAAS,EAAE;oBAC9B,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC,IAAI,CAAC;oBAC3B,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,QAAQ,CAAC;oBACtC,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC,QAAQ,CAAC;oBACvC,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC,QAAQ,CAAC;oBACvC,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;iBACvC;gBAED,cAAc;gBACd,IAAI,KAAK,CAAC,IAAI,GAAG,IAAI,CAAC,iBAAiB,EAAE;oBACvC,OAAO;iBACR;gBAED,qBAAqB;gBACrB,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;gBACvB,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,oBAAoB;gBACtC,MAAM,oBAAoB,GAAG,KAAK,CAAC,IAAI,GAAG,IAAI,CAAC,iBAAiB,CAAC;gBACjE,MAAM,wBAAwB,GAAG,GAAG,GAAG,IAAI,CAAC,mBAAmB,CAAC;gBAChE,IAAI,GAAG,GAAG,wBAAwB,IAAI,oBAAoB,GAAG,GAAG,EAAE;oBAChE,IAAI,CAAC,aAAa,CAAC,kBAAkB,CAAC,GAAG;wBACvC,KAAK,EAAE,CAAC,IAAI,CAAC,aAAa,CAAC,kBAAkB,CAAC,EAAE,KAAK,IAAI,CAAC,CAAC,GAAG,CAAC;wBAC/D,EAAE,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;wBAC5B,oBAAoB;wBACpB,wBAAwB;qBACzB,CAAC;oBACF,OAAO;iBACR;gBACD,IAAI,CAAC,mBAAmB,GAAG,GAAG,CAAC;gBAE/B,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC;gBAC3C,MAAM,KAAK,GAAG,IAAA,OAAG,EAAC,OAAO,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,IAAI,CAAC,YAAY,CAAC;gBACrE,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC;oBACjB,KAAK;oBACL,IAAI;iBACL,CAAC,CAAC;gBAEH,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,GAAG,IAAI,CAAC,iBAAiB,CAAC;gBACjD,IAAI,IAAI,IAAI,IAAI,CAAC,QAAQ,EAAE;oBACzB,MAAM,KAAK,GAAG,IAAA,OAAG,EAAC,IAAI,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;oBAClD,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC,gBAAgB,CAAC;oBAC/C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,EAAE,CAAC,EAAE,EAAE;wBAC9B,IAAI,CAAC,WAAW,EAAE,CAAC;wBACnB,IAAI,CAAC,iBAAiB,IAAI,IAAI,CAAC,cAAc,CAAC;qBAC/C;iBACF;gBACD,OAAO;YACT,CAAC;WAAC;QAvKA,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,YAAY,IAAI,GAAG,CAAC;QACjD,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;QACxC,IAAI,CAAC,YAAY,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,YAAY,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC;QAC1D,IAAI,CAAC,WAAW,GAAG,CAAC,GAAG,IAAI,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;QACnE,IAAI,CAAC,WAAW,GAAG,CAAC,GAAG,IAAI,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;QACnE,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,IAAI,EAAE,CAAC;IACxC,CAAC;IAED,MAAM;QACJ,OAAO;YACL,GAAG,IAAI,CAAC,aAAa;YACrB,EAAE,EAAE,IAAI,CAAC,EAAE;YACX,iBAAiB,EAAE,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM;YAC5D,iBAAiB,EAAE,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM;YAC5D,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,oBAAoB,EAAE,IAAI,CAAC,iBAAiB,GAAG,IAAI;SACpD,CAAC;IACJ,CAAC;IAEO,WAAW;QACjB,MAAM,WAAW,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CACnD,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,GAAG,CAAC,CAAC,KAAK,CAAC,IAAI,CACtC,CAAC;QAEF,IAAI,IAAI,CAAC,OAAO,CAAC,oBAAoB,EAAE;YACrC,MAAM,IAAI,GAAG,WAAW,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;YAChC,MAAM,MAAM,GAAG,IAAI;gBACjB,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI;oBACf,SAAS;oBACT,IAAI,CAAC,KAAK;gBACZ,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC;YAE1B,sCAAsC;YACtC,+DAA+D;YAC/D,MAAM,SAAS,GAAG,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,cAAc,GAAG,MAAM,CAAC;YACvE,IAAI,SAAS,GAAG,CAAC,EAAE;gBACjB,KACE,IAAI,IAAI,GAAG,MAAM,EACjB,IAAI,GAAG,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,cAAc,EAClD,IAAI,IAAI,IAAI,CAAC,KAAK,EAClB;oBACA,WAAW,CAAC,IAAI,CAAC;wBACf,KAAK,EAAE;4BACL,IAAI;4BACJ,IAAI,EAAE,IAAI,CAAC,OAAO,CAAC,oBAAoB;4BACvC,UAAU,EAAE,IAAI;yBACjB;wBACD,IAAI,EAAE,OAAO;qBACd,CAAC,CAAC;oBACH,IAAI,CAAC,aAAa,CAAC,iBAAiB,CAAC,GAAG;wBACtC,KAAK,EAAE,CAAC,IAAI,CAAC,aAAa,CAAC,iBAAiB,CAAC,EAAE,KAAK,IAAI,CAAC,CAAC,GAAG,CAAC;wBAC9D,EAAE,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;wBAC5B,IAAI;qBACL,CAAC;iBACH;aACF;SACF;QACD,IAAI,CAAC,gBAAgB,IAAI,IAAI,CAAC,cAAc,CAAC;QAC7C,MAAM,MAAM,GAAG,CAAC,GAAG,WAAW,EAAE,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CACrE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,IAAI,IAAI,CAAC,iBAAiB,CAC9C,CAAC;QACF,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,GAAG,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAClE,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC;QAClC,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC;QAElC,KAAK,MAAM,MAAM,IAAI,MAAM,EAAE;YAC3B,IAAI,MAAM,CAAC,IAAI,KAAK,OAAO,EAAE;gBAC3B,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;aAC1B;iBAAM;gBACL,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;aAC1B;YACD,IAAI,CAAC,aAAa,CAAC,SAAS,CAAC,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;YACzD,IAAI,CAAC,iBAAiB,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC;SAC5C;QAED,IAAI,CAAC,KAAK,EAAE,CAAC;QACb,IAAI,IAAI,CAAC,KAAK,KAAK,IAAI,CAAC,YAAY,EAAE;YACpC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC;SAChB;IACH,CAAC;IAEO,IAAI;QACV,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACpB,IAAI,CAAC,WAAW,GAAG,EAAE,CAAC;QACtB,IAAI,CAAC,WAAW,GAAG,EAAE,CAAC;IACxB,CAAC;CAmFF;AAvMD,kCAuMC","sourcesContent":["import { randomUUID } from \"crypto\";\n\nimport { int } from \"..\";\nimport { CodecFrame } from \"./depacketizer\";\nimport { AVProcessor } from \"./interface\";\nimport { MediaKind } from \"./webm\";\n\nexport type LipsyncInput = {\n  frame?: CodecFrame;\n  eol?: boolean;\n};\n\nexport type LipsyncOutput = {\n  frame?: CodecFrame;\n  eol?: boolean;\n};\n\nexport class LipsyncBase implements AVProcessor<LipsyncInput> {\n  private id = randomUUID();\n  bufferLength: number;\n  /**ms */\n  baseTime?: number;\n  audioBuffer: {\n    frame: CodecFrame;\n    kind: MediaKind;\n    [key: string]: any;\n  }[][];\n  videoBuffer: { frame: CodecFrame; kind: MediaKind }[][];\n  stopped = false;\n  /**ms */\n  private interval: number;\n  /**ms */\n  private bufferDuration: number;\n  private ptime: number;\n  private index = 0;\n  private currentTimestamp!: number;\n  /**ms */\n  private lastCommittedTime = 0;\n  private lastExecutionTime = 0;\n  private internalStats = {};\n  /**ms */\n  private lastFrameReceivedAt = 0;\n\n  constructor(\n    private audioOutput: (output: LipsyncOutput) => void,\n    private videoOutput: (output: LipsyncOutput) => void,\n    private options: Partial<LipSyncOptions> = {}\n  ) {\n    this.interval = this.options.syncInterval ?? 500;\n    this.bufferDuration = this.interval / 2;\n    this.bufferLength = (this.options.bufferLength ?? 10) * 2;\n    this.audioBuffer = [...new Array(this.bufferLength)].map(() => []);\n    this.videoBuffer = [...new Array(this.bufferLength)].map(() => []);\n    this.ptime = this.options.ptime ?? 20;\n  }\n\n  toJSON(): Record<string, any> {\n    return {\n      ...this.internalStats,\n      id: this.id,\n      audioBufferLength: this.audioBuffer.flatMap((v) => v).length,\n      videoBufferLength: this.videoBuffer.flatMap((v) => v).length,\n      baseTime: this.baseTime,\n      lastCommittedTimeSec: this.lastCommittedTime / 1000,\n    };\n  }\n\n  private executeTask() {\n    const audioBuffer = this.audioBuffer[this.index].sort(\n      (a, b) => a.frame.time - b.frame.time\n    );\n\n    if (this.options.fillDummyAudioPacket) {\n      const last = audioBuffer.at(-1);\n      const expect = last\n        ? last.frame.time +\n          // offset\n          this.ptime\n        : this.currentTimestamp;\n\n      // パケット間の損失/muteはdtxプラグインでダミーパケットを挿入する\n      // interval中のパケットが途中から無いもしくはinterval中にパケットが無い場合はここでダミーパケットを挿入する\n      const audioDiff = this.currentTimestamp + this.bufferDuration - expect;\n      if (audioDiff > 0) {\n        for (\n          let time = expect;\n          time < this.currentTimestamp + this.bufferDuration;\n          time += this.ptime\n        ) {\n          audioBuffer.push({\n            frame: {\n              time,\n              data: this.options.fillDummyAudioPacket,\n              isKeyframe: true,\n            },\n            kind: \"audio\",\n          });\n          this.internalStats[\"pushDummyPacket\"] = {\n            count: (this.internalStats[\"pushDummyPacket\"]?.count ?? 0) + 1,\n            at: new Date().toISOString(),\n            time,\n          };\n        }\n      }\n    }\n    this.currentTimestamp += this.bufferDuration;\n    const joined = [...audioBuffer, ...this.videoBuffer[this.index]].filter(\n      (b) => b.frame.time >= this.lastCommittedTime\n    );\n    const sorted = joined.sort((a, b) => a.frame.time - b.frame.time);\n    this.audioBuffer[this.index] = [];\n    this.videoBuffer[this.index] = [];\n\n    for (const output of sorted) {\n      if (output.kind === \"audio\") {\n        this.audioOutput(output);\n      } else {\n        this.videoOutput(output);\n      }\n      this.internalStats[\"lipsync\"] = new Date().toISOString();\n      this.lastCommittedTime = output.frame.time;\n    }\n\n    this.index++;\n    if (this.index === this.bufferLength) {\n      this.index = 0;\n    }\n  }\n\n  private stop() {\n    this.stopped = true;\n    this.audioBuffer = [];\n    this.videoBuffer = [];\n  }\n\n  processAudioInput = ({ frame, eol }: LipsyncInput) => {\n    if (!frame) {\n      this.audioOutput({ eol });\n      this.stop();\n      this.audioOutput = undefined as any;\n      return;\n    }\n\n    this.processInput(frame, this.audioBuffer, \"audio\");\n  };\n\n  processVideoInput = ({ frame, eol }: LipsyncInput) => {\n    if (!frame) {\n      this.videoOutput({ eol });\n      this.stop();\n      this.videoOutput = undefined as any;\n      return;\n    }\n\n    this.processInput(frame, this.videoBuffer, \"video\");\n  };\n\n  private processInput = (\n    frame: CodecFrame,\n    buffer: {\n      frame: CodecFrame;\n      kind: MediaKind;\n    }[][],\n    kind: MediaKind\n  ) => {\n    if (this.stopped) {\n      return;\n    }\n\n    if (this.baseTime == undefined) {\n      this.baseTime = frame.time;\n      this.currentTimestamp = this.baseTime;\n      this.lastExecutionTime = this.baseTime;\n      this.lastCommittedTime = this.baseTime;\n      this.lastFrameReceivedAt = Date.now();\n    }\n\n    // 過去のフレームを捨てる\n    if (frame.time < this.lastCommittedTime) {\n      return;\n    }\n\n    // NTPの同期ずれが疑われるので捨てる\n    const now = Date.now();\n    const gap = 5000; // RTCP SR interval;\n    const lastCommittedElapsed = frame.time - this.lastCommittedTime;\n    const lastFrameReceivedElapsed = now - this.lastFrameReceivedAt;\n    if (gap < lastFrameReceivedElapsed && lastCommittedElapsed < gap) {\n      this.internalStats[\"invalidFrameTime\"] = {\n        count: (this.internalStats[\"invalidFrameTime\"]?.count ?? 0) + 1,\n        at: new Date().toISOString(),\n        lastCommittedElapsed,\n        lastFrameReceivedElapsed,\n      };\n      return;\n    }\n    this.lastFrameReceivedAt = now;\n\n    const elapsed = frame.time - this.baseTime;\n    const index = int(elapsed / this.bufferDuration) % this.bufferLength;\n    buffer[index].push({\n      frame,\n      kind,\n    });\n\n    const diff = frame.time - this.lastExecutionTime;\n    if (diff >= this.interval) {\n      const times = int(diff / this.bufferDuration) - 1;\n      this.lastExecutionTime = this.currentTimestamp;\n      for (let i = 0; i < times; i++) {\n        this.executeTask();\n        this.lastExecutionTime += this.bufferDuration;\n      }\n    }\n    return;\n  };\n}\n\nexport interface LipSyncOptions {\n  /**ms */\n  syncInterval: number;\n  /**\n   * int\n   * @description syncInterval * bufferLength = max packet lifetime\n   * */\n  bufferLength: number;\n  fillDummyAudioPacket: Buffer;\n  ptime: number;\n}\n"]}

package/lib/rtp/src/processor/lipsyncCallback.d.ts

@@ -1,10 +1,13 @@
-import { LipsyncBase, LipSyncOptions, LipsyncOutput } from "./lipsync";
+import { LipsyncBase, LipsyncInput, LipSyncOptions, LipsyncOutput } from "./lipsync";
 export declare class LipsyncCallback extends LipsyncBase {
     private audioCb?;
+    private audioDestructor?;
     private videoCb?;
+    private videoDestructor?;
     constructor(options?: Partial<LipSyncOptions>);
-    pipeAudio: (cb: (input: LipsyncOutput) => void) => void;
-    pipeVideo: (cb: (input: LipsyncOutput) => void) => void;
-    inputAudio: ({ frame, eol }: import("./lipsync").LipsyncInput) => void;
-    inputVideo: ({ frame, eol }: import("./lipsync").LipsyncInput) => void;
+    pipeAudio: (cb: (input: LipsyncOutput) => void, destructor?: () => void) => void;
+    pipeVideo: (cb: (input: LipsyncOutput) => void, destructor?: () => void) => void;
+    inputAudio: (input: LipsyncInput) => void;
+    inputVideo: (input: LipsyncInput) => void;
+    destroy: () => void;
 }

package/lib/rtp/src/processor/lipsyncCallback.js

@@ -19,39 +19,74 @@ class LipsyncCallback extends lipsync_1.LipsyncBase {
             writable: true,
             value: void 0
         });
+        Object.defineProperty(this, "audioDestructor", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
         Object.defineProperty(this, "videoCb", {
             enumerable: true,
             configurable: true,
             writable: true,
             value: void 0
         });
+        Object.defineProperty(this, "videoDestructor", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
         Object.defineProperty(this, "pipeAudio", {
             enumerable: true,
             configurable: true,
             writable: true,
-            value: (cb) => {
+            value: (cb, destructor) => {
                 this.audioCb = cb;
+                this.audioDestructor = destructor;
             }
         });
         Object.defineProperty(this, "pipeVideo", {
             enumerable: true,
             configurable: true,
             writable: true,
-            value: (cb) => {
+            value: (cb, destructor) => {
                 this.videoCb = cb;
+                this.videoDestructor = destructor;
             }
         });
         Object.defineProperty(this, "inputAudio", {
             enumerable: true,
             configurable: true,
             writable: true,
-            value: this.processAudioInput
+            value: (input) => {
+                this.processAudioInput(input);
+            }
         });
         Object.defineProperty(this, "inputVideo", {
             enumerable: true,
             configurable: true,
             writable: true,
-            value: this.processVideoInput
+            value: (input) => {
+                this.processVideoInput(input);
+            }
+        });
+        Object.defineProperty(this, "destroy", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: () => {
+                if (this.audioDestructor) {
+                    this.audioDestructor();
+                    this.audioDestructor = undefined;
+                }
+                if (this.videoDestructor) {
+                    this.videoDestructor();
+                    this.videoDestructor = undefined;
+                }
+                this.audioCb = undefined;
+                this.videoCb = undefined;
+            }
         });
     }
 }

package/lib/rtp/src/processor/lipsyncCallback.js.map

@@ -1 +1 @@
-{"version":3,"file":"lipsyncCallback.js","sourceRoot":"","sources":["../../../../../rtp/src/processor/lipsyncCallback.ts"],"names":[],"mappings":";;;AAAA,uCAAuE;AAEvE,MAAa,eAAgB,SAAQ,qBAAW;IAG9C,YAAY,UAAmC,EAAE;QAC/C,KAAK,CACH,CAAC,MAAM,EAAE,EAAE;YACT,IAAI,IAAI,CAAC,OAAO,EAAE;gBAChB,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;aACtB;QACH,CAAC,EACD,CAAC,MAAM,EAAE,EAAE;YACT,IAAI,IAAI,CAAC,OAAO,EAAE;gBAChB,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;aACtB;QACH,CAAC,EACD,OAAO,CACR,CAAC;QAfJ;;;;;WAAiD;QACjD;;;;;WAAiD;QAiBjD;;;;mBAAY,CAAC,EAAkC,EAAE,EAAE;gBACjD,IAAI,CAAC,OAAO,GAAG,EAAE,CAAC;YACpB,CAAC;WAAC;QACF;;;;mBAAY,CAAC,EAAkC,EAAE,EAAE;gBACjD,IAAI,CAAC,OAAO,GAAG,EAAE,CAAC;YACpB,CAAC;WAAC;QAEF;;;;mBAAa,IAAI,CAAC,iBAAiB;WAAC;QACpC;;;;mBAAa,IAAI,CAAC,iBAAiB;WAAC;IAVpC,CAAC;CAWF;AA5BD,0CA4BC","sourcesContent":["import { LipsyncBase, LipSyncOptions, LipsyncOutput } from \"./lipsync\";\n\nexport class LipsyncCallback extends LipsyncBase {\n  private audioCb?: (input: LipsyncOutput) => void;\n  private videoCb?: (input: LipsyncOutput) => void;\n  constructor(options: Partial<LipSyncOptions> = {}) {\n    super(\n      (output) => {\n        if (this.audioCb) {\n          this.audioCb(output);\n        }\n      },\n      (output) => {\n        if (this.videoCb) {\n          this.videoCb(output);\n        }\n      },\n      options\n    );\n  }\n\n  pipeAudio = (cb: (input: LipsyncOutput) => void) => {\n    this.audioCb = cb;\n  };\n  pipeVideo = (cb: (input: LipsyncOutput) => void) => {\n    this.videoCb = cb;\n  };\n\n  inputAudio = this.processAudioInput;\n  inputVideo = this.processVideoInput;\n}\n"]}
+{"version":3,"file":"lipsyncCallback.js","sourceRoot":"","sources":["../../../../../rtp/src/processor/lipsyncCallback.ts"],"names":[],"mappings":";;;AAAA,uCAKmB;AAEnB,MAAa,eAAgB,SAAQ,qBAAW;IAK9C,YAAY,UAAmC,EAAE;QAC/C,KAAK,CACH,CAAC,MAAM,EAAE,EAAE;YACT,IAAI,IAAI,CAAC,OAAO,EAAE;gBAChB,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;aACtB;QACH,CAAC,EACD,CAAC,MAAM,EAAE,EAAE;YACT,IAAI,IAAI,CAAC,OAAO,EAAE;gBAChB,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;aACtB;QACH,CAAC,EACD,OAAO,CACR,CAAC;QAjBJ;;;;;WAAiD;QACjD;;;;;WAAqC;QACrC;;;;;WAAiD;QACjD;;;;;WAAqC;QAiBrC;;;;mBAAY,CAAC,EAAkC,EAAE,UAAuB,EAAE,EAAE;gBAC1E,IAAI,CAAC,OAAO,GAAG,EAAE,CAAC;gBAClB,IAAI,CAAC,eAAe,GAAG,UAAU,CAAC;YACpC,CAAC;WAAC;QACF;;;;mBAAY,CAAC,EAAkC,EAAE,UAAuB,EAAE,EAAE;gBAC1E,IAAI,CAAC,OAAO,GAAG,EAAE,CAAC;gBAClB,IAAI,CAAC,eAAe,GAAG,UAAU,CAAC;YACpC,CAAC;WAAC;QAEF;;;;mBAAa,CAAC,KAAmB,EAAE,EAAE;gBACnC,IAAI,CAAC,iBAAiB,CAAC,KAAK,CAAC,CAAC;YAChC,CAAC;WAAC;QACF;;;;mBAAa,CAAC,KAAmB,EAAE,EAAE;gBACnC,IAAI,CAAC,iBAAiB,CAAC,KAAK,CAAC,CAAC;YAChC,CAAC;WAAC;QAEF;;;;mBAAU,GAAG,EAAE;gBACb,IAAI,IAAI,CAAC,eAAe,EAAE;oBACxB,IAAI,CAAC,eAAe,EAAE,CAAC;oBACvB,IAAI,CAAC,eAAe,GAAG,SAAS,CAAC;iBAClC;gBACD,IAAI,IAAI,CAAC,eAAe,EAAE;oBACxB,IAAI,CAAC,eAAe,EAAE,CAAC;oBACvB,IAAI,CAAC,eAAe,GAAG,SAAS,CAAC;iBAClC;gBACD,IAAI,CAAC,OAAO,GAAG,SAAS,CAAC;gBACzB,IAAI,CAAC,OAAO,GAAG,SAAS,CAAC;YAC3B,CAAC;WAAC;IA7BF,CAAC;CA8BF;AAjDD,0CAiDC","sourcesContent":["import {\n  LipsyncBase,\n  LipsyncInput,\n  LipSyncOptions,\n  LipsyncOutput,\n} from \"./lipsync\";\n\nexport class LipsyncCallback extends LipsyncBase {\n  private audioCb?: (input: LipsyncOutput) => void;\n  private audioDestructor?: () => void;\n  private videoCb?: (input: LipsyncOutput) => void;\n  private videoDestructor?: () => void;\n  constructor(options: Partial<LipSyncOptions> = {}) {\n    super(\n      (output) => {\n        if (this.audioCb) {\n          this.audioCb(output);\n        }\n      },\n      (output) => {\n        if (this.videoCb) {\n          this.videoCb(output);\n        }\n      },\n      options\n    );\n  }\n\n  pipeAudio = (cb: (input: LipsyncOutput) => void, destructor?: () => void) => {\n    this.audioCb = cb;\n    this.audioDestructor = destructor;\n  };\n  pipeVideo = (cb: (input: LipsyncOutput) => void, destructor?: () => void) => {\n    this.videoCb = cb;\n    this.videoDestructor = destructor;\n  };\n\n  inputAudio = (input: LipsyncInput) => {\n    this.processAudioInput(input);\n  };\n  inputVideo = (input: LipsyncInput) => {\n    this.processVideoInput(input);\n  };\n\n  destroy = () => {\n    if (this.audioDestructor) {\n      this.audioDestructor();\n      this.audioDestructor = undefined;\n    }\n    if (this.videoDestructor) {\n      this.videoDestructor();\n      this.videoDestructor = undefined;\n    }\n    this.audioCb = undefined;\n    this.videoCb = undefined;\n  };\n}\n"]}

package/lib/rtp/src/processor/mute.d.ts

@@ -8,18 +8,31 @@ export declare class MuteHandlerBase implements Processor<MuteInput, MuteOutput>
     private props;
     readonly id: string;
     private buffer;
-    private started;
+    private index;
     private ended;
-    private intervalId;
+    private baseTime?;
     private currentTimestamp;
+    private internalStats;
+    /**ms */
+    private lastCommittedTime;
+    private lastExecutionTime;
+    /**ms */
+    private interval;
+    private bufferDuration;
+    private bufferLength;
+    /**ms */
+    private lastFrameReceivedAt;
     constructor(output: (o: MuteOutput) => void, props: {
         ptime: number;
         dummyPacket: Buffer;
+        /**ms
+         * @description intervalごとに無音区間に空パケットを挿入する
+         */
         interval: number;
-        sort?: boolean;
+        bufferLength: number;
     });
     toJSON(): Record<string, any>;
-    private startIfNeed;
+    private executeTask;
     private stop;
     processInput: ({ frame, eol }: MuteInput) => MuteOutput[];
 }