xgplayer-transmuxer 3.0.0-alpha.2

package/es/flv/fixer.js

@@ -0,0 +1,437 @@
+import _classCallCheck from '@babel/runtime/helpers/classCallCheck';
+import _createClass from '@babel/runtime/helpers/createClass';
+import 'core-js/modules/web.dom-collections.for-each.js';
+import 'core-js/modules/es.number.is-finite.js';
+import 'core-js/modules/es.number.constructor.js';
+import 'core-js/modules/es.array.splice.js';
+import 'core-js/modules/es.array.iterator.js';
+import 'core-js/modules/es.object.to-string.js';
+import 'core-js/modules/es.typed-array.uint8-array.js';
+import 'core-js/modules/es.typed-array.copy-within.js';
+import 'core-js/modules/es.typed-array.every.js';
+import 'core-js/modules/es.typed-array.fill.js';
+import 'core-js/modules/es.typed-array.filter.js';
+import 'core-js/modules/es.typed-array.find.js';
+import 'core-js/modules/es.typed-array.find-index.js';
+import 'core-js/modules/es.typed-array.for-each.js';
+import 'core-js/modules/es.typed-array.includes.js';
+import 'core-js/modules/es.typed-array.index-of.js';
+import 'core-js/modules/es.typed-array.iterator.js';
+import 'core-js/modules/es.typed-array.join.js';
+import 'core-js/modules/es.typed-array.last-index-of.js';
+import 'core-js/modules/es.typed-array.map.js';
+import 'core-js/modules/es.typed-array.reduce.js';
+import 'core-js/modules/es.typed-array.reduce-right.js';
+import 'core-js/modules/es.typed-array.reverse.js';
+import 'core-js/modules/es.typed-array.set.js';
+import 'core-js/modules/es.typed-array.slice.js';
+import 'core-js/modules/es.typed-array.some.js';
+import 'core-js/modules/es.typed-array.sort.js';
+import 'core-js/modules/es.typed-array.subarray.js';
+import 'core-js/modules/es.typed-array.to-locale-string.js';
+import 'core-js/modules/es.typed-array.to-string.js';
+import { WarningType, AudioCodecType } from '../model/types.js';
+import { AAC } from '../codec/aac.js';
+import { isSafari } from '../utils/env.js';
+import { AudioSample } from '../model/audio-sample.js';
+
+var LARGE_AV_FIRST_FRAME_GAP = 500; // ms
+
+var AUDIO_GAP_OVERLAP_THRESHOLD_COUNT = 3;
+var MAX_SILENT_FRAME_DURATION = 1000; // ms
+
+var AUDIO_EXCETION_LOG_EMIT_DURATION = 5000; // 5s
+
+var MAX_VIDEO_FRAME_DURATION = 1000; // ms
+
+var MAX_DTS_DELTA_WITH_NEXT_CHUNK = 200; // ms
+
+var VIDEO_EXCETION_LOG_EMIT_DURATION = 5000; // 5s
+
+var TRACK_BREACKED_CHECK_TIME = 5;
+var FlvFixer = /*#__PURE__*/function () {
+  function FlvFixer(videoTrack, audioTrack, metadataTrack) {
+    _classCallCheck(this, FlvFixer);
+
+    this.videoTrack = videoTrack;
+    this.audioTrack = audioTrack;
+    this.metadataTrack = metadataTrack;
+    this._baseDts = -1;
+    this._baseDtsInited = false;
+    this._audioNextPts = undefined;
+    this._videoNextDts = undefined;
+    this._audioTimestampBreak = 0;
+    this._videoTimestampBreak = 0;
+    this._lastAudioExceptionGapDot = -Infinity;
+    this._lastAudioExceptionOverlapDot = -Infinity;
+    this._lastAudioExceptionLargeGapDot = -Infinity;
+    this._lastVideoExceptionLargeGapDot = -Infinity;
+    this._lastVideoExceptionChunkFirstDtsDot = -Infinity;
+  }
+  /**
+   * @param {number} startTime 点播seek到的时间点
+   * @param {boolean} discontinuity 是否换流
+   * @param {boolean} contiguous 前后chunk时间戳是否连续
+   */
+
+
+  _createClass(FlvFixer, [{
+    key: "fix",
+    value: function fix() {
+      var _this = this;
+
+      var startTime = arguments.length > 0 && arguments[0] !== undefined ? arguments[0] : 0;
+      var discontinuity = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : false;
+      var contiguous = arguments.length > 2 && arguments[2] !== undefined ? arguments[2] : true;
+      startTime = Math.round(startTime * 1000);
+      var videoTrack = this.videoTrack;
+      var audioTrack = this.audioTrack;
+
+      if (discontinuity || !contiguous) {
+        this._videoLastSample = null;
+        this._audioNextPts = undefined;
+        this._videoNextDts = undefined;
+        this._audioTimestampBreak = 0;
+        this._videoTimestampBreak = 0;
+        this._lastAudioExceptionGapDot = -Infinity;
+        this._lastAudioExceptionOverlapDot = -Infinity;
+        this._lastAudioExceptionLargeGapDot = -Infinity;
+        this._lastVideoExceptionLargeGapDot = -Infinity;
+        this._lastVideoExceptionChunkFirstDtsDot = -Infinity;
+      }
+
+      if (discontinuity && !contiguous) {
+        this._baseDtsInited = false;
+      }
+
+      if (!this._baseDtsInited) {
+        this._calculateBaseDts(audioTrack, videoTrack);
+      }
+
+      if (!this._baseDtsInited) return;
+
+      if (!contiguous && startTime) {
+        this._audioNextPts = this._videoNextDts = startTime;
+      }
+
+      this._resetBaseDtsWhenStreamBreaked();
+
+      this._fixAudio(audioTrack);
+
+      this._fixVideo(videoTrack);
+
+      if (this.metadataTrack.exist()) {
+        var timescale = this.metadataTrack.timescale;
+        this.metadataTrack.seiSamples.forEach(function (s) {
+          s.pts = s.originPts - _this._baseDts;
+          s.time = Math.max(0, s.pts) / timescale;
+        });
+        this.metadataTrack.flvScriptSamples.forEach(function (s) {
+          s.pts = s.originPts - _this._baseDts;
+          s.time = Math.max(0, s.pts) / timescale;
+        });
+      }
+
+      if (videoTrack.samples.length) {
+        videoTrack.baseMediaDecodeTime = videoTrack.samples[0].dts;
+      }
+
+      if (audioTrack.samples.length) {
+        audioTrack.baseMediaDecodeTime = audioTrack.samples[0].pts * audioTrack.timescale / 1000;
+      }
+    }
+  }, {
+    key: "_fixVideo",
+    value: function _fixVideo(videoTrack) {
+      var _this2 = this;
+
+      var samples = videoTrack.samples;
+      if (!samples.length) return;
+      samples.forEach(function (x) {
+        x.dts -= _this2._baseDts;
+        x.pts -= _this2._baseDts;
+      });
+      var refSampleDurationInt;
+
+      if (videoTrack.fpsNum && videoTrack.fpsDen) {
+        refSampleDurationInt = videoTrack.timescale * (videoTrack.fpsDen / videoTrack.fpsNum);
+      } else {
+        var first = videoTrack.samples[0];
+        var last = videoTrack.samples[samples.length - 1];
+        refSampleDurationInt = Math.floor((last.dts - first.dts) / (samples.length - 1));
+      }
+
+      var lastSample = samples.pop();
+
+      if (this._videoLastSample) {
+        samples.unshift(this._videoLastSample);
+      }
+
+      this._videoLastSample = lastSample;
+      if (!samples.length) return;
+
+      if (this._videoNextDts === undefined) {
+        var samp0 = samples[0];
+        this._videoNextDts = samp0.dts;
+      }
+
+      var len = samples.length;
+      var sampleDuration = 0;
+      var firstSample = samples[0];
+      var vDelta = this._videoNextDts - firstSample.dts;
+
+      if (Math.abs(vDelta) > MAX_DTS_DELTA_WITH_NEXT_CHUNK) {
+        // emit large delta of first sample with expect
+        if (Math.abs(firstSample.dts - this._lastVideoExceptionChunkFirstDtsDot) > VIDEO_EXCETION_LOG_EMIT_DURATION) {
+          var _samples$;
+
+          this._lastVideoExceptionChunkFirstDtsDot = firstSample.dts;
+          videoTrack.warnings.push({
+            type: WarningType.LARGE_VIDEO_GAP_BETWEEN_CHUNK,
+            nextDts: this._videoNextDts,
+            firstSampleDts: firstSample.dts,
+            nextSampleDts: (_samples$ = samples[1]) === null || _samples$ === void 0 ? void 0 : _samples$.dts,
+            sampleDuration: vDelta
+          });
+        } // only video breaked
+
+
+        if (this._videoTimestampBreak >= TRACK_BREACKED_CHECK_TIME) {
+          this._videoNextDts = firstSample.dts;
+          this._videoTimestampBreak = 0;
+        } else {
+          // resolve first frame only
+          firstSample.dts += vDelta;
+          firstSample.pts += vDelta;
+        }
+      }
+
+      for (var i = 0; i < len; i++) {
+        var dts = samples[i].dts;
+        var nextSample = samples[i + 1];
+
+        if (i < len - 1) {
+          sampleDuration = nextSample.dts - dts;
+        } else if (lastSample) {
+          sampleDuration = lastSample.dts - dts;
+        } else {
+          sampleDuration = refSampleDurationInt;
+        }
+
+        if (sampleDuration > MAX_VIDEO_FRAME_DURATION || sampleDuration < 0) {
+          this._videoTimestampBreak++; // emit stream breaked
+
+          if (Math.abs(dts - this._lastVideoExceptionLargeGapDot) > VIDEO_EXCETION_LOG_EMIT_DURATION) {
+            this._lastVideoExceptionLargeGapDot = dts;
+            videoTrack.warnings.push({
+              type: WarningType.LARGE_VIDEO_GAP,
+              time: dts / videoTrack.timescale,
+              dts: dts,
+              originDts: samples[i].originDts,
+              nextDts: this._videoNextDts,
+              sampleDuration: sampleDuration,
+              refSampleDuration: refSampleDurationInt
+            });
+          }
+
+          sampleDuration = refSampleDurationInt;
+        }
+
+        samples[i].duration = sampleDuration;
+        this._videoNextDts += sampleDuration;
+      }
+    }
+  }, {
+    key: "_fixAudio",
+    value: function _fixAudio(audioTrack) {
+      var _this3 = this;
+
+      var samples = audioTrack.samples;
+      if (!samples.length) return; // offset origin timestamp
+
+      samples.forEach(function (x) {
+        x.dts = x.pts -= _this3._baseDts;
+      });
+
+      this._doFixAudioInternal(audioTrack, samples, 1000);
+    }
+  }, {
+    key: "_calculateBaseDts",
+    value: function _calculateBaseDts(audioTrack, videoTrack) {
+      var audioSamps = audioTrack.samples;
+      var videoSamps = videoTrack.samples;
+
+      if (!audioSamps.length && !videoSamps.length) {
+        return false;
+      }
+
+      var audioBasePts = Infinity;
+      var videoBaseDts = Infinity;
+
+      if (audioSamps.length) {
+        audioTrack.baseDts = audioBasePts = audioSamps[0].pts;
+      }
+
+      if (videoSamps.length) {
+        videoTrack.baseDts = videoBaseDts = videoSamps[0].dts;
+      }
+
+      this._baseDts = Math.min(audioBasePts, videoBaseDts);
+      var delta = videoBaseDts - audioBasePts;
+
+      if (Number.isFinite(delta) && Math.abs(delta) > LARGE_AV_FIRST_FRAME_GAP) {
+        videoTrack.warnings.push({
+          type: WarningType.LARGE_AV_SHIFT,
+          videoBaseDts: videoBaseDts,
+          audioBasePts: audioBasePts,
+          baseDts: this._baseDts,
+          delta: delta
+        });
+      }
+
+      this._baseDtsInited = true;
+      return true;
+    }
+  }, {
+    key: "_resetBaseDtsWhenStreamBreaked",
+    value: function _resetBaseDtsWhenStreamBreaked() {
+      if (this._baseDtsInited && this._videoTimestampBreak && this._audioTimestampBreak) {
+        /**
+         * timestamp breaked
+         *                     _audioNextDts
+         *  ---------------------|
+         * (_baseDts)          _videoNextDts
+         * ----------------------|
+         *                        <----------------
+         *                                       nextVideo.dts
+         * ----------------------------------------|
+         *                                       nextAudio.dts
+         * ---------------------------------------|
+         */
+        // calc baseDts base on new samples
+        var calc = this._calculateBaseDts(this.audioTrack, this.videoTrack);
+
+        if (!calc) return; // consider the expect dts for next frame
+
+        this._baseDts -= Math.min(this._audioNextPts, this._videoNextDts);
+        this._videoLastSample = null;
+        this._videoTimestampBreak = 0;
+        this._audioTimestampBreak = 0;
+      }
+    }
+  }, {
+    key: "_doFixAudioInternal",
+    value: function _doFixAudioInternal(audioTrack, samples, timescale) {
+      if (!audioTrack.sampleDuration) {
+        audioTrack.sampleDuration = audioTrack.codecType === AudioCodecType.AAC ? AAC.getFrameDuration(audioTrack.timescale, timescale) : this._getG711Duration(audioTrack);
+      }
+
+      var refSampleDuration = audioTrack.sampleDuration;
+      var sampleDurationInSampleRate = audioTrack.codecType === AudioCodecType.AAC ? 1024 : refSampleDuration * audioTrack.timescale / 1000;
+
+      if (this._audioNextPts === undefined) {
+        var samp0 = samples[0];
+        this._audioNextPts = samp0.pts;
+      }
+
+      for (var i = 0; i < samples.length; i++) {
+        var nextPts = this._audioNextPts;
+        var sample = samples[i];
+        var delta = sample.pts - nextPts; // only audio breaked
+
+        if (i === 0 && this._audioTimestampBreak >= TRACK_BREACKED_CHECK_TIME) {
+          nextPts = this._audioNextPts = sample.dts;
+          delta = 0;
+          this._audioTimestampBreak = 0;
+        } // fill frames
+        // delta >= 3 * refSampleDurationInt
+        // delta <= 500s
+
+
+        if (!this._audioTimestampBreak && delta >= AUDIO_GAP_OVERLAP_THRESHOLD_COUNT * refSampleDuration && delta <= MAX_SILENT_FRAME_DURATION && !isSafari) {
+          var silentFrame = this._getSilentFrame(audioTrack) || samples[0].data.subarray();
+          var count = Math.floor(delta / refSampleDuration);
+
+          if (Math.abs(sample.pts - this._lastAudioExceptionGapDot) > AUDIO_EXCETION_LOG_EMIT_DURATION) {
+            this._lastAudioExceptionGapDot = sample.pts;
+            audioTrack.warnings.push({
+              type: WarningType.AUDIO_FILLED,
+              pts: sample.pts,
+              originPts: sample.originPts,
+              count: count,
+              nextPts: nextPts,
+              refSampleDuration: refSampleDuration
+            });
+          }
+
+          for (var j = 0; j < count; j++) {
+            var silentSample = new AudioSample(Math.floor(this._audioNextPts + refSampleDuration) - Math.floor(this._audioNextPts), silentFrame, sampleDurationInSampleRate);
+            silentSample.originPts = Math.floor(this._baseDts + nextPts);
+            samples.splice(i, 0, silentSample);
+            this._audioNextPts += refSampleDuration;
+            i++;
+          }
+
+          i--; // delta  <= -3 * refSampleDurationInt
+          // delta  >= -500ms
+        } else if (delta <= -AUDIO_GAP_OVERLAP_THRESHOLD_COUNT * refSampleDuration && delta >= -1 * MAX_SILENT_FRAME_DURATION) {
+          // need discard frames
+          if (Math.abs(sample.pts - this._lastAudioExceptionOverlapDot) > AUDIO_EXCETION_LOG_EMIT_DURATION) {
+            this._lastAudioExceptionOverlapDot = sample.pts;
+            audioTrack.warnings.push({
+              type: WarningType.AUDIO_DROPPED,
+              pts: sample.pts,
+              originPts: sample.originPts,
+              nextPts: nextPts,
+              refSampleDuration: refSampleDuration
+            });
+          }
+
+          samples.splice(i, 1);
+          i--;
+        } else {
+          if (Math.abs(delta) > MAX_SILENT_FRAME_DURATION) {
+            this._audioTimestampBreak++;
+
+            if (Math.abs(sample.pts - this._lastAudioExceptionLargeGapDot) > AUDIO_EXCETION_LOG_EMIT_DURATION) {
+              this._lastAudioExceptionLargeGapDot = sample.pts;
+              audioTrack.warnings.push({
+                type: WarningType.LARGE_AUDIO_GAP,
+                time: sample.pts / 1000,
+                pts: sample.pts,
+                originPts: sample.originPts,
+                nextPts: nextPts,
+                sampleDuration: delta,
+                refSampleDuration: refSampleDuration
+              });
+            }
+          }
+
+          sample.dts = sample.pts = nextPts;
+          sample.duration = sampleDurationInSampleRate;
+          this._audioNextPts += refSampleDuration;
+        }
+      }
+    }
+  }, {
+    key: "_getG711Duration",
+    value: function _getG711Duration(track) {
+      var sampleSize = track.sampleSize,
+          channelCount = track.channelCount,
+          sampleRate = track.sampleRate;
+      var samp0 = track.samples[0];
+      if (!samp0) return;
+      return samp0.data.byteLength * 2 / channelCount / (sampleSize / 8) / sampleRate * 1000;
+    }
+  }, {
+    key: "_getSilentFrame",
+    value: function _getSilentFrame(track) {
+      if (track.codecType === AudioCodecType.AAC) return AAC.getSilentFrame(track.codec, track.channelCount);
+      return new Uint8Array(8 * track.sampleDuration * track.channelCount);
+    }
+  }]);
+
+  return FlvFixer;
+}();
+
+export { FlvFixer };

package/es/flv/index.d.ts

@@ -0,0 +1,65 @@
+/**
+ * @typedef {Object} DemuxResult
+ * @property {VideoTrack} videoTrack
+ * @property {AudioTrack} audioTrack
+ * @property {MetadataTrack} metadataTrack
+ */
+export class FlvDemuxer {
+    static AUDIO_RATE: number[];
+    /**
+     * @param { Uint8Array } data
+     * @returns {boolean}
+     */
+    static probe(data: Uint8Array): boolean;
+    /**
+     * @param {VideoTrack} [videoTrack]
+     * @param {AudioTrack} [audioTrack]
+     * @param {MetadataTrack} [metadataTrack]
+     */
+    constructor(videoTrack?: VideoTrack, audioTrack?: AudioTrack, metadataTrack?: MetadataTrack);
+    _headerParsed: boolean;
+    _remainingData: any;
+    _gopId: number;
+    _needAddMetaBeforeKeyFrameNal: boolean;
+    videoTrack: VideoTrack;
+    audioTrack: AudioTrack;
+    metadataTrack: MetadataTrack;
+    _fixer: FlvFixer;
+    /**
+     * @param {Uint8Array} data
+     * @param {boolean} [discontinuity=false]
+     * @param {boolean} [contiguous=true]
+     * @returns {DemuxResult}
+     */
+    demux(data: Uint8Array, discontinuity?: boolean, contiguous?: boolean): DemuxResult;
+    /**
+     * @param {number} [startTime=0]
+     * @param {boolean} [discontinuity=false]
+     * @param {boolean} [contiguous=true]
+     * @returns {DemuxResult}
+     */
+    fix(startTime?: number, discontinuity?: boolean, contiguous?: boolean): DemuxResult;
+    /**
+     * @param {Uint8Array} data
+     * @param {boolean} [discontinuity=false]
+     * @param {boolean} [contiguous=true]
+     * @param {number} [startTime=0]
+     * @returns {DemuxResult}
+     */
+    demuxAndFix(data: Uint8Array, discontinuity?: boolean, contiguous?: boolean, startTime?: number): DemuxResult;
+    _parseAudio(data: any, pts: any): void;
+    _parseG711(data: any, pts: any, format: any): void;
+    _parseAac(data: any, pts: any): void;
+    _parseVideo(data: any, dts: any): void;
+    _checkAddMetaNalToUnits(hevc: any, units: any, track: any): any;
+    _parseScript(data: any, pts: any): void;
+}
+export type DemuxResult = {
+    videoTrack: VideoTrack;
+    audioTrack: AudioTrack;
+    metadataTrack: MetadataTrack;
+};
+import { VideoTrack } from "../model";
+import { AudioTrack } from "../model";
+import { MetadataTrack } from "../model/metadata-track";
+import { FlvFixer } from "./fixer";