shaka-player 4.4.3 → 4.6.0

package/lib/media/segment_utils.js

@@ -0,0 +1,343 @@
+/*! @license
+ * Shaka Player
+ * Copyright 2016 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+goog.provide('shaka.media.SegmentUtils');
+
+goog.require('goog.asserts');
+goog.require('shaka.log');
+goog.require('shaka.media.ClosedCaptionParser');
+goog.require('shaka.util.BufferUtils');
+goog.require('shaka.util.MimeUtils');
+goog.require('shaka.util.Mp4BoxParsers');
+goog.require('shaka.util.Mp4Parser');
+goog.require('shaka.util.TsParser');
+
+
+/**
+ * @summary Utility functions for segment parsing.
+ */
+shaka.media.SegmentUtils = class {
+  /**
+   * @param {string} mimeType
+   * @return {shaka.media.SegmentUtils.BasicInfo}
+   */
+  static getBasicInfoFromMimeType(mimeType) {
+    const baseMimeType = shaka.util.MimeUtils.getBasicType(mimeType);
+    const type = baseMimeType.split('/')[0];
+    const codecs = shaka.util.MimeUtils.getCodecs(mimeType);
+    return {
+      type: type,
+      mimeType: baseMimeType,
+      codecs: codecs,
+      language: null,
+      height: null,
+      width: null,
+      channelCount: null,
+      sampleRate: null,
+      closedCaptions: new Map(),
+    };
+  }
+
+  /**
+   * @param {!BufferSource} data
+   * @return {?shaka.media.SegmentUtils.BasicInfo}
+   */
+  static getBasicInfoFromTs(data) {
+    const uint8ArrayData = shaka.util.BufferUtils.toUint8(data);
+    const tsParser = new shaka.util.TsParser().parse(uint8ArrayData);
+    const tsCodecs = tsParser.getCodecs();
+    const videoInfo = tsParser.getVideoInfo();
+    const codecs = [];
+    let hasAudio = false;
+    let hasVideo = false;
+    switch (tsCodecs.audio) {
+      case 'aac':
+        codecs.push('mp4a.40.2');
+        hasAudio = true;
+        break;
+      case 'mp3':
+        codecs.push('mp4a.40.34');
+        hasAudio = true;
+        break;
+      case 'ac3':
+        codecs.push('ac-3');
+        hasAudio = true;
+        break;
+      case 'ec3':
+        codecs.push('ec-3');
+        hasAudio = true;
+        break;
+    }
+    switch (tsCodecs.video) {
+      case 'avc':
+        if (videoInfo.codec) {
+          codecs.push(videoInfo.codec);
+        } else {
+          codecs.push('avc1.42E01E');
+        }
+        hasVideo = true;
+        break;
+      case 'hvc':
+        if (videoInfo.codec) {
+          codecs.push(videoInfo.codec);
+        } else {
+          codecs.push('hvc1.1.6.L93.90');
+        }
+        hasVideo = true;
+        break;
+    }
+    if (!codecs.length) {
+      return null;
+    }
+    const onlyAudio = hasAudio && !hasVideo;
+    const closedCaptions = new Map();
+    if (hasVideo) {
+      const captionParser = new shaka.media.ClosedCaptionParser('video/mp2t');
+      captionParser.parseFrom(data);
+      for (const stream of captionParser.getStreams()) {
+        closedCaptions.set(stream, stream);
+      }
+      captionParser.reset();
+    }
+    return {
+      type: onlyAudio ? 'audio' : 'video',
+      mimeType: 'video/mp2t',
+      codecs: codecs.join(', '),
+      language: null,
+      height: videoInfo.height,
+      width: videoInfo.width,
+      channelCount: null,
+      sampleRate: null,
+      closedCaptions: closedCaptions,
+    };
+  }
+
+  /**
+   * @param {?BufferSource} initData
+   * @param {!BufferSource} data
+   * @return {?shaka.media.SegmentUtils.BasicInfo}
+   */
+  static getBasicInfoFromMp4(initData, data) {
+    const Mp4Parser = shaka.util.Mp4Parser;
+    const SegmentUtils = shaka.media.SegmentUtils;
+
+    const codecs = [];
+
+    let hasAudio = false;
+    let hasVideo = false;
+
+    const addCodec = (codec) => {
+      const codecLC = codec.toLowerCase();
+      switch (codecLC) {
+        case 'avc1':
+        case 'avc3':
+          codecs.push(codecLC + '.42E01E');
+          hasVideo = true;
+          break;
+        case 'hev1':
+        case 'hvc1':
+          codecs.push(codecLC + '.1.6.L93.90');
+          hasVideo = true;
+          break;
+        case 'dvh1':
+        case 'dvhe':
+          codecs.push(codecLC + '.05.04');
+          hasVideo = true;
+          break;
+        case 'vp09':
+          codecs.push(codecLC + '.00.10.08');
+          hasVideo = true;
+          break;
+        case 'av01':
+          codecs.push(codecLC + '.0.01M.08');
+          hasVideo = true;
+          break;
+        case 'mp4a':
+          // We assume AAC, but this can be wrong since mp4a supports
+          // others codecs
+          codecs.push('mp4a.40.2');
+          hasAudio = true;
+          break;
+        case 'ac-3':
+        case 'ec-3':
+        case 'opus':
+        case 'flac':
+          codecs.push(codecLC);
+          hasAudio = true;
+          break;
+      }
+    };
+
+    const codecBoxParser = (box) => addCodec(box.name);
+
+    /** @type {?string} */
+    let language = null;
+    /** @type {?string} */
+    let height = null;
+    /** @type {?string} */
+    let width = null;
+    /** @type {?number} */
+    let channelCount = null;
+    /** @type {?number} */
+    let sampleRate = null;
+
+    new Mp4Parser()
+        .box('moov', Mp4Parser.children)
+        .box('trak', Mp4Parser.children)
+        .fullBox('tkhd', (box) => {
+          goog.asserts.assert(
+              box.version != null,
+              'TKHD is a full box and should have a valid version.');
+          const parsedTKHDBox = shaka.util.Mp4BoxParsers.parseTKHD(
+              box.reader, box.version);
+          height = String(parsedTKHDBox.height);
+          width = String(parsedTKHDBox.width);
+        })
+        .box('mdia', Mp4Parser.children)
+        .fullBox('mdhd', (box) => {
+          goog.asserts.assert(
+              box.version != null,
+              'MDHD is a full box and should have a valid version.');
+          const parsedMDHDBox = shaka.util.Mp4BoxParsers.parseMDHD(
+              box.reader, box.version);
+          language = parsedMDHDBox.language;
+        })
+        .box('minf', Mp4Parser.children)
+        .box('stbl', Mp4Parser.children)
+        .fullBox('stsd', Mp4Parser.sampleDescription)
+
+        // AUDIO
+        // These are the various boxes that signal a codec.
+        .box('mp4a', (box) => {
+          const parsedMP4ABox = shaka.util.Mp4BoxParsers.parseMP4A(box.reader);
+          channelCount = parsedMP4ABox.channelCount;
+          sampleRate = parsedMP4ABox.sampleRate;
+          if (box.reader.hasMoreData()) {
+            Mp4Parser.children(box);
+          } else {
+            codecBoxParser(box);
+          }
+        })
+        .box('esds', (box) => {
+          const parsedESDSBox = shaka.util.Mp4BoxParsers.parseESDS(box.reader);
+          codecs.push(parsedESDSBox.codec);
+          hasAudio = true;
+        })
+        .box('ac-3', codecBoxParser)
+        .box('ec-3', codecBoxParser)
+        .box('opus', codecBoxParser)
+        .box('Opus', codecBoxParser)
+        .box('fLaC', codecBoxParser)
+
+        // VIDEO
+        // These are the various boxes that signal a codec.
+        .box('avc1', (box) => {
+          const parsedAVCBox =
+              shaka.util.Mp4BoxParsers.parseAVC(box.reader, box.name);
+          codecs.push(parsedAVCBox.codec);
+          hasVideo = true;
+        })
+        .box('avc3', (box) => {
+          const parsedAVCBox =
+              shaka.util.Mp4BoxParsers.parseAVC(box.reader, box.name);
+          codecs.push(parsedAVCBox.codec);
+          hasVideo = true;
+        })
+        .box('hev1', codecBoxParser)
+        .box('hvc1', codecBoxParser)
+        .box('dvh1', codecBoxParser)
+        .box('dvhe', codecBoxParser)
+        .box('vp09', codecBoxParser)
+        .box('av01', codecBoxParser)
+
+        // This signals an encrypted sample, which we can go inside of to
+        // find the codec used.
+        // Note: If encrypted, you can only have audio or video, not both.
+        .box('enca', Mp4Parser.visualSampleEntry)
+        .box('encv', Mp4Parser.visualSampleEntry)
+        .box('sinf', Mp4Parser.children)
+        .box('frma', (box) => {
+          const {codec} = shaka.util.Mp4BoxParsers.parseFRMA(box.reader);
+          addCodec(codec);
+        })
+
+        .parse(initData || data, /* partialOkay= */ true);
+    if (!codecs.length) {
+      return null;
+    }
+    const onlyAudio = hasAudio && !hasVideo;
+    const closedCaptions = new Map();
+    if (hasVideo) {
+      const captionParser = new shaka.media.ClosedCaptionParser('video/mp4');
+      if (initData) {
+        captionParser.init(initData);
+      }
+      captionParser.parseFrom(data);
+      for (const stream of captionParser.getStreams()) {
+        closedCaptions.set(stream, stream);
+      }
+      captionParser.reset();
+    }
+    return {
+      type: onlyAudio ? 'audio' : 'video',
+      mimeType: onlyAudio ? 'audio/mp4' : 'video/mp4',
+      codecs: SegmentUtils.filterDuplicateCodecs_(codecs).join(', '),
+      language: language,
+      height: height,
+      width: width,
+      channelCount: channelCount,
+      sampleRate: sampleRate,
+      closedCaptions: closedCaptions,
+    };
+  }
+
+  /**
+   * @param {!Array.<string>} codecs
+   * @return {!Array.<string>} codecs
+   * @private
+   */
+  static filterDuplicateCodecs_(codecs) {
+    // Filter out duplicate codecs.
+    const seen = new Set();
+    const ret = [];
+    for (const codec of codecs) {
+      const shortCodec = shaka.util.MimeUtils.getCodecBase(codec);
+      if (!seen.has(shortCodec)) {
+        ret.push(codec);
+        seen.add(shortCodec);
+      } else {
+        shaka.log.debug('Ignoring duplicate codec');
+      }
+    }
+    return ret;
+  }
+};
+
+
+/**
+ * @typedef {{
+ *   type: string,
+ *   mimeType: string,
+ *   codecs: string,
+ *   language: ?string,
+ *   height: ?string,
+ *   width: ?string,
+ *   channelCount: ?number,
+ *   sampleRate: ?number,
+ *   closedCaptions: Map.<string, string>
+ * }}
+ *
+ * @property {string} type
+ * @property {string} mimeType
+ * @property {string} codecs
+ * @property {?string} language
+ * @property {?string} height
+ * @property {?string} width
+ * @property {?number} channelCount
+ * @property {?number} sampleRate
+ * @property {Map.<string, string>} closedCaptions
+ */
+shaka.media.SegmentUtils.BasicInfo;

package/lib/media/stall_detector.js

@@ -174,9 +174,6 @@ shaka.media.StallDetector.MediaElementImplementation = class {
     if (this.mediaElement_.paused) {
       return false;
     }
-    if (this.mediaElement_.playbackRate == 0) {
-      return false;
-    }
 
     // If we have don't have enough content, we are not stalled, we are
     // buffering.

package/lib/media/streaming_engine.js

@@ -1054,6 +1054,14 @@ shaka.media.StreamingEngine = class {
       this.playerInterface_.mediaSourceEngine.clearSelectedClosedCaptionId();
     }
 
+    if (!this.playerInterface_.mediaSourceEngine.isStreamingAllowed() &&
+        mediaState.type != ContentType.TEXT) {
+      // It is not allowed to add segments yet, so we schedule an update to
+      // check again later. So any prediction we make now could be terribly
+      // invalid soon.
+      return this.config_.updateIntervalSeconds / 2;
+    }
+
     const logPrefix = shaka.media.StreamingEngine.logPrefix_(mediaState);
 
     // Compute how far we've buffered ahead of the playhead.
@@ -1163,6 +1171,12 @@ shaka.media.StreamingEngine = class {
     }
 
     if (mediaState.segmentPrefetch && mediaState.segmentIterator) {
+      const initSegmentReference = reference.initSegmentReference;
+      if (initSegmentReference && (!mediaState.lastSegmentReference ||
+          !shaka.media.InitSegmentReference.equal(
+              initSegmentReference, mediaState.lastInitSegmentReference))) {
+        mediaState.segmentPrefetch.prefetchInitSegment(initSegmentReference);
+      }
       mediaState.segmentPrefetch.prefetchSegments(reference);
     }
 
@@ -1373,6 +1387,11 @@ shaka.media.StreamingEngine = class {
             await this.append_(
                 mediaState, presentationTime, stream, reference, dataToAppend,
                 /* isChunkedData= */ true);
+
+            if (mediaState.segmentPrefetch && mediaState.segmentIterator) {
+              mediaState.segmentPrefetch.prefetchSegments(
+                  reference, /* skipFirst= */ true);
+            }
           }
         };
 
@@ -1403,6 +1422,11 @@ shaka.media.StreamingEngine = class {
           await this.append_(
               mediaState, presentationTime, stream, reference, result);
         }
+
+        if (mediaState.segmentPrefetch && mediaState.segmentIterator) {
+          mediaState.segmentPrefetch.prefetchSegments(
+              reference, /* skipFirst= */ true);
+        }
       } else {
         if (this.config_.lowLatencyMode && !isReadableStreamSupported) {
           shaka.log.warning('Low latency streaming mode is enabled, but ' +
@@ -1498,12 +1522,12 @@ shaka.media.StreamingEngine = class {
 
   /**
    * @param {!BufferSource} rawResult
-   * @param {shaka.extern.HlsAes128Key} aes128Key
-   * @param {!shaka.media.SegmentIterator} iter
+   * @param {shaka.extern.aes128Key} aes128Key
+   * @param {number} position
    * @return {!Promise.<!BufferSource>} finalResult
    * @private
    */
-  async aes128Decrypt_(rawResult, aes128Key, iter) {
+  async aes128Decrypt_(rawResult, aes128Key, position) {
     const key = aes128Key;
     if (!key.cryptoKey) {
       goog.asserts.assert(key.fetchKey, 'If AES-128 cryptoKey was not ' +
@@ -1514,7 +1538,7 @@ shaka.media.StreamingEngine = class {
     let iv = key.iv;
     if (!iv) {
       iv = shaka.util.BufferUtils.toUint8(new ArrayBuffer(16));
-      let sequence = key.firstMediaSequenceNumber + iter.currentPosition();
+      let sequence = key.firstMediaSequenceNumber + position;
       for (let i = iv.byteLength - 1; i >= 0; i--) {
         iv[i] = sequence & 0xff;
         sequence >>= 8;
@@ -1684,22 +1708,21 @@ shaka.media.StreamingEngine = class {
       shaka.log.v1(logPrefix,
           'setting append window end to ' + appendWindowEnd);
 
-      if ((mediaState.lastCodecs && codecs != mediaState.lastCodecs) ||
-          (mediaState.lastMimeType && mimeType != mediaState.lastMimeType)) {
+      const isResetMediaSourceNecessary =
+          mediaState.lastCodecs && mediaState.lastMimeType &&
+          this.playerInterface_.mediaSourceEngine.isResetMediaSourceNecessary(
+              mediaState.type, mediaState.stream);
+      if (isResetMediaSourceNecessary) {
+        let otherState = null;
         if (mediaState.type === ContentType.VIDEO) {
-          /** @type {shaka.media.StreamingEngine.MediaState_} */
-          const audioState = this.mediaStates_.get(ContentType.AUDIO);
-          if (audioState) {
-            audioState.lastInitSegmentReference = null;
-            this.abortOperations_(audioState);
-          }
+          otherState = this.mediaStates_.get(ContentType.AUDIO);
         } else if (mediaState.type === ContentType.AUDIO) {
-          /** @type {shaka.media.StreamingEngine.MediaState_} */
-          const videoState = this.mediaStates_.get(ContentType.VIDEO);
-          if (videoState) {
-            videoState.lastInitSegmentReference = null;
-            this.abortOperations_(videoState);
-          }
+          otherState = this.mediaStates_.get(ContentType.VIDEO);
+        }
+        if (otherState) {
+          otherState.lastInitSegmentReference = null;
+          this.forceClearBuffer_(otherState);
+          this.abortOperations_(otherState).catch(() => {});
         }
       }
 
@@ -1729,6 +1752,19 @@ shaka.media.StreamingEngine = class {
               mediaState.type, timestampOffset, appendWindowStart,
               appendWindowEnd, ignoreTimestampOffset,
               mediaState.stream, streamsByType);
+          if (isResetMediaSourceNecessary) {
+            let otherState = null;
+            if (mediaState.type === ContentType.VIDEO) {
+              otherState = this.mediaStates_.get(ContentType.AUDIO);
+            } else if (mediaState.type === ContentType.AUDIO) {
+              otherState = this.mediaStates_.get(ContentType.VIDEO);
+            }
+            if (otherState) {
+              otherState.waitingToClearBuffer = false;
+              otherState.performingUpdate = false;
+              this.forceClearBuffer_(otherState);
+            }
+          }
         } catch (error) {
           mediaState.lastAppendWindowStart = null;
           mediaState.lastAppendWindowEnd = null;
@@ -1738,7 +1774,10 @@ shaka.media.StreamingEngine = class {
           throw error;
         }
       };
-      operations.push(setProperties());
+      // Dispatching init asynchronously causes the sourceBuffers in
+      // the MediaSourceEngine to become detached do to race conditions
+      // with mediaSource and sourceBuffers being created simultaneously.
+      await setProperties();
     }
 
     if (!shaka.media.InitSegmentReference.equal(
@@ -1749,8 +1788,7 @@ shaka.media.StreamingEngine = class {
         shaka.log.v1(logPrefix, 'fetching init segment');
 
         const fetchInit =
-            this.fetch_(mediaState, reference.initSegmentReference,
-            /* streamDataCallback= */ undefined, /* isInit= */ true);
+            this.fetch_(mediaState, reference.initSegmentReference);
         const append = async () => {
           try {
             const initSegment = await fetchInit;
@@ -2154,13 +2192,12 @@ shaka.media.StreamingEngine = class {
    * @param {(!shaka.media.InitSegmentReference|!shaka.media.SegmentReference)}
    *   reference
    * @param {?function(BufferSource):!Promise=} streamDataCallback
-   * @param {boolean=} isInit
    *
    * @return {!Promise.<BufferSource>}
    * @private
    * @suppress {strictMissingProperties}
    */
-  async fetch_(mediaState, reference, streamDataCallback, isInit) {
+  async fetch_(mediaState, reference, streamDataCallback) {
     if (reference instanceof shaka.media.InitSegmentReference) {
       const segmentData = reference.getSegmentData();
       if (segmentData) {
@@ -2168,27 +2205,26 @@ shaka.media.StreamingEngine = class {
       }
     }
     let op = null;
-    if (
-      mediaState.segmentPrefetch &&
-      reference instanceof shaka.media.SegmentReference
-    ) {
+    if (mediaState.segmentPrefetch) {
       op = mediaState.segmentPrefetch.getPrefetchedSegment(
           reference, streamDataCallback);
     }
     if (!op) {
       op = this.dispatchFetch_(
-          reference, mediaState.stream, streamDataCallback, isInit,
-      );
+          reference, mediaState.stream, streamDataCallback);
+    }
+
+    let position = 0;
+    if (mediaState.segmentIterator) {
+      position = mediaState.segmentIterator.currentPosition();
     }
 
     mediaState.operation = op;
     const response = await op.promise;
     mediaState.operation = null;
     let result = response.data;
-    if (reference.hlsAes128Key) {
-      const iter = mediaState.segmentIterator;
-      goog.asserts.assert(iter, 'mediaState.segmentIterator should exist');
-      result = await this.aes128Decrypt_(result, reference.hlsAes128Key, iter);
+    if (reference.aes128Key) {
+      result = await this.aes128Decrypt_(result, reference.aes128Key, position);
     }
     return result;
   }
@@ -2200,18 +2236,17 @@ shaka.media.StreamingEngine = class {
    * @param {(!shaka.media.InitSegmentReference|!shaka.media.SegmentReference)}
    *   reference
    * @param {?function(BufferSource):!Promise=} streamDataCallback
-   * @param {boolean=} isInit
    *
    * @return {!shaka.net.NetworkingEngine.PendingRequest}
    * @private
    */
-  dispatchFetch_(reference, stream, streamDataCallback, isInit) {
+  dispatchFetch_(reference, stream, streamDataCallback) {
     const requestType = shaka.net.NetworkingEngine.RequestType.SEGMENT;
-    const type = isInit ?
-        shaka.net.NetworkingEngine.AdvancedRequestType.INIT_SEGMENT :
-        shaka.net.NetworkingEngine.AdvancedRequestType.MEDIA_SEGMENT;
     const segment = reference instanceof shaka.media.SegmentReference ?
         reference : undefined;
+    const type = segment ?
+        shaka.net.NetworkingEngine.AdvancedRequestType.MEDIA_SEGMENT :
+        shaka.net.NetworkingEngine.AdvancedRequestType.INIT_SEGMENT;
     const request = shaka.util.Networking.createSegmentRequest(
         reference.getUris(),
         reference.startByte,

package/lib/mss/mss_parser.js

@@ -213,6 +213,22 @@ shaka.mss.MssParser = class {
     // No-op
   }
 
+  /**
+   * @override
+   * @exportInterface
+   */
+  onInitialVariantChosen(variant) {
+    // No-op
+  }
+
+  /**
+   * @override
+   * @exportInterface
+   */
+  banLocation(uri) {
+    // No-op
+  }
+
   /**
    * Makes a network request for the manifest and parses the resulting data.
    *
@@ -500,6 +516,7 @@ shaka.mss.MssParser = class {
     const stream = {
       id: id,
       originalId: streamIndex.getAttribute('Name') || String(id),
+      groupId: null,
       createSegmentIndex: () => Promise.resolve(),
       closeSegmentIndex: () => Promise.resolve(),
       segmentIndex: null,
@@ -528,6 +545,7 @@ shaka.mss.MssParser = class {
       spatialAudio: false,
       closedCaptions: null,
       hdr: undefined,
+      videoLayout: undefined,
       tilesLayout: undefined,
       matchedStreams: [],
       mssPrivateData: {
@@ -537,6 +555,7 @@ shaka.mss.MssParser = class {
       },
       accessibilityPurpose: null,
       external: false,
+      fastSwitching: false,
     };
 
     // This is specifically for text tracks.
@@ -1076,7 +1095,5 @@ shaka.mss.MssParser.Context;
  */
 shaka.mss.MssParser.TimeRange;
 
-shaka.media.ManifestParser.registerParserByExtension(
-    'ism', () => new shaka.mss.MssParser());
 shaka.media.ManifestParser.registerParserByMime(
     'application/vnd.ms-sstr+xml', () => new shaka.mss.MssParser());