shaka-player 4.4.2 → 4.5.0

package/lib/media/media_source_engine.js

@@ -8,6 +8,7 @@ goog.provide('shaka.media.MediaSourceEngine');
 
 goog.require('goog.asserts');
 goog.require('shaka.log');
+goog.require('shaka.config.CodecSwitchingStrategy');
 goog.require('shaka.media.Capabilities');
 goog.require('shaka.media.ContentWorkarounds');
 goog.require('shaka.media.ClosedCaptionParser');
@@ -31,7 +32,7 @@ goog.require('shaka.util.Mp4Parser');
 goog.require('shaka.util.Platform');
 goog.require('shaka.util.PublicPromise');
 goog.require('shaka.util.TsParser');
-goog.require('shaka.lcevc.Dil');
+goog.require('shaka.lcevc.Dec');
 
 
 /**
@@ -53,10 +54,10 @@ shaka.media.MediaSourceEngine = class {
    *    MediaSourceEngine is destroyed, it will destroy the displayer.
    * @param {!function(!Array.<shaka.extern.ID3Metadata>, number, ?number)=}
    *    onMetadata
-   * @param {?shaka.lcevc.Dil} [lcevcDil] Optional -  LCEVC Dil Object
+   * @param {?shaka.lcevc.Dec} [lcevcDec] Optional -  LCEVC Decoder Object
    *
    */
-  constructor(video, textDisplayer, onMetadata, lcevcDil) {
+  constructor(video, textDisplayer, onMetadata, lcevcDec) {
     /** @private {HTMLMediaElement} */
     this.video_ = video;
 
@@ -91,8 +92,8 @@ shaka.media.MediaSourceEngine = class {
                     number, ?number)} */
     this.onMetadata_ = onMetadata || onMetadataNoOp;
 
-    /** @private {?shaka.lcevc.Dil} */
-    this.lcevcDil_ = lcevcDil || null;
+    /** @private {?shaka.lcevc.Dec} */
+    this.lcevcDec_ = lcevcDec || null;
 
     /**
      * @private {!Object.<string,
@@ -115,6 +116,9 @@ shaka.media.MediaSourceEngine = class {
     /** @private {string} */
     this.url_ = '';
 
+    /** @private {boolean} */
+    this.playbackHasBegun_ = false;
+
     /** @private {MediaSource} */
     this.mediaSource_ = this.createMediaSource(this.mediaSourceOpen_);
 
@@ -138,6 +142,9 @@ shaka.media.MediaSourceEngine = class {
 
     /** @private {!shaka.util.PublicPromise.<number>} */
     this.textSequenceModeOffset_ = new shaka.util.PublicPromise();
+
+    /** @private {boolean} */
+    this.needSplitMuxedContent_ = false;
   }
 
   /**
@@ -156,6 +163,11 @@ shaka.media.MediaSourceEngine = class {
     this.eventManager_.listenOnce(
         mediaSource, 'sourceopen', () => this.onSourceOpen_(p));
 
+    // Correctly set when playback has begun.
+    this.eventManager_.listenOnce(this.video_, 'playing', () => {
+      this.playbackHasBegun_ = true;
+    });
+
     // Store the object URL for releasing it later.
     this.url_ = shaka.media.MediaSourceEngine.createObjectURL(mediaSource);
 
@@ -341,7 +353,7 @@ shaka.media.MediaSourceEngine = class {
     this.queues_ = {};
 
     // This object is owned by Player
-    this.lcevcDil_ = null;
+    this.lcevcDec_ = null;
   }
 
   /**
@@ -379,6 +391,8 @@ shaka.media.MediaSourceEngine = class {
   async init(streamsByType, sequenceMode=false,
       manifestType=shaka.media.ManifestParser.UNKNOWN,
       ignoreManifestTimestampsInSegmentsMode=false) {
+    const ContentType = shaka.util.ManifestParserUtils.ContentType;
+
     await this.mediaSourceOpen_;
 
     this.sequenceMode_ = sequenceMode;
@@ -392,8 +406,13 @@ shaka.media.MediaSourceEngine = class {
 
     for (const contentType of streamsByType.keys()) {
       const stream = streamsByType.get(contentType);
-      this.initSourceBuffer_(contentType, stream);
-      this.queues_[contentType] = [];
+      this.initSourceBuffer_(contentType, stream, stream.codecs);
+      if (this.needSplitMuxedContent_) {
+        this.queues_[ContentType.AUDIO] = [];
+        this.queues_[ContentType.VIDEO] = [];
+      } else {
+        this.queues_[contentType] = [];
+      }
     }
   }
 
@@ -402,9 +421,10 @@ shaka.media.MediaSourceEngine = class {
    *
    * @param {shaka.util.ManifestParserUtils.ContentType} contentType
    * @param {shaka.extern.Stream} stream
+   * @param {string} codecs
    * @private
    */
-  initSourceBuffer_(contentType, stream) {
+  initSourceBuffer_(contentType, stream, codecs) {
     const ContentType = shaka.util.ManifestParserUtils.ContentType;
 
     goog.asserts.assert(
@@ -412,7 +432,7 @@ shaka.media.MediaSourceEngine = class {
         'Type negotiation should happen before MediaSourceEngine.init!');
 
     let mimeType = shaka.util.MimeUtils.getFullType(
-        stream.mimeType, stream.codecs);
+        stream.mimeType, codecs);
     if (contentType == ContentType.TEXT) {
       this.reinitText(mimeType, this.sequenceMode_, stream.external);
     } else {
@@ -424,9 +444,19 @@ shaka.media.MediaSourceEngine = class {
       }
       const mimeTypeWithAllCodecs =
           shaka.util.MimeUtils.getFullTypeWithAllCodecs(
-              stream.mimeType, stream.codecs);
+              stream.mimeType, codecs);
       const TransmuxerEngine = shaka.transmuxer.TransmuxerEngine;
       if (needTransmux) {
+        const audioCodec = shaka.util.ManifestParserUtils.guessCodecsSafe(
+            ContentType.AUDIO, (codecs || '').split(','));
+        const videoCodec = shaka.util.ManifestParserUtils.guessCodecsSafe(
+            ContentType.VIDEO, (codecs || '').split(','));
+        if (audioCodec && videoCodec) {
+          this.needSplitMuxedContent_ = true;
+          this.initSourceBuffer_(ContentType.AUDIO, stream, audioCodec);
+          this.initSourceBuffer_(ContentType.VIDEO, stream, videoCodec);
+          return;
+        }
         const transmuxerPlugin =
             TransmuxerEngine.findTransmuxer(mimeTypeWithAllCodecs);
         if (transmuxerPlugin) {
@@ -513,7 +543,8 @@ shaka.media.MediaSourceEngine = class {
    * @return {?number} The timestamp in seconds, or null if nothing is buffered.
    */
   bufferStart(contentType) {
-    if (this.reloadingMediaSource_) {
+    if (this.reloadingMediaSource_ ||
+          !Object.keys(this.sourceBuffers_).length) {
       return null;
     }
     const ContentType = shaka.util.ManifestParserUtils.ContentType;
@@ -743,7 +774,7 @@ shaka.media.MediaSourceEngine = class {
    */
   async appendBuffer(
       contentType, data, reference, stream, hasClosedCaptions, seeked = false,
-      adaptation = false, isChunkedData = false) {
+      adaptation = false, isChunkedData = false, fromSplit = false) {
     const ContentType = shaka.util.ManifestParserUtils.ContentType;
 
     if (contentType == ContentType.TEXT) {
@@ -759,6 +790,21 @@ shaka.media.MediaSourceEngine = class {
       return;
     }
 
+    if (!fromSplit && this.needSplitMuxedContent_) {
+      await this.appendBuffer(ContentType.AUDIO, data, reference, stream,
+          hasClosedCaptions, seeked, adaptation, isChunkedData,
+          /* fromSplit= */ true);
+      await this.appendBuffer(ContentType.VIDEO, data, reference, stream,
+          hasClosedCaptions, seeked, adaptation, isChunkedData,
+          /* fromSplit= */ true);
+      return;
+    }
+
+    if (!this.sourceBuffers_[contentType]) {
+      shaka.log.warning('Attempted to restore a non-existent source buffer');
+      return;
+    }
+
     let timestampOffset = this.sourceBuffers_[contentType].timestampOffset;
 
     let mimeType = this.sourceBufferTypes_[contentType];
@@ -850,7 +896,7 @@ shaka.media.MediaSourceEngine = class {
       if (goog.DEBUG && reference && !reference.isPreload() && !isChunkedData) {
         bufferedBefore = this.getBuffered_(contentType);
       }
-      this.append_(contentType, data);
+      this.append_(contentType, data, timestampOffset);
     });
 
     if (goog.DEBUG && reference && !reference.isPreload() && !isChunkedData) {
@@ -910,6 +956,11 @@ shaka.media.MediaSourceEngine = class {
       await this.enqueueOperation_(
           contentType,
           () => this.remove_(contentType, startTime, endTime));
+      if (this.needSplitMuxedContent_) {
+        await this.enqueueOperation_(
+            ContentType.AUDIO,
+            () => this.remove_(ContentType.AUDIO, startTime, endTime));
+      }
     }
   }
 
@@ -931,6 +982,12 @@ shaka.media.MediaSourceEngine = class {
       await this.enqueueOperation_(
           contentType,
           () => this.remove_(contentType, 0, this.mediaSource_.duration));
+      if (this.needSplitMuxedContent_) {
+        await this.enqueueOperation_(
+            ContentType.AUDIO,
+            () => this.remove_(
+                ContentType.AUDIO, 0, this.mediaSource_.duration));
+      }
     }
   }
 
@@ -961,6 +1018,11 @@ shaka.media.MediaSourceEngine = class {
     await this.enqueueOperation_(
         contentType,
         () => this.flush_(contentType));
+    if (this.needSplitMuxedContent_) {
+      await this.enqueueOperation_(
+          ContentType.AUDIO,
+          () => this.flush_(ContentType.AUDIO));
+    }
   }
 
   /**
@@ -1014,16 +1076,33 @@ shaka.media.MediaSourceEngine = class {
       operations.push(this.enqueueOperation_(
           contentType,
           () => this.abort_(contentType)));
+      if (this.needSplitMuxedContent_) {
+        operations.push(this.enqueueOperation_(
+            ContentType.AUDIO,
+            () => this.abort_(ContentType.AUDIO)));
+      }
     }
     if (!ignoreTimestampOffset) {
       operations.push(this.enqueueOperation_(
           contentType,
           () => this.setTimestampOffset_(contentType, timestampOffset)));
+      if (this.needSplitMuxedContent_) {
+        operations.push(this.enqueueOperation_(
+            ContentType.AUDIO,
+            () => this.setTimestampOffset_(
+                ContentType.AUDIO, timestampOffset)));
+      }
     }
     operations.push(this.enqueueOperation_(
         contentType,
         () => this.setAppendWindow_(
             contentType, appendWindowStart, appendWindowEnd)));
+    if (this.needSplitMuxedContent_) {
+      operations.push(this.enqueueOperation_(
+          ContentType.AUDIO,
+          () => this.setAppendWindow_(
+              ContentType.AUDIO, appendWindowStart, appendWindowEnd)));
+    }
 
     await Promise.all(operations);
   }
@@ -1054,9 +1133,19 @@ shaka.media.MediaSourceEngine = class {
     this.enqueueOperation_(
         contentType,
         () => this.abort_(contentType));
+    if (this.needSplitMuxedContent_) {
+      this.enqueueOperation_(
+          ContentType.AUDIO,
+          () => this.abort_(ContentType.AUDIO));
+    }
     await this.enqueueOperation_(
         contentType,
         () => this.setTimestampOffset_(contentType, timestampOffset));
+    if (this.needSplitMuxedContent_) {
+      await this.enqueueOperation_(
+          ContentType.AUDIO,
+          () => this.setTimestampOffset_(ContentType.AUDIO, timestampOffset));
+    }
   }
 
   /**
@@ -1120,16 +1209,17 @@ shaka.media.MediaSourceEngine = class {
    * Append data to the SourceBuffer.
    * @param {shaka.util.ManifestParserUtils.ContentType} contentType
    * @param {BufferSource} data
+   * @param {number} timestampOffset
    * @private
    */
-  append_(contentType, data) {
+  append_(contentType, data, timestampOffset) {
     const ContentType = shaka.util.ManifestParserUtils.ContentType;
 
-    // Append only video data to the LCEVC Dil.
-    if (contentType == ContentType.VIDEO && this.lcevcDil_) {
-      // Append video buffers to the LCEVC Dil for parsing and storing
+    // Append only video data to the LCEVC Dec.
+    if (contentType == ContentType.VIDEO && this.lcevcDec_) {
+      // Append video buffers to the LCEVC Dec for parsing and storing
       // of LCEVC data.
-      this.lcevcDil_.appendBuffer(data);
+      this.lcevcDec_.appendBuffer(data, timestampOffset);
     }
 
     // This will trigger an 'updateend' event.
@@ -1292,7 +1382,6 @@ shaka.media.MediaSourceEngine = class {
    */
   enqueueOperation_(contentType, start) {
     this.destroyer_.ensureNotDestroyed();
-
     const operation = {
       start: start,
       p: new shaka.util.PublicPromise(),
@@ -1563,8 +1652,24 @@ shaka.media.MediaSourceEngine = class {
    */
   async reset_(streamsByType) {
     const Functional = shaka.util.Functional;
+    const ContentType = shaka.util.ManifestParserUtils.ContentType;
     this.reloadingMediaSource_ = true;
+    this.needSplitMuxedContent_ = false;
     const currentTime = this.video_.currentTime;
+
+    // When codec switching if the user is currently paused we don't want
+    // to trigger a play when switching codec.
+    // Playing can also end up in a paused state after a codec switch
+    // so we need to remember the current states.
+    const previousAutoPlayState = this.video_.autoplay;
+    const previousPausedState = this.video_.paused;
+    if (this.playbackHasBegun_) {
+      // Only set autoplay to false if the video playback has already begun.
+      // When a codec switch happens before playback has begun this can cause
+      // autoplay not to work as expected.
+      this.video_.autoplay = false;
+    }
+
     try {
       this.eventManager_.removeAll();
 
@@ -1598,7 +1703,6 @@ shaka.media.MediaSourceEngine = class {
       }
       await Promise.all(cleanup);
       this.transmuxers_ = {};
-      this.queues_ = {};
       this.sourceBuffers_ = {};
 
       const previousDuration = this.mediaSource_.duration;
@@ -1622,7 +1726,13 @@ shaka.media.MediaSourceEngine = class {
 
       for (const contentType of streamsByType.keys()) {
         const stream = streamsByType.get(contentType);
-        this.initSourceBuffer_(contentType, stream);
+        this.initSourceBuffer_(contentType, stream, stream.codecs);
+        if (this.needSplitMuxedContent_) {
+          this.queues_[ContentType.AUDIO] = [];
+          this.queues_[ContentType.VIDEO] = [];
+        } else {
+          this.queues_[contentType] = [];
+        }
       }
 
       // Fake a seek to catchup the playhead.
@@ -1631,6 +1741,17 @@ shaka.media.MediaSourceEngine = class {
       await sourceBufferAdded;
     } finally {
       this.reloadingMediaSource_ = false;
+
+      this.destroyer_.ensureNotDestroyed();
+
+      this.eventManager_.listenOnce(this.video_, 'canplay', () => {
+        this.destroyer_.ensureNotDestroyed();
+
+        this.video_.autoplay = previousAutoPlayState;
+        if (!previousPausedState) {
+          this.video_.play();
+        }
+      });
     }
   }
 
@@ -1696,7 +1817,10 @@ shaka.media.MediaSourceEngine = class {
       return false;
     }
 
-    if (shaka.media.Capabilities.isChangeTypeSupported()) {
+    if (this.config_.codecSwitchingStrategy ===
+        shaka.config.CodecSwitchingStrategy.SMOOTH &&
+          shaka.media.Capabilities.isChangeTypeSupported() &&
+            !this.needSplitMuxedContent_) {
       await this.changeType(contentType, newMimeType, transmuxer);
     } else {
       if (transmuxer) {
@@ -1708,11 +1832,75 @@ shaka.media.MediaSourceEngine = class {
   }
 
   /**
-   * Update LCEVC DIL object when ready for LCEVC Decodes
-   * @param {?shaka.lcevc.Dil} lcevcDil
+   * Returns true if it's necessary codec switch to load the new stream.
+   *
+   * @param {shaka.util.ManifestParserUtils.ContentType} contentType
+   * @param {shaka.extern.Stream} stream
+   * @return {boolean}
+   * @private
+   */
+  isCodecSwitchNecessary_(contentType, stream) {
+    const MimeUtils = shaka.util.MimeUtils;
+    const currentCodec = MimeUtils.getCodecBase(
+        MimeUtils.getCodecs(this.sourceBufferTypes_[contentType]));
+    const currentBasicType = MimeUtils.getBasicType(
+        this.sourceBufferTypes_[contentType]);
+
+    let newMimeType = shaka.util.MimeUtils.getFullType(
+        stream.mimeType, stream.codecs);
+    let needTransmux = this.config_.forceTransmux;
+    if (!shaka.media.Capabilities.isTypeSupported(newMimeType) ||
+        (!this.sequenceMode_ &&
+        shaka.util.MimeUtils.RAW_FORMATS.includes(newMimeType))) {
+      needTransmux = true;
+    }
+    const newMimeTypeWithAllCodecs =
+        shaka.util.MimeUtils.getFullTypeWithAllCodecs(
+            stream.mimeType, stream.codecs);
+    const TransmuxerEngine = shaka.transmuxer.TransmuxerEngine;
+    if (needTransmux) {
+      const transmuxerPlugin =
+          TransmuxerEngine.findTransmuxer(newMimeTypeWithAllCodecs);
+      if (transmuxerPlugin) {
+        const transmuxer = transmuxerPlugin();
+        newMimeType =
+            transmuxer.convertCodecs(contentType, newMimeTypeWithAllCodecs);
+        transmuxer.destroy();
+      }
+    }
+
+    const newCodec = MimeUtils.getCodecBase(
+        MimeUtils.getCodecs(newMimeType));
+    const newBasicType = MimeUtils.getBasicType(newMimeType);
+
+    return currentCodec !== newCodec || currentBasicType !== newBasicType;
+  }
+
+  /**
+   * Returns true if it's necessary reset the media source to load the
+   * new stream.
+   *
+   * @param {shaka.util.ManifestParserUtils.ContentType} contentType
+   * @param {shaka.extern.Stream} stream
+   * @return {boolean}
+   */
+  isResetMediaSourceNecessary(contentType, stream) {
+    if (!this.isCodecSwitchNecessary_(contentType, stream)) {
+      return false;
+    }
+
+    return this.config_.codecSwitchingStrategy !==
+        shaka.config.CodecSwitchingStrategy.SMOOTH ||
+        !shaka.media.Capabilities.isChangeTypeSupported() ||
+        this.needSplitMuxedContent_;
+  }
+
+  /**
+   * Update LCEVC Decoder object when ready for LCEVC Decode.
+   * @param {?shaka.lcevc.Dec} lcevcDec
    */
-  updateLcevcDil(lcevcDil) {
-    this.lcevcDil_ = lcevcDil;
+  updateLcevcDec(lcevcDec) {
+    this.lcevcDec_ = lcevcDec;
   }
 };
 

package/lib/media/segment_index.js

@@ -379,7 +379,7 @@ shaka.media.SegmentIndex = class {
             lastReference.tileDuration,
             lastReference.syncTime,
             lastReference.status,
-            lastReference.hlsAes128Key,
+            lastReference.aes128Key,
         );
     this.references[this.references.length - 1].discontinuitySequence =
         lastReference.discontinuitySequence;
@@ -708,7 +708,7 @@ shaka.media.MetaSegmentIndex = class extends shaka.media.SegmentIndex {
       }
 
       numPassedInEarlierIndexes += index.numEvicted_ +
-        index.references.length;
+        index.getNumReferences();
     }
 
     return null;
@@ -732,9 +732,9 @@ shaka.media.MetaSegmentIndex = class extends shaka.media.SegmentIndex {
         return reference;
       }
 
-      numPassedInEarlierIndexes += index.numEvicted_ +
-        index.references.length;
-      sawSegments = sawSegments || index.references.length != 0;
+      const num = index.getNumReferences();
+      numPassedInEarlierIndexes += index.numEvicted_ + num;
+      sawSegments = sawSegments || num != 0;
     }
 
     return null;

package/lib/media/segment_reference.js

@@ -32,9 +32,11 @@ shaka.media.InitSegmentReference = class {
    *   the quality of the media associated with this init segment.
    * @param {(null|number)=} timescale
    * @param {(null|BufferSource)=} segmentData
+   * @param {?shaka.extern.aes128Key=} aes128Key
+   *  The segment's AES-128-CBC full segment encryption key and iv.
    */
   constructor(uris, startByte, endByte, mediaQuality = null, timescale = null,
-      segmentData = null) {
+      segmentData = null, aes128Key = null) {
     /** @type {function():!Array.<string>} */
     this.getUris = uris;
 
@@ -52,6 +54,9 @@ shaka.media.InitSegmentReference = class {
 
     /** @type {BufferSource|null} */
     this.segmentData = segmentData;
+
+    /** @type {?shaka.extern.aes128Key} */
+    this.aes128Key = aes128Key;
   }
 
   /**
@@ -191,7 +196,7 @@ shaka.media.SegmentReference = class {
    * @param {shaka.media.SegmentReference.Status=} status
    *  The segment status is used to indicate that a segment does not exist or is
    *  not available.
-   * @param {?shaka.extern.HlsAes128Key=} hlsAes128Key
+   * @param {?shaka.extern.aes128Key=} aes128Key
    *  The segment's AES-128-CBC full segment encryption key and iv.
    */
   constructor(
@@ -199,7 +204,7 @@ shaka.media.SegmentReference = class {
       timestampOffset, appendWindowStart, appendWindowEnd,
       partialReferences = [], tilesLayout = '', tileDuration = null,
       syncTime = null, status = shaka.media.SegmentReference.Status.AVAILABLE,
-      hlsAes128Key = null) {
+      aes128Key = null) {
     // A preload hinted Partial Segment has the same startTime and endTime.
     goog.asserts.assert(startTime <= endTime,
         'startTime must be less than or equal to endTime');
@@ -271,8 +276,8 @@ shaka.media.SegmentReference = class {
     /** @type {boolean} */
     this.byterangeOptimization = false;
 
-    /** @type {?shaka.extern.HlsAes128Key} */
-    this.hlsAes128Key = hlsAes128Key;
+    /** @type {?shaka.extern.aes128Key} */
+    this.aes128Key = aes128Key;
 
     /** @type {?shaka.media.SegmentReference.ThumbnailSprite} */
     this.thumbnailSprite = null;