shaka-player 3.1.4 → 3.3.0

package/lib/hls/hls_parser.js

@@ -25,6 +25,7 @@ goog.require('shaka.net.DataUriPlugin');
 goog.require('shaka.net.NetworkingEngine');
 goog.require('shaka.util.ArrayUtils');
 goog.require('shaka.util.BufferUtils');
+goog.require('shaka.util.CmcdManager');
 goog.require('shaka.util.DataViewReader');
 goog.require('shaka.util.Error');
 goog.require('shaka.util.FakeEvent');
@@ -92,7 +93,8 @@ shaka.hls.HlsParser = class {
      * timestamps, offsets, and to handle TS rollover.
      *
      * During parsing, used to avoid duplicates in the async methods
-     * createStreamInfoFromMediaTag_ and createStreamInfoFromVariantTag_.
+     * createStreamInfoFromMediaTag_, createStreamInfoFromImageTag_ and
+     * createStreamInfoFromVariantTag_.
      *
      * During parsing of updates, used by getStartTime_ to determine the start
      * time of the first segment from existing segment references.
@@ -331,7 +333,8 @@ shaka.hls.HlsParser = class {
     const segments = await this.createSegments_(
         streamInfo.verbatimMediaPlaylistUri, playlist, stream.type,
         stream.mimeType, streamInfo.mediaSequenceToStartTime, mediaVariables,
-        streamInfo.discontinuityToMediaSequence);
+        streamInfo.discontinuityToMediaSequence, stream.codecs,
+        stream.bandwidth);
 
     stream.segmentIndex.mergeAndEvict(
         segments, this.presentationTimeline_.getSegmentAvailabilityStart());
@@ -403,6 +406,9 @@ shaka.hls.HlsParser = class {
     /** @type {!Array.<!shaka.hls.Tag>} */
     const variantTags = Utils.filterTagsByName(
         playlist.tags, 'EXT-X-STREAM-INF');
+    /** @type {!Array.<!shaka.hls.Tag>} */
+    const imageTags = Utils.filterTagsByName(
+        playlist.tags, 'EXT-X-IMAGE-STREAM-INF');
 
     this.parseCodecs_(variantTags);
 
@@ -437,6 +443,7 @@ shaka.hls.HlsParser = class {
     this.parseClosedCaptions_(mediaTags);
     const variants = await this.createVariantsForTags_(variantTags);
     const textStreams = await this.parseTexts_(mediaTags);
+    const imageStreams = await this.parseImages_(imageTags);
 
     // Make sure that the parser has not been destroyed.
     if (!this.playerInterface_) {
@@ -555,7 +562,7 @@ shaka.hls.HlsParser = class {
       presentationTimeline: this.presentationTimeline_,
       variants,
       textStreams,
-      imageStreams: [],
+      imageStreams,
       offlineSessionIds: [],
       minBufferTime: 0,
     };
@@ -688,6 +695,34 @@ shaka.hls.HlsParser = class {
     return textStreams.filter((s) => s);
   }
 
+  /**
+   * @param {!Array.<!shaka.hls.Tag>} imageTags from the playlist.
+   * @return {!Promise.<!Array.<!shaka.extern.Stream>>}
+   * @private
+   */
+  async parseImages_(imageTags) {
+    // Create image stream for each image tag.
+    const imageStreamPromises = imageTags.map(async (tag) => {
+      const disableThumbnails = this.config_.disableThumbnails;
+      if (disableThumbnails) {
+        return null;
+      }
+      try {
+        const streamInfo = await this.createStreamInfoFromImageTag_(tag);
+        goog.asserts.assert(
+            streamInfo, 'Should always have a streamInfo for image');
+        return streamInfo.stream;
+      } catch (e) {
+        if (this.config_.hls.ignoreImageStreamFailures) {
+          return null;
+        }
+        throw e;
+      }
+    });
+    const imageStreams = await Promise.all(imageStreamPromises);
+    return imageStreams.filter((s) => s);
+  }
+
   /**
    * @param {!Array.<!shaka.hls.Tag>} mediaTags Media tags from the playlist.
    * @private
@@ -730,7 +765,7 @@ shaka.hls.HlsParser = class {
       const videoRange = tag.getAttributeValue('VIDEO-RANGE');
 
       const streamInfos = await this.createStreamInfosForVariantTag_(tag,
-          resolution, frameRate);
+          resolution, frameRate, bandwidth);
 
       if (streamInfos) {
         goog.asserts.assert(streamInfos.audio.length ||
@@ -764,10 +799,11 @@ shaka.hls.HlsParser = class {
    * @param {!shaka.hls.Tag} tag
    * @param {?string} resolution
    * @param {?string} frameRate
+   * @param {number} bandwidth
    * @return {!Promise.<?shaka.hls.HlsParser.StreamInfos>}
    * @private
    */
-  async createStreamInfosForVariantTag_(tag, resolution, frameRate) {
+  async createStreamInfosForVariantTag_(tag, resolution, frameRate, bandwidth) {
     const ContentType = shaka.util.ManifestParserUtils.ContentType;
     /** @type {!Array.<string>} */
     let allCodecs = this.getCodecsForVariantTag_(tag);
@@ -837,7 +873,8 @@ shaka.hls.HlsParser = class {
     let streamInfo;
     if (!ignoreStream) {
       streamInfo =
-          await this.createStreamInfoFromVariantTag_(tag, allCodecs, type);
+          await this.createStreamInfoFromVariantTag_(tag, allCodecs, type,
+              bandwidth);
     }
     if (streamInfo) {
       res[streamInfo.stream.type] = [streamInfo];
@@ -860,15 +897,12 @@ shaka.hls.HlsParser = class {
    */
   getCodecsForVariantTag_(tag) {
     // These are the default codecs to assume if none are specified.
-    // The video codec is H.264, with baseline profile and level 3.0.
-    // http://blog.pearce.org.nz/2013/11/what-does-h264avc1-codecs-parameters.html
-    // The audio codec is "low-complexity" AAC.
     const defaultCodecsArray = [];
     if (!this.config_.disableVideo) {
-      defaultCodecsArray.push('avc1.42E01E');
+      defaultCodecsArray.push(this.config_.hls.defaultVideoCodec);
     }
     if (!this.config_.disableAudio) {
-      defaultCodecsArray.push('mp4a.40.2');
+      defaultCodecsArray.push(this.config_.hls.defaultAudioCodec);
     }
     const defaultCodecs = defaultCodecsArray.join(',');
 
@@ -1206,16 +1240,62 @@ shaka.hls.HlsParser = class {
     return streamInfo;
   }
 
+  /**
+   * Parse EXT-X-MEDIA media tag into a Stream object.
+   *
+   * @param {shaka.hls.Tag} tag
+   * @return {!Promise.<?shaka.hls.HlsParser.StreamInfo>}
+   * @private
+   */
+  async createStreamInfoFromImageTag_(tag) {
+    goog.asserts.assert(tag.name == 'EXT-X-IMAGE-STREAM-INF',
+        'Should only be called on image tags!');
+    /** @type {string} */
+    const type = shaka.util.ManifestParserUtils.ContentType.IMAGE;
+
+    const verbatimImagePlaylistUri = this.variableSubstitution_(
+        tag.getRequiredAttrValue('URI'), this.globalVariables_);
+    const codecs = tag.getAttributeValue('CODECS', 'jpeg') || '';
+
+    // Check if the stream has already been created as part of another Variant
+    // and return it if it has.
+    if (this.uriToStreamInfosMap_.has(verbatimImagePlaylistUri)) {
+      return this.uriToStreamInfosMap_.get(verbatimImagePlaylistUri);
+    }
+
+    const language = this.getLanguage_(tag);
+    const name = tag.getAttributeValue('NAME');
+
+    const characteristics = tag.getAttributeValue('CHARACTERISTICS');
+
+    const streamInfo = await this.createStreamInfo_(
+        verbatimImagePlaylistUri, codecs, type, language, /* primary= */ false,
+        name, /* channelsCount= */ null, /* closedCaptions= */ null,
+        characteristics, /* forced= */ false, /* spatialAudio= */ false);
+    if (streamInfo == null) {
+      return null;
+    }
+
+    // TODO: This check is necessary because of the possibility of multiple
+    // calls to createStreamInfoFromImageTag_ before either has resolved.
+    if (this.uriToStreamInfosMap_.has(verbatimImagePlaylistUri)) {
+      return this.uriToStreamInfosMap_.get(verbatimImagePlaylistUri);
+    }
+    this.uriToStreamInfosMap_.set(verbatimImagePlaylistUri, streamInfo);
+    return streamInfo;
+  }
+
   /**
    * Parse an EXT-X-STREAM-INF media tag into a Stream object.
    *
    * @param {!shaka.hls.Tag} tag
    * @param {!Array.<string>} allCodecs
    * @param {string} type
+   * @param {number} bandwidth
    * @return {!Promise.<?shaka.hls.HlsParser.StreamInfo>}
    * @private
    */
-  async createStreamInfoFromVariantTag_(tag, allCodecs, type) {
+  async createStreamInfoFromVariantTag_(tag, allCodecs, type, bandwidth) {
     goog.asserts.assert(tag.name == 'EXT-X-STREAM-INF',
         'Should only be called on variant tags!');
     const verbatimMediaPlaylistUri = this.variableSubstitution_(
@@ -1231,7 +1311,7 @@ shaka.hls.HlsParser = class {
         codecs, type, /* language= */ 'und', /* primary= */ false,
         /* name= */ null, /* channelcount= */ null, closedCaptions,
         /* characteristics= */ null, /* forced= */ false,
-        /* spatialAudio= */ false);
+        /* spatialAudio= */ false, bandwidth);
     if (streamInfo == null) {
       return null;
     }
@@ -1258,12 +1338,13 @@ shaka.hls.HlsParser = class {
    * @param {?string} characteristics
    * @param {boolean} forced
    * @param {boolean} spatialAudio
+   * @param {(number|undefined)} bandwidth
    * @return {!Promise.<?shaka.hls.HlsParser.StreamInfo>}
    * @private
    */
   async createStreamInfo_(verbatimMediaPlaylistUri, codecs, type, language,
       primary, name, channelsCount, closedCaptions, characteristics, forced,
-      spatialAudio) {
+      spatialAudio, bandwidth = undefined) {
     // TODO: Refactor, too many parameters
     let absoluteMediaPlaylistUri = shaka.hls.Utils.constructAbsoluteUri(
         this.masterPlaylistUri_, verbatimMediaPlaylistUri);
@@ -1278,7 +1359,8 @@ shaka.hls.HlsParser = class {
         response.data, absoluteMediaPlaylistUri);
 
     if (playlist.type != shaka.hls.PlaylistType.MEDIA) {
-      // EXT-X-MEDIA tags should point to media playlists.
+      // EXT-X-MEDIA and EXT-X-IMAGE-STREAM-INF tags should point to media
+      // playlists.
       throw new shaka.util.Error(
           shaka.util.Error.Severity.CRITICAL,
           shaka.util.Error.Category.MANIFEST,
@@ -1380,7 +1462,7 @@ shaka.hls.HlsParser = class {
     try {
       segments = await this.createSegments_(verbatimMediaPlaylistUri,
           playlist, type, mimeType, mediaSequenceToStartTime, mediaVariables,
-          discontinuityToMediaSequence);
+          discontinuityToMediaSequence, codecs, bandwidth);
     } catch (error) {
       if (error.code == shaka.util.Error.Code.HLS_INTERNAL_SKIP_STREAM) {
         shaka.log.alwaysWarn('Skipping unsupported HLS stream',
@@ -1635,11 +1717,12 @@ shaka.hls.HlsParser = class {
    * @param {!Map.<string, string>} variables
    * @param {string} absoluteMediaPlaylistUri
    * @return {!shaka.media.SegmentReference}
+   * @param {string} type
    * @private
    */
   createSegmentReference_(
       initSegmentReference, previousReference, hlsSegment, startTime,
-      timestampOffset, variables, absoluteMediaPlaylistUri) {
+      timestampOffset, variables, absoluteMediaPlaylistUri, type) {
     const tags = hlsSegment.tags;
     const absoluteSegmentUri = this.variableSubstitution_(
         hlsSegment.absoluteUri, variables);
@@ -1730,6 +1813,17 @@ shaka.hls.HlsParser = class {
       endByte = partialSegmentRefs[partialSegmentRefs.length - 1].endByte;
     }
 
+    let tilesLayout = '';
+    if (type == shaka.util.ManifestParserUtils.ContentType.IMAGE) {
+      // By default in HLS the tilesLayout is 1x1
+      tilesLayout = '1x1';
+      const tilesTag =
+          shaka.hls.Utils.getFirstTagWithName(tags, 'EXT-X-TILES');
+      if (tilesTag) {
+        tilesLayout = tilesTag.getRequiredAttrValue('LAYOUT');
+      }
+    }
+
     return new shaka.media.SegmentReference(
         startTime,
         endTime,
@@ -1741,6 +1835,7 @@ shaka.hls.HlsParser = class {
         /* appendWindowStart= */ 0,
         /* appendWindowEnd= */ Infinity,
         partialSegmentRefs,
+        tilesLayout,
     );
   }
 
@@ -1795,11 +1890,14 @@ shaka.hls.HlsParser = class {
    * @param {!Map.<number, number>} mediaSequenceToStartTime
    * @param {!Map.<string, string>} variables
    * @param {!Map.<number, number>} discontinuityToMediaSequence
+   * @param {string} codecs
+   * @param {(number|undefined)} bandwidth
    * @return {!Promise<!Array.<!shaka.media.SegmentReference>>}
    * @private
    */
   async createSegments_(verbatimMediaPlaylistUri, playlist, type, mimeType,
-      mediaSequenceToStartTime, variables, discontinuityToMediaSequence) {
+      mediaSequenceToStartTime, variables, discontinuityToMediaSequence,
+      codecs, bandwidth) {
     /** @type {Array.<!shaka.hls.Segment>} */
     const hlsSegments = playlist.segments;
     goog.asserts.assert(hlsSegments.length, 'Playlist should have segments!');
@@ -1836,12 +1934,13 @@ shaka.hls.HlsParser = class {
         initSegmentRef = this.getInitSegmentReference_(
             playlist.absoluteUri, hlsSegments[0].tags, variables);
         goog.asserts.assert(
-            type != shaka.util.ManifestParserUtils.ContentType.TEXT,
+            type != shaka.util.ManifestParserUtils.ContentType.TEXT &&
+            type != shaka.util.ManifestParserUtils.ContentType.IMAGE,
             'Should only get start time from audio or video streams');
         this.playlistStartTime_ = await this.getStartTime_(
             verbatimMediaPlaylistUri, initSegmentRef, mimeType,
             position, /* isDiscontinuity= */ false,
-            hlsSegments[0], variables);
+            hlsSegments[0], variables, type, codecs, bandwidth);
       }
       firstStartTime = this.playlistStartTime_;
     }
@@ -1894,7 +1993,8 @@ shaka.hls.HlsParser = class {
         // eslint-disable-next-line no-await-in-loop
         timestampOffset = await this.getTimestampOffset_(
             discontintuitySequenceNum, verbatimMediaPlaylistUri, initSegmentRef,
-            mimeType, position, item, variables, startTime);
+            mimeType, position, item, variables, startTime, type, codecs,
+            bandwidth);
       }
 
       // If the stream is low latency and the user has not configured the
@@ -1919,7 +2019,8 @@ shaka.hls.HlsParser = class {
             startTime,
             timestampOffset,
             variables,
-            playlist.absoluteUri);
+            playlist.absoluteUri,
+            type);
 
         references.push(reference);
       } else if (!this.lowLatencyMode_) {
@@ -1946,13 +2047,17 @@ shaka.hls.HlsParser = class {
    * @param {!shaka.hls.Segment} segment
    * @param {!Map.<string, string>} variables
    * @param {number} startTime
+   * @param {string} type
+   * @param {string} codecs
+   * @param {number|undefined} bandwidth
    * @return {!Promise.<number>}
    * @throws {shaka.util.Error}
    * @private
    */
   async getTimestampOffset_(discontintuitySequenceNum,
       verbatimMediaPlaylistUri, initSegmentRef,
-      mimeType, mediaSequenceNumber, segment, variables, startTime) {
+      mimeType, mediaSequenceNumber, segment, variables, startTime, type,
+      codecs, bandwidth) {
     let timestampOffset = 0;
     if (this.discontinuityToTso_.has(discontintuitySequenceNum)) {
       timestampOffset =
@@ -1961,7 +2066,7 @@ shaka.hls.HlsParser = class {
       const mediaStartTime = await this.getStartTime_(
           verbatimMediaPlaylistUri, initSegmentRef, mimeType,
           mediaSequenceNumber, /* isDiscontinuity= */ true, segment,
-          variables);
+          variables, type, codecs, bandwidth);
       timestampOffset = startTime - mediaStartTime;
       shaka.log.v1('Segment timestampOffset =', timestampOffset);
       this.discontinuityToTso_.set(
@@ -1975,10 +2080,14 @@ shaka.hls.HlsParser = class {
    * segment if we have to.
    *
    * @param {!shaka.media.AnySegmentReference} reference
+   * @param {string} type
+   * @param {string} mimeType
+   * @param {string} codecs
+   * @param {number|undefined} bandwidth
    * @return {!Promise.<shaka.extern.Response>}
    * @private
    */
-  async fetchStartOfSegment_(reference) {
+  async fetchStartOfSegment_(reference, type, mimeType, codecs, bandwidth) {
     const requestType = shaka.net.NetworkingEngine.RequestType.SEGMENT;
 
     // Create two requests:
@@ -1992,6 +2101,20 @@ shaka.hls.HlsParser = class {
         reference.endByte,
         this.config_.retryParameters);
 
+    // We can only add partial CMCD data here because the stream
+    // and manifest objects are still being created
+    this.playerInterface_.modifySegmentRequest(
+        fullRequest,
+        {
+          type: type,
+          init: reference instanceof shaka.media.InitSegmentReference,
+          duration: reference.endTime - reference.startTime,
+          mimeType: mimeType,
+          codecs: codecs,
+          bandwidth: bandwidth,
+        },
+    );
+
     if (this.config_.hls.useFullSegmentsForStartTime) {
       return this.makeNetworkRequest_(fullRequest, requestType);
     }
@@ -2002,6 +2125,18 @@ shaka.hls.HlsParser = class {
         reference.startByte + shaka.hls.HlsParser.START_OF_SEGMENT_SIZE_ - 1,
         this.config_.retryParameters);
 
+    this.playerInterface_.modifySegmentRequest(
+        partialRequest,
+        {
+          type: type,
+          init: reference instanceof shaka.media.InitSegmentReference,
+          duration: reference.endTime - reference.startTime,
+          mimeType: mimeType,
+          codecs: codecs,
+          bandwidth: bandwidth,
+        },
+    );
+
     // TODO(vaage): The need to do fall back requests is not likely to be unique
     //    to here. It would be nice if the fallback(s) could be included into
     //    the same abortable operation as the original request.
@@ -2048,12 +2183,15 @@ shaka.hls.HlsParser = class {
    * @param {boolean} isDiscontinuity
    * @param {!shaka.hls.Segment} segment
    * @param {!Map.<string, string>} variables
+   * @param {string} type
+   * @param {string} codecs
+   * @param {number|undefined} bandwidth
    * @return {!Promise.<number>}
    * @private
    */
   async getStartTime_(
       verbatimMediaPlaylistUri, initSegmentRef, mimeType, mediaSequenceNumber,
-      isDiscontinuity, segment, variables) {
+      isDiscontinuity, segment, variables, type, codecs, bandwidth) {
     const segmentRef = this.createSegmentReference_(
         initSegmentRef,
         /* previousReference= */ null,
@@ -2061,7 +2199,8 @@ shaka.hls.HlsParser = class {
         /* startTime= */ 0,
         /* timestampOffset= */ 0,
         variables,
-        /* absoluteMediaPlaylistUri= */ '');
+        /* absoluteMediaPlaylistUri= */ '',
+        type);
     // If we are updating the manifest, we can usually skip fetching the segment
     // by examining the references we already have.  This won't be possible if
     // there was some kind of lag or delay updating the manifest on the server,
@@ -2120,10 +2259,22 @@ shaka.hls.HlsParser = class {
     if (mimeType == 'video/mp4' || mimeType == 'audio/mp4') {
       // We also need the init segment to get the correct timescale. But if the
       // stream is self-initializing, use the same response for both.
-      const fetches = [this.fetchStartOfSegment_(segmentRef)];
+      const fetches = [this.fetchStartOfSegment_(
+          segmentRef,
+          type,
+          mimeType,
+          codecs,
+          bandwidth,
+      )];
 
       if (initSegmentRef) {
-        fetches.push(this.fetchStartOfSegment_(initSegmentRef));
+        fetches.push(this.fetchStartOfSegment_(
+            initSegmentRef,
+            type,
+            mimeType,
+            codecs,
+            bandwidth,
+        ));
       }
 
       const responses = await Promise.all(fetches);
@@ -2139,7 +2290,13 @@ shaka.hls.HlsParser = class {
     }
 
     if (mimeType == 'video/mp2t') {
-      const response = await this.fetchStartOfSegment_(segmentRef);
+      const response = await this.fetchStartOfSegment_(
+          segmentRef,
+          type,
+          mimeType,
+          codecs,
+          bandwidth,
+      );
       goog.asserts.assert(response.data, 'Should have a response body!');
       return this.getStartTimeFromTsSegment_(
           verbatimMediaPlaylistUri, response.uri, response.data);
@@ -2437,6 +2594,12 @@ shaka.hls.HlsParser = class {
       }
     }
 
+    if (contentType == ContentType.IMAGE) {
+      if (!codecs || codecs == 'jpeg') {
+        return 'image/jpeg';
+      }
+    }
+
     // If unable to guess mime type, request a segment and try getting it
     // from the response.
     const headRequest = shaka.net.NetworkingEngine.makeRequest(
@@ -2511,6 +2674,9 @@ shaka.hls.HlsParser = class {
     const request = shaka.net.NetworkingEngine.makeRequest(
         [absoluteUri], this.config_.retryParameters);
 
+    const format = shaka.util.CmcdManager.StreamingFormat.HLS;
+    this.playerInterface_.modifyManifestRequest(request, {format: format});
+
     return this.makeNetworkRequest_(request, requestType);
   }
 
@@ -2807,6 +2973,19 @@ shaka.hls.HlsParser.TEXT_EXTENSIONS_TO_MIME_TYPES_ = {
 };
 
 
+/**
+ * @const {!Object.<string, string>}
+ * @private
+ */
+shaka.hls.HlsParser.IMAGE_EXTENSIONS_TO_MIME_TYPES_ = {
+  'jpg': 'image/jpeg',
+  'png': 'image/png',
+  'svg': 'image/svg+xml',
+  'webp': 'image/webp',
+  'avif': 'image/avif',
+};
+
+
 /**
  * @const {!Object.<string, !Object.<string, string>>}
  * @private
@@ -2815,6 +2994,7 @@ shaka.hls.HlsParser.EXTENSION_MAP_BY_CONTENT_TYPE_ = {
   'audio': shaka.hls.HlsParser.AUDIO_EXTENSIONS_TO_MIME_TYPES_,
   'video': shaka.hls.HlsParser.VIDEO_EXTENSIONS_TO_MIME_TYPES_,
   'text': shaka.hls.HlsParser.TEXT_EXTENSIONS_TO_MIME_TYPES_,
+  'image': shaka.hls.HlsParser.IMAGE_EXTENSIONS_TO_MIME_TYPES_,
 };
 
 

package/lib/media/closed_caption_parser.js

@@ -4,14 +4,13 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-goog.provide('shaka.media.IClosedCaptionParser');
 goog.provide('shaka.media.ClosedCaptionParser');
+goog.provide('shaka.media.IClosedCaptionParser');
 
 goog.require('shaka.cea.CeaDecoder');
 goog.require('shaka.cea.Mp4CeaParser');
 goog.require('shaka.util.BufferUtils');
 goog.requireType('shaka.cea.ICaptionDecoder');
-goog.requireType('shaka.cea.ICaptionDecoder.ClosedCaption');
 goog.requireType('shaka.cea.ICeaParser');