shaka-player 4.11.13 → 4.12.1

package/lib/offline/storage.js

@@ -11,6 +11,8 @@ goog.require('shaka.Player');
 goog.require('shaka.log');
 goog.require('shaka.media.DrmEngine');
 goog.require('shaka.media.ManifestParser');
+goog.require('shaka.media.SegmentIndex');
+goog.require('shaka.media.SegmentReference');
 goog.require('shaka.net.NetworkingEngine');
 goog.require('shaka.net.NetworkingUtils');
 goog.require('shaka.offline.DownloadInfo');
@@ -20,13 +22,16 @@ goog.require('shaka.offline.SessionDeleter');
 goog.require('shaka.offline.StorageMuxer');
 goog.require('shaka.offline.StoredContentUtils');
 goog.require('shaka.offline.StreamBandwidthEstimator');
+goog.require('shaka.text.TextEngine');
 goog.require('shaka.util.AbortableOperation');
 goog.require('shaka.util.ArrayUtils');
+goog.require('shaka.util.BufferUtils');
 goog.require('shaka.util.ConfigUtils');
 goog.require('shaka.util.Destroyer');
 goog.require('shaka.util.Error');
 goog.require('shaka.util.IDestroyable');
 goog.require('shaka.util.Iterables');
+goog.require('shaka.util.ManifestParserUtils');
 goog.require('shaka.util.MimeUtils');
 goog.require('shaka.util.Platform');
 goog.require('shaka.util.PlayerConfiguration');
@@ -142,6 +147,13 @@ shaka.offline.Storage = class {
       this.config_ = null;
       this.networkingEngine_ = null;
     });
+
+    /**
+     * Contains an ID for use with creating streams.  The manifest parser should
+     * start with small IDs, so this starts with a large one.
+     * @private {number}
+     */
+    this.nextExternalStreamId_ = 1e9;
   }
 
 
@@ -249,8 +261,12 @@ shaka.offline.Storage = class {
    *   application-specific metadata you need associated with the stored
    *   content.  For details on the data types that can be stored here, please
    *   refer to {@link https://bit.ly/StructClone}
-   * @param {string=} mimeType
+   * @param {?string=} mimeType
    *   The mime type for the content |manifestUri| points to.
+   * @param {?Array.<string>=} externalThumbnails
+   *   The external thumbnails to store along the main content.
+   * @param {?Array.<shaka.extern.ExtraText>=} externalText
+   *   The external text to store along the main content.
    * @return {!shaka.extern.IAbortableOperation.<shaka.extern.StoredContent>}
    *   An AbortableOperation that resolves with a structure representing what
    *   was stored.  The "offlineUri" member is the URI that should be given to
@@ -260,7 +276,7 @@ shaka.offline.Storage = class {
    *   AbortableOperation.
    * @export
    */
-  store(uri, appMetadata, mimeType) {
+  store(uri, appMetadata, mimeType, externalThumbnails, externalText) {
     goog.asserts.assert(
         this.networkingEngine_,
         'Cannot call |store| after calling |destroy|.');
@@ -290,7 +306,8 @@ shaka.offline.Storage = class {
     this.openDownloadManagers_.push(downloader);
 
     const storeOp = this.store_(
-        uri, appMetadata || {}, getParser, config, downloader);
+        uri, appMetadata || {}, externalThumbnails || [], externalText || [],
+        getParser, config, downloader);
     const abortableStoreOp = new shaka.util.AbortableOperation(storeOp, () => {
       return downloader.abortAll();
     });
@@ -306,13 +323,16 @@ shaka.offline.Storage = class {
    *
    * @param {string} uri
    * @param {!Object} appMetadata
+   * @param {!Array.<string>} externalThumbnails
+   * @param {!Array.<shaka.extern.ExtraText>} externalText
    * @param {function():!Promise.<shaka.extern.ManifestParser>} getParser
    * @param {shaka.extern.PlayerConfiguration} config
    * @param {!shaka.offline.DownloadManager} downloader
    * @return {!Promise.<shaka.extern.StoredContent>}
    * @private
    */
-  async store_(uri, appMetadata, getParser, config, downloader) {
+  async store_(uri, appMetadata, externalThumbnails, externalText,
+      getParser, config, downloader) {
     this.requireSupport_();
 
     // Since we will need to use |parser|, |drmEngine|, |activeHandle|, and
@@ -357,6 +377,23 @@ shaka.offline.Storage = class {
             uri);
       }
 
+      for (const thumbnailUri of externalThumbnails) {
+        const imageStream =
+            // eslint-disable-next-line no-await-in-loop
+            await this.createExternalImageStream_(thumbnailUri, manifest);
+        manifest.imageStreams.push(imageStream);
+        this.ensureNotDestroyed_();
+      }
+
+      for (const text of externalText) {
+        const textStream =
+            // eslint-disable-next-line no-await-in-loop
+            await this.createExternalTextStream_(manifest,
+                text.uri, text.language, text.kind, text.mime, text.codecs);
+        manifest.textStreams.push(textStream);
+        this.ensureNotDestroyed_();
+      }
+
       // Create the DRM engine, and load the keys in the manifest.
       drmEngine = await this.createDrmEngine(
           manifest,
@@ -674,10 +711,11 @@ shaka.offline.Storage = class {
     const preferredDecodingAttributes = config.preferredDecodingAttributes;
     const preferredVideoCodecs = config.preferredVideoCodecs;
     const preferredAudioCodecs = config.preferredAudioCodecs;
+    const preferredTextFormats = config.preferredTextFormats;
 
     shaka.util.StreamUtils.chooseCodecsAndFilterManifest(
         manifest, preferredVideoCodecs, preferredAudioCodecs,
-        preferredDecodingAttributes);
+        preferredDecodingAttributes, preferredTextFormats);
 
     for (const variant of manifest.variants) {
       goog.asserts.assert(
@@ -1216,6 +1254,238 @@ shaka.offline.Storage = class {
     return manifest;
   }
 
+  /**
+   * @param {string} uri
+   * @param {shaka.extern.Manifest} manifest
+   * @return {!Promise.<shaka.extern.Stream>}
+   * @private
+   */
+  async createExternalImageStream_(uri, manifest) {
+    const mimeType = await this.getTextMimetype_(uri);
+
+    if (mimeType != 'text/vtt') {
+      throw new shaka.util.Error(
+          shaka.util.Error.Severity.RECOVERABLE,
+          shaka.util.Error.Category.TEXT,
+          shaka.util.Error.Code.UNSUPPORTED_EXTERNAL_THUMBNAILS_URI,
+          uri);
+    }
+
+    goog.asserts.assert(
+        this.networkingEngine_, 'Need networking engine.');
+    const buffer = await this.getTextData_(uri,
+        this.networkingEngine_,
+        this.config_.streaming.retryParameters);
+
+    const factory = shaka.text.TextEngine.findParser(mimeType);
+    if (!factory) {
+      throw new shaka.util.Error(
+          shaka.util.Error.Severity.CRITICAL,
+          shaka.util.Error.Category.TEXT,
+          shaka.util.Error.Code.MISSING_TEXT_PLUGIN,
+          mimeType);
+    }
+    const TextParser = factory();
+    const time = {
+      periodStart: 0,
+      segmentStart: 0,
+      segmentEnd: manifest.presentationTimeline.getDuration(),
+      vttOffset: 0,
+    };
+    const data = shaka.util.BufferUtils.toUint8(buffer);
+    const cues = TextParser.parseMedia(data, time, uri, /* images= */ []);
+
+    const references = [];
+    for (const cue of cues) {
+      let uris = null;
+      const getUris = () => {
+        if (uris == null) {
+          uris = shaka.util.ManifestParserUtils.resolveUris(
+              [uri], [cue.payload]);
+        }
+        return uris || [];
+      };
+      const reference = new shaka.media.SegmentReference(
+          cue.startTime,
+          cue.endTime,
+          getUris,
+          /* startByte= */ 0,
+          /* endByte= */ null,
+          /* initSegmentReference= */ null,
+          /* timestampOffset= */ 0,
+          /* appendWindowStart= */ 0,
+          /* appendWindowEnd= */ Infinity,
+      );
+      if (cue.payload.includes('#xywh')) {
+        const spriteInfo = cue.payload.split('#xywh=')[1].split(',');
+        if (spriteInfo.length === 4) {
+          reference.setThumbnailSprite({
+            height: parseInt(spriteInfo[3], 10),
+            positionX: parseInt(spriteInfo[0], 10),
+            positionY: parseInt(spriteInfo[1], 10),
+            width: parseInt(spriteInfo[2], 10),
+          });
+        }
+      }
+      references.push(reference);
+    }
+
+    let segmentMimeType = mimeType;
+    if (references.length) {
+      segmentMimeType = await shaka.net.NetworkingUtils.getMimeType(
+          references[0].getUris()[0],
+          this.networkingEngine_, this.config_.manifest.retryParameters);
+    }
+
+    return {
+      id: this.nextExternalStreamId_++,
+      originalId: null,
+      groupId: null,
+      createSegmentIndex: () => Promise.resolve(),
+      segmentIndex: new shaka.media.SegmentIndex(references),
+      mimeType: segmentMimeType || '',
+      codecs: '',
+      kind: '',
+      encrypted: false,
+      drmInfos: [],
+      keyIds: new Set(),
+      language: 'und',
+      originalLanguage: null,
+      label: null,
+      type: shaka.util.ManifestParserUtils.ContentType.IMAGE,
+      primary: false,
+      trickModeVideo: null,
+      emsgSchemeIdUris: null,
+      roles: [],
+      forced: false,
+      channelsCount: null,
+      audioSamplingRate: null,
+      spatialAudio: false,
+      closedCaptions: null,
+      tilesLayout: '1x1',
+      accessibilityPurpose: null,
+      external: true,
+      fastSwitching: false,
+      fullMimeTypes: new Set([shaka.util.MimeUtils.getFullType(
+          segmentMimeType || '', '')]),
+      isAudioMuxedInVideo: false,
+    };
+  }
+
+  /**
+   * @param {shaka.extern.Manifest} manifest
+   * @param {string} uri
+   * @param {string} language
+   * @param {string} kind
+   * @param {string=} mimeType
+   * @param {string=} codec
+   * @private
+   */
+  async createExternalTextStream_(manifest, uri, language, kind, mimeType,
+      codec) {
+    if (!mimeType) {
+      mimeType = await this.getTextMimetype_(uri);
+    }
+
+    /** @type {shaka.extern.Stream} */
+    const stream = {
+      id: this.nextExternalStreamId_++,
+      originalId: null,
+      groupId: null,
+      createSegmentIndex: () => Promise.resolve(),
+      segmentIndex: shaka.media.SegmentIndex.forSingleSegment(
+          /* startTime= */ 0,
+          /* duration= */ manifest.presentationTimeline.getDuration(),
+          /* uris= */ [uri]),
+      mimeType: mimeType || '',
+      codecs: codec || '',
+      kind: kind,
+      encrypted: false,
+      drmInfos: [],
+      keyIds: new Set(),
+      language: language,
+      originalLanguage: language,
+      label: null,
+      type: shaka.util.ManifestParserUtils.ContentType.TEXT,
+      primary: false,
+      trickModeVideo: null,
+      emsgSchemeIdUris: null,
+      roles: [],
+      forced: false,
+      channelsCount: null,
+      audioSamplingRate: null,
+      spatialAudio: false,
+      closedCaptions: null,
+      accessibilityPurpose: null,
+      external: true,
+      fastSwitching: false,
+      fullMimeTypes: new Set([shaka.util.MimeUtils.getFullType(
+          mimeType || '', codec || '')]),
+      isAudioMuxedInVideo: false,
+    };
+
+    const fullMimeType = shaka.util.MimeUtils.getFullType(
+        stream.mimeType, stream.codecs);
+    const supported = shaka.text.TextEngine.isTypeSupported(fullMimeType);
+    if (!supported) {
+      throw new shaka.util.Error(
+          shaka.util.Error.Severity.CRITICAL,
+          shaka.util.Error.Category.TEXT,
+          shaka.util.Error.Code.MISSING_TEXT_PLUGIN,
+          mimeType);
+    }
+
+    return stream;
+  }
+
+  /**
+   * @param {string} uri
+   * @return {!Promise.<string>}
+   * @private
+   */
+  async getTextMimetype_(uri) {
+    let mimeType;
+    try {
+      goog.asserts.assert(
+          this.networkingEngine_, 'Need networking engine.');
+      // eslint-disable-next-line require-atomic-updates
+      mimeType = await shaka.net.NetworkingUtils.getMimeType(uri,
+          this.networkingEngine_,
+          this.config_.streaming.retryParameters);
+    } catch (error) {}
+
+    if (mimeType) {
+      return mimeType;
+    }
+
+    shaka.log.error(
+        'The mimeType has not been provided and it could not be deduced ' +
+        'from its uri.');
+    throw new shaka.util.Error(
+        shaka.util.Error.Severity.RECOVERABLE,
+        shaka.util.Error.Category.TEXT,
+        shaka.util.Error.Code.TEXT_COULD_NOT_GUESS_MIME_TYPE,
+        uri);
+  }
+
+  /**
+   * @param {string} uri
+   * @param {!shaka.net.NetworkingEngine} netEngine
+   * @param {shaka.extern.RetryParameters} retryParams
+   * @return {!Promise.<BufferSource>}
+   * @private
+   */
+  async getTextData_(uri, netEngine, retryParams) {
+    const type = shaka.net.NetworkingEngine.RequestType.SEGMENT;
+
+    const request = shaka.net.NetworkingEngine.makeRequest([uri], retryParams);
+    request.method = 'GET';
+
+    const response = await netEngine.request(type, request).promise;
+
+    return response.data;
+  }
+
   /**
    * This method is public so that it can be override in testing.
    *
@@ -1340,6 +1610,7 @@ shaka.offline.Storage = class {
       tilesLayout: stream.tilesLayout,
       external: stream.external,
       fastSwitching: stream.fastSwitching,
+      isAudioMuxedInVideo: stream.isAudioMuxedInVideo,
     };
 
     const startTime =
@@ -1396,6 +1667,7 @@ shaka.offline.Storage = class {
         dataKey: 0,
         mimeType: segment.mimeType,
         codecs: segment.codecs,
+        thumbnailSprite: segment.thumbnailSprite,
       };
       streamDb.segments.push(segmentDB);
       if (numberOfParallelDownloads !== 0) {