shaka-player 3.2.2 → 3.3.2

package/lib/offline/storage.js

@@ -13,6 +13,7 @@ goog.require('shaka.log');
 goog.require('shaka.media.DrmEngine');
 goog.require('shaka.media.ManifestParser');
 goog.require('shaka.net.NetworkingEngine');
+goog.require('shaka.offline.DownloadInfo');
 goog.require('shaka.offline.DownloadManager');
 goog.require('shaka.offline.OfflineUri');
 goog.require('shaka.offline.SessionDeleter');
@@ -28,11 +29,10 @@ goog.require('shaka.util.Functional');
 goog.require('shaka.util.IDestroyable');
 goog.require('shaka.util.Iterables');
 goog.require('shaka.util.MimeUtils');
-goog.require('shaka.util.Networking');
+goog.require('shaka.util.Mutex');
 goog.require('shaka.util.Platform');
 goog.require('shaka.util.PlayerConfiguration');
 goog.require('shaka.util.StreamUtils');
-goog.requireType('shaka.media.InitSegmentReference');
 goog.requireType('shaka.media.SegmentReference');
 goog.requireType('shaka.offline.StorageCellHandle');
 
@@ -95,14 +95,6 @@ shaka.offline.Storage = class {
       this.networkingEngine_ = new shaka.net.NetworkingEngine();
     }
 
-    /**
-     * A list of segment ids for all the segments that were added during the
-     * current store. If the store fails or is aborted, these need to be
-     * removed from storage.
-     * @private {!Array.<number>}
-     */
-    this.segmentsFromStore_ = [];
-
     /**
      * A list of open operations that are being performed by this instance of
      * |shaka.offline.Storage|.
@@ -285,7 +277,7 @@ shaka.offline.Storage = class {
   store(uri, appMetadata, mimeType) {
     goog.asserts.assert(
         this.networkingEngine_,
-        'Cannot call |downloadManifest_| after calling |destroy|.');
+        'Cannot call |store| after calling |destroy|.');
 
     // Get a copy of the current config.
     const config = this.getConfiguration();
@@ -371,6 +363,8 @@ shaka.offline.Storage = class {
     const muxer = new shaka.offline.StorageMuxer();
     /** @type {?shaka.offline.StorageCellHandle} */
     let activeHandle = null;
+    /** @type {?number} */
+    let manifestId = null;
 
     // This will be used to store any errors from drm engine. Whenever drm
     // engine is passed to another function to do work, we should check if this
@@ -398,6 +392,7 @@ shaka.offline.Storage = class {
             uri);
       }
 
+      // Create the DRM engine, and load the keys in the manifest.
       drmEngine = await this.createDrmEngine(
           manifest,
           (e) => { drmError = drmError || e; },
@@ -422,38 +417,38 @@ shaka.offline.Storage = class {
       this.ensureNotDestroyed_();
 
       goog.asserts.assert(drmEngine, 'drmEngine should be non-null here.');
+      const {manifestDB, toDownload} = this.makeManifestDB_(
+          drmEngine, manifest, uri, appMetadata, config, downloader);
 
-      const manifestDB = await this.downloadManifest_(
-          activeHandle.cell, drmEngine, manifest, uri, appMetadata, config,
-          downloader);
+      // Store the empty manifest, before downloading the segments.
+      const ids = await activeHandle.cell.addManifests([manifestDB]);
+      this.ensureNotDestroyed_();
+      manifestId = ids[0];
+
+      goog.asserts.assert(drmEngine, 'drmEngine should be non-null here.');
       this.ensureNotDestroyed_();
       if (drmError) {
         throw drmError;
       }
 
-      const ids = await activeHandle.cell.addManifests([manifestDB]);
+      await this.downloadSegments_(toDownload, manifestId, manifestDB,
+          downloader, config, activeHandle.cell, manifest, drmEngine);
       this.ensureNotDestroyed_();
 
       const offlineUri = shaka.offline.OfflineUri.manifest(
-          activeHandle.path.mechanism, activeHandle.path.cell, ids[0]);
+          activeHandle.path.mechanism, activeHandle.path.cell, manifestId);
 
       return shaka.offline.StoredContentUtils.fromManifestDB(
           offlineUri, manifestDB);
     } catch (e) {
-      // If we did start saving some data, we need to remove it all to avoid
-      // wasting storage. However if the muxer did not manage to initialize,
-      // then we won't have an active cell to remove the segments from.
-      if (activeHandle) {
-        await activeHandle.cell.removeSegments(
-            this.segmentsFromStore_, () => {});
+      if (manifestId != null) {
+        await shaka.offline.Storage.cleanStoredManifest(manifestId);
       }
 
       // If we already had an error, ignore this error to avoid hiding
       // the original error.
       throw drmError || e;
     } finally {
-      this.segmentsFromStore_ = [];
-
       await muxer.destroy();
 
       if (parser) {
@@ -466,6 +461,215 @@ shaka.offline.Storage = class {
     }
   }
 
+  /**
+   * Download and then store the contents of each segment.
+   * The promise this returns will wait for local downloads.
+   *
+   * @param {!Array.<!shaka.offline.DownloadInfo>} toDownload
+   * @param {number} manifestId
+   * @param {shaka.extern.ManifestDB} manifestDB
+   * @param {!shaka.offline.DownloadManager} downloader
+   * @param {shaka.extern.PlayerConfiguration} config
+   * @param {shaka.extern.StorageCell} storage
+   * @param {shaka.extern.Manifest} manifest
+   * @param {!shaka.media.DrmEngine} drmEngine
+   * @return {!Promise}
+   * @private
+   */
+  async downloadSegments_(
+      toDownload, manifestId, manifestDB, downloader, config, storage,
+      manifest, drmEngine) {
+    /**
+     * @param {!Array.<!shaka.offline.DownloadInfo>} toDownload
+     * @param {boolean} updateDRM
+     */
+    const download = async (toDownload, updateDRM) => {
+      const throwIfAbortedFn = () => {
+        this.ensureNotDestroyed_();
+      };
+      for (const download of toDownload) {
+        /** @param {?BufferSource} data */
+        let data;
+        const request = download.makeSegmentRequest(config);
+        const estimateId = download.estimateId;
+        const isInitSegment = download.isInitSegment;
+        const onDownloaded = (d) => {
+          data = d;
+          return Promise.resolve();
+        };
+        downloader.queue(download.groupId,
+            request, estimateId, isInitSegment, onDownloaded);
+        downloader.queueWork(download.groupId, async () => {
+          goog.asserts.assert(data, 'We should have loaded data by now');
+          goog.asserts.assert(data instanceof ArrayBuffer,
+              'The data should be an ArrayBuffer');
+
+          const ref = /** @type {!shaka.media.SegmentReference} */ (
+            download.ref);
+          manifestDB = (await shaka.offline.Storage.assignStreamToManifest(
+              manifestId, ref, {data}, throwIfAbortedFn)) || manifestDB;
+        });
+      }
+      await downloader.waitToFinish();
+
+      if (updateDRM) {
+        // Re-store the manifest, to attach session IDs.
+        // These were (maybe) discovered inside the downloader; we can only add
+        // them now, at the end, since the manifestDB is in flux during the
+        // process of downloading and storing, and assignStreamToManifest does
+        // not know about the DRM engine.
+        this.ensureNotDestroyed_();
+        this.setManifestDrmFields_(manifest, manifestDB, drmEngine, config);
+        await storage.updateManifest(manifestId, manifestDB);
+      }
+    };
+
+    const usingBgFetch = false; // TODO: Get.
+
+    if (this.getManifestIsEncrypted_(manifest) && usingBgFetch &&
+        !this.getManifestIncludesInitData_(manifest)) {
+      // Background fetch can't make DRM sessions, so if we have to get the
+      // init data from the init segments, download those first before anything
+      // else.
+      await download(toDownload.filter((info) => info.isInitSegment), true);
+      this.ensureNotDestroyed_();
+      toDownload = toDownload.filter((info) => !info.isInitSegment);
+    }
+
+    if (!usingBgFetch) {
+      await download(toDownload, false);
+      this.ensureNotDestroyed_();
+
+      goog.asserts.assert(
+          !manifestDB.isIncomplete, 'The manifest should be complete by now');
+    } else {
+      // TODO: Send the request to the service worker. Don't await the result.
+    }
+  }
+
+  /**
+   * Removes all of the contents for a given manifest, statelessly.
+   *
+   * @param {number} manifestId
+   * @return {!Promise}
+   */
+  static async cleanStoredManifest(manifestId) {
+    const muxer = new shaka.offline.StorageMuxer();
+    await muxer.init();
+    const activeHandle = await muxer.getActive();
+    const uri = shaka.offline.OfflineUri.manifest(
+        activeHandle.path.mechanism,
+        activeHandle.path.cell,
+        manifestId);
+    await muxer.destroy();
+    const storage = new shaka.offline.Storage();
+    await storage.remove(uri.toString());
+  }
+
+  /**
+   * Load the given manifest, modifies it by assigning the given data to the
+   * segments corresponding to "ref", then re-stores the manifest.
+   * The parts of this function that modify the manifest are protected by a
+   * mutex, to prevent race conditions; specifically, it prevents two parallel
+   * instances of this method from both loading the manifest into memory at the
+   * same time, which would result in the slower/later call overwriting the
+   * changes of the other.
+   *
+   * @param {number} manifestId
+   * @param {!shaka.media.SegmentReference} ref
+   * @param {shaka.extern.SegmentDataDB} data
+   * @param {function()} throwIfAbortedFn  A function that should throw if the
+   *   download has been aborted.
+   * @return {!Promise.<?shaka.extern.ManifestDB>}
+   */
+  static async assignStreamToManifest(manifestId, ref, data, throwIfAbortedFn) {
+    /** @type {shaka.offline.StorageMuxer} */
+    const muxer = new shaka.offline.StorageMuxer();
+
+    const idForRef = shaka.offline.DownloadInfo.idForSegmentRef(ref);
+    let manifestUpdated = false;
+    let dataKey;
+    let activeHandle;
+    /** @type {!shaka.extern.ManifestDB} */
+    let manifestDB;
+
+    let mutexId = 0;
+
+    try {
+      await muxer.init();
+      activeHandle = await muxer.getActive();
+
+      // Store the data.
+      const dataKeys = await activeHandle.cell.addSegments([data]);
+      dataKey = dataKeys[0];
+      throwIfAbortedFn();
+
+      // Acquire the mutex before accessing the manifest, since there could be
+      // multiple instances of this method running at once.
+      mutexId = await shaka.offline.Storage.mutex_.acquire();
+      throwIfAbortedFn();
+
+      // Load the manifest.
+      const manifests = await activeHandle.cell.getManifests([manifestId]);
+      throwIfAbortedFn();
+      manifestDB = manifests[0];
+
+      // Assign the stored data to the manifest.
+      let complete = true;
+      for (const stream of manifestDB.streams) {
+        for (const segment of stream.segments) {
+          if (segment.pendingSegmentRefId == idForRef) {
+            segment.dataKey = dataKey;
+            // Now that the segment has been associated with the appropriate
+            // dataKey, the pendingSegmentRefId is no longer necessary.
+            segment.pendingSegmentRefId = undefined;
+          }
+          if (segment.pendingInitSegmentRefId == idForRef) {
+            segment.initSegmentKey = dataKey;
+            // Now that the init segment has been associated with the
+            // appropriate initSegmentKey, the pendingInitSegmentRefId is no
+            // longer necessary.
+            segment.pendingInitSegmentRefId = undefined;
+          }
+          if (segment.pendingSegmentRefId) {
+            complete = false;
+          }
+          if (segment.pendingInitSegmentRefId) {
+            complete = false;
+          }
+        }
+      }
+
+      // Update the size of the manifest.
+      manifestDB.size += data.data.byteLength;
+
+      // Mark the manifest as complete, if all segments are downloaded.
+      if (complete) {
+        manifestDB.isIncomplete = false;
+      }
+
+      // Re-store the manifest.
+      await activeHandle.cell.updateManifest(manifestId, manifestDB);
+      manifestUpdated = true;
+      throwIfAbortedFn();
+    } catch (e) {
+      await shaka.offline.Storage.cleanStoredManifest(manifestId);
+
+      if (activeHandle && !manifestUpdated && dataKey) {
+        // The cleanStoredManifest method will not "see" any segments that have
+        // been downloaded but not assigned to the manifest yet. So un-store
+        // them separately.
+        await activeHandle.cell.removeSegments([dataKey], (key) => {});
+      }
+
+      throw e;
+    } finally {
+      await muxer.destroy();
+      shaka.offline.Storage.mutex_.release(mutexId);
+    }
+    return manifestDB;
+  }
+
   /**
    * Filter |manifest| such that it will only contain the variants and text
    * streams that we want to store and can actually play.
@@ -477,16 +681,15 @@ shaka.offline.Storage = class {
    * @private
    */
   async filterManifest_(manifest, drmEngine, config) {
-    const StreamUtils = shaka.util.StreamUtils;
-
     // Filter the manifest based on the restrictions given in the player
     // configuration.
     const maxHwRes = {width: Infinity, height: Infinity};
-    StreamUtils.filterByRestrictions(manifest, config.restrictions, maxHwRes);
+    shaka.util.StreamUtils.filterByRestrictions(
+        manifest, config.restrictions, maxHwRes);
 
     // Filter the manifest based on what we know MediaCapabilities will be able
     // to play later (no point storing something we can't play).
-    StreamUtils.filterManifestByMediaCapabilities(
+    await shaka.util.StreamUtils.filterManifestByMediaCapabilities(
         manifest, config.offline.usePersistentLicense);
 
     // Gather all tracks.
@@ -498,24 +701,24 @@ shaka.offline.Storage = class {
     const preferredVideoCodecs = config.preferredVideoCodecs;
     const preferredAudioCodecs = config.preferredAudioCodecs;
 
-    StreamUtils.chooseCodecsAndFilterManifest(
+    shaka.util.StreamUtils.chooseCodecsAndFilterManifest(
         manifest, preferredVideoCodecs, preferredAudioCodecs,
         preferredAudioChannelCount, preferredDecodingAttributes);
 
     for (const variant of manifest.variants) {
       goog.asserts.assert(
-          StreamUtils.isPlayable(variant),
+          shaka.util.StreamUtils.isPlayable(variant),
           'We should have already filtered by "is playable"');
 
-      allTracks.push(StreamUtils.variantToTrack(variant));
+      allTracks.push(shaka.util.StreamUtils.variantToTrack(variant));
     }
 
     for (const text of manifest.textStreams) {
-      allTracks.push(StreamUtils.textStreamToTrack(text));
+      allTracks.push(shaka.util.StreamUtils.textStreamToTrack(text));
     }
 
     for (const image of manifest.imageStreams) {
-      allTracks.push(StreamUtils.imageStreamToTrack(image));
+      allTracks.push(shaka.util.StreamUtils.imageStreamToTrack(image));
     }
 
     // Let the application choose which tracks to store.
@@ -584,19 +787,21 @@ shaka.offline.Storage = class {
 
   /**
    * Create a download manager and download the manifest.
+   * This also sets up download infos for each segment to be downloaded.
    *
-   * @param {shaka.extern.StorageCell} storage
    * @param {!shaka.media.DrmEngine} drmEngine
    * @param {shaka.extern.Manifest} manifest
    * @param {string} uri
    * @param {!Object} metadata
    * @param {shaka.extern.PlayerConfiguration} config
    * @param {!shaka.offline.DownloadManager} downloader
-   * @return {!Promise.<shaka.extern.ManifestDB>}
+   * @return {{
+   *   manifestDB: shaka.extern.ManifestDB,
+   *   toDownload: !Array.<!shaka.offline.DownloadInfo>
+   * }}
    * @private
    */
-  async downloadManifest_(
-      storage, drmEngine, manifest, uri, metadata, config, downloader) {
+  makeManifestDB_(drmEngine, manifest, uri, metadata, config, downloader) {
     const pendingContent = shaka.offline.StoredContentUtils.fromManifest(
         uri, manifest, /* size= */ 0, metadata);
     // In https://github.com/google/shaka-player/issues/2652, we found that this
@@ -619,12 +824,73 @@ shaka.offline.Storage = class {
     };
     downloader.setCallbacks(onProgress, onInitData);
 
-    const isEncrypted = manifest.variants.some((variant) => {
+    const needsInitData = this.getManifestIsEncrypted_(manifest) &&
+                          !this.getManifestIncludesInitData_(manifest);
+
+    let currentSystemId = null;
+    if (needsInitData) {
+      const drmInfo = drmEngine.getDrmInfo();
+      currentSystemId =
+          shaka.offline.Storage.defaultSystemIds_.get(drmInfo.keySystem);
+    }
+
+    // Make the estimator, which is used to make the download registries.
+    const estimator = new shaka.offline.StreamBandwidthEstimator();
+    for (const stream of manifest.textStreams) {
+      estimator.addText(stream);
+    }
+    for (const stream of manifest.imageStreams) {
+      estimator.addImage(stream);
+    }
+    for (const variant of manifest.variants) {
+      estimator.addVariant(variant);
+    }
+    const {streams, toDownload} = this.createStreams_(
+        downloader, estimator, drmEngine, manifest, config);
+
+    const drmInfo = drmEngine.getDrmInfo();
+    const usePersistentLicense = config.offline.usePersistentLicense;
+    if (drmInfo && usePersistentLicense) {
+      // Don't store init data, since we have stored sessions.
+      drmInfo.initData = [];
+    }
+
+    const manifestDB = {
+      creationTime: Date.now(),
+      originalManifestUri: uri,
+      duration: manifest.presentationTimeline.getDuration(),
+      size: 0,
+      expiration: drmEngine.getExpiration(),
+      streams,
+      sessionIds: usePersistentLicense ? drmEngine.getSessionIds() : [],
+      drmInfo,
+      appMetadata: metadata,
+      isIncomplete: true,
+    };
+
+    return {manifestDB, toDownload};
+  }
+
+  /**
+   * @param {shaka.extern.Manifest} manifest
+   * @return {boolean}
+   * @private
+   */
+  getManifestIsEncrypted_(manifest) {
+    return manifest.variants.some((variant) => {
       const videoEncrypted = variant.video && variant.video.encrypted;
       const audioEncrypted = variant.audio && variant.audio.encrypted;
       return videoEncrypted || audioEncrypted;
     });
-    const includesInitData = manifest.variants.some((variant) => {
+  }
+
+  /**
+   * @param {shaka.extern.Manifest} manifest
+   * @return {boolean}
+   * @private
+   */
+  getManifestIncludesInitData_(manifest) {
+    return manifest.variants.some((variant) => {
       const videoDrmInfos = variant.video ? variant.video.drmInfos : [];
       const audioDrmInfos = variant.audio ? variant.audio.drmInfos : [];
       const drmInfos = videoDrmInfos.concat(audioDrmInfos);
@@ -632,36 +898,28 @@ shaka.offline.Storage = class {
         return drmInfos.initData && drmInfos.initData.length;
       });
     });
-    const needsInitData = isEncrypted && !includesInitData;
-
-    let currentSystemId = null;
-    if (needsInitData) {
-      const drmInfo = drmEngine.getDrmInfo();
-      currentSystemId =
-          shaka.offline.Storage.defaultSystemIds_.get(drmInfo.keySystem);
-    }
-
-    try {
-      const manifestDB = this.createOfflineManifest_(
-          downloader, storage, drmEngine, manifest, uri, metadata, config);
+  }
 
-      manifestDB.size = await downloader.waitToFinish();
-      manifestDB.expiration = drmEngine.getExpiration();
-      const sessions = drmEngine.getSessionIds();
-      manifestDB.sessionIds = config.offline.usePersistentLicense ?
-          sessions : [];
+  /**
+   * @param {shaka.extern.Manifest} manifest
+   * @param {shaka.extern.ManifestDB} manifestDB
+   * @param {!shaka.media.DrmEngine} drmEngine
+   * @param {shaka.extern.PlayerConfiguration} config
+   * @private
+   */
+  setManifestDrmFields_(manifest, manifestDB, drmEngine, config) {
+    manifestDB.expiration = drmEngine.getExpiration();
 
-      if (isEncrypted && config.offline.usePersistentLicense &&
-          !sessions.length) {
-        throw new shaka.util.Error(
-            shaka.util.Error.Severity.CRITICAL,
-            shaka.util.Error.Category.STORAGE,
-            shaka.util.Error.Code.NO_INIT_DATA_FOR_OFFLINE);
-      }
+    const sessions = drmEngine.getSessionIds();
+    manifestDB.sessionIds = config.offline.usePersistentLicense ?
+        sessions : [];
 
-      return manifestDB;
-    } finally {
-      await downloader.destroy();
+    if (this.getManifestIsEncrypted_(manifest) &&
+        config.offline.usePersistentLicense && !sessions.length) {
+      throw new shaka.util.Error(
+          shaka.util.Error.Severity.CRITICAL,
+          shaka.util.Error.Category.STORAGE,
+          shaka.util.Error.Code.NO_INIT_DATA_FOR_OFFLINE);
     }
   }
 
@@ -764,7 +1022,7 @@ shaka.offline.Storage = class {
    * @return {!Promise}
    * @private
    */
-  removeFromStorage_( storage, uri, manifest) {
+  removeFromStorage_(storage, uri, manifest) {
     /** @type {!Array.<number>} */
     const segmentIds = shaka.offline.Storage.getAllSegmentIds_(manifest);
 
@@ -921,6 +1179,10 @@ shaka.offline.Storage = class {
     const playerInterface = {
       networkingEngine: networkingEngine,
 
+      // No need to apply CMCD data for offline requests
+      modifyManifestRequest: (request, manifestInfo) => {},
+      modifySegmentRequest: (request, segmentInfo) => {},
+
       // Don't bother filtering now. We will do that later when we have all the
       // information we need to filter.
       filter: () => Promise.resolve(),
@@ -1007,100 +1269,23 @@ shaka.offline.Storage = class {
   }
 
   /**
-   * Creates an offline 'manifest' for the real manifest.  This does not store
-   * the segments yet, only adds them to the download manager through
-   * createStreams_.
-   *
-   * @param {!shaka.offline.DownloadManager} downloader
-   * @param {shaka.extern.StorageCell} storage
-   * @param {!shaka.media.DrmEngine} drmEngine
-   * @param {shaka.extern.Manifest} manifest
-   * @param {string} originalManifestUri
-   * @param {!Object} metadata
-   * @param {shaka.extern.PlayerConfiguration} config
-   * @return {shaka.extern.ManifestDB}
-   * @private
-   */
-  createOfflineManifest_(
-      downloader, storage, drmEngine, manifest, originalManifestUri, metadata,
-      config) {
-    const estimator = new shaka.offline.StreamBandwidthEstimator();
-
-    const streams = this.createStreams_(
-        downloader, storage, estimator, drmEngine, manifest, config);
-
-    const usePersistentLicense = config.offline.usePersistentLicense;
-    const drmInfo = drmEngine.getDrmInfo();
-
-    if (drmInfo && usePersistentLicense) {
-      // Don't store init data, since we have stored sessions.
-      drmInfo.initData = [];
-    }
-
-    return {
-      creationTime: Date.now(),
-      originalManifestUri: originalManifestUri,
-      duration: manifest.presentationTimeline.getDuration(),
-      size: 0,
-      expiration: drmEngine.getExpiration(),
-      streams: streams,
-      sessionIds: usePersistentLicense ? drmEngine.getSessionIds() : [],
-      drmInfo: drmInfo,
-      appMetadata: metadata,
-    };
-  }
-
-  /**
-   * Converts manifest Streams to database Streams.  This will use the current
-   * configuration to get the tracks to use, then it will search each segment
-   * index and add all the segments to the download manager through
-   * createStream_.
+   * Converts manifest Streams to database Streams.
    *
    * @param {!shaka.offline.DownloadManager} downloader
-   * @param {shaka.extern.StorageCell} storage
    * @param {shaka.offline.StreamBandwidthEstimator} estimator
    * @param {!shaka.media.DrmEngine} drmEngine
    * @param {shaka.extern.Manifest} manifest
    * @param {shaka.extern.PlayerConfiguration} config
-   * @return {!Array.<shaka.extern.StreamDB>}
+   * @return {{
+   *   streams: !Array.<shaka.extern.StreamDB>,
+   *   toDownload: !Array.<!shaka.offline.DownloadInfo>
+   * }}
    * @private
    */
-  createStreams_(downloader, storage, estimator, drmEngine, manifest, config) {
-    // Pass all variants and text streams to the estimator so that we can
-    // get the best estimate for each stream later.
-    for (const variant of manifest.variants) {
-      estimator.addVariant(variant);
-    }
-    for (const text of manifest.textStreams) {
-      estimator.addText(text);
-    }
-    for (const image of manifest.imageStreams) {
-      estimator.addImage(image);
-    }
-
-    // TODO(joeyparrish): Break out stack-based state and method params into a
-    // separate class to clean up.  See:
-    // https://github.com/google/shaka-player/issues/2781#issuecomment-678438039
-
-    /**
-     * A cache mapping init segment references to Promises to their DB key.
-     *
-     * @type {!Map.<shaka.media.InitSegmentReference, !Promise.<?number>>}
-     */
-    const initSegmentDbKeyCache = new Map();
-
-    // A null init segment reference always maps to a null DB key.
-    initSegmentDbKeyCache.set(
-        null, /** @type {!Promise.<?number>} */(Promise.resolve(null)));
-
-    /**
-     * A cache mapping equivalent segment references to Promises to their DB
-     * key.  The key in this map is a string of the form
-     * "<URI>-<startByte>-<endByte>".
-     *
-     * @type {!Map.<string, !Promise.<number>>}
-     */
-    const segmentDbKeyCache = new Map();
+  createStreams_(downloader, estimator, drmEngine, manifest, config) {
+    // Download infos are stored based on their refId, to dedup them.
+    /** @type {!Map.<string, !shaka.offline.DownloadInfo>} */
+    const toDownload = new Map();
 
     // Find the streams we want to download and create a stream db instance
     // for each of them.
@@ -1110,8 +1295,7 @@ shaka.offline.Storage = class {
 
     for (const stream of streamSet) {
       const streamDB = this.createStream_(
-          downloader, storage, estimator, manifest, stream, config,
-          initSegmentDbKeyCache, segmentDbKeyCache);
+          downloader, estimator, manifest, stream, config, toDownload);
       streamDBs.set(stream.id, streamDB);
     }
 
@@ -1125,27 +1309,26 @@ shaka.offline.Storage = class {
       }
     }
 
-    return Array.from(streamDBs.values());
+    return {
+      streams: Array.from(streamDBs.values()),
+      toDownload: Array.from(toDownload.values()),
+    };
   }
 
   /**
    * Converts a manifest stream to a database stream.  This will search the
-   * segment index and add all the segments to the download manager.
+   * segment index and add all the segments to the download infos.
    *
    * @param {!shaka.offline.DownloadManager} downloader
-   * @param {shaka.extern.StorageCell} storage
    * @param {shaka.offline.StreamBandwidthEstimator} estimator
    * @param {shaka.extern.Manifest} manifest
    * @param {shaka.extern.Stream} stream
    * @param {shaka.extern.PlayerConfiguration} config
-   * @param {!Map.<shaka.media.InitSegmentReference, !Promise.<?number>>}
-   *   initSegmentDbKeyCache
-   * @param {!Map.<string, !Promise.<number>>} segmentDbKeyCache
+   * @param {!Map.<string, !shaka.offline.DownloadInfo>} toDownload
    * @return {shaka.extern.StreamDB}
    * @private
    */
-  createStream_(downloader, storage, estimator, manifest, stream, config,
-      initSegmentDbKeyCache, segmentDbKeyCache) {
+  createStream_(downloader, estimator, manifest, stream, config, toDownload) {
     /** @type {shaka.extern.StreamDB} */
     const streamDb = {
       id: stream.id,
@@ -1175,138 +1358,60 @@ shaka.offline.Storage = class {
       tilesLayout: stream.tilesLayout,
     };
 
-    // Download each stream in parallel.
-    const downloadGroup = stream.id;
-
     const startTime =
         manifest.presentationTimeline.getSegmentAvailabilityStart();
 
     shaka.offline.Storage.forEachSegment_(stream, startTime, (segment) => {
-      const initSegmentKeyPromise = this.getInitSegmentDbKey_(
-          downloader, downloadGroup, stream.id, storage, estimator,
-          segment.initSegmentReference, config, initSegmentDbKeyCache);
-
-      const segmentKeyPromise = this.getSegmentDbKey_(
-          downloader, downloadGroup, stream.id, storage, estimator, segment,
-          config, segmentDbKeyCache);
-
-      downloader.queueWork(downloadGroup, async () => {
-        const initSegmentKey = await initSegmentKeyPromise;
-        const dataKey = await segmentKeyPromise;
-
-        streamDb.segments.push({
-          initSegmentKey,
-          startTime: segment.startTime,
-          endTime: segment.endTime,
-          appendWindowStart: segment.appendWindowStart,
-          appendWindowEnd: segment.appendWindowEnd,
-          timestampOffset: segment.timestampOffset,
-          tilesLayout: segment.tilesLayout,
-          dataKey,
-        });
-      });
-    });
-
-    return streamDb;
-  }
-
-  /**
-   * Get a Promise to the DB key for a given init segment reference.
-   *
-   * The return values will be cached so that multiple calls with the same init
-   * segment reference will only trigger one request.
-   *
-   * @param {!shaka.offline.DownloadManager} downloader
-   * @param {number} downloadGroup
-   * @param {number} streamId
-   * @param {shaka.extern.StorageCell} storage
-   * @param {shaka.offline.StreamBandwidthEstimator} estimator
-   * @param {shaka.media.InitSegmentReference} initSegmentReference
-   * @param {shaka.extern.PlayerConfiguration} config
-   * @param {!Map.<shaka.media.InitSegmentReference, !Promise.<?number>>}
-   *   initSegmentDbKeyCache
-   * @return {!Promise.<?number>}
-   * @private
-   */
-  getInitSegmentDbKey_(
-      downloader, downloadGroup, streamId, storage, estimator,
-      initSegmentReference, config, initSegmentDbKeyCache) {
-    if (initSegmentDbKeyCache.has(initSegmentReference)) {
-      return initSegmentDbKeyCache.get(initSegmentReference);
-    }
-
-    const request = shaka.util.Networking.createSegmentRequest(
-        initSegmentReference.getUris(),
-        initSegmentReference.startByte,
-        initSegmentReference.endByte,
-        config.streaming.retryParameters);
-
-    const promise = downloader.queue(
-        downloadGroup,
-        request,
-        estimator.getInitSegmentEstimate(streamId),
-        /* isInitSegment= */ true,
-        async (data) => {
-          /** @type {!Array.<number>} */
-          const ids = await storage.addSegments([{data: data}]);
-          this.segmentsFromStore_.push(ids[0]);
-          return ids[0];
-        });
-
-    initSegmentDbKeyCache.set(initSegmentReference, promise);
-    return promise;
-  }
+      const pendingSegmentRefId =
+          shaka.offline.DownloadInfo.idForSegmentRef(segment);
+      let pendingInitSegmentRefId = undefined;
+
+      // Set up the download for the segment, which will be downloaded later,
+      // perhaps in a service worker.
+      if (!toDownload.has(pendingSegmentRefId)) {
+        const estimateId = downloader.addDownloadEstimate(
+            estimator.getSegmentEstimate(stream.id, segment));
+        const segmentDownload = new shaka.offline.DownloadInfo(
+            segment,
+            estimateId,
+            stream.id,
+            /* isInitSegment= */ false);
+        toDownload.set(pendingSegmentRefId, segmentDownload);
+      }
 
-  /**
-   * Get a Promise to the DB key for a given segment reference.
-   *
-   * The return values will be cached so that multiple calls with the same
-   * segment reference will only trigger one request.
-   *
-   * @param {!shaka.offline.DownloadManager} downloader
-   * @param {number} downloadGroup
-   * @param {number} streamId
-   * @param {shaka.extern.StorageCell} storage
-   * @param {shaka.offline.StreamBandwidthEstimator} estimator
-   * @param {shaka.media.SegmentReference} segmentReference
-   * @param {shaka.extern.PlayerConfiguration} config
-   * @param {!Map.<string, !Promise.<number>>} segmentDbKeyCache
-   * @return {!Promise.<number>}
-   * @private
-   */
-  getSegmentDbKey_(
-      downloader, downloadGroup, streamId, storage, estimator,
-      segmentReference, config, segmentDbKeyCache) {
-    const mapKey = [
-      segmentReference.getUris()[0],
-      segmentReference.startByte,
-      segmentReference.endByte,
-    ].join('-');
-
-    if (segmentDbKeyCache.has(mapKey)) {
-      return segmentDbKeyCache.get(mapKey);
-    }
+      // Set up the download for the init segment, similarly, if there is one.
+      if (segment.initSegmentReference) {
+        pendingInitSegmentRefId = shaka.offline.DownloadInfo.idForSegmentRef(
+            segment.initSegmentReference);
+        if (!toDownload.has(pendingInitSegmentRefId)) {
+          const estimateId = downloader.addDownloadEstimate(
+              estimator.getInitSegmentEstimate(stream.id));
+          const initDownload = new shaka.offline.DownloadInfo(
+              segment.initSegmentReference,
+              estimateId,
+              stream.id,
+              /* isInitSegment= */ true);
+          toDownload.set(pendingInitSegmentRefId, initDownload);
+        }
+      }
 
-    const request = shaka.util.Networking.createSegmentRequest(
-        segmentReference.getUris(),
-        segmentReference.startByte,
-        segmentReference.endByte,
-        config.streaming.retryParameters);
-
-    const promise = downloader.queue(
-        downloadGroup,
-        request,
-        estimator.getSegmentEstimate(streamId, segmentReference),
-        /* isInitSegment= */ false,
-        async (data) => {
-          /** @type {!Array.<number>} */
-          const ids = await storage.addSegments([{data: data}]);
-          this.segmentsFromStore_.push(ids[0]);
-          return ids[0];
-        });
+      /** @type {!shaka.extern.SegmentDB} */
+      const segmentDB = {
+        pendingInitSegmentRefId,
+        initSegmentKey: pendingInitSegmentRefId ? 0 : null,
+        startTime: segment.startTime,
+        endTime: segment.endTime,
+        appendWindowStart: segment.appendWindowStart,
+        appendWindowEnd: segment.appendWindowEnd,
+        timestampOffset: segment.timestampOffset,
+        tilesLayout: segment.tilesLayout,
+        pendingSegmentRefId,
+        dataKey: 0,
+      };
+      streamDb.segments.push(segmentDB);
+    });
 
-    segmentDbKeyCache.set(mapKey, promise);
-    return promise;
+    return streamDb;
   }
 
   /**
@@ -1408,21 +1513,21 @@ shaka.offline.Storage = class {
    * @private
    */
   static getAllSegmentIds_(manifest) {
-    /** @type {!Array.<number>} */
-    const ids = [];
+    /** @type {!Set.<number>} */
+    const ids = new Set();
 
     // Get every segment for every stream in the manifest.
     for (const stream of manifest.streams) {
       for (const segment of stream.segments) {
         if (segment.initSegmentKey != null) {
-          ids.push(segment.initSegmentKey);
+          ids.add(segment.initSegmentKey);
         }
 
-        ids.push(segment.dataKey);
+        ids.add(segment.dataKey);
       }
     }
 
-    return ids;
+    return Array.from(ids);
   }
 
   /**
@@ -1557,6 +1662,9 @@ shaka.offline.Storage = class {
   }
 };
 
+/** @private {!shaka.util.Mutex} */
+shaka.offline.Storage.mutex_ = new shaka.util.Mutex();
+
 shaka.offline.Storage.defaultSystemIds_ = new Map()
     .set('org.w3.clearkey', '1077efecc0b24d02ace33c1e52e2fb4b')
     .set('com.widevine.alpha', 'edef8ba979d64acea3c827dcd51d21ed')