serwist 9.0.0-preview.24

package/src/plugins/expiration/models/CacheTimestampsModel.ts

@@ -0,0 +1,184 @@
+/*
+  Copyright 2018 Google LLC
+
+  Use of this source code is governed by an MIT-style
+  license that can be found in the LICENSE file or at
+  https://opensource.org/licenses/MIT.
+*/
+
+import type { DBSchema, IDBPDatabase } from "idb";
+import { deleteDB, openDB } from "idb";
+
+const DB_NAME = "serwist-expiration";
+const CACHE_OBJECT_STORE = "cache-entries";
+
+const normalizeURL = (unNormalizedUrl: string) => {
+  const url = new URL(unNormalizedUrl, location.href);
+  url.hash = "";
+
+  return url.href;
+};
+
+interface CacheTimestampsModelEntry {
+  id: string;
+  cacheName: string;
+  url: string;
+  timestamp: number;
+}
+
+interface CacheDbSchema extends DBSchema {
+  "cache-entries": {
+    key: string;
+    value: CacheTimestampsModelEntry;
+    indexes: { cacheName: string; timestamp: number };
+  };
+}
+
+/**
+ * Returns the timestamp model.
+ *
+ * @private
+ */
+export class CacheTimestampsModel {
+  private readonly _cacheName: string;
+  private _db: IDBPDatabase<CacheDbSchema> | null = null;
+
+  /**
+   *
+   * @param cacheName
+   *
+   * @private
+   */
+  constructor(cacheName: string) {
+    this._cacheName = cacheName;
+  }
+
+  /**
+   * Takes a URL and returns an ID that will be unique in the object store.
+   *
+   * @param url
+   * @returns
+   * @private
+   */
+  private _getId(url: string): string {
+    return `${this._cacheName}|${normalizeURL(url)}`;
+  }
+
+  /**
+   * Performs an upgrade of indexedDB.
+   *
+   * @param db
+   *
+   * @private
+   */
+  private _upgradeDb(db: IDBPDatabase<CacheDbSchema>) {
+    const objStore = db.createObjectStore(CACHE_OBJECT_STORE, {
+      keyPath: "id",
+    });
+
+    // TODO(philipwalton): once we don't have to support EdgeHTML, we can
+    // create a single index with the keyPath `['cacheName', 'timestamp']`
+    // instead of doing both these indexes.
+    objStore.createIndex("cacheName", "cacheName", { unique: false });
+    objStore.createIndex("timestamp", "timestamp", { unique: false });
+  }
+
+  /**
+   * Performs an upgrade of indexedDB and deletes deprecated DBs.
+   *
+   * @param db
+   *
+   * @private
+   */
+  private _upgradeDbAndDeleteOldDbs(db: IDBPDatabase<CacheDbSchema>) {
+    this._upgradeDb(db);
+    if (this._cacheName) {
+      void deleteDB(this._cacheName);
+    }
+  }
+
+  /**
+   * @param url
+   * @param timestamp
+   *
+   * @private
+   */
+  async setTimestamp(url: string, timestamp: number): Promise<void> {
+    url = normalizeURL(url);
+
+    const entry = {
+      id: this._getId(url),
+      cacheName: this._cacheName,
+      url,
+      timestamp,
+    } satisfies CacheTimestampsModelEntry;
+    const db = await this.getDb();
+    const tx = db.transaction(CACHE_OBJECT_STORE, "readwrite", {
+      durability: "relaxed",
+    });
+    await tx.store.put(entry);
+    await tx.done;
+  }
+
+  /**
+   * Returns the timestamp stored for a given URL.
+   *
+   * @param url
+   * @returns
+   * @private
+   */
+  async getTimestamp(url: string): Promise<number | undefined> {
+    const db = await this.getDb();
+    const entry = await db.get(CACHE_OBJECT_STORE, this._getId(url));
+    return entry?.timestamp;
+  }
+
+  /**
+   * Iterates through all the entries in the object store (from newest to
+   * oldest) and removes entries once either `maxCount` is reached or the
+   * entry's timestamp is less than `minTimestamp`.
+   *
+   * @param minTimestamp
+   * @param maxCount
+   * @returns
+   * @private
+   */
+  async expireEntries(minTimestamp: number, maxCount?: number): Promise<string[]> {
+    const db = await this.getDb();
+    let cursor = await db.transaction(CACHE_OBJECT_STORE, "readwrite").store.index("timestamp").openCursor(null, "prev");
+    const urlsDeleted: string[] = [];
+    let entriesNotDeletedCount = 0;
+    while (cursor) {
+      const result = cursor.value;
+      // TODO(philipwalton): once we can use a multi-key index, we
+      // won't have to check `cacheName` here.
+      if (result.cacheName === this._cacheName) {
+        // Delete an entry if it's older than the max age or
+        // if we already have the max number allowed.
+        if ((minTimestamp && result.timestamp < minTimestamp) || (maxCount && entriesNotDeletedCount >= maxCount)) {
+          cursor.delete();
+          urlsDeleted.push(result.url);
+        } else {
+          entriesNotDeletedCount++;
+        }
+      }
+      cursor = await cursor.continue();
+    }
+
+    return urlsDeleted;
+  }
+
+  /**
+   * Returns an open connection to the database.
+   *
+   * @private
+   */
+  private async getDb() {
+    if (!this._db) {
+      this._db = await openDB(DB_NAME, 1, {
+        upgrade: this._upgradeDbAndDeleteOldDbs.bind(this),
+      });
+    }
+    return this._db;
+  }
+}

package/src/plugins/googleAnalytics/constants.ts

@@ -0,0 +1,22 @@
+/*
+  Copyright 2018 Google LLC
+
+  Use of this source code is governed by an MIT-style
+  license that can be found in the LICENSE file or at
+  https://opensource.org/licenses/MIT.
+*/
+
+export const QUEUE_NAME = "serwist-google-analytics";
+export const MAX_RETENTION_TIME = 60 * 48; // Two days in minutes
+export const GOOGLE_ANALYTICS_HOST = "www.google-analytics.com";
+export const GTM_HOST = "www.googletagmanager.com";
+export const ANALYTICS_JS_PATH = "/analytics.js";
+export const GTAG_JS_PATH = "/gtag/js";
+export const GTM_JS_PATH = "/gtm.js";
+export const COLLECT_DEFAULT_PATH = "/collect";
+
+// This RegExp matches all known Measurement Protocol single-hit collect
+// endpoints. Most of the time the default path (/collect) is used, but
+// occasionally an experimental endpoint is used when testing new features,
+// (e.g. /r/collect or /j/collect)
+export const COLLECT_PATHS_REGEX = /^\/(\w+\/)?collect/;

package/src/plugins/googleAnalytics/initialize.ts

@@ -0,0 +1,209 @@
+/*
+  Copyright 2018 Google LLC
+
+  Use of this source code is governed by an MIT-style
+  license that can be found in the LICENSE file or at
+  https://opensource.org/licenses/MIT.
+*/
+
+import type { RouteMatchCallbackOptions } from "../../types.js";
+import { getFriendlyURL } from "../../utils/getFriendlyURL.js";
+import { logger } from "../../utils/logger.js";
+import { cacheNames as privateCacheNames } from "../../utils/cacheNames.js";
+import { Route } from "../../Route.js";
+import { NetworkFirst } from "../../strategies/NetworkFirst.js";
+import { NetworkOnly } from "../../strategies/NetworkOnly.js";
+import { BackgroundSyncPlugin } from "../backgroundSync/BackgroundSyncPlugin.js";
+import type { Queue, QueueEntry } from "../backgroundSync/Queue.js";
+import {
+  ANALYTICS_JS_PATH,
+  COLLECT_PATHS_REGEX,
+  GOOGLE_ANALYTICS_HOST,
+  GTAG_JS_PATH,
+  GTM_HOST,
+  GTM_JS_PATH,
+  MAX_RETENTION_TIME,
+  QUEUE_NAME,
+} from "./constants.js";
+import type { Serwist } from "../../Serwist.js";
+
+export interface GoogleAnalyticsInitializeOptions {
+  serwist: Serwist;
+  /**
+   * The cache name to store and retrieve analytics.js. Defaults to Serwist's default cache names.
+   */
+  cacheName?: string;
+  /**
+   * [Measurement Protocol parameters](https://developers.google.com/analytics/devguides/collection/protocol/v1/parameters),
+   * expressed as key/value pairs, to be added to replayed Google Analytics
+   * requests. This can be used to, e.g., set a custom dimension indicating
+   * that the request was replayed.
+   */
+  parameterOverrides?: { [paramName: string]: string };
+  /**
+   * A function that allows you to modify the hit parameters prior to replaying
+   * the hit. The function is invoked with the original hit's URLSearchParams
+   * object as its only argument.
+   */
+  hitFilter?: (params: URLSearchParams) => void;
+}
+
+/**
+ * Creates the requestWillDequeue callback to be used with the background
+ * sync plugin. The callback takes the failed request and adds the
+ * `qt` param based on the current time, as well as applies any other
+ * user-defined hit modifications.
+ *
+ * @param config
+ * @returns The requestWillDequeue callback function.
+ * @private
+ */
+const createOnSyncCallback = (config: Pick<GoogleAnalyticsInitializeOptions, "parameterOverrides" | "hitFilter">) => {
+  return async ({ queue }: { queue: Queue }) => {
+    let entry: QueueEntry | undefined = undefined;
+    while ((entry = await queue.shiftRequest())) {
+      const { request, timestamp } = entry;
+      const url = new URL(request.url);
+
+      try {
+        // Measurement protocol requests can set their payload parameters in
+        // either the URL query string (for GET requests) or the POST body.
+        const params = request.method === "POST" ? new URLSearchParams(await request.clone().text()) : url.searchParams;
+
+        // Calculate the qt param, accounting for the fact that an existing
+        // qt param may be present and should be updated rather than replaced.
+        const originalHitTime = timestamp! - (Number(params.get("qt")) || 0);
+        const queueTime = Date.now() - originalHitTime;
+
+        // Set the qt param prior to applying hitFilter or parameterOverrides.
+        params.set("qt", String(queueTime));
+
+        // Apply `parameterOverrides`, if set.
+        if (config.parameterOverrides) {
+          for (const param of Object.keys(config.parameterOverrides)) {
+            const value = config.parameterOverrides[param];
+            params.set(param, value);
+          }
+        }
+
+        // Apply `hitFilter`, if set.
+        if (typeof config.hitFilter === "function") {
+          config.hitFilter.call(null, params);
+        }
+
+        // Retry the fetch. Ignore URL search params from the URL as they're
+        // now in the post body.
+        await fetch(
+          new Request(url.origin + url.pathname, {
+            body: params.toString(),
+            method: "POST",
+            mode: "cors",
+            credentials: "omit",
+            headers: { "Content-Type": "text/plain" },
+          }),
+        );
+
+        if (process.env.NODE_ENV !== "production") {
+          logger.log(`Request for '${getFriendlyURL(url.href)}' has been replayed`);
+        }
+      } catch (err) {
+        await queue.unshiftRequest(entry);
+
+        if (process.env.NODE_ENV !== "production") {
+          logger.log(`Request for '${getFriendlyURL(url.href)}' failed to replay, putting it back in the queue.`);
+        }
+        throw err;
+      }
+    }
+    if (process.env.NODE_ENV !== "production") {
+      logger.log("All Google Analytics request successfully replayed; " + "the queue is now empty!");
+    }
+  };
+};
+
+/**
+ * Creates GET and POST routes to catch failed Measurement Protocol hits.
+ *
+ * @param bgSyncPlugin
+ * @returns The created routes.
+ * @private
+ */
+const createCollectRoutes = (bgSyncPlugin: BackgroundSyncPlugin) => {
+  const match = ({ url }: RouteMatchCallbackOptions) => url.hostname === GOOGLE_ANALYTICS_HOST && COLLECT_PATHS_REGEX.test(url.pathname);
+
+  const handler = new NetworkOnly({
+    plugins: [bgSyncPlugin],
+  });
+
+  return [new Route(match, handler, "GET"), new Route(match, handler, "POST")];
+};
+
+/**
+ * Creates a route with a network first strategy for the analytics.js script.
+ *
+ * @param cacheName
+ * @returns The created route.
+ * @private
+ */
+const createAnalyticsJsRoute = (cacheName: string) => {
+  const match = ({ url }: RouteMatchCallbackOptions) => url.hostname === GOOGLE_ANALYTICS_HOST && url.pathname === ANALYTICS_JS_PATH;
+
+  const handler = new NetworkFirst({ cacheName });
+
+  return new Route(match, handler, "GET");
+};
+
+/**
+ * Creates a route with a network first strategy for the gtag.js script.
+ *
+ * @param cacheName
+ * @returns The created route.
+ * @private
+ */
+const createGtagJsRoute = (cacheName: string) => {
+  const match = ({ url }: RouteMatchCallbackOptions) => url.hostname === GTM_HOST && url.pathname === GTAG_JS_PATH;
+
+  const handler = new NetworkFirst({ cacheName });
+
+  return new Route(match, handler, "GET");
+};
+
+/**
+ * Creates a route with a network first strategy for the gtm.js script.
+ *
+ * @param cacheName
+ * @returns The created route.
+ * @private
+ */
+const createGtmJsRoute = (cacheName: string) => {
+  const match = ({ url }: RouteMatchCallbackOptions) => url.hostname === GTM_HOST && url.pathname === GTM_JS_PATH;
+
+  const handler = new NetworkFirst({ cacheName });
+
+  return new Route(match, handler, "GET");
+};
+
+/**
+ * Initialize Serwist's offline Google Analytics v3 support.
+ *
+ * @param options
+ */
+export const initialize = ({ serwist, cacheName, ...options }: GoogleAnalyticsInitializeOptions): void => {
+  const resolvedCacheName = privateCacheNames.getGoogleAnalyticsName(cacheName);
+
+  const bgSyncPlugin = new BackgroundSyncPlugin(QUEUE_NAME, {
+    maxRetentionTime: MAX_RETENTION_TIME,
+    onSync: createOnSyncCallback(options),
+  });
+
+  const routes = [
+    createGtmJsRoute(resolvedCacheName),
+    createAnalyticsJsRoute(resolvedCacheName),
+    createGtagJsRoute(resolvedCacheName),
+    ...createCollectRoutes(bgSyncPlugin),
+  ];
+
+  for (const route of routes) {
+    serwist.registerRoute(route);
+  }
+};

package/src/plugins/precaching/PrecacheFallbackPlugin.ts

@@ -0,0 +1,83 @@
+/*
+  Copyright 2020 Google LLC
+
+  Use of this source code is governed by an MIT-style
+  license that can be found in the LICENSE file or at
+  https://opensource.org/licenses/MIT.
+*/
+
+import type { HandlerDidErrorCallbackParam, SerwistPlugin } from "../../types.js";
+import type { Serwist } from "../../Serwist.js";
+
+export interface PrecacheFallbackEntry {
+  /**
+   * A function that checks whether the fallback entry can be used
+   * for a request.
+   */
+  matcher: (param: HandlerDidErrorCallbackParam) => boolean;
+  /**
+   * A precached URL to be used as a fallback.
+   */
+  url: string;
+}
+
+export interface PrecacheFallbackPluginOptions {
+  /**
+   * Precached URLs to be used as the fallback
+   * if the associated strategy can't generate a response.
+   */
+  fallbackUrls: (string | PrecacheFallbackEntry)[];
+  /**
+   * Your `Serwist` instance.
+   */
+  serwist: Serwist;
+}
+
+/**
+ * `PrecacheFallbackPlugin` allows you to specify offline fallbacks
+ * to be used when a given strategy is unable to generate a response.
+ *
+ * It does this by intercepting the `handlerDidError` plugin callback
+ * and returning a precached response, taking the expected revision parameter
+ * into account automatically.
+ *
+ * Unless you explicitly pass in a `PrecacheController` instance to the
+ * constructor, the default instance will be used. Generally speaking, most
+ * developers will end up using the default.
+ */
+export class PrecacheFallbackPlugin implements SerwistPlugin {
+  private readonly _fallbackUrls: (string | PrecacheFallbackEntry)[];
+  private readonly _serwist: Serwist;
+
+  /**
+   * Constructs a new `PrecacheFallbackPlugin` with the associated `fallbackUrls`.
+   *
+   * @param config
+   */
+  constructor({ fallbackUrls, serwist }: PrecacheFallbackPluginOptions) {
+    this._fallbackUrls = fallbackUrls;
+    this._serwist = serwist;
+  }
+
+  /**
+   * @returns The precache response for one of the fallback URLs, or `undefined` if
+   * nothing satisfies the conditions.
+   * @private
+   */
+  async handlerDidError(param: HandlerDidErrorCallbackParam) {
+    for (const fallback of this._fallbackUrls) {
+      if (typeof fallback === "string") {
+        const fallbackResponse = await this._serwist.matchPrecache(fallback);
+        if (fallbackResponse !== undefined) {
+          return fallbackResponse;
+        }
+      } else if (fallback.matcher(param)) {
+        const fallbackResponse = await this._serwist.matchPrecache(fallback.url);
+        if (fallbackResponse !== undefined) {
+          return fallbackResponse;
+        }
+      }
+    }
+    return undefined;
+  }
+}

package/src/plugins/rangeRequests/RangeRequestsPlugin.ts

@@ -0,0 +1,38 @@
+/*
+  Copyright 2018 Google LLC
+
+  Use of this source code is governed by an MIT-style
+  license that can be found in the LICENSE file or at
+  https://opensource.org/licenses/MIT.
+*/
+
+import type { SerwistPlugin } from "../../types.js";
+import { createPartialResponse } from "./createPartialResponse.js";
+
+/**
+ * The range request plugin makes it easy for a request with a 'Range' header to
+ * be fulfilled by a cached response.
+ *
+ * It does this by intercepting the `cachedResponseWillBeUsed` plugin callback
+ * and returning the appropriate subset of the cached response body.
+ */
+export class RangeRequestsPlugin implements SerwistPlugin {
+  /**
+   * @param options
+   * @returns If request contains a 'Range' header, then a
+   * new response with status 206 whose body is a subset of `cachedResponse` is
+   * returned. Otherwise, `cachedResponse` is returned as-is.
+   * @private
+   */
+  cachedResponseWillBeUsed: SerwistPlugin["cachedResponseWillBeUsed"] = async ({ request, cachedResponse }) => {
+    // Only return a sliced response if there's something valid in the cache,
+    // and there's a Range: header in the request.
+    if (cachedResponse && request.headers.has("range")) {
+      return await createPartialResponse(request, cachedResponse);
+    }
+
+    // If there was no Range: header, or if cachedResponse wasn't valid, just
+    // pass it through as-is.
+    return cachedResponse;
+  };
+}

package/src/plugins/rangeRequests/createPartialResponse.ts

@@ -0,0 +1,93 @@
+/*
+  Copyright 2018 Google LLC
+
+  Use of this source code is governed by an MIT-style
+  license that can be found in the LICENSE file or at
+  https://opensource.org/licenses/MIT.
+*/
+
+import { assert } from "../../utils/assert.js";
+import { SerwistError } from "../../utils/SerwistError.js";
+import { logger } from "../../utils/logger.js";
+import { calculateEffectiveBoundaries } from "./utils/calculateEffectiveBoundaries.js";
+import { parseRangeHeader } from "./utils/parseRangeHeader.js";
+
+/**
+ * Given a `Request` and `Response` objects as input, this will return a
+ * promise for a new `Response`.
+ *
+ * If the original `Response` already contains partial content (i.e. it has
+ * a status of 206), then this assumes it already fulfills the `Range:`
+ * requirements, and will return it as-is.
+ *
+ * @param request A request, which should contain a Range:
+ * header.
+ * @param originalResponse A response.
+ * @returns Either a `206 Partial Content` response, with
+ * the response body set to the slice of content specified by the request's
+ * `Range:` header, or a `416 Range Not Satisfiable` response if the
+ * conditions of the `Range:` header can't be met.
+ */
+export const createPartialResponse = async (request: Request, originalResponse: Response): Promise<Response> => {
+  try {
+    if (process.env.NODE_ENV !== "production") {
+      assert!.isInstance(request, Request, {
+        moduleName: "@serwist/range-requests",
+        funcName: "createPartialResponse",
+        paramName: "request",
+      });
+
+      assert!.isInstance(originalResponse, Response, {
+        moduleName: "@serwist/range-requests",
+        funcName: "createPartialResponse",
+        paramName: "originalResponse",
+      });
+    }
+
+    if (originalResponse.status === 206) {
+      // If we already have a 206, then just pass it through as-is;
+      // see https://github.com/GoogleChrome/workbox/issues/1720
+      return originalResponse;
+    }
+
+    const rangeHeader = request.headers.get("range");
+    if (!rangeHeader) {
+      throw new SerwistError("no-range-header");
+    }
+
+    const boundaries = parseRangeHeader(rangeHeader);
+    const originalBlob = await originalResponse.blob();
+
+    const effectiveBoundaries = calculateEffectiveBoundaries(originalBlob, boundaries.start, boundaries.end);
+
+    const slicedBlob = originalBlob.slice(effectiveBoundaries.start, effectiveBoundaries.end);
+    const slicedBlobSize = slicedBlob.size;
+
+    const slicedResponse = new Response(slicedBlob, {
+      // Status code 206 is for a Partial Content response.
+      // See https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/206
+      status: 206,
+      statusText: "Partial Content",
+      headers: originalResponse.headers,
+    });
+
+    slicedResponse.headers.set("Content-Length", String(slicedBlobSize));
+    slicedResponse.headers.set("Content-Range", `bytes ${effectiveBoundaries.start}-${effectiveBoundaries.end - 1}/` + `${originalBlob.size}`);
+
+    return slicedResponse;
+  } catch (error) {
+    if (process.env.NODE_ENV !== "production") {
+      logger.warn("Unable to construct a partial response; returning a " + "416 Range Not Satisfiable response instead.");
+      logger.groupCollapsed("View details here.");
+      logger.log(error);
+      logger.log(request);
+      logger.log(originalResponse);
+      logger.groupEnd();
+    }
+
+    return new Response("", {
+      status: 416,
+      statusText: "Range Not Satisfiable",
+    });
+  }
+};

package/src/plugins/rangeRequests/utils/calculateEffectiveBoundaries.ts

@@ -0,0 +1,59 @@
+/*
+  Copyright 2018 Google LLC
+
+  Use of this source code is governed by an MIT-style
+  license that can be found in the LICENSE file or at
+  https://opensource.org/licenses/MIT.
+*/
+
+import { assert } from "../../../utils/assert.js";
+import { SerwistError } from "../../../utils/SerwistError.js";
+
+/**
+ * @param blob A source blob.
+ * @param start The offset to use as the start of the
+ * slice.
+ * @param end The offset to use as the end of the slice.
+ * @returns An object with `start` and `end` properties, reflecting
+ * the effective boundaries to use given the size of the blob.
+ * @private
+ */
+export const calculateEffectiveBoundaries = (blob: Blob, start?: number, end?: number): { start: number; end: number } => {
+  if (process.env.NODE_ENV !== "production") {
+    assert!.isInstance(blob, Blob, {
+      moduleName: "@serwist/range-requests",
+      funcName: "calculateEffectiveBoundaries",
+      paramName: "blob",
+    });
+  }
+
+  const blobSize = blob.size;
+
+  if ((end && end > blobSize) || (start && start < 0)) {
+    throw new SerwistError("range-not-satisfiable", {
+      size: blobSize,
+      end,
+      start,
+    });
+  }
+
+  let effectiveStart: number;
+  let effectiveEnd: number;
+
+  if (start !== undefined && end !== undefined) {
+    effectiveStart = start;
+    // Range values are inclusive, so add 1 to the value.
+    effectiveEnd = end + 1;
+  } else if (start !== undefined && end === undefined) {
+    effectiveStart = start;
+    effectiveEnd = blobSize;
+  } else if (end !== undefined && start === undefined) {
+    effectiveStart = blobSize - end;
+    effectiveEnd = blobSize;
+  }
+
+  return {
+    start: effectiveStart!,
+    end: effectiveEnd!,
+  };
+};