phantomas 2.0.0 → 2.4.0

package/core/modules/requestsMonitor/requestsMonitor.js

@@ -24,28 +24,31 @@ function lowerCaseHeaders(headers) {
 
 // parse given URL to get protocol and domain
 function parseEntryUrl(entry) {
-  const parseUrl = require("url").parse;
   var parsed;
 
   // asset type
   entry.type = "other";
 
-  if (entry.url.indexOf("data:") !== 0) {
-    // @see http://nodejs.org/api/url.html#url_url
-    parsed = parseUrl(entry.url) || {};
+  if (entry.url.indexOf("data:") === 0) {
+    // base64 encoded data
+    entry.domain = false;
+    entry.protocol = false;
+    entry.isBase64 = true;
+  } else if (entry.url.indexOf("blob:") === 0) {
+    // blob image or video
+    entry.domain = false;
+    entry.protocol = false;
+    entry.isBlob = true;
+  } else {
+    parsed = new URL(entry.url) || {};
 
-    entry.protocol = parsed.protocol.replace(":", "");
+    entry.protocol = parsed.protocol.replace(":", ""); // e.g. "http:"
     entry.domain = parsed.hostname;
-    entry.query = parsed.query;
+    entry.query = parsed.search.substring(1);
 
     if (entry.protocol === "https") {
       entry.isSSL = true;
     }
-  } else {
-    // base64 encoded data
-    entry.domain = false;
-    entry.protocol = false;
-    entry.isBase64 = true;
   }
 
   return entry;
@@ -103,6 +106,7 @@ function addContentType(headerValue, entry) {
       break;
 
     case "video/webm":
+    case "video/mp4":
       entry.type = "video";
       entry.isVideo = true;
       break;
@@ -189,6 +193,11 @@ module.exports = function (phantomas) {
       const resId = resp._requestId,
         request = requests[resId];
 
+      if (resp.fromDiskCache === true) {
+        phantomas.log("response from disk cache ignored: %j", resp);
+        return;
+      }
+
       var entry = {
         id: resId,
         url: resp.url,
@@ -226,7 +235,7 @@ module.exports = function (phantomas) {
        *
        * "Throughout this work, time is measured in milliseconds"
        */
-      if (!entry.isBase64) {
+      if (!entry.isBase64 && !entry.isBlob) {
         // resp.timing is empty when handling data:image/gif;base64,R0lGODlhAQABAIABAAAAAP///yH5BAEAAAEALAAAAAABAAEAQAICTAEAOw%3D%3D
         assert(
           typeof resp.timing !== "undefined",
@@ -314,8 +323,14 @@ module.exports = function (phantomas) {
           break;
       }
 
+      // HTTP and TLS protocols version
+      entry.httpVersion = resp.protocol;
+      if (resp.securityDetails) {
+        entry.tlsVersion = resp.securityDetails.protocol;
+      }
+
       // requests stats
-      if (!entry.isBase64) {
+      if (!entry.isBase64 && !entry.isBlob) {
         phantomas.incrMetric("requests");
         phantomas.addOffender("requests", {
           url: entry.url,
@@ -343,6 +358,8 @@ module.exports = function (phantomas) {
 
       if (entry.isBase64) {
         phantomas.emit("base64recv", entry, resp); // @desc base64-encoded "response" has been received
+      } else if (entry.isBlob) {
+        // Do nothing
       } else {
         phantomas.log(
           "recv: HTTP %d <%s> [%s]",

package/core/scope.js

@@ -3,7 +3,8 @@
  *
  * Code below is executed in page's "scope" (injected by lib/browser.js)
  */
-(function (scope) {
+/* istanbul ignore next */
+(function coreScope(scope) {
   "use strict";
 
   // create a scope

package/extensions/cookies/cookies.js

@@ -54,8 +54,7 @@ module.exports = function (phantomas) {
 
   phantomas.on("init", async (page) => {
     const url = phantomas.getParam("url"),
-      // https://nodejs.org/docs/latest/api/url.html#url_legacy_url_api
-      domain = require("url").parse(url).hostname;
+      domain = new URL(url).hostname;
 
     // domain field in cookies needs to be set
     // https://github.com/miyakogi/pyppeteer/issues/94#issuecomment-403261859

package/extensions/filmStrip/filmStrip.js

@@ -28,7 +28,7 @@ module.exports = function (phantomas) {
       .replace(/\/+$/, ""),
     filmStripPrefix = phantomas
       .getParam("film-strip-prefix", "screenshot", "string")
-      .replace(/[^a-z0-9\-]+/gi, "-");
+      .replace(/[^a-z0-9-]+/gi, "-");
 
   var zoomFactor = phantomas.getParam("film-strip-zoom-factor", 0.5);
   phantomas.setZoom(zoomFactor);

package/extensions/pageSource/pageSource.js

@@ -33,6 +33,8 @@ module.exports = (phantomas) => {
     return new Promise(async (resolve) => {
       // https://github.com/GoogleChrome/puppeteer/blob/v1.11.0/docs/api.md#pageevaluatepagefunction-args
       const bodyHandle = await page.$("body");
+
+      /* istanbul ignore next */
       const html = await page.evaluate((body) => body.innerHTML, bodyHandle);
 
       // phantomas.log(html);

package/extensions/pageStorage/pageStorage.js

@@ -0,0 +1,82 @@
+/**
+ * Support for session-/localStorage injection.
+ */
+"use strict";
+
+module.exports = function (phantomas) {
+  const SESSION_STORAGE = "session-storage",
+    LOCAL_STORAGE = "local-storage";
+
+  phantomas.on("init", async (page) => {
+    let sessionStorage = phantomas.getParam(SESSION_STORAGE, false);
+    let localStorage = phantomas.getParam(LOCAL_STORAGE, false);
+
+    // Mapping given "storage-string" to json object if string has been given
+    if (typeof sessionStorage == "string" || sessionStorage instanceof String) {
+      sessionStorage = parseStorage(sessionStorage);
+    }
+
+    if (typeof localStorage == "string" || localStorage instanceof String) {
+      localStorage = parseStorage(localStorage);
+    }
+
+    if (sessionStorage) {
+      phantomas.log(
+        "Injecting sessionStorage: %j",
+        JSON.stringify(sessionStorage)
+      );
+      await injectStorage(page, sessionStorage, SESSION_STORAGE);
+    }
+    if (localStorage) {
+      phantomas.log("Injecting localStorag: %j", JSON.stringify(localStorage));
+      await injectStorage(page, localStorage, LOCAL_STORAGE);
+    }
+  });
+
+  function parseStorage(storageString) {
+    // --sessionStorage='bar=foo;domain=url'
+    // --localStorage='bar=fooLocal;domain=urlLocal'
+    var storageMap = {};
+    storageString.split(";").forEach(function (singleEntry) {
+      var entryKeyValue = singleEntry.split("=");
+      storageMap[entryKeyValue[0]] = entryKeyValue[1];
+    });
+    return storageMap;
+  }
+
+  /**
+   * Inject the given storage into the specified page storage.
+   * Either localStorage or sessionStorage
+   *
+   * @param {Page} page in which page the storage should be injected
+   * @param {Object} storage the JSON object consisting of the storage keys and values
+   * @param {string} storageType either localStorage or sessionStorage
+   */
+  async function injectStorage(page, storage, storageType) {
+    if (!page || !storage || !storageType) {
+      return;
+    }
+
+    /* istanbul ignore next */
+    await page.evaluateOnNewDocument(
+      (storage, storageType, SESSION_STORAGE, LOCAL_STORAGE) => {
+        const keys = Object.keys(storage);
+        const values = Object.values(storage);
+        if (storageType === SESSION_STORAGE) {
+          for (let i = 0; i < keys.length; i++) {
+            sessionStorage.setItem(keys[i], values[i]);
+          }
+        }
+        if (storageType === LOCAL_STORAGE) {
+          for (let i = 0; i < keys.length; i++) {
+            localStorage.setItem(keys[i], values[i]);
+          }
+        }
+      },
+      storage,
+      storageType,
+      SESSION_STORAGE,
+      LOCAL_STORAGE
+    );
+  }
+};

package/extensions/screenshot/screenshot.js

@@ -5,16 +5,24 @@
 
 module.exports = function (phantomas) {
   const workingDirectory = require("process").cwd(),
-    param = phantomas.getParam("screenshot");
+    param = phantomas.getParam("screenshot"),
+    fullPage = phantomas.getParam("fullPageScreenshot");
   var path = "";
 
   if (typeof param === "undefined") {
     phantomas.log(
-      "Screenshot: to enable screenshot of the fully loaded page run phantomas with --screenshot option"
+      "Screenshot: to enable screenshot of the page run phantomas with --screenshot option"
     );
     return;
   }
 
+  if (fullPage === true) {
+    // the full page option is now disabled by default (bug #853)
+    phantomas.log(
+      "Screenshot: --full-page-screenshot option enabled. Please note that the option can cause layout bugs and lazyloadableImagesUnderTheFold miscount"
+    );
+  }
+
   // --screenshot
   if (param === true) {
     // defaults to "2013-12-07T20:15:01.521Z.png"
@@ -27,7 +35,11 @@ module.exports = function (phantomas) {
     path = param;
   }
 
-  path = workingDirectory + "/" + path;
+  // deal with relative paths
+  if (path.indexOf("/") !== 0) {
+    path = workingDirectory + "/" + path;
+  }
+
   phantomas.log("Screenshot will be saved in %s", path);
 
   phantomas.on("beforeClose", (page) => {
@@ -36,17 +48,23 @@ module.exports = function (phantomas) {
       const options = {
         path: path,
         type: "png",
-        fullPage: true, // takes a screenshot of the full scrollable page
+        fullPage: fullPage === true,
       };
       phantomas.log("Will take screenshot, options: %j", options);
 
-      // https://github.com/GoogleChrome/puppeteer/blob/v1.11.0/docs/api.md#pagescreenshotoptions
-      await page.screenshot(options);
+      try {
+        // https://github.com/GoogleChrome/puppeteer/blob/v1.11.0/docs/api.md#pagescreenshotoptions
+        await page.screenshot(options);
+
+        phantomas.log("Screenshot stored in %s", path);
 
-      phantomas.log("Screenshot stored in %s", path);
+        // let clients know that we stored the page source in a file
+        phantomas.emit("screenshot", path);
+      } catch (err) {
+        phantomas.log("Error while taking the screenshot");
+        phantomas.log(err);
+      }
 
-      // let clients know that we stored the page source in a file
-      phantomas.emit("screenshot", path);
       resolve();
     });
   });

package/extensions/scroll/scroll.js

@@ -3,7 +3,6 @@
  *
  * Pass --scroll as an option in CLI mode
  */
-/* global document: true, window: true */
 "use strict";
 
 module.exports = function (phantomas) {
@@ -22,7 +21,10 @@ module.exports = function (phantomas) {
     return new Promise(async (resolve) => {
       phantomas.log("Scrolling the page...");
 
+      /* istanbul ignore next */
       await page.evaluate(() => document.body.scrollIntoView(false));
+
+      /* istanbul ignore next */
       const scrollOffset = await page.evaluate(() => document.body.scrollTop);
 
       // wait for lazy loading to do its job

package/extensions/userAgent/userAgent.js

@@ -0,0 +1,55 @@
+/**
+ * Sets a user agent according to --user-agent or --phone or --tablet options
+ */
+"use strict";
+
+module.exports = function (phantomas) {
+  // the user-agent template we use for all emulated devices
+  let userAgent =
+    "Mozilla/5.0 (<Platform>) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/<BrowserVersion> Safari/537.36 Phantomas/<PhantomasVersion>";
+
+  // default platform for desktop
+  let platform = "Windows NT 10.0; Win64; x64";
+
+  // --phone option overwrites the default platform
+  if (
+    phantomas.getParam("phone") === true ||
+    phantomas.getParam("phone-landscape") === true
+  ) {
+    platform = "Linux; Android 10; SM-G981B";
+  }
+
+  // --tablet option overwrites the default platform
+  if (
+    phantomas.getParam("tablet") === true ||
+    phantomas.getParam("tablet-landscape") === true
+  ) {
+    platform = "Linux; Android 10; SM-T870";
+  }
+
+  // if --user-agent option is set, it overwrites --phone and --tablet
+  // it can contain <Platform>, <BrowserVersion> and <PhantomasVersion> if needed
+  const param = phantomas.getParam("user-agent");
+  if (typeof param !== "undefined") {
+    phantomas.log(
+      "userAgent: --user-agent option detected with value %s",
+      param
+    );
+    userAgent = param;
+  }
+
+  phantomas.on("init", async (page, browser) => {
+    const browserVersion = await browser.version();
+    // browserVersion will look like HeadlessChrome/88.0.4298.0
+    // let's keep the number only:
+    const versionNumber = browserVersion.split("/")[1];
+
+    userAgent = userAgent.replace("<Platform>", platform);
+    userAgent = userAgent.replace("<BrowserVersion>", versionNumber);
+    userAgent = userAgent.replace("<PhantomasVersion>", phantomas.getVersion());
+
+    // @see // https://github.com/GoogleChrome/puppeteer/blob/v1.11.0/docs/api.md#pagesetuseragentuseragent
+    await page.setUserAgent(userAgent);
+    phantomas.log("userAgent set to %s", userAgent);
+  });
+};

package/extensions/viewport/viewport.js

@@ -1,7 +1,7 @@
 /**
  * Provides the --viewport option to set any device resolution and pixel density ratio.
  * If the user sets a viewport size as well as a device option (--phone, --tablet, ...),
- * we assume that he/she wants to overwrite the device values.
+ * we assume that he or she wants to overwrite the device values.
  *
  * Two syntaxes are supported:
  *  - 1200x800 will set a 1x pixel density ratio

package/extensions/waitForSelector/waitForSelector.js

@@ -1,7 +1,6 @@
 /**
  * Delays report generation until given CSS selector can be resolved (i.e. given element exists)
  */
-/* global document: true */
 "use strict";
 
 module.exports = function (phantomas) {

package/hooks/build

@@ -0,0 +1,3 @@
+#!/bin/bash
+# see https://docs.docker.com/docker-hub/builds/advanced/#build-hook-examples
+docker build --build-arg GITHUB_SHA=$SOURCE_COMMIT -f $DOCKERFILE_PATH -t $IMAGE_NAME .

package/lib/browser.js

@@ -2,8 +2,7 @@
  * Expose puppeteer API and events emitter object for lib/index.js
  */
 const debug = require("debug")("phantomas:browser"),
-  puppeteer = require("puppeteer"),
-  VERSION = require("../package.json").version;
+  puppeteer = require("puppeteer");
 
 function Browser() {
   this.browser = null;
@@ -17,7 +16,7 @@ function Browser() {
 Browser.prototype.bind = (events) => (this.events = events);
 
 // initialize puppeter instance
-Browser.prototype.init = async () => {
+Browser.prototype.init = async (phantomasOptions) => {
   const networkDebug = require("debug")("phantomas:network"),
     env = require("process").env;
 
@@ -26,10 +25,23 @@ Browser.prototype.init = async () => {
       // page.evaluate throw "Protocol error (Runtime.callFunctionOn): Target closed." without the following
       // https://github.com/GoogleChrome/puppeteer/issues/1175#issuecomment-369728215
       "--disable-dev-shm-usage",
+
+      // enable http/3 support
+      // https://www.bram.us/2020/04/08/how-to-enable-http3-in-chrome-firefox-safari/
+      "--enable-quic",
+      "--quic-version=h3-29",
     ],
   };
 
+  // handle Phantomas options
+  //
+  // --ignore-ssl-errors               ignores SSL errors, such as expired or self-signed certificate errors
+  if (phantomasOptions["ignore-ssl-errors"]) {
+    options["ignoreHTTPSErrors"] = true;
+  }
+
   // customize path to Chromium binary
+  /* istanbul ignore next */
   if (env["PHANTOMAS_CHROMIUM_EXECUTABLE"]) {
     options.executablePath = env["PHANTOMAS_CHROMIUM_EXECUTABLE"];
   }
@@ -37,6 +49,7 @@ Browser.prototype.init = async () => {
   // detect that we run inside a container
   // @see https://github.com/jessfraz/dockerfiles/issues/65
   // @see https://github.com/Zenika/alpine-chrome#-the-best-with-seccomp
+  /* istanbul ignore next */
   if (env["DOCKERIZED"]) {
     debug("Docker environment detected");
     debug(
@@ -44,13 +57,27 @@ Browser.prototype.init = async () => {
     );
   }
 
+  // detect that we run inside an Amazon Lambda machine
+  // (note: the LAMBDA_TASK_ROOT env variable is automatically set by AWS)
+  // @see https://docs.aws.amazon.com/lambda/latest/dg/configuration-envvars.html#configuration-envvars-runtime
+  /* istanbul ignore next */
+  if (env["LAMBDA_TASK_ROOT"]) {
+    // Chrome then requires some more flags
+    options.args.push(
+      "--no-sandbox",
+      "--no-zygote",
+      "--disable-gpu",
+      "--single-process"
+    );
+  }
+
   debug("Launching Puppeteer: %j", options);
 
   try {
-    // https://github.com/GoogleChrome/puppeteer/blob/v1.11.0/docs/api.md#puppeteerlaunchoptions
+    // https://github.com/puppeteer/puppeteer/blob/main/docs/api.md#puppeteerlaunchoptions
     this.browser = await puppeteer.launch(options);
     this.page = await this.browser.newPage();
-  } catch (ex) {
+  } catch (ex) /* istanbul ignore next */ {
     debug("Puppeteer failed to launch: %s", ex);
     throw ex;
   }
@@ -60,13 +87,6 @@ Browser.prototype.init = async () => {
 
   debug("Using binary from: %s", this.browser.process().spawnfile);
 
-  // set a custom user agent, e.g. "phantomas/2.0.0-beta (HeadlessChrome/72.0.3617.0)"
-  // https://github.com/GoogleChrome/puppeteer/blob/v1.11.0/docs/api.md#pagesetuseragentuseragent
-  const browserVersion = await this.browser.version();
-  await this.page.setUserAgent(
-    "phantomas/" + VERSION + " (" + browserVersion + ")"
-  );
-
   // https://github.com/GoogleChrome/puppeteer/blob/v1.11.0/docs/api.md#browserversion
   debug("Original browser: %s", await this.browser.userAgent());
   debug("Viewport: %j", await this.page.viewport());
@@ -153,8 +173,19 @@ Browser.prototype.init = async () => {
   });
 
   this.onRequestLoaded = (eventName, data) => {
-    var meta = responses[data.requestId],
-      response = meta.response;
+    var meta = responses[data.requestId];
+
+    /* istanbul ignore if */
+    if (typeof meta === "undefined") {
+      // the browser sometimes looses trace of a request, let's ignore.
+      networkDebug(
+        "Can't find request id %d in previous requests",
+        data.requestId
+      );
+      return;
+    }
+
+    var response = meta.response;
 
     // errorText: 'net::ERR_FAILED' - request is blocked (meta.response will be empty)
     // errorText: 'net::ERR_ABORTED' - HTTP 404
@@ -181,17 +212,29 @@ Browser.prototype.init = async () => {
     // https://github.com/GoogleChrome/puppeteer/blob/v1.11.0/docs/api.md#cdpsessionsendmethod-params
     response.getContent = (async () => {
       networkDebug("Getting content for #%s", data.requestId);
-      const resp = await this.cdp.send("Network.getResponseBody", {
-        requestId: data.requestId,
-      });
-      networkDebug(
-        "Content for #%s received (%d bytes)",
-        data.requestId,
-        resp.body.length
-      );
+      let body = null;
+
+      try {
+        const resp = await this.cdp.send("Network.getResponseBody", {
+          requestId: data.requestId,
+        });
+        networkDebug(
+          "Content for #%s received (%d bytes)",
+          data.requestId,
+          resp.body.length
+        );
+
+        body = resp.body;
+      } catch (err) {
+        // In case the resource was dumped after a redirect
+        // https://github.com/puppeteer/puppeteer/issues/2258
+        networkDebug(
+          "Could not read the content of #%s. It was probably removed from the browser's buffer by a redirect.",
+          data.requestId
+        );
+      }
 
-      //console.log('getContent()', resp);
-      return resp.body;
+      return body;
     }).bind(this);
 
     networkDebug(
@@ -229,9 +272,10 @@ Browser.prototype.init = async () => {
   // Fired when data chunk was received over the network
   this.cdp.on("Network.dataReceived", (data) => {
     networkDebug("Network.dataReceived: %j", data);
-
-    responses[data.requestId]._chunks++;
-    responses[data.requestId]._dataLength += data.dataLength;
+    if (responses[data.requestId]) {
+      responses[data.requestId]._chunks++;
+      responses[data.requestId]._dataLength += data.dataLength;
+    }
   });
 
   return this.page;
@@ -255,11 +299,12 @@ Browser.prototype.visit = (url, waitUntil, timeout) => {
         waitUntil: waitUntil,
         timeout: (timeout || 30) * 1000, // defaults to 30 seconds, provide in miliseconds!
       });
+
+      debug("URL opened: <%s>", url);
     } catch (ex) {
       debug("Opening URL failed: " + ex);
       return reject(ex);
     }
-    debug("URL opened: <%s>", url);
 
     // https://github.com/GoogleChrome/puppeteer/issues/1325#issuecomment-382003386
     // bind to this event when getting "Protocol error (Runtime.callFunctionOn): Target closed."
@@ -271,7 +316,7 @@ Browser.prototype.visit = (url, waitUntil, timeout) => {
       debug("Metrics: %s", JSON.stringify(metrics));
 
       this.events.emit("metrics", metrics); // @desc Emitted when Chromuim's page.metrics() has been called
-    } catch (ex) {
+    } catch (ex) /* istanbul ignore next */ {
       debug("Get metrics failed: " + ex);
       return reject(ex);
     }
@@ -284,17 +329,20 @@ Browser.prototype.close = async () => {
   try {
     // Allow the beforeunload event to be executed
     // https://github.com/puppeteer/puppeteer/blob/v1.11.0/docs/api.md#pagecloseoptions
-    if (this.page) await this.page.close({ runBeforeUnload: true });
+    if (this.page && !this.page.isClosed())
+      await this.page.close({ runBeforeUnload: true });
 
     // The page is closed, let's close the browser
-    if (this.browser) await this.browser.close();
-  } catch (ex) {
+    if (this.browser && this.browser.isConnected()) await this.browser.close();
+  } catch (ex) /* istanbul ignore next */ {
     debug("An exception was raised in Browser.prototype.close(): " + ex);
-    throw ex;
+    debug(ex);
   }
 
   this.events.emit("close"); // @desc Chromium has been closed
   debug("Browser closed");
 };
 
+Browser.prototype.getPuppeteerBrowser = () => this.browser;
+
 module.exports = Browser;