phantomas 2.2.0 → 2.5.0

package/Dockerfile

@@ -1,26 +1,30 @@
 # https://hub.docker.com/_/node
-FROM node:lts-alpine3.13
-
-# Installs latest Chromium package.
-# https://pkgs.alpinelinux.org/package/edge/community/x86_64/chromium
-ENV CHROMIUM_VERSION 88.0.4324.182-r0
-
-RUN echo "http://dl-cdn.alpinelinux.org/alpine/edge/main" > /etc/apk/repositories \
-  && echo "http://dl-cdn.alpinelinux.org/alpine/edge/community" >> /etc/apk/repositories \
-  && echo "http://dl-cdn.alpinelinux.org/alpine/edge/testing" >> /etc/apk/repositories \
-  && echo "http://dl-cdn.alpinelinux.org/alpine/v3.13/main" >> /etc/apk/repositories \
-  && apk upgrade -U -a \
-  && apk add \
-    chromium \
-    ca-certificates \
-    freetype \
-    freetype-dev \
-    harfbuzz \
-    nss \
-    ttf-freefont
-
-RUN which chromium-browser
-RUN chromium-browser --no-sandbox --version
+FROM node:lts-bullseye-slim
+
+# install dependencies of Chrome binary that will be fetched by npm ci
+RUN apt-get update \
+  && apt-get install -y --no-install-recommends \
+      fonts-liberation \
+      libasound2 \
+      libatk-bridge2.0-0  \
+      libatk1.0-0 \
+      libatspi2.0-0 \
+      libc6 \
+      libcairo2 \
+      libcups2 \
+      libdbus-1-3 \
+      libfreetype6 \
+      libgbm1 \
+      libharfbuzz0b  \
+      libnss3 \
+      libpango-1.0-0 \
+      libx11-6 \
+      libxext6 \
+      libxkbcommon0 \
+      x11-utils \
+      xdg-utils \
+      zlib1g \
+  && rm -rf /var/lib/apt/lists/*
 
 # Set up a working directory
 ENV HOME /opt/phantomas
@@ -30,11 +34,6 @@ RUN chown -R nobody:nogroup .
 # Run everything after as non-privileged user.
 USER nobody
 
-# Tell Puppeteer to skip installing Chrome. We'll be using the installed binary
-ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true
-
-# Tell phantomas where Chromium binary is and that we're in docker
-ENV PHANTOMAS_CHROMIUM_EXECUTABLE /usr/bin/chromium-browser
 ENV DOCKERIZED yes
 
 # Install dependencies
@@ -42,8 +41,12 @@ COPY package.json .
 COPY package-lock.json .
 RUN npm ci
 
-ARG COMMIT_SHA="dev"
-ENV COMMIT_SHA ${COMMIT_SHA}
+# TODO: find the chrome binary and symlink it to the PATH
+RUN ldd $(find . -wholename '*chrome-linux/chrome') && \
+  $(find . -wholename '*chrome-linux/chrome') --version
+
+ARG GITHUB_SHA="dev"
+ENV COMMIT_SHA ${GITHUB_SHA}
 
 # label the image with branch name and commit hash
 LABEL maintainer="maciej.brencz@gmail.com"

package/README.md

@@ -1,11 +1,16 @@
-phantomas [![npm](https://img.shields.io/npm/dt/phantomas.svg)]() [![Inline docs](http://inch-ci.org/github/macbre/phantomas.svg?branch=phantomas-v2)](http://inch-ci.org/github/macbre/phantomas) [![code style: prettier](https://img.shields.io/badge/code_style-prettier-ff69b4.svg?style=flat-square)](https://github.com/prettier/prettier)
+phantomas
+[![npm](https://img.shields.io/npm/dt/phantomas.svg)]()
+[![Inline docs](http://inch-ci.org/github/macbre/phantomas.svg?branch=phantomas-v2)](http://inch-ci.org/github/macbre/phantomas)
+[![code style: prettier](https://img.shields.io/badge/code_style-prettier-ff69b4.svg?style=flat-square)](https://github.com/prettier/prettier)
+[![Coverage Status](https://coveralls.io/repos/github/macbre/phantomas/badge.svg?branch=devel&1)](https://coveralls.io/github/macbre/phantomas?branch=devel)
+[![CodeFactor](https://www.codefactor.io/repository/github/macbre/phantomas/badge)](https://www.codefactor.io/repository/github/macbre/phantomas)
 =========
 
 [Headless Chromium](https://chromium.googlesource.com/chromium/src/+/lkgr/headless/README.md)-based modular web performance metrics collector. And why phantomas? Well, [because](http://en.wikipedia.org/wiki/Fantômas) :)
 
 ## Requirements
 
-* [NodeJS](http://nodejs.org) 12+
+* [NodeJS](http://nodejs.org) 14+
 
 ## Installation
 
@@ -27,6 +32,12 @@ You can use [phantomas Docker image](https://hub.docker.com/r/macbre/phantomas):
 docker pull macbre/phantomas:latest
 ```
 
+Or you can fetch from [GitHub's Containers registry](https://github.com/macbre/phantomas/pkgs/container/phantomas)
+
+```
+docker pull ghcr.io/macbre/phantomas:latest
+```
+
 ## Support
 
 [![Foo](https://raw.githubusercontent.com/macbre/phantomas/devel/docs/phantomas-banner.png)](https://xscode.com/macbre/phantomas)

package/bin/program.js

@@ -68,6 +68,14 @@ function getProgram() {
       "--cookies-file <file>",
       "specifies the file name to store the persistent Cookies"
     )
+    .option(
+      "--local-storage <values>",
+      'ability to set a local storage, key-value pairs (e.g. "bar=foo;domain=url")'
+    )
+    .option(
+      "--session-storage <values>",
+      'ability to set a session storage, key-value pairs (e.g. "bar=foo;domain=url")'
+    )
     .option(
       "--ignore-ssl-errors",
       "ignores SSL errors, such as expired or self-signed certificate errors"

package/core/modules/navigationTiming/scope.js

@@ -1,4 +1,5 @@
-(function (phantomas) {
+/* istanbul ignore next */
+(function navigationTimingScope(phantomas) {
   // Prevent events from being sent from an iframe. Only from the main document.
   if (window.parent !== window || window.location.href === "about:blank") {
     return;

package/core/scope.js

@@ -3,7 +3,8 @@
  *
  * Code below is executed in page's "scope" (injected by lib/browser.js)
  */
-(function (scope) {
+/* istanbul ignore next */
+(function coreScope(scope) {
   "use strict";
 
   // create a scope

package/extensions/pageSource/pageSource.js

@@ -33,6 +33,8 @@ module.exports = (phantomas) => {
     return new Promise(async (resolve) => {
       // https://github.com/GoogleChrome/puppeteer/blob/v1.11.0/docs/api.md#pageevaluatepagefunction-args
       const bodyHandle = await page.$("body");
+
+      /* istanbul ignore next */
       const html = await page.evaluate((body) => body.innerHTML, bodyHandle);
 
       // phantomas.log(html);

package/extensions/pageStorage/pageStorage.js

@@ -0,0 +1,82 @@
+/**
+ * Support for session-/localStorage injection.
+ */
+"use strict";
+
+module.exports = function (phantomas) {
+  const SESSION_STORAGE = "session-storage",
+    LOCAL_STORAGE = "local-storage";
+
+  phantomas.on("init", async (page) => {
+    let sessionStorage = phantomas.getParam(SESSION_STORAGE, false);
+    let localStorage = phantomas.getParam(LOCAL_STORAGE, false);
+
+    // Mapping given "storage-string" to json object if string has been given
+    if (typeof sessionStorage == "string" || sessionStorage instanceof String) {
+      sessionStorage = parseStorage(sessionStorage);
+    }
+
+    if (typeof localStorage == "string" || localStorage instanceof String) {
+      localStorage = parseStorage(localStorage);
+    }
+
+    if (sessionStorage) {
+      phantomas.log(
+        "Injecting sessionStorage: %j",
+        JSON.stringify(sessionStorage)
+      );
+      await injectStorage(page, sessionStorage, SESSION_STORAGE);
+    }
+    if (localStorage) {
+      phantomas.log("Injecting localStorag: %j", JSON.stringify(localStorage));
+      await injectStorage(page, localStorage, LOCAL_STORAGE);
+    }
+  });
+
+  function parseStorage(storageString) {
+    // --sessionStorage='bar=foo;domain=url'
+    // --localStorage='bar=fooLocal;domain=urlLocal'
+    var storageMap = {};
+    storageString.split(";").forEach(function (singleEntry) {
+      var entryKeyValue = singleEntry.split("=");
+      storageMap[entryKeyValue[0]] = entryKeyValue[1];
+    });
+    return storageMap;
+  }
+
+  /**
+   * Inject the given storage into the specified page storage.
+   * Either localStorage or sessionStorage
+   *
+   * @param {Page} page in which page the storage should be injected
+   * @param {Object} storage the JSON object consisting of the storage keys and values
+   * @param {string} storageType either localStorage or sessionStorage
+   */
+  async function injectStorage(page, storage, storageType) {
+    if (!page || !storage || !storageType) {
+      return;
+    }
+
+    /* istanbul ignore next */
+    await page.evaluateOnNewDocument(
+      (storage, storageType, SESSION_STORAGE, LOCAL_STORAGE) => {
+        const keys = Object.keys(storage);
+        const values = Object.values(storage);
+        if (storageType === SESSION_STORAGE) {
+          for (let i = 0; i < keys.length; i++) {
+            sessionStorage.setItem(keys[i], values[i]);
+          }
+        }
+        if (storageType === LOCAL_STORAGE) {
+          for (let i = 0; i < keys.length; i++) {
+            localStorage.setItem(keys[i], values[i]);
+          }
+        }
+      },
+      storage,
+      storageType,
+      SESSION_STORAGE,
+      LOCAL_STORAGE
+    );
+  }
+};

package/extensions/scroll/scroll.js

@@ -21,7 +21,10 @@ module.exports = function (phantomas) {
     return new Promise(async (resolve) => {
       phantomas.log("Scrolling the page...");
 
+      /* istanbul ignore next */
       await page.evaluate(() => document.body.scrollIntoView(false));
+
+      /* istanbul ignore next */
       const scrollOffset = await page.evaluate(() => document.body.scrollTop);
 
       // wait for lazy loading to do its job

package/hooks/build

@@ -1,3 +1,3 @@
 #!/bin/bash
 # see https://docs.docker.com/docker-hub/builds/advanced/#build-hook-examples
-docker build --build-arg COMMIT_SHA=$SOURCE_COMMIT -f $DOCKERFILE_PATH -t $IMAGE_NAME .
+docker build --build-arg GITHUB_SHA=$SOURCE_COMMIT -f $DOCKERFILE_PATH -t $IMAGE_NAME .

package/lib/browser.js

@@ -1,3 +1,4 @@
+// @ts-check
 /**
  * Expose puppeteer API and events emitter object for lib/index.js
  */
@@ -11,7 +12,7 @@ function Browser() {
 
 /**
  * Use the provided events emitter
- * @param {EventEmitter} events
+ * @param {puppeteer.EventEmitter} events
  */
 Browser.prototype.bind = (events) => (this.events = events);
 
@@ -36,6 +37,7 @@ Browser.prototype.init = async (phantomasOptions) => {
   }
 
   // customize path to Chromium binary
+  /* istanbul ignore next */
   if (env["PHANTOMAS_CHROMIUM_EXECUTABLE"]) {
     options.executablePath = env["PHANTOMAS_CHROMIUM_EXECUTABLE"];
   }
@@ -43,6 +45,7 @@ Browser.prototype.init = async (phantomasOptions) => {
   // detect that we run inside a container
   // @see https://github.com/jessfraz/dockerfiles/issues/65
   // @see https://github.com/Zenika/alpine-chrome#-the-best-with-seccomp
+  /* istanbul ignore next */
   if (env["DOCKERIZED"]) {
     debug("Docker environment detected");
     debug(
@@ -53,6 +56,7 @@ Browser.prototype.init = async (phantomasOptions) => {
   // detect that we run inside an Amazon Lambda machine
   // (note: the LAMBDA_TASK_ROOT env variable is automatically set by AWS)
   // @see https://docs.aws.amazon.com/lambda/latest/dg/configuration-envvars.html#configuration-envvars-runtime
+  /* istanbul ignore next */
   if (env["LAMBDA_TASK_ROOT"]) {
     // Chrome then requires some more flags
     options.args.push(
@@ -69,13 +73,13 @@ Browser.prototype.init = async (phantomasOptions) => {
     // https://github.com/puppeteer/puppeteer/blob/main/docs/api.md#puppeteerlaunchoptions
     this.browser = await puppeteer.launch(options);
     this.page = await this.browser.newPage();
-  } catch (ex) {
+  } catch (ex) /* istanbul ignore next */ {
     debug("Puppeteer failed to launch: %s", ex);
     throw ex;
   }
 
   // A Chrome Devtools Protocol session attached to the target
-  this.cdp = this.page._client;
+  this.cdp = this.page._client();
 
   debug("Using binary from: %s", this.browser.process().spawnfile);
 
@@ -94,13 +98,14 @@ Browser.prototype.init = async (phantomasOptions) => {
   this.page.on("dialog", async (dialog) => {
     // https://github.com/GoogleChrome/puppeteer/blob/v1.11.0/docs/api.md#class-dialog
     const message = dialog.message();
-    debug("dialog: %s [%s]", dialog._type, message);
+    const type = dialog.type();
+    debug("dialog: %s [%s]", type, message);
 
-    switch (dialog._type) {
+    switch (type) {
       case "alert":
       case "confirm":
       case "prompt":
-        this.events.emit(dialog._type, message); // @desc Emitted when a JavaScript dialog appears: alert, prompt or confirm
+        this.events.emit(type, message); // @desc Emitted when a JavaScript dialog appears: alert, prompt or confirm
         break;
     }
 
@@ -167,6 +172,7 @@ Browser.prototype.init = async (phantomasOptions) => {
   this.onRequestLoaded = (eventName, data) => {
     var meta = responses[data.requestId];
 
+    /* istanbul ignore if */
     if (typeof meta === "undefined") {
       // the browser sometimes looses trace of a request, let's ignore.
       networkDebug(
@@ -307,7 +313,7 @@ Browser.prototype.visit = (url, waitUntil, timeout) => {
       debug("Metrics: %s", JSON.stringify(metrics));
 
       this.events.emit("metrics", metrics); // @desc Emitted when Chromuim's page.metrics() has been called
-    } catch (ex) {
+    } catch (ex) /* istanbul ignore next */ {
       debug("Get metrics failed: " + ex);
       return reject(ex);
     }
@@ -325,7 +331,7 @@ Browser.prototype.close = async () => {
 
     // The page is closed, let's close the browser
     if (this.browser && this.browser.isConnected()) await this.browser.close();
-  } catch (ex) {
+  } catch (ex) /* istanbul ignore next */ {
     debug("An exception was raised in Browser.prototype.close(): " + ex);
     debug(ex);
   }

package/lib/index.js

@@ -29,7 +29,7 @@ function phantomas(url, opts) {
   debug("phantomas: %s", VERSION);
   debug(
     "Puppeteer: preferred revision r%s",
-    puppeteer._launcher._preferredRevision
+    puppeteer.default._preferredRevision
   );
   debug("URL: <%s>", url);
 
@@ -39,7 +39,7 @@ function phantomas(url, opts) {
 
   debug("Options: %s", JSON.stringify(options));
 
-  events.setMaxListeners(100); // MaxListenersExceededWarning: Possible EventEmitter memory leak detected.
+  events.setMaxListeners(250); // MaxListenersExceededWarning: Possible EventEmitter memory leak detected.
 
   var results = new Results();
   results.setUrl(url);
@@ -82,7 +82,9 @@ function phantomas(url, opts) {
         evaluate: page.evaluate.bind(page),
 
         // @see https://github.com/GoogleChrome/puppeteer/blob/v1.11.0/docs/api.md#pageselector-1
-        querySelectorAll: async (selector) => {
+        querySelectorAll: async function querySelectorAll(
+          selector
+        ) /* istanbul ignore next */ {
           debug('querySelectorAll("%s")', selector);
           return page.$$(selector);
         },
@@ -106,6 +108,7 @@ function phantomas(url, opts) {
 
       // pass phantomas options to page scope
       // https://github.com/GoogleChrome/puppeteer/blob/v1.11.0/docs/api.md#pageevaluateonnewdocumentpagefunction-args
+      /* istanbul ignore next */
       await page.evaluateOnNewDocument((options) => {
         window.__phantomas_options = options;
       }, options);
@@ -132,6 +135,7 @@ function phantomas(url, opts) {
             scope[type].apply(scope, args);
             break;
 
+          /* istanbul ignore next */
           default:
             debug("Unrecognized event type: " + type);
         }

package/lib/metadata/metadata.json

@@ -63,7 +63,7 @@
     "init": {
       "file": "/lib/index.js",
       "desc": "Browser's scope and modules are set up, the page is about to be loaded",
-      "arguments": "page, browser"
+      "arguments": "page, browser.getPuppeteerBrowser("
     },
     "beforeClose": {
       "file": "/lib/index.js",
@@ -102,6 +102,11 @@
       "desc": "Saves the source of page being loaded to a file\nPlease note that saving each file takes a few ms.\nConsider increasing default timeout.\nRun phantomas with --page-source option to use this module.",
       "options": []
     },
+    "pageStorage": {
+      "file": "/extensions/pageStorage/pageStorage.js",
+      "desc": "",
+      "options": []
+    },
     "postLoadDelay": {
       "file": "/extensions/postLoadDelay/postLoadDelay.js",
       "desc": "Delays report generation for a given time",
@@ -117,6 +122,11 @@
       "desc": "Allow page to be scrolled after it is loaded\nPass --scroll as an option in CLI mode",
       "options": []
     },
+    "userAgent": {
+      "file": "/extensions/userAgent/userAgent.js",
+      "desc": "Sets a user agent according to --user-agent or --phone or --tablet options",
+      "options": []
+    },
     "viewport": {
       "file": "/extensions/viewport/viewport.js",
       "desc": "Provides the --viewport option to set any device resolution and pixel density ratio.\nIf the user sets a viewport size as well as a device option (--phone, --tablet, ...),\nwe assume that he or she wants to overwrite the device values.\nTwo syntaxes are supported:\n - 1200x800 will set a 1x pixel density ratio\n - 1200x800x2 will set the given ratio (float values such as 1.5 are accepted)",
@@ -1050,13 +1060,13 @@
       "offenders": true,
       "unit": "number",
       "module": "assetsTypes",
-      "testsCovered": false
+      "testsCovered": true
     },
     "videoSize": {
       "desc": "size of video responses (with compression)",
       "unit": "bytes",
       "module": "assetsTypes",
-      "testsCovered": false
+      "testsCovered": true
     },
     "base64Count": {
       "desc": "number of base64 encoded \"responses\" (no HTTP request was actually made)",
@@ -1206,7 +1216,7 @@
       "testsCovered": false
     },
     "recalcStyleDuration": {
-      "desc": "combined duration of all page style recalculations",
+      "desc": "combined duration of style recalculations",
       "unit": "ms",
       "module": "cpuTasks",
       "testsCovered": false
@@ -1620,7 +1630,7 @@
       "testsCovered": true
     },
     "oldTlsProtocol": {
-      "desc": "number of domains using TLS 1.1 or 1.2",
+      "desc": "number of domains using TLS 1.2",
       "offenders": true,
       "unit": "number",
       "module": "protocols",
@@ -1868,6 +1878,6 @@
   },
   "metricsCount": 187,
   "modulesCount": 36,
-  "extensionsCount": 12,
-  "version": "2.0.0"
+  "extensionsCount": 14,
+  "version": "2.4.0"
 }

package/modules/ajaxRequests/scope.js

@@ -1,4 +1,4 @@
-(function (phantomas) {
+(function ajaxRequestsScope(phantomas) {
   phantomas.spy(
     window.XMLHttpRequest.prototype,
     "open",