puppeteer-pro 1.5.3 → 1.5.6

package/.env.example

@@ -0,0 +1,2 @@
+# wit.ai
+WIT_AI_ACCESS_TOKEN=

package/.eslintignore

@@ -0,0 +1,6 @@
+*.js
+!index.js
+!src/plugins/*/injections/*.js
+
+!test/*.js
+!src/plugins/*/*.js

package/.eslintrc.json

@@ -0,0 +1,39 @@
+{
+  "extends": "eslint:recommended",
+  // "extends": "google",
+  "parserOptions": {
+    "ecmaVersion": 2018
+  },
+  "env": {
+    "node": true,
+    "browser": true,
+    "es6": true,
+    "mocha": true
+  },
+  "rules": {
+    "comma-dangle": [
+      "error",
+      {
+        "arrays": "only-multiline"
+      }
+    ],
+    "linebreak-style": [
+      "error",
+      "windows"
+    ],
+    "max-len": [
+      "error",
+      {
+        "code": 1e10
+      }
+    ],
+    "no-console": "off",
+    "require-await": [
+      "error"
+    ],
+    "semi": [
+      "error",
+      "always"
+    ]
+  }
+}

package/.github/dependabot.yml

@@ -0,0 +1,16 @@
+version: 2
+updates:
+- package-ecosystem: npm
+  directory: "/"
+  schedule:
+    interval: daily
+    time: "03:00"
+    timezone: America/New_York
+  open-pull-requests-limit: 10
+  versioning-strategy: increase
+  labels:
+  - dependencies
+  commit-message:
+    prefix: fix
+    prefix-development: chore
+    include: scope

package/.github/workflows/master.cron.code-analyze.yml

@@ -0,0 +1,58 @@
+name: CodeQL
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+  schedule:
+    - cron: '0 9 * * 1'
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'javascript' ]
+        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
+        # Learn more about CodeQL language support at https://git.io/codeql-language-support
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v1
+      with:
+        languages: ${{ matrix.language }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+        # queries: ./path/to/local/query, your-org/your-repo/queries@main
+
+    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v1
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 https://git.io/JvXDl
+
+    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
+    #    and modify them (or add more) to build your code if your project
+    #    uses a compiled language
+
+    #- run: |
+    #   make bootstrap
+    #   make release
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v1

package/.github/workflows/master.cron.publish.yml

@@ -0,0 +1,34 @@
+name: Publish
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: '0 9 * * *'
+
+jobs:
+  run:
+    name: Publish
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v1
+      - uses: actions/setup-node@v1
+        with:
+          node-version: 16
+      - run: npm install
+
+      - uses: DamianReeves/write-file-action@v1.0
+        with:
+          path: .env
+          contents: ${{ secrets.ENV }}
+          write-mode: overwrite
+          
+      - run: npm run build --if-present
+      - run: npm test
+      
+      - uses: ItzRabbs/semantic-release-action@master
+        with:
+          directory: 'dist'
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          NPM_TOKEN: ${{ secrets.NPM_AUTH_TOKEN }}
+        continue-on-error: true

package/.github/workflows/master.pr.build.yml

@@ -0,0 +1,31 @@
+name: Build on PR
+
+on:
+  pull_request_target:
+    branches:
+      - master
+
+jobs:
+  run:
+    name: Build on PR
+    runs-on: ubuntu-latest
+    if: github.actor == 'dependabot[bot]'
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+
+      - uses: actions/setup-node@v1
+        with:
+          node-version: 16
+      - run: npm install
+
+      - uses: DamianReeves/write-file-action@v1.0
+        with:
+          path: .env
+          contents: ${{ secrets.ENV }}
+          write-mode: overwrite
+          
+      - run: npm run build --if-present
+      - run: npm test

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "puppeteer-pro",
-  "version": "1.5.3",
+  "version": "1.5.6",
   "description": "A simple puppeteer wrapper to enable useful plugins with ease",
   "main": "index.js",
   "scripts": {
@@ -36,20 +36,20 @@
   },
   "dependencies": {
     "axios": "^0.27.2",
-    "ghost-cursor": "^1.1.6",
-    "user-agents": "^1.0.1029"
+    "ghost-cursor": "^1.1.11",
+    "user-agents": "^1.0.1042"
   },
   "devDependencies": {
-    "@types/node": "^17.0.35",
+    "@types/node": "^17.0.41",
     "@types/user-agents": "^1.0.2",
     "chai": "^4.3.6",
     "copyfiles": "^2.4.1",
     "dotenv-safe": "^8.2.0",
-    "eslint": "^8.16.0",
+    "eslint": "^8.17.0",
     "mocha": "^10.0.0",
-    "puppeteer": "^14.1.1",
+    "puppeteer": "^14.3.0",
     "rimraf": "^3.0.2",
     "tslint": "^6.1.3",
-    "typescript": "^4.7.2"
+    "typescript": "^4.7.3"
   }
-}
+}

package/src/index.ts

@@ -0,0 +1,279 @@
+import * as events from 'events';
+
+// Puppeteer Defaults
+import * as Puppeteer from 'puppeteer';
+
+const browserEvents = new events.EventEmitter();
+
+export async function connect(options?: Puppeteer.ConnectOptions): Promise<Puppeteer.Browser> {
+  const browser = await Puppeteer.connect(options || {});
+
+  for (const plugin of plugins) {
+    await plugin.init(browser);
+  }
+
+  const _close = browser.close;
+  browser.close = async () => {
+    await _close.apply(browser);
+    browserEvents.emit('close');
+  };
+
+  return browser;
+}
+
+/** The method launches a browser instance with given arguments. The browser will be closed when the parent node.js process is closed. */
+export async function launch(options?: Puppeteer.LaunchOptions & Puppeteer.BrowserLaunchArgumentOptions & Puppeteer.BrowserConnectOptions): Promise<Puppeteer.Browser> {
+  if (!options) options = {};
+
+  const browser = await Puppeteer.launch({ defaultViewport: undefined, ...options });
+
+  for (const plugin of plugins) {
+    await plugin.init(browser);
+  }
+
+  const _close = browser.close;
+  browser.close = async () => {
+    await _close.apply(browser);
+    browserEvents.emit('close');
+  };
+
+  return browser;
+}
+
+// PuppeteerPro
+let interceptions = 0;
+export class Plugin {
+  protected browser: Puppeteer.Browser | null = null;
+  private initialized = false;
+  private startCounter = 0;
+  protected dependencies: Plugin[] = [];
+  protected requiresInterception = false;
+
+  get isInitialized() { return this.initialized; }
+  get isStopped() { return this.startCounter === 0; }
+
+  protected async addDependency(plugin: Plugin) {
+    this.dependencies.push(plugin);
+  }
+
+  async init(browser: Puppeteer.Browser) {
+    if (this.initialized) return;
+
+    this.browser = browser;
+
+    const offOnClose: (() => void)[] = [];
+    browserEvents.once('close', async () => {
+      offOnClose.forEach(fn => fn());
+
+      this.browser = null;
+      this.initialized = false;
+      this.startCounter = 0;
+
+      await this.onClose();
+    });
+
+    this.startCounter++;
+
+    const thisOnTargetCreated = this.onTargetCreated.bind(this);
+    browser.on('targetcreated', thisOnTargetCreated);
+    offOnClose.push(() => browser.off('targetcreated', thisOnTargetCreated));
+
+    this.initialized = true;
+
+    this.dependencies.forEach(x => x.init(browser));
+
+    return this.afterLaunch(browser);
+  }
+
+  protected async afterLaunch(_browser: Puppeteer.Browser) { }
+  protected async onClose() { }
+
+  protected async onTargetCreated(target: Puppeteer.Target) {
+    if (this.isStopped) return;
+
+    if (target.type() !== 'page') return;
+    const page = await target.page() as Puppeteer.Page;
+    if (page.isClosed()) return;
+
+    const offOnClose: (() => void)[] = [];
+    page.once('close', async () => {
+      offOnClose.forEach(fn => fn());
+    });
+
+    const requestHandlers: ((request: any) => void)[] = [];
+    page.on('request', request => {
+      const _respond = request.respond;
+      let responded = 0;
+      let respondArgs: IArguments;
+
+      const _abort = request.abort;
+      let aborted = 0;
+      let abortArgs: IArguments;
+
+      const _continue = request.continue;
+      let continued = 0;
+      let continueArgs: IArguments;
+
+      // tslint:disable-next-line: only-arrow-functions
+      const handleRequest = async function () {
+        const total = responded + aborted + continued;
+
+        if (!(request as any)._interceptionHandled) {
+          if (responded === 1) await _respond.apply(request, respondArgs);
+          else if (responded === 0 && aborted >= 1 && total === requestHandlers.length) await _abort.apply(request, abortArgs);
+          else if (continued === requestHandlers.length) await _continue.apply(request, continueArgs);
+        }
+      };
+
+      // tslint:disable-next-line: only-arrow-functions
+      request.respond = async function () { responded++; respondArgs = respondArgs || arguments; await handleRequest(); };
+      // tslint:disable-next-line: only-arrow-functions
+      request.abort = async function () { aborted++; abortArgs = abortArgs || arguments; await handleRequest(); };
+      // tslint:disable-next-line: only-arrow-functions
+      request.continue = async function () { continued++; continueArgs = continueArgs || arguments; await handleRequest(); };
+
+      requestHandlers.forEach(handler => handler(request));
+    });
+
+    const _pageOn = page.on;
+    page.on = function async(eventName, handler) {
+      if (eventName === 'request') {
+        requestHandlers.push(handler);
+
+        return page;
+      } else {
+        return _pageOn.call(page, eventName, handler);
+      }
+    };
+
+    if (this.requiresInterception) {
+      await page.setRequestInterception(true);
+
+      const thisOnRequest = this.onRequest.bind(this);
+      page.on('request', thisOnRequest);
+      offOnClose.push(() => page.off('request', thisOnRequest));
+    }
+
+    const thisOnDialog = this.onDialog.bind(this);
+    page.on('dialog', thisOnDialog);
+    offOnClose.push(() => page.off('dialog', thisOnDialog));
+
+    await this.onPageCreated(page);
+  }
+  protected async onPageCreated(_page: Puppeteer.Page) { }
+
+  protected async onRequest(request: Puppeteer.HTTPRequest) {
+    const interceptionHandled = (request as any)._interceptionHandled;
+    if (interceptionHandled) return;
+    if (this.isStopped) return request.continue();
+
+    await this.processRequest(request);
+  }
+  protected async processRequest(_request: Puppeteer.HTTPRequest) { }
+
+  protected async onDialog(dialog: Puppeteer.Dialog) {
+    const handled = (dialog as any)._handled;
+
+    if (handled) return;
+    if (this.isStopped) return;
+
+    await this.processDialog(dialog);
+  }
+  protected async processDialog(_dialog: Puppeteer.Dialog) { }
+
+  protected async beforeRestart() { }
+  async restart() {
+    await this.beforeRestart();
+
+    this.startCounter++;
+    if (this.requiresInterception) interceptions++;
+
+    this.dependencies.forEach(x => x.restart());
+
+    await this.afterRestart();
+  }
+  protected async afterRestart() { }
+
+  protected async beforeStop() { }
+  async stop() {
+    await this.beforeStop();
+
+    this.startCounter--;
+    if (this.requiresInterception) interceptions--;
+
+    if (interceptions === 0 && this.browser) {
+      const pages = await this.browser.pages();
+
+      pages.filter(x => !x.isClosed()).forEach(async (page: Puppeteer.Page) => {
+        await page.setRequestInterception(false);
+      });
+    }
+
+    this.dependencies.forEach(x => x.stop());
+
+    await this.afterStop();
+  }
+  protected async afterStop() { }
+
+  protected async getFirstPage() {
+    if (!this.browser) return null;
+
+    const pages = await this.browser.pages();
+    const openPages = pages.filter(x => !x.isClosed());
+    const activePages = pages.filter(x => x.url() !== 'about:blank');
+
+    return activePages[0] || openPages[0];
+  }
+}
+
+let plugins: Plugin[] = [];
+export function addPlugin(plugin: Plugin) { plugins.push(plugin); }
+export async function clearPlugins() {
+  plugins.forEach(async plugin => {
+    await plugin.stop();
+  });
+
+  plugins = [];
+}
+
+import { AnonymizeUserAgentPlugin } from './plugins/anonymize.user.agent/index';
+export function anonymizeUserAgent(): AnonymizeUserAgentPlugin {
+  const plugin = new AnonymizeUserAgentPlugin();
+  plugins.push(plugin);
+  return plugin;
+}
+
+import { AvoidDetectionPlugin } from './plugins/avoid.detection';
+export function avoidDetection(): AvoidDetectionPlugin {
+  const plugin = new AvoidDetectionPlugin();
+  plugins.push(plugin);
+  return plugin;
+}
+
+import { BlockResourcesPlugin, Resource } from './plugins/block.resources';
+export function blockResources(...resources: Resource[]): BlockResourcesPlugin {
+  const plugin = new BlockResourcesPlugin(resources);
+  plugins.push(plugin);
+  return plugin;
+}
+
+import { DisableDialogsPlugin } from './plugins/disable.dialogs';
+export function disableDialogs(logMessages = false): DisableDialogsPlugin {
+  const plugin = new DisableDialogsPlugin(logMessages);
+  plugins.push(plugin);
+  return plugin;
+}
+
+import { ManageCookiesPlugin, ManageCookiesOption } from './plugins/manage.cookies';
+export function manageCookies(opts: ManageCookiesOption): ManageCookiesPlugin {
+  const plugin = new ManageCookiesPlugin(opts);
+  plugins.push(plugin);
+  return plugin;
+}
+
+import { SolveRecaptchaPlugin } from './plugins/solve.recaptcha';
+export function solveRecaptchas(accessToken: string): SolveRecaptchaPlugin {
+  const plugin = new SolveRecaptchaPlugin(accessToken);
+  plugins.push(plugin);
+  return plugin;
+}

package/src/plugins/anonymize.user.agent/index.ts

@@ -0,0 +1,66 @@
+import * as Puppeteer from 'puppeteer';
+import UserAgent = require('user-agents');
+
+import { Plugin } from '../../index';
+
+const sleep = (time: number) => { return new Promise(resolve => { setTimeout(resolve, time); }); };
+
+interface PageUserAgent {
+  target: Puppeteer.Page;
+  userAgent: string;
+  newUserAgent: string;
+}
+
+export class AnonymizeUserAgentPlugin extends Plugin {
+  private pages: PageUserAgent[] = [];
+  private userAgent?: string;
+
+  constructor() {
+    super();
+
+    try {
+      this.userAgent = new UserAgent({ vendor: 'Google Inc.', platform: 'Win32' }).toString();
+    }
+    catch (ex) {
+      console.warn('Could not create a random user agent');
+    }
+  }
+
+  protected async afterLaunch(browser: Puppeteer.Browser) {
+    const _newPage = browser.newPage;
+    browser.newPage = async (): Promise<Puppeteer.Page> => {
+      const page = await _newPage.apply(browser);
+      await sleep(100); // Sleep to allow user agent to set
+      return page;
+    };
+  }
+
+  protected async onClose() {
+    this.pages = [];
+  }
+
+  protected async onPageCreated(page: Puppeteer.Page) {
+    const userAgent = await page.browser().userAgent();
+    const newUserAgent = this.userAgent || userAgent.replace('HeadlessChrome/', 'Chrome/').replace(/\(([^)]+)\)/, '(Windows NT 10.0; Win64; x64)');
+
+    this.pages.push({ target: page, userAgent, newUserAgent });
+
+    await page.setUserAgent(newUserAgent);
+  }
+
+  protected async beforeRestart() {
+    for (const page of this.pages) {
+      if (page.target.isClosed()) continue;
+
+      await page.target.setUserAgent(page.newUserAgent);
+    }
+  }
+
+  protected async afterStop() {
+    for (const page of this.pages) {
+      if (page.target.isClosed()) continue;
+
+      await page.target.setUserAgent(page.userAgent);
+    }
+  }
+}

package/src/plugins/anonymize.user.agent/test.js

@@ -0,0 +1,35 @@
+const PuppeteerPro = require('../../../dist/index');
+const chai = require('chai');
+
+const expect = chai.expect;
+
+const sleep = time => { return new Promise(resolve => { setTimeout(resolve, time); }); };
+
+module.exports = plugin => async browserWSEndpoint => {
+  const browser = browserWSEndpoint ? await PuppeteerPro.connect({ browserWSEndpoint }) : await PuppeteerPro.launch();
+  let page;
+
+  try {
+    page = await browser.newPage();
+
+    const getResult = async () => {
+      await page.goto('https://httpbin.org/headers');
+      await sleep(100);
+
+      const data = await page.evaluate(() => JSON.parse(document.body.innerText));
+      return data.headers['User-Agent'];
+    };
+
+    expect(await getResult()).to.not.contain('Headless');
+
+    await plugin.stop();
+    expect(await getResult()).to.contain('Headless');
+
+    await plugin.restart();
+    expect(await getResult()).to.not.contain('Headless');
+  }
+  finally {
+    if (page) await page.close();
+    if (browser) await browser.close();
+  }
+};

package/src/plugins/avoid.detection/index.ts

@@ -0,0 +1,19 @@
+import * as fs from 'fs';
+import * as path from 'path';
+import * as Puppeteer from 'puppeteer';
+
+import { Plugin } from '../../index';
+import { AnonymizeUserAgentPlugin } from './../anonymize.user.agent/index';
+
+const injectionsFolder = path.resolve(`${__dirname}/injections`);
+const injections = fs.readdirSync(injectionsFolder).map(fileName => require(`${injectionsFolder}/${fileName}`));
+
+export class AvoidDetectionPlugin extends Plugin {
+  dependencies = [new AnonymizeUserAgentPlugin()];
+
+  protected async onPageCreated(page: Puppeteer.Page) {
+    for (const injection of injections) {
+      if (!this.isStopped && !page.isClosed()) await page.evaluateOnNewDocument(injection);
+    }
+  }
+}

package/src/plugins/avoid.detection/test.js

@@ -0,0 +1,40 @@
+const PuppeteerPro = require('../../../dist/index');
+const chai = require('chai');
+
+const expect = chai.expect;
+
+const sleep = time => { return new Promise(resolve => { setTimeout(resolve, time); }); };
+
+module.exports = plugin => async browserWSEndpoint => {
+  const browser = browserWSEndpoint ? await PuppeteerPro.connect({ browserWSEndpoint }) : await PuppeteerPro.launch();
+  let page;
+
+  try {
+    page = await browser.newPage();
+
+    const getResult = async () => {
+      try {
+        await page.goto('https://bot.sannysoft.com');
+        await sleep(1000);
+
+        // Disable hairline as it seems there is a race condition. Test results keep changing after every run even though the detection is running.
+        return await page.evaluate(() => document.querySelector('table').querySelectorAll('.failed:not(#hairline-feature)').length === 0);
+      }
+      catch (ex) {
+        return false;
+      }
+    };
+
+    expect(await getResult()).to.be.true;
+
+    await plugin.stop();
+    expect(await getResult()).to.be.false;
+
+    await plugin.restart();
+    expect(await getResult()).to.be.true;
+  }
+  finally {
+    if (page) await page.close();
+    if (browser) await browser.close();
+  }
+};