regressify 1.0.0

package/.vscode/settings.json

@@ -0,0 +1,57 @@
+{
+  "files.exclude": {
+    ".idea": true,
+    "**/tsconfig.tsbuildinfo": true,
+    "**/package-lock.json": true,
+    "**/*.css.map": true,
+    "**/node_modules": true,
+    "**/.turbo": true,
+    ".vscode": true
+  },
+  "markdown.extension.tableFormatter.normalizeIndentation": true,
+  "markdown.extension.orderedList.marker": "one",
+  "markdownlint.ignore": [
+    "**/SUMMARY.md"
+  ],
+  "compile-hero.typescriptx-output-toggle": false,
+  "compile-hero.sass-output-toggle": false,
+  "compile-hero.pug-output-toggle": false,
+  "compile-hero.less-output-toggle": false,
+  "compile-hero.javascript-output-toggle": false,
+  "compile-hero.jade-output-toggle": false,
+  "compile-hero.disable-compile-files-on-did-save-code": false,
+  "compile-hero.typescript-output-toggle": false,
+  "compile-hero.scss-output-directory": ".",
+  "path-intellisense.showHiddenFiles": true,
+  "editor.formatOnSave": true,
+  "editor.formatOnPaste": true,
+  "markdown.extension.list.indentationSize": "inherit",
+  "editor.tabSize": 2,
+  "editor.rulers": [
+    80,
+    120
+  ],
+  "editor.minimap.enabled": false,
+  "markdown.extension.print.absoluteImgPath": false,
+  "markdown.extension.print.imgToBase64": true,
+  "markdown.extension.print.theme": "dark",
+  "liveSassCompile.settings.generateMap": false,
+  "search.exclude": {
+    "**/public/assets/css": true,
+    "**/public/assets/js": true,
+    "**/public/assets/vendor": true
+  },
+  "json.schemas": [
+    {
+      "fileMatch": [
+        "/*.tests.json"
+      ],
+      "url": "./.engine_scripts/test-schema.json"
+    }
+  ],
+  "yaml.schemas": {
+    "./.engine_scripts/test-schema.json": "/*.tests.{yaml,yml}",
+    "./.engine_scripts/replacement-profiles-schema.json": "/_replacement-profiles.{yaml,yml}"
+  },
+  "git.branchValidationRegex": "^(main|master|develop|stage|fe-develop|fe-release|(features|bugfixes|infra|refactor)\\/.+)$"
+}

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 Tuyen Pham <tuyen-at-work>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,32 @@
+# Visual Regression Test
+
+Please check [Documentation](https://tuyen.blog/optimizely-cms/testing/get-started/) for the instructions.
+
+## Use
+
+1. Install:
+
+   ```bash
+   npm i @eshn/visual-regression-tests
+   ```
+
+1. Manual Set up all test and config files in the **visual_tests** folder and place it at the root of the project, or automatically add it using the command:
+   ```bash
+   npx eshn-visual generate
+   ```
+1. Add to scripts in package.json:
+
+   ```bash
+    "ref": "eshn-visual ref",
+    "approve": "eshn-visual approve",
+    "test": "eshn-visual test"
+   ```
+
+1. Use new command aliases:
+
+   | Command                                             | Alias 1 | Alias 2       | Description         |
+   | --------------------------------------------------- | ------- | ------------- | ------------------- |
+   | npm run ref -- --test-suite alloy                   | r alloy | ref alloy     |                     |
+   | npm run approve -- --test-suite alloy               | a alloy | approve alloy |                     |
+   | npm run ref -- --test-suite alloy                   | t alloy | test alloy    |                     |
+   | npm run ref -- --test-suite sign-in --requiredLogin | t alloy | test alloy    | Run with login mode |

package/alias.ps1

@@ -0,0 +1,44 @@
+function Approve-TestSuite {
+  [alias("a")]
+  [alias("approve")]
+  param (
+    # Test suite name
+    [Parameter(
+      ValueFromPipeline,
+      Mandatory)]
+    [string]
+    $name
+  )
+  
+  npm run approve -- --test-suite $name
+}
+
+function Reference-TestSuite {  
+  [alias("r")]
+  [alias("ref")]
+  param (
+    # Test suite name
+    [Parameter(
+      ValueFromPipeline,
+      Mandatory)]
+    [string]
+    $name
+  )
+  
+  npm run ref -- --test-suite $name
+}
+
+function Test-TestSuite {
+  [alias("t")]
+  [alias("test")]
+  param (
+    # Test suite name
+    [Parameter(
+      ValueFromPipeline,
+      Mandatory)]
+    [string]
+    $name
+  )
+  
+  npm run test -- --test-suite $name
+}

package/cli.js

@@ -0,0 +1,76 @@
+#!/usr/bin/env node
+import fs from 'fs';
+import path, { dirname } from 'path';
+import { exec } from 'child_process';
+import { fileURLToPath, pathToFileURL } from 'url';
+import chalk from 'chalk';
+
+function getLibraryPath() {
+  const __filename = fileURLToPath(import.meta.url);
+  const __dirname = dirname(__filename);
+
+  let currentDir = __dirname;
+  while (!fs.existsSync(path.join(currentDir, 'package.json'))) {
+    const parentDir = path.dirname(currentDir);
+    if (parentDir === currentDir) {
+      return null;
+    }
+    currentDir = parentDir;
+  }
+  const packageJsonPath = path.join(currentDir, 'package.json');
+  const packageJson = JSON.parse(fs.readFileSync(packageJsonPath, 'utf8'));
+  return `node_modules/${packageJson.name}`;
+}
+
+function runCommand(command) {
+  const childProcess = exec(command, { env: { ...process.env, FORCE_COLOR: '1' } });
+
+  childProcess.stdout.on('data', (data) => {
+    process.stdout.write(data);
+  });
+
+  childProcess.stderr.on('data', (data) => {
+    process.stderr.write(data);
+  });
+
+  childProcess.on('close', (code) => {
+    if (code !== 0) {
+      console.log(chalk.red(`Command exited with code ${code}`));
+    }
+  });
+
+  childProcess.on('error', (err) => {
+    console.log(chalk.red(`Failed to start command: ${err.message}`));
+  });
+}
+
+const args = process.argv.slice(2);
+let commandBase = `tsx ${getLibraryPath()}/src/index.ts`;
+
+if (args[0] === 'generate') {
+  const __filename = fileURLToPath(import.meta.url);
+  const __dirname = dirname(__filename);
+  const postInstallPath = pathToFileURL(path.join(__dirname, 'generate_tests.js'));
+  if (fs.existsSync(postInstallPath)) {
+    console.log(chalk.yellow('generate folder visual_tests ...'));
+    await import(postInstallPath);
+  } else {
+    console.log(chalk.red('generate_tests.js not found!'));
+  }
+} else if (args[0] === 'ref') {
+  const command = `${commandBase} --command test --ref ${args.slice(1).join(' ')}`;
+  console.log(chalk.yellow(`Running command: ${command}`));
+  runCommand(command);
+} else if (args[0] === 'approve') {
+  const command = `${commandBase} --command approve ${args.slice(1).join(' ')}`;
+  console.log(chalk.yellow(`Running command: ${command}`));
+  runCommand(command);
+} else if (args[0] === 'test') {
+  const command = `${commandBase} --command test ${args.slice(1).join(' ')}`;
+  console.log(chalk.yellow(`Running command: ${command}`));
+  runCommand(command);
+} else {
+  console.log(
+    chalk.red("Invalid command. Use one of the following: 'eshn-visual generate' 'eshn-visual ref', 'eshn-visual approve', 'eshn-visual test'.")
+  );
+}

package/generate_tests.js

@@ -0,0 +1,39 @@
+import fs from 'fs';
+import path from 'path';
+import inquirer from 'inquirer';
+import { fileURLToPath } from 'url';
+import pkg from 'ncp';
+import chalk from 'chalk';
+
+const { ncp } = pkg;
+
+async function askUser() {
+  const response = await inquirer.prompt([
+    {
+      type: 'confirm',
+      name: 'addFolder',
+      message: 'Do you want to add the "visual_tests" folder to the root of your project?',
+      default: true,
+    },
+  ]);
+
+  if (response.addFolder) {
+    const sourceFolder = path.join(path.dirname(fileURLToPath(import.meta.url)), 'visual_tests');
+    const destinationFolder = path.join(process.cwd(), 'visual_tests');
+    if (!fs.existsSync(destinationFolder)) {
+      ncp(sourceFolder, destinationFolder, function (err) {
+        if (err) {
+          console.log(chalk.red('Error copying folder:'), err);
+        } else {
+          console.log(chalk.green('Folder "visual_tests" has been copied to your project!'));
+        }
+      });
+    } else {
+      console.log(chalk.yellow('Folder "visual_tests" already exists.'));
+    }
+  } else {
+    console.log('No folder was added.');
+  }
+}
+
+askUser().catch((err) => console.error('Error:', err));

package/package.json

@@ -0,0 +1,44 @@
+{
+  "name": "regressify",
+  "version": "1.0.0",
+  "description": "Visual regression tests support",
+  "main": "index.js",
+  "type": "module",
+  "bin": {
+    "eshn-visual": "./cli.js"
+  },
+  "scripts": {
+    "ref": "tsx src/index.ts --command test --ref",
+    "approve": "tsx src/index.ts --command approve",
+    "test": "tsx src/index.ts --command test"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/precise-alloy/regression-test.git"
+  },
+  "keywords": [
+    "cli",
+    "visual",
+    "test",
+    "regression"
+  ],
+  "author": "donghoang",
+  "license": "ISC",
+  "dependencies": {
+    "backstopjs": "^6.2.2",
+    "chalk": "^5.3.0",
+    "inquirer": "^12.2.0",
+    "js-yaml": "^4.1.0",
+    "ncp": "^2.0.0",
+    "tsx": "^4.19.2"
+  },
+  "devDependencies": {
+    "@types/backstopjs": "^6.1.2",
+    "@types/js-yaml": "^4.0.8",
+    "@types/node": "^20.8.10",
+    "typescript": "^5.2.2"
+  },
+  "peerDependencies": {
+    "tsx": "^4.19.2"
+  }
+}

package/src/config.ts

@@ -0,0 +1,172 @@
+import fs from 'fs';
+import { Config, Scenario, ViewportNext } from 'backstopjs';
+import { createScenario } from './scenarios.js';
+import path from 'path';
+import { getFlagArg, getStringArg, parseDataFromFile, getLibraryPath } from './helpers.js';
+import { fileURLToPath } from 'url';
+import { TestSuiteModel, ScenarioModel } from './types.js';
+import chalk from 'chalk';
+import { exit } from 'process';
+import YAML from 'js-yaml';
+import { getTestUrl } from './replacements.js';
+
+const engine: 'puppeteer' | 'playwright' = 'playwright';
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const testSuite = getStringArg('--test-suite');
+const isRef = getFlagArg('--ref');
+const globalRequiredLogin = getFlagArg('--requiredLogin');
+if (globalRequiredLogin) {
+  console.log('force run all scenarios in login mode');
+}
+
+const scenarios: Scenario[] = [];
+
+const getScriptPath = (path: string, engine: 'puppeteer' | 'playwright') => {
+  return (engine == 'puppeteer' ? 'puppet' : 'playwright') + path;
+};
+
+if (!testSuite) {
+  console.log(chalk.red('Argument `--test-suite` must be set.'));
+  console.log(chalk.red('Sample command: npm run <command> -- --test-suite <test-suite>'));
+  console.log(chalk.red('Command is either `ref`, `approve` or `test`.'));
+  exit(1);
+}
+
+const getData = (testSuite: String): TestSuiteModel | undefined => {
+  let extensions: { ext: string; parse: (content: string) => unknown }[] = [
+    {
+      ext: 'yaml',
+      parse: YAML.load,
+    },
+    {
+      ext: 'yaml',
+      parse: YAML.load,
+    },
+    {
+      ext: 'json',
+      parse: JSON.parse,
+    },
+  ];
+
+  for (let i = 0; i < extensions.length; i++) {
+    const dataPath = path.resolve(__dirname, `../visual_tests/${testSuite}.tests.${extensions[i].ext}`);
+
+    if (fs.existsSync(dataPath)) {
+      console.log('Data path: ', dataPath);
+      const content = fs.readFileSync(dataPath, 'utf-8');
+      return extensions[i].parse(content) as TestSuiteModel;
+    }
+  }
+};
+
+const expandScenarios = (model: ScenarioModel, scenarios: ScenarioModel[], level: number) => {
+  if (level > 100) {
+    throw 'Level is too large';
+  }
+
+  if (!model.needs) {
+    return;
+  }
+
+  const neededActions: string[] = [];
+  if (typeof model.needs === 'string') {
+    neededActions.push(model.needs);
+  } else {
+    model.needs.forEach((n) => neededActions.push(n));
+  }
+
+  neededActions.reverse().forEach((n) => {
+    const targetScenarios = scenarios.filter((s) => !!s.id && s.id.toLowerCase() == n.toLowerCase());
+    if (targetScenarios.length !== 1) {
+      throw `The test suite must contains exactly ONE scenario with id: ${n}`;
+    }
+
+    var targetScenario = targetScenarios[0];
+    expandScenarios(targetScenario, scenarios, level + 1);
+    if (!!targetScenario.actions) {
+      if (!model.actions) {
+        model.actions = [];
+      }
+      model.actions = [...targetScenario.actions, ...model.actions];
+    }
+  });
+
+  model.needs = undefined;
+};
+
+const data = getData(testSuite);
+const viewports = parseDataFromFile(data?.viewportsPath ?? 'visual_tests/_viewports.yaml') as ViewportNext[];
+if (data) {
+  [].forEach.call(data.scenarios, (s: ScenarioModel) => {
+    expandScenarios(s, data.scenarios, 0);
+  });
+
+  const getTestUrlLocal = (url: string) => getTestUrl(url, isRef);
+
+  const pad = String(data?.scenarios.length).length;
+  data.scenarios.forEach((s, index) => {
+    const opts: ScenarioModel = {
+      ...s,
+      requiredLogin: globalRequiredLogin || s.requiredLogin,
+      getTestUrl: getTestUrlLocal,
+      url: isRef ? s.url : getTestUrl(s.url, isRef),
+      index: String(index + 1).padStart(pad, ' '),
+      total: data.scenarios.length,
+      delay: s.delay ?? 1000,
+      hideSelectors: s.hideSelectors ?? data.hideSelectors,
+      removeSelectors: s.removeSelectors ?? data.removeSelectors,
+      useCssOverride: s.useCssOverride ?? data.useCssOverride,
+      jsOnReadyPath: s.jsOnReadyPath,
+      viewports: !!s.viewportNames
+        ? typeof s.viewportNames === 'string'
+          ? viewports.filter((v) => v.label.toLowerCase() == (s.viewportNames as string).trim().toLowerCase())
+          : viewports.filter((v) => s.viewportNames?.includes(v.label))
+        : !!data.viewportNames
+        ? typeof data.viewportNames === 'string'
+          ? viewports.filter((v) => v.label.toLowerCase() == (data.viewportNames as string).trim().toLowerCase())
+          : viewports.filter((v) => data.viewportNames?.includes(v.label))
+        : undefined,
+      referenceUrl: !isRef ? s.url : undefined,
+      misMatchThreshold: s.misMatchThreshold ?? data.misMatchThreshold ?? 0.1,
+      postInteractionWait: s.postInteractionWait ?? data.postInteractionWait ?? 1,
+    };
+
+    const scenario = createScenario(opts);
+    scenarios.push(scenario);
+  });
+}
+
+export const config: Config = {
+  id: testSuite,
+  viewports,
+  onBeforeScript: getScriptPath('/onBefore.js', engine),
+  onReadyScript: getScriptPath('/onReady.js', engine),
+  scenarios,
+  paths: {
+    bitmaps_reference: '.backstop/' + testSuite + '/bitmaps_reference',
+    bitmaps_test: '.backstop/' + testSuite + '/bitmaps_test',
+    engine_scripts: `${getLibraryPath()}/.engine_scripts`,
+    html_report: '.backstop/' + testSuite + '/html_report',
+    ci_report: '.backstop/' + testSuite + '/ci_report',
+  },
+  report: [isRef ? 'CI' : 'browser'],
+  engine,
+  engineOptions: {
+    args: [
+      '--disable-infobars',
+      '--disable-setuid-sandbox',
+      '--ignore-certifcate-errors',
+      '--ignore-certifcate-errors-spki-list',
+      '--no-sandbox',
+      '--window-position=0,0',
+    ],
+    browser: data?.browser ?? 'chromium',
+  },
+  asyncCaptureLimit: data?.asyncCaptureLimit ?? 5,
+  asyncCompareLimit: data?.asyncCompareLimit ?? 50,
+  debug: false,
+  debugWindow: data?.debug,
+};
+
+export default config;

package/src/helpers.ts

@@ -0,0 +1,40 @@
+import fs from 'fs';
+import YAML from 'js-yaml';
+import path, { dirname } from 'path';
+import { fileURLToPath } from 'url';
+export const getStringArg = (key: string): string | undefined => {
+  const index = process.argv.indexOf(key);
+  return index >= 0 && index < process.argv.length - 1 && !process.argv[index + 1].startsWith('-') ? process.argv[index + 1] : undefined;
+};
+
+export const getFlagArg = (key: string): boolean => {
+  return process.argv.indexOf(key) >= 0;
+};
+
+export const parseDataFromFile = (dataPath: string, type: 'yaml' | 'json' = 'yaml'): unknown | undefined => {
+  if (!!dataPath && fs.existsSync(dataPath)) {
+    let content = fs.readFileSync(dataPath, 'utf-8');
+    if (type === 'json') {
+      return JSON.parse(content);
+    } else if (type === 'yaml') {
+      return YAML.load(content);
+    }
+  }
+};
+
+export function getLibraryPath() {
+  const __filename = fileURLToPath(import.meta.url);
+  const __dirname = dirname(__filename);
+
+  let currentDir = __dirname;
+  while (!fs.existsSync(path.join(currentDir, 'package.json'))) {
+    const parentDir = path.dirname(currentDir);
+    if (parentDir === currentDir) {
+      return null;
+    }
+    currentDir = parentDir;
+  }
+  const packageJsonPath = path.join(currentDir, 'package.json');
+  const packageJson = JSON.parse(fs.readFileSync(packageJsonPath, 'utf8'));
+  return `node_modules/${packageJson.name}`;
+}

package/src/index.ts

@@ -0,0 +1,55 @@
+import chalk from 'chalk';
+import backstop from 'backstopjs';
+import config from './config.js';
+import { getStringArg } from './helpers.js';
+import path from 'path';
+import fs from 'fs';
+import { fileURLToPath } from 'url';
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+
+const command = getStringArg('--command') as 'approve' | 'init' | 'reference' | 'test' | undefined;
+if (!command) {
+  throw '`command` must be set';
+}
+
+const PATCH_START = '<!-- PATCH START -->';
+const PATCH_END = '<!-- PATCH END -->';
+
+const customStyle = `
+${PATCH_START}
+<style>
+  [id^="test"] > div[display="true"] > p[display] {
+    white-space: pre;
+    overflow-x: auto;
+  }
+</style>
+${PATCH_END}
+`;
+
+const packCompare = () => {
+  const projectRootDir = process.cwd();
+  const reportIndex = path.resolve(projectRootDir, 'node_modules/backstopjs/compare/output/index.html');
+  if (fs.existsSync(reportIndex)) {
+    let html = fs.readFileSync(reportIndex, 'utf-8');
+    const patchStartIndex = html.indexOf(PATCH_START);
+    const patchEndIndex = html.indexOf(PATCH_END);
+    if (patchStartIndex > 0 && patchEndIndex > patchStartIndex) {
+      html = html.replace(new RegExp(PATCH_START + '.*' + patchEndIndex, 'gi'), customStyle);
+    } else {
+      html = html.replace('</head>', customStyle + '</head>');
+    }
+    fs.writeFileSync(reportIndex, html);
+  } else {
+    console.log(chalk.red('File does not exist: ' + reportIndex));
+  }
+};
+
+packCompare();
+backstop(command, { config })
+  .then(() => {
+    console.log(chalk.green(command.toUpperCase() + ' FINISHED SUCCESSFULLY'));
+  })
+  .catch(() => {
+    console.log(chalk.red(command.toUpperCase() + ' FAILED'));
+  });

package/src/replacements.ts

@@ -0,0 +1,34 @@
+import fs from 'fs';
+import path from 'path';
+import { ReplacementModel, ReplacementsModel } from './types';
+import YAML from 'js-yaml';
+import { getStringArg } from './helpers.js';
+import { fileURLToPath } from 'url';
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+
+const getReplacementProfile = (): ReplacementModel[] | undefined => {
+  const replacementProfileName = getStringArg('replacement-profile') ?? process.env.REPLACEMENT_PROFILE;
+  if (!!replacementProfileName) {
+    const replacementProfilePath = path.resolve(__dirname, '../visual_tests/_replacement-profiles.yaml');
+    if (!fs.existsSync(replacementProfilePath)) {
+      throw "Replacement profile doesn't exist: " + replacementProfilePath;
+    }
+
+    const profiles = YAML.load(fs.readFileSync(replacementProfilePath, 'utf-8')) as ReplacementsModel;
+    return profiles.profiles[replacementProfileName];
+  }
+};
+
+const replacementProfile = getReplacementProfile();
+
+export const getTestUrl = (url: string, isRef: boolean) => {
+  if (isRef || !replacementProfile) {
+    return url;
+  }
+
+  let testUrl = url;
+  replacementProfile.forEach((e) => (testUrl = testUrl.replace(e.ref, e.test)));
+
+  return testUrl;
+};

package/src/scenarios.ts

@@ -0,0 +1,21 @@
+import { ScenarioModel } from './types';
+
+export const createScenario = (opts: ScenarioModel): ScenarioModel => {
+  const parsedUrl = new URL(opts.url);
+
+  return {
+    ...opts,
+    label: opts.label ?? `${opts.index} of ${opts.total}: ${parsedUrl.pathname}`,
+    cookiePath: opts.cookiePath ?? 'visual_tests/_cookies.yaml',
+    cssOverridePath: opts.cssOverridePath ?? 'visual_tests/_override.css',
+    jsOnReadyPath: opts.jsOnReadyPath ?? 'visual_tests/_on-ready.js',
+    referenceUrl: opts.referenceUrl ?? '',
+    readyEvent: '',
+    hideSelectors: opts.hideSelectors ?? [],
+    removeSelectors: opts.removeSelectors ?? [],
+    selectors: [],
+    selectorExpansion: true,
+    expect: 0,
+    requireSameDimensions: true,
+  };
+};