ui5-test-runner 4.5.1 → 5.0.0

package/README.md

@@ -34,18 +34,15 @@ A self-sufficient test runner for UI5 applications enabling parallel execution o
 
 ## ⚠️ Breaking changes
 
-### v4
-* Dropping support of Node.js 16
-
-### v3
-* Dropping support of Node.js 14
-
-### v2
-
-* Command line **parameters** as well as configuration file **syntax** have changed
-* Dependencies are installed **on demand**
-* Browser instantiation command evolved in an **incompatible way** (see [documentation](https://arnaudbuchholz.github.io/ui5-test-runner/browser.html)).
-* Output is different (report, traces)
+| Version | Reason | 
+|-|-|
+| **5**.0.0 | • Some coverage reports now includes **all** files, leading to a potential decrease of coverage |
+| **4**.0.0 | • Drop support of Node.js 16 |
+| **3**.0.0 | • Drop support of Node.js 14 |
+| **2**.0.0 | • Command line **parameters** as well as configuration file **syntax** have changed |
+|| • Dependencies are installed **on demand** |
+|| • Browser instantiation command evolved in an **incompatible way** (see [documentation](https://arnaudbuchholz.github.io/ui5-test-runner/browser.html)) |
+|| • Output is different (report, traces) |
 
 ## ✒ Contributors
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ui5-test-runner",
-  "version": "4.5.1",
+  "version": "5.0.0",
   "description": "Standalone test runner for UI5",
   "main": "index.js",
   "bin": {
@@ -51,7 +51,7 @@
   },
   "devDependencies": {
     "@openui5/types": "^1.123.0",
-    "@ui5/cli": "^3.9.2",
+    "@ui5/cli": "^3.10.0",
     "@ui5/middleware-code-coverage": "^1.1.1",
     "dotenv": "^16.4.5",
     "jest": "^29.7.0",
@@ -61,7 +61,7 @@
     "standard": "^17.1.0",
     "start-server-and-test": "^2.0.3",
     "typescript": "^5.4.5",
-    "ui5-tooling-transpile": "^3.3.7"
+    "ui5-tooling-transpile": "^3.4.0"
   },
   "optionalDependencies": {
     "fsevents": "^2.3.3"

package/src/browsers.js

@@ -116,8 +116,9 @@ async function start (job, url, scripts = []) {
     }
   }
   if (resolvedScripts.length) {
-    resolvedScripts.unshift(`window['ui5-test-runner/base-host'] = 'http://localhost:${job.port}'
-`)
+    resolvedScripts.unshift(`(function () {
+  window['ui5-test-runner/base-host'] = 'http://localhost:${job.port}'
+}())`)
   }
   const progress = newProgress(job, url)
   const pageBrowser = {

package/src/coverage.js

@@ -1,11 +1,11 @@
 'use strict'
 
-const { join, dirname, isAbsolute } = require('path')
+const { join, dirname, isAbsolute, relative, sep } = require('path')
 const { fork } = require('child_process')
 const { cleanDir, createDir, filename, download, allocPromise } = require('./tools')
 const { readdir, readFile, stat, writeFile, access, constants } = require('fs').promises
 const { Readable } = require('stream')
-const { getOutput } = require('./output')
+const { getOutput, newProgress } = require('./output')
 const { resolvePackage } = require('./npm')
 const { promisify } = require('util')
 const { UTRError } = require('./error')
@@ -64,7 +64,7 @@ async function instrument (job) {
   await cleanDir(job.coverageTempDir)
   await createDir(join(job.coverageTempDir, 'settings'))
   const settings = JSON.parse((await readFile(job.coverageSettings)).toString())
-  settings.cwd = job.cwd
+  settings.cwd = job.webapp
   if (!settings.exclude) {
     settings.exclude = []
   }
@@ -75,6 +75,7 @@ async function instrument (job) {
   settings.exclude.push(join(job.reportDir, '**'))
   settings.exclude.push(join(job.coverageReportDir, '**'))
   await writeFile(job[$nycSettingsPath], JSON.stringify(settings))
+  job.nycSettings = settings
   if (job.mode === 'url') {
     if (!job[$remoteOnLegacy]) {
       job[$coverageRemote] = true
@@ -86,6 +87,92 @@ async function instrument (job) {
   await nyc(job, 'instrument', job.webapp, join(job.coverageTempDir, 'instrumented'), '--nycrc-path', job[$nycSettingsPath])
 }
 
+function getUrlOrigin (job) {
+  const { origin } = new URL(job.url[0])
+  if (job.url.some(url => new URL(url).origin !== origin)) {
+    getOutput(job).assumingOneOrigin()
+  }
+  return origin
+}
+
+async function buildAllIndex (job) {
+  async function scanFs (path, onFolder, onFile) {
+    const items = await readdir(path)
+    await onFolder(items.length)
+    for (const item of items) {
+      const itemPath = join(path, item)
+      const itemStat = await stat(itemPath)
+      if (itemStat.isDirectory()) {
+        await scanFs(itemPath, onFolder, onFile)
+      } else {
+        await onFile(itemPath, (await readFile(itemPath)).toString())
+      }
+    }
+  }
+
+  const output = getOutput(job)
+  output.debug('coverage', 'Build index for all files...')
+  const progress = newProgress(job, 'Build index for all files', 1, 0)
+
+  try {
+    const index = []
+    let scan
+    let start
+    if (job.mode === 'legacy' || job[$remoteOnLegacy]) {
+      scan = scanFs
+      start = join(job.coverageTempDir, 'instrumented')
+    } else {
+      scan = require(job.coverageRemoteScanner)
+      start = getUrlOrigin(job)
+    }
+
+    await scan(
+      start,
+      count => {
+        progress.total += count
+        ++progress.count
+      },
+      async (file, source) => {
+        if (file.endsWith('.js') || file.endsWith('.ts')) {
+          output.debug('coverage', file)
+          try {
+            const coverageData = source
+              .match(/coverageData\s*=\s*({[^;]*});/)[1]
+              .replace(/([^"])(\w+):/g, (_, before, name) => `${before}"${name}":`)
+            const [, coveragePath] = coverageData.match(/"path"\s*:\s*"([^"]+)"/)
+            const UNDEFINED = '__undefined__'
+            const validatedCoverageData = JSON.stringify(
+              JSON.parse(coverageData.replace(/\bundefined\b/g, `"${UNDEFINED}"`)),
+              (key, value) => {
+                if (value === UNDEFINED) {
+                  return undefined
+                }
+                return value
+              }
+            )
+            index.push(`"${coveragePath}": ${validatedCoverageData}`)
+          } catch (e) {
+            output.debug('coverage', `Error when extracting all coverage for ${file}`, e)
+          }
+        } else {
+          output.debug('coverage', `Ignore all coverage for ${file}`)
+        }
+        ++progress.count
+      }
+    )
+    if (index.length === 0) {
+      output.noInfoForAllCoverage()
+    } else {
+      await writeFile(join(job.coverageTempDir, 'all-index.json'), `{${index.join(',')}}`)
+    }
+  } catch (e) {
+    output.genericError(e)
+    output.noInfoForAllCoverage()
+  } finally {
+    progress.done()
+  }
+}
+
 async function getReadableSource (job, pathOrUrl) {
   if (isAbsolute(pathOrUrl)) {
     try {
@@ -99,16 +186,59 @@ async function getReadableSource (job, pathOrUrl) {
     return filePath
   } catch (e) {}
   try {
-    // Assuming all files are coming from the same server
-    const { origin } = new URL(job.testPageUrls[0])
-    const filePath = join(job.coverageTempDir, 'sources', pathOrUrl)
+    const origin = getUrlOrigin(job)
+    if (!job.coverageSourceDir) {
+      job.coverageSourceDir = join(job.coverageTempDir, 'sources')
+    }
+    const filePath = join(job.coverageSourceDir, pathOrUrl)
     await download(origin + pathOrUrl, filePath)
     return filePath
   } catch (e) {}
 }
 
+async function checkAllSourcesAreAvailable (job, coverageFilename) {
+  const output = getOutput(job)
+  job.status = 'Checking remote source files'
+  output.debug('coverage', 'Checking remote source files...')
+  const coverageData = require(coverageFilename)
+  const filenames = Object.keys(coverageData)
+  let changes = 0
+  let basePath
+  for (const filename of filenames) {
+    const fileData = coverageData[filename]
+    const filePath = await getReadableSource(job, fileData.path)
+    if (!filePath) {
+      // TODO this will compromise coverage report generation
+      continue
+    }
+    if (filePath && filePath !== fileData.path) {
+      fileData.path = filePath
+      ++changes
+    }
+    const fileFolder = dirname(filePath)
+    if (basePath === undefined) {
+      basePath = fileFolder
+    } else {
+      const diff = relative(basePath, fileFolder).split(sep)
+      while (diff.shift() === '..') {
+        basePath = dirname(basePath)
+      }
+    }
+  }
+  if (basePath !== job.nycSettings.cwd) {
+    job.nycSettings.cwd = basePath
+    await writeFile(job[$nycSettingsPath], JSON.stringify(job.nycSettings))
+  }
+  if (changes > 0) {
+    await writeFile(coverageFilename, JSON.stringify(coverageData))
+  }
+}
+
 async function generateCoverageReport (job) {
   job.status = 'Generating coverage report'
+  if (job.nycSettings.all) {
+    await buildAllIndex(job)
+  }
   const output = getOutput(job)
   output.debug('coverage', 'Generating coverage report...')
   await cleanDir(job.coverageReportDir)
@@ -116,23 +246,8 @@ async function generateCoverageReport (job) {
   await createDir(coverageMergedDir)
   const coverageFilename = join(coverageMergedDir, 'coverage.json')
   await nyc(job, 'merge', job.coverageTempDir, coverageFilename)
-  if (job[$coverageRemote] && !job.coverageProxy) {
-    job.status = 'Checking remote source files'
-    output.debug('coverage', 'Checking remote source files...')
-    const coverageData = require(coverageFilename)
-    const filenames = Object.keys(coverageData)
-    let changes = 0
-    for (const filename of filenames) {
-      const fileData = coverageData[filename]
-      const filePath = await getReadableSource(job, fileData.path)
-      if (filePath && filePath !== fileData.path) {
-        fileData.path = filePath
-        ++changes
-      }
-    }
-    if (changes > 0) {
-      await writeFile(coverageFilename, JSON.stringify(coverageData))
-    }
+  if (job[$coverageRemote]) {
+    await checkAllSourcesAreAvailable(job, coverageFilename)
   }
   const reporters = job.coverageReporters.map(reporter => `--reporter=${reporter}`)
   if (!job.coverageReporters.includes('text')) {
@@ -201,7 +316,7 @@ module.exports = {
       return [{
         match: /(.*\.js)(\?.*)?$/,
         custom: async (request, response, url) => {
-          if (!url.match(job.coverageProxyInclude) || url.match(/\bresources\b/) || url.match(job.coverageProxyExclude)) {
+          if (!url.match(job.coverageProxyInclude) || url.match(job.coverageProxyExclude)) {
             getOutput(job).debug('coverage', 'coverage_proxy ignore', url)
             return
           }
@@ -210,21 +325,20 @@ module.exports = {
             await access(instrumentedSourcePath, constants.R_OK)
             return
           } catch (e) {}
-          const instrumenting = sources[url]
-          if (instrumenting) {
-            await instrumenting
-            return // ok
+          if (!sources[url]) {
+            sources[url] = (async () => {
+              const sourcePath = await getReadableSource(job, url)
+              getOutput(job).debug('coverage', 'coverage_proxy instrument', url, sourcePath)
+              if (sourcePath) {
+                const source = (await readFile(sourcePath)).toString()
+                const instrumentedSource = await instrument(source, sourcePath)
+                await createDir(dirname(instrumentedSourcePath))
+                await writeFile(instrumentedSourcePath, instrumentedSource)
+                delete sources[url]
+              }
+            })()
           }
-          sources[url] = (async () => {
-            const sourcePath = await getReadableSource(job, url)
-            getOutput(job).debug('coverage', 'coverage_proxy instrument', url, sourcePath)
-            if (sourcePath) {
-              const source = (await readFile(sourcePath)).toString()
-              const instrumentedSource = await instrument(source, sourcePath)
-              await createDir(dirname(instrumentedSourcePath))
-              await writeFile(instrumentedSourcePath, instrumentedSource)
-            }
-          })()
+          await sources[url]
         }
       },
       instrumentedMapping,

package/src/defaults/junit-xml-report.js

@@ -39,7 +39,8 @@ async function main () {
           time = (new Date(test.end) - new Date(test.start)) / 1000
         }
         o(`    <testcase
-      name="${xmlEscape(test.name)}" ${
+      name="${xmlEscape(test.name)}"
+      classname="${xmlEscape(module.name)}" ${
         time === undefined
         ? ''
         : `

package/src/defaults/puppeteer.js

@@ -8,17 +8,22 @@ require('./browser')({
     name: 'puppeteer',
     options: [
       ['--visible [flag]', 'Show the browser', false],
+      ['--firefox [flag]', 'Use firefox instead of chrome', false],
+      ['--binary <binary>', 'Binary path'],
       ['-w, --viewport-width <width>', 'Viewport width', 1920],
       ['-h, --viewport-height <height>', 'Viewport height', 1080],
       ['-l, --language <lang...>', 'Language(s)', ['en-US']],
       ['-u, --unsecure', 'Disable security features', false],
       ['--basic-auth-username <username>', 'Username for basic authentication', ''],
       ['--basic-auth-password <password>', 'Password for basic authentication', '']
-    ],
-    capabilities: {
+    ]
+  },
+
+  async capabilities ({ settings, options }) {
+    return {
       modules: ['puppeteer'],
       screenshot: '.png',
-      scripts: true,
+      scripts: !options.firefox,
       traces: ['console', 'network']
     }
   },
@@ -68,7 +73,14 @@ require('./browser')({
       )
     }
 
+    let product
+    if (options.firefox) {
+      product = 'firefox'
+    }
+
     browser = await puppeteer.launch({
+      product,
+      executablePath: options.binary,
       headless: options.visible ? false : 'new',
       defaultViewport: null,
       args

package/src/defaults/scan-ui5.js

@@ -0,0 +1,20 @@
+module.exports = async function scanUI5 (url, onFolder, onFile) {
+  if (url.match(/\/((?:test-)?resources\/.*)/)) {
+    return // ignore UI5 resources
+  }
+  const html = await (await fetch(url)).text()
+  const items = [...html.matchAll(/<a href="([^"]+)" class="icon/ig)]
+    .map(([_, item]) => item)
+    .filter(item => item.endsWith('/') || item.endsWith('.js') || item.endsWith('.ts'))
+  await onFolder(items.length)
+  for (const item of items) {
+    const itemUrl = new URL(item, url).toString()
+    if (item.endsWith('/')) {
+      await scanUI5(itemUrl, onFolder, onFile)
+    } else if (item.endsWith('.ts')) {
+      await onFile(itemUrl, await (await fetch(itemUrl.replace(/\.ts$/, '.js'))).text())
+    } else {
+      await onFile(itemUrl, await (await fetch(itemUrl + '?instrument=true')).text())
+    }
+  }
+}

package/src/defaults/selenium-webdriver/edge.js

@@ -4,8 +4,7 @@ module.exports = async ({
   seleniumWebdriver,
   settings,
   options,
-  loggingPreferences,
-  $capabilities
+  loggingPreferences
 }) => {
   const { Browser, Builder } = seleniumWebdriver
   const edge = require(join(settings.modules['selenium-webdriver'], 'edge'))

package/src/defaults/selenium-webdriver/firefox.js

@@ -4,15 +4,14 @@ module.exports = async ({
   seleniumWebdriver,
   settings,
   options,
-  loggingPreferences,
-  $capabilities
+  loggingPreferences
 }) => {
   const { Browser, Builder } = seleniumWebdriver
   const firefox = require(join(settings.modules['selenium-webdriver'], 'firefox'))
 
   const firefoxOptions = new firefox.Options()
   if (!options.visible) {
-    firefoxOptions.headless = true
+    firefoxOptions.addArguments('-headless')
   }
   firefoxOptions.setLoggingPrefs(loggingPreferences)
   if (options.binary) {

package/src/defaults/selenium-webdriver.js

@@ -7,7 +7,7 @@ const { writeFile } = require('fs/promises')
 let logging
 let driver
 
-function browser (value, defaultValue) {
+function browser (value) {
   if (value === undefined) {
     return 'chrome'
   }

package/src/defaults/webdriverio.js

@@ -0,0 +1,89 @@
+'use strict'
+
+const { InvalidArgumentError } = require('commander')
+
+let browserio
+
+function browser (value) {
+  if (value === undefined) {
+    return 'chrome'
+  }
+  if (!['chrome', 'firefox'].includes(value)) {
+    throw new InvalidArgumentError('Browser name')
+  }
+  return value
+}
+
+require('./browser')({
+  metadata: {
+    name: 'webdriverio',
+    options: [
+      ['--visible [flag]', 'Show the browser', false],
+      ['-b, --browser <name>', 'Browser driver', browser, 'chrome'],
+      ['--binary <binary>', 'Binary path']
+    ]
+  },
+
+  async capabilities ({ settings, options }) {
+    return {
+      modules: ['webdriverio'],
+      screenshot: '.png',
+      scripts: true,
+      traces: []
+    }
+  },
+
+  async screenshot ({ filename }) {
+    if (browserio) {
+      await browserio.saveScreenshot(filename)
+      return true
+    }
+  },
+
+  async beforeExit () {
+    if (browserio) {
+      await browserio.deleteSession()
+    }
+  },
+
+  async run ({
+    settings: { url, scripts, modules },
+    options,
+    consoleWriter,
+    networkWriter
+  }) {
+    const { remote } = require(modules.webdriverio)
+
+    const [browserOptions, args] = {
+      chrome: [
+        'goog:chromeOptions',
+        options.visible ? [] : ['--headless=new', '--log-level=3', '--disable-gpu']
+      ],
+      firefox: [
+        'moz:firefoxOptions',
+        options.visible ? [] : ['-headless']
+      ]
+    }[options.browser]
+
+    browserio = await remote({
+      capabilities: {
+        browserName: options.browser,
+        webSocketUrl: true,
+        [browserOptions]: {
+          args,
+          binary: options.binary
+        }
+      }
+    })
+
+    if (scripts && scripts.length) {
+      for (const script of scripts) {
+        await browserio.scriptAddPreloadScript({
+          functionDeclaration: `() => ${script}`
+        })
+      }
+    }
+
+    await browserio.url(url)
+  }
+})

package/src/job.js

@@ -132,6 +132,7 @@ function getCommand (cwd) {
     .option('-ccf, --coverage-check-functions <percent>', '[💻🔗] What % of functions must be covered', percent, 0)
     .option('-ccl, --coverage-check-lines <percent>', '[💻🔗] What % of lines must be covered', percent, 0)
     .option('-ccs, --coverage-check-statements <percent>', '[💻🔗] What % of statements must be covered', percent, 0)
+    .option('-crs, --coverage-remote-scanner <path>', '[💻🔗] Scan for files when all coverage is requested', '$/scan-ui5.js')
     .option('-s, --serve-only [flag]', '[💻🔗] Serve only', boolean, false)
 
     // Specific to legacy (and might be used with url if pointing to local project)
@@ -146,8 +147,8 @@ function getCommand (cwd) {
 
     // Specific to coverage in url mode (experimental)
     .option('-cp, --coverage-proxy [flag]', `[🔗] ${EXPERIMENTAL_OPTION} use internal proxy to instrument remote files`, boolean, false)
-    .option('-cpi, --coverage-proxy-include <regexp>', `[🔗] ${EXPERIMENTAL_OPTION} urls to instrument for coverage`, regex, regex('.*'))
-    .option('-cpe, --coverage-proxy-exclude <regexp>', `[🔗] ${EXPERIMENTAL_OPTION} urls to ignore for coverage`, regex, regex('/((test-)?resources|tests?)/.*'))
+    .option('-cpi, --coverage-proxy-include <regexp>', `[🔗] ${EXPERIMENTAL_OPTION} urls to instrument for coverage`, regex, '.*')
+    .option('-cpe, --coverage-proxy-exclude <regexp>', `[🔗] ${EXPERIMENTAL_OPTION} urls to ignore for coverage`, regex, '/((test-)?resources|tests?)/')
 
     .addOption(new Option('--debug-probe-only', DEBUG_OPTION, boolean).hideHelp())
     .addOption(new Option('--debug-keep-browser-open', DEBUG_OPTION, boolean).hideHelp())
@@ -219,7 +220,7 @@ function finalize (job) {
   function updateToAbsolute (member, from = job.cwd) {
     job[member] = toAbsolute(job[member], from)
   }
-  'browser,coverageSettings,progressPage'
+  'browser,coverageSettings,coverageRemoteScanner,progressPage'
     .split(',')
     .forEach(setting => { job[setting] = checkDefault(job[setting]) })
   updateToAbsolute('cwd', job.initialCwd)
@@ -299,6 +300,8 @@ function finalize (job) {
     overrideDirIfNotSet('coverageReportDir', settings['report-dir'])
     overrideDirIfNotSet('coverageTempDir', settings['temp-dir'])
     overrideIfNotSet('coverageReporters', settings.reporter)
+
+    checkAccess({ path: job.coverageRemoteScanner, label: 'coverage remote scanner', file: true })
   }
 
   if (job.mode === 'url') {

package/src/output.js

@@ -364,7 +364,7 @@ function build (job) {
     },
 
     packageNotLatest (name, latestVersion) {
-      wrap(() => log(job, `/!\\ latest version of ${name} is ${latestVersion}`))()
+      wrap(() => log(job, `⚠️ [PKGVRS] latest version of ${name} is ${latestVersion}`))()
     },
 
     browserStart (url) {
@@ -472,7 +472,15 @@ function build (job) {
     }),
 
     instrumentationSkipped: wrap(() => {
-      log(job, p80()`Skipping nyc instrumentation (--url)`)
+      log(job, p80()`⚠️ [SKPNYC] Skipping nyc instrumentation (--url)`)
+    }),
+
+    assumingOneOrigin: wrap(() => {
+      log(job, p80()`⚠️ [COVORG] Considering only one origin`)
+    }),
+
+    noInfoForAllCoverage: wrap(() => {
+      log(job, p80()`⚠️ [COVALL] Unable to process all coverage, report might be incomplete`)
     }),
 
     endpointError: wrap(({ api, url, data, error }) => {
@@ -548,7 +556,7 @@ function build (job) {
     }),
 
     unhandled: wrap(() => {
-      warn(job, p80()`Some requests are not handled properly, check the unhandled.txt report for more info`)
+      warn(job, p80()`⚠️ [UNHAND] Some requests are not handled properly, check the unhandled.txt report for more info`)
     }),
 
     reportGeneratorFailed: wrap((generator, exitCode, buffers) => {

package/src/reserve.js

@@ -13,7 +13,8 @@ module.exports = async job => check({
     ...job.mappings ?? [],
     ...job.serveOnly ? [] : endpoints(job),
     ...ui5(job),
-    ...await coverage(job), {
+    ...job.serveOnly ? [] : await coverage(job),
+    {
       // Project mapping
       match: /^\/(.*)/,
       file: join(job.webapp, '$1'),