ui5-test-runner 4.0.0 → 4.1.1

package/README.md

@@ -17,7 +17,6 @@ A self-sufficient test runner for UI5 applications enabling parallel execution o
 
 ## 📚 [Documentation](https://arnaudbuchholz.github.io/ui5-test-runner/)
 
-
 ## 💿 How to install
 
 * Works with [Node.js](https://nodejs.org/en/download/) >= 18
@@ -30,20 +29,6 @@ A self-sufficient test runner for UI5 applications enabling parallel execution o
 
 **NOTE** : additional packages might be needed during the execution (`puppeteer`, `selenium-webdriver`, `nyc`...) . If they are found installed **locally** in the tested project, they are used. Otherwise, they are installed **globally**.
 
-## 🖥️ How to demo
-
-* Clone the project [training-ui5con18-opa](https://github.com/ArnaudBuchholz/training-ui5con18-opa) and run `npm install`
-* Use `npm run karma` to test with the karma runner
-* *Serving the application (a.k.a. legacy mode)*
-  * `npx ui5-test-runner --port 8081 --ui5 https://ui5.sap.com/1.109.0/ --cache .ui5 --keep-alive`
-  * Follow the progress of the test executions using http://localhost:8081/_/progress.html
-  * When the tests are completed, check the code coverage with http://localhost:8081/_/coverage/lcov-report/index.html
-* *Serving the application with `@ui5/cli`*
-  * Use `npm start` to serve the application with `@ui5/cli`
-  * `npx ui5-test-runner --port 8081 --url http://localhost:8080/test/testsuite.qunit.html --keep-alive`
-  * Follow the progress of the test executions using http://localhost:8081/_/progress.html
-
-
 ## ⚖️ License
 [![FOSSA Status](https://app.fossa.com/api/projects/git%2Bgithub.com%2FArnaudBuchholz%2Fui5-test-runner.svg?type=large)](https://app.fossa.com/projects/git%2Bgithub.com%2FArnaudBuchholz%2Fui5-test-runner?ref=badge_large)
 

package/package.json

@@ -1,13 +1,8 @@
 {
   "name": "ui5-test-runner",
-  "version": "4.0.0",
+  "version": "4.1.1",
   "description": "Standalone test runner for UI5",
   "main": "index.js",
-  "files": [
-    "defaults/*",
-    "src/**/*",
-    "*.js"
-  ],
   "bin": {
     "ui5-test-runner": "./index.js"
   },
@@ -74,16 +69,16 @@
     "reserve": "^1.15.6"
   },
   "devDependencies": {
-    "@openui5/types": "^1.120.6",
-    "@ui5/cli": "^3.9.0",
+    "@openui5/types": "^1.121.0",
+    "@ui5/cli": "^3.9.1",
     "@ui5/middleware-code-coverage": "^1.1.1",
     "jest": "^29.7.0",
-    "nock": "^13.5.1",
+    "nock": "^13.5.3",
     "nyc": "^15.1.0",
     "standard": "^17.1.0",
     "start-server-and-test": "^2.0.3",
     "typescript": "^5.3.3",
-    "ui5-tooling-transpile": "^3.3.3"
+    "ui5-tooling-transpile": "^3.3.4"
   },
   "optionalDependencies": {
     "fsevents": "^2.3.3"

package/src/coverage.js

@@ -86,8 +86,31 @@ async function instrument (job) {
   await nyc(job, 'instrument', job.webapp, join(job.coverageTempDir, 'instrumented'), '--nycrc-path', job[$nycSettingsPath])
 }
 
+async function getReadableSource (job, pathOrUrl) {
+  if (isAbsolute(pathOrUrl)) {
+    try {
+      await access(pathOrUrl, constants.R_OK)
+      return pathOrUrl
+    } catch (e) {}
+  }
+  try {
+    const filePath = join(job.webapp, pathOrUrl)
+    await access(filePath, constants.R_OK)
+    return filePath
+  } catch (e) {}
+  try {
+    // Assuming all files are coming from the same server
+    const { origin } = new URL(job.testPageUrls[0])
+    const filePath = join(job.coverageTempDir, 'sources', pathOrUrl)
+    await download(origin + pathOrUrl, filePath)
+    return filePath
+  } catch (e) {}
+}
+
 async function generateCoverageReport (job) {
   job.status = 'Generating coverage report'
+  const output = getOutput(job)
+  output.debug('coverage', 'Generating coverage report...')
   await cleanDir(job.coverageReportDir)
   const coverageMergedDir = join(job.coverageTempDir, 'merged')
   await createDir(coverageMergedDir)
@@ -95,25 +118,17 @@ async function generateCoverageReport (job) {
   await nyc(job, 'merge', job.coverageTempDir, coverageFilename)
   if (job[$coverageRemote] && !job.coverageProxy) {
     job.status = 'Checking remote source files'
-    // Assuming all files are coming from the same server
-    const { origin } = new URL(job.testPageUrls[0])
-    const sourcesBasePath = join(job.coverageTempDir, 'sources')
+    output.debug('coverage', 'Checking remote source files...')
     const coverageData = require(coverageFilename)
     const filenames = Object.keys(coverageData)
     let changes = 0
     for (const filename of filenames) {
       const fileData = coverageData[filename]
-      const { path } = fileData
-      if (isAbsolute(path)) {
-        try {
-          await access(path, constants.R_OK)
-          continue
-        } catch (e) {}
+      const filePath = await getReadableSource(job, fileData.path)
+      if (filePath && filePath !== fileData.path) {
+        fileData.path = filePath
+        ++changes
       }
-      const filePath = join(sourcesBasePath, path)
-      fileData.path = filePath
-      await download(origin + path, filePath)
-      ++changes
     }
     if (changes > 0) {
       await writeFile(coverageFilename, JSON.stringify(coverageData))
@@ -151,9 +166,7 @@ module.exports = {
   async collect (job, url, coverageData) {
     job[$coverageFileIndex] = (job[$coverageFileIndex] || 0) + 1
     const coverageFileName = join(job.coverageTempDir, `${filename(url)}_${job[$coverageFileIndex]}.json`)
-    if (job.debugCoverage) {
-      getOutput(job).wrap(() => console.log('coverage', coverageFileName))
-    }
+    getOutput(job).debug('coverage', `saved coverage in '${coverageFileName}'`)
     await writeFile(coverageFileName, JSON.stringify(coverageData))
   },
   generateCoverageReport: job => job.coverage ? generateCoverageReport(job) : Promise.resolve(),
@@ -175,8 +188,8 @@ module.exports = {
     }
     if (job.mode === 'url' && job.coverageProxy) {
       await setupNyc(job)
+      // Assuming all files are coming from the same server
       const { origin } = new URL(job.url[0])
-      const sourcesBasePath = join(job.coverageTempDir, 'sources')
       const { createInstrumenter } = require(join(await nycInstallationPath, 'node_modules/istanbul-lib-instrument'))
       const instrumenter = createInstrumenter({
         produceSourceMap: true,
@@ -188,34 +201,30 @@ module.exports = {
       return [{
         match: /(.*\.js)(\?.*)?$/,
         custom: async (request, response, url) => {
-          if (!url.match(job.coverageProxyInclude) || url.match(job.coverageProxyExclude)) {
-            if (job.debugCoverage) {
-              getOutput(job).wrap(() => console.log('coverage_proxy ignore', url))
-            }
-            return // Ignore
+          if (!url.match(job.coverageProxyInclude) || url.match(/\bresources\b/) || url.match(job.coverageProxyExclude)) {
+            getOutput(job).debug('coverage', 'coverage_proxy ignore', url)
+            return
           }
-          const sourcePath = join(sourcesBasePath, url)
+          const instrumentedSourcePath = join(instrumentedBasePath, url)
           try {
-            await access(sourcePath, constants.R_OK)
-          } catch (e) {
-            try {
-              if (sources[url]) {
-                await sources[url]
-              } else {
-                if (job.debugCoverage) {
-                  getOutput(job).wrap(() => console.log('coverage_proxy instrument', url))
-                }
-                sources[url] = await download(origin + url, sourcePath)
-              }
-            } catch (statusCode) {
-              return statusCode
-            }
+            await access(instrumentedSourcePath, constants.R_OK)
+            return
+          } catch (e) {}
+          const instrumenting = sources[url]
+          if (instrumenting) {
+            await instrumenting
+            return // ok
           }
-          const source = (await readFile(sourcePath)).toString()
-          const instrumentedSource = await instrument(source, sourcePath)
-          const instrumentedSourcePath = join(instrumentedBasePath, url)
-          await createDir(dirname(instrumentedSourcePath))
-          await writeFile(instrumentedSourcePath, instrumentedSource)
+          sources[url] = (async () => {
+            const sourcePath = await getReadableSource(job, url)
+            getOutput(job).debug('coverage', 'coverage_proxy instrument', url, sourcePath)
+            if (sourcePath) {
+              const source = (await readFile(sourcePath)).toString()
+              const instrumentedSource = await instrument(source, sourcePath)
+              await createDir(dirname(instrumentedSourcePath))
+              await writeFile(instrumentedSourcePath, instrumentedSource)
+            }
+          })()
         }
       },
       instrumentedMapping,

package/src/defaults/report/progress.js

@@ -4,6 +4,16 @@
   report.ready.then(update => {
     let lastState = {}
 
+    async function retry () {
+      try {
+        await fetch('/_/progress', { method: 'INFO' })
+        location.hash = ''
+        refresh()
+      } catch (e) {
+        setTimeout(retry, 250)
+      }
+    }
+
     async function refresh () {
       const [, page, test] = location.hash.match(/#?([^-]*)(?:-(.*))?/)
       let url = '/_/progress'
@@ -22,6 +32,7 @@
           ...lastState,
           disconnected: true
         })
+        retry()
         return
       }
       if (test) {

package/src/endpoints.js

@@ -141,6 +141,14 @@ module.exports = job => {
         match: '^/_/punyexpr.js',
         file: punyexprBinPath
       }, {
+      // Endpoint to retry on progress
+        method: 'INFO',
+        match: '^/_/progress',
+        custom: (request, response) => {
+          response.writeHead(204)
+          response.end()
+        }
+      }, {
       // Endpoint to follow progress
         match: '^/_/progress(?:\\?page=([^&]*)(?:&test=([^&]*))?)?',
         custom: (request, response, pageId, testId) => getJobProgress(job, request, response, pageId, testId)

package/src/get-job-progress.js

@@ -1,5 +1,7 @@
 'use strict'
 
+const { $proxifiedUrls } = require('./symbols')
+
 const send = (response, obj) => {
   let json
   if (typeof obj !== 'string') {
@@ -60,6 +62,13 @@ module.exports = {
       ...job,
       status: job.status
     }, (key, value) => {
+      if (key === 'qunitPages' && job[$proxifiedUrls]) {
+        const unproxifiedQunitPages = {}
+        for (const url of Object.keys(job.qunitPages)) {
+          unproxifiedQunitPages[job[$proxifiedUrls][url] || url] = job.qunitPages[url]
+        }
+        return unproxifiedQunitPages
+      }
       if (key === 'modules') {
         return undefined
       }

package/src/inject/opa-iframe-coverage.js

@@ -2,7 +2,6 @@
   'use strict'
 
   const MODULE = 'ui5-test-runner/opa-iframe-coverage'
-
   if (window[MODULE]) {
     return // already installed
   }

package/src/inject/post.js

@@ -1,9 +1,9 @@
 (function () {
   'use strict'
 
-  const POST = 'ui5-test-runner/post'
-  if (window[POST]) {
-    return
+  const MODULE = 'ui5-test-runner/post'
+  if (window[MODULE]) {
+    return // already installed
   }
 
   const base = window['ui5-test-runner/base-host'] || ''
@@ -19,7 +19,7 @@
     }
     const ui5Summary = obj => {
       const id = obj.getId && obj.getId()
-      const className = obj.getMetadata().getName()
+      const className = obj.getMetadata && obj.getMetadata() && obj.getMetadata().getName()
       return {
         'ui5:class': className,
         'ui5:id': id
@@ -71,7 +71,7 @@
 
   window['ui5-test-runner/stringify'] = stringify
 
-  window[POST] = function post (url, data) {
+  window[MODULE] = function post (url, data) {
     function request () {
       return new Promise(function (resolve, reject) {
         const xhr = new XMLHttpRequest()
@@ -87,11 +87,10 @@
       })
     }
     lastPost = lastPost
+      .then(request)
       .then(undefined, function (reason) {
         console.error('Failed to POST to ' + url + '\nreason: ' + reason.toString())
-        throw new Error('failed')
       })
-      .then(request)
     return lastPost
   }
 }())

package/src/inject/qunit-hooks.js

@@ -18,18 +18,34 @@
     }
   }
 
+  function getModules () {
+    if (QUnit.config && QUnit.config.modules) {
+      return QUnit.config.modules.map(({ name, tests }) => ({
+        name,
+        tests: tests.map(({ name, testId, skip }) => ({ name, testId, skip }))
+      }))
+    }
+    return []
+  }
+
+  function extend (details) {
+    details.isOpa = isOpa()
+    details.modules = getModules()
+    return details
+  }
+
   QUnit.begin(function (details) {
     details.isOpa = isOpa()
     return post('QUnit/begin', details)
   })
 
   QUnit.testStart(function (details) {
-    return post('QUnit/testStart', details)
+    return post('QUnit/testStart', extend(details))
   })
 
   QUnit.log(function (log) {
     let ready = false
-    post('QUnit/log', log)
+    post('QUnit/log', extend(log))
       .then(undefined, function () {
         console.error('Failed to POST to QUnit/log (no timestamp)', log)
       })

package/src/inject/qunit-intercept.js

@@ -2,6 +2,12 @@
 (function () {
   'use strict'
 
+  const MODULE = 'ui5-test-runner/qunit-intercept'
+  if (window[MODULE]) {
+    return // already installed
+  }
+  window[MODULE] = true
+
   const callbacks = {}
   const mock = new Proxy({}, {
     get: function (instance, property) {

package/src/inject/ui5-coverage.js

@@ -1,6 +1,12 @@
 (function () {
   'use strict'
 
+  const MODULE = 'ui5-test-runner/ui5-coverage'
+  if (window[MODULE]) {
+    return // already installed
+  }
+  window[MODULE] = true
+
   // inspired from ui5/resources/sap/ui/qunit/qunit-coverage-istanbul-dbg.js
 
   function appendUrlParameter (url) {

package/src/job-mode.js

@@ -39,7 +39,6 @@ function buildAndCheckMode (job) {
       'libs',
       'mappings',
       'cache',
-      'webapp',
       'watch',
       'testsuite'
     ])

package/src/job.js

@@ -105,9 +105,10 @@ function getCommand (cwd) {
     .option('--no-npm-install', '[💻🔗🧪] Prevent any NPM install (execution may fail if a dependency is missing)')
     .option('-bt, --browser-close-timeout <timeout>', '[💻🔗🧪] Maximum waiting time for browser close', timeout, 2000)
     .option('-br, --browser-retry <count>', '[💻🔗🧪] Browser instantiation retries : if the command fails unexpectedly, it is re-executed (0 means no retry)', 1)
+    .option('-oi, --output-interval <interval>', '[💻🔗🧪] Interval for reporting progress on non interactive output (CI/CD) (0 means no output)', timeout, 30000)
 
     // Common to legacy and url
-    .option('-qs, --qunit-strict', '[💻🔗] Strict mode on qunit execution (fails if no modules declared)', boolean, false)
+    .option('--webapp <path>', '[💻🔗] Base folder of the web application (relative to cwd)', 'webapp')
     .option('-pf, --page-filter <regexp>', '[💻🔗] Filter out pages not matching the regexp')
     .option('-pp, --page-params <params>', '[💻🔗] Add parameters to page URL')
     .option('-t, --global-timeout <timeout>', '[💻🔗] Limit the pages execution time, fail the page if it takes longer than the timeout (0 means no timeout)', timeout, 0)
@@ -135,7 +136,6 @@ function getCommand (cwd) {
     .option('--libs <lib...>', '[💻] Library mapping (<relative>=<path> or <path>)', arrayOf(lib))
     .option('--mappings <mapping...>', '[💻] Custom mapping (<match>=<file|url>(<config>))', arrayOf(mapping))
     .option('--cache <path>', '[💻] Cache UI5 resources locally in the given folder (empty to disable)')
-    .option('--webapp <path>', '[💻] Base folder of the web application (relative to cwd)', 'webapp')
     .option('--testsuite <path>', '[💻] Path of the testsuite file (relative to webapp, URL parameters are supported)', 'test/testsuite.qunit.html')
     .option('-w, --watch [flag]', '[💻] Monitor the webapp folder and re-execute tests on change', boolean, false)
 
@@ -150,7 +150,6 @@ function getCommand (cwd) {
     .addOption(new Option('--debug-keep-report', DEBUG_OPTION, boolean).hideHelp())
     .addOption(new Option('--debug-capabilities-test <name>', DEBUG_OPTION).hideHelp())
     .addOption(new Option('--debug-capabilities-no-timeout', DEBUG_OPTION, boolean).hideHelp())
-    .addOption(new Option('--debug-coverage', DEBUG_OPTION, boolean).hideHelp())
     .addOption(new Option('--debug-coverage-no-custom-fs', DEBUG_OPTION, boolean).hideHelp())
     .addOption(new Option('--debug-verbose <module...>', DEBUG_OPTION, arrayOf(string), []).hideHelp())
 
@@ -264,9 +263,7 @@ function finalize (job) {
   if (job.coverage) {
     function overrideIfNotSet (option, valueFromSettings) {
       if (valueFromSettings && job[$valueSources][option] !== 'cli') {
-        if (job.debugCoverage) {
-          output.wrap(() => console.log(`${option} extracted from nyc settings : ${valueFromSettings}`))
-        }
+        output.debug('coverage', `${option} extracted from nyc settings : ${valueFromSettings}`)
         job[option] = valueFromSettings
       }
     }

package/src/output.js

@@ -14,6 +14,7 @@ const { filename, noop, pad } = require('./tools')
 const inJest = typeof jest !== 'undefined'
 const interactive = process.stdout.columns !== undefined && !inJest
 const $output = Symbol('output')
+const $outputStart = Symbol('output-start')
 
 if (!interactive) {
   const UTF8_BOM_CODE = '\ufeff'
@@ -31,6 +32,15 @@ if (inJest) {
   cons = console
 }
 
+const formatTime = duration => {
+  duration = Math.ceil(duration / 1000)
+  const seconds = duration % 60
+  const minutes = (duration - seconds) / 60
+  return minutes.toString().padStart(2, '0') + ':' + seconds.toString().padStart(2, '0')
+}
+
+const getElapsed = job => formatTime(Date.now() - job[$outputStart])
+
 const write = (...parts) => parts.forEach(part => process.stdout.write(part))
 
 function clean (job) {
@@ -84,8 +94,16 @@ function bar (ratio, msg) {
 const TICKS = ['\u280b', '\u2819', '\u2839', '\u2838', '\u283c', '\u2834', '\u2826', '\u2827', '\u2807', '\u280f']
 
 function progress (job, cleanFirst = true) {
-  if (cleanFirst) {
-    clean(job)
+  if (interactive) {
+    if (cleanFirst) {
+      clean(job)
+    }
+  } else {
+    if (job[$browsers]) {
+      write(`${getElapsed(job)} │ Progress\n──────┴──────────\n`)
+    } else {
+      return
+    }
   }
   const output = job[$output]
   output.lines = 1
@@ -131,11 +149,26 @@ function progress (job, cleanFirst = true) {
   }
 }
 
-function output (job, ...texts) {
-  writeFileSync(join(job.reportDir, 'output.txt'), texts.map(t => t.toString()).join(' ') + '\n', {
-    encoding: 'utf-8',
-    flag: 'a'
-  })
+function output (job, ...args) {
+  writeFileSync(
+    join(job.reportDir, 'output.txt'),
+    args.map(arg => {
+      if (typeof arg === 'object') {
+        return JSON.stringify(arg, undefined, 2)
+      }
+      if (arg === undefined) {
+        return 'undefined'
+      }
+      if (arg === null) {
+        return 'null'
+      }
+      return arg.toString()
+    }).join(' ') + '\n',
+    {
+      encoding: 'utf-8',
+      flag: 'a'
+    }
+  )
 }
 
 function log (job, ...texts) {
@@ -185,13 +218,6 @@ function browserIssue (job, { type, url, code, dir }) {
   log(job, p`└──────────${pad.x('─')}┘`)
 }
 
-const formatTime = duration => {
-  duration = Math.ceil(duration / 1000)
-  const seconds = duration % 60
-  const minutes = (duration - seconds) / 60
-  return minutes.toString().padStart(2, '0') + ':' + seconds.toString().padStart(2, '0')
-}
-
 function build (job) {
   let wrap
   if (interactive) {
@@ -206,8 +232,7 @@ function build (job) {
   } else {
     wrap = method => method
   }
-  const outputStart = Date.now()
-  const getElapsed = () => formatTime(Date.now() - outputStart)
+  job[$outputStart] = Date.now()
 
   return {
     lastTick: 0,
@@ -228,6 +253,7 @@ function build (job) {
     debug: wrap((module, ...args) => {
       if (job.debugVerbose && job.debugVerbose.includes(module)) {
         console.log(`🐞${module}`, ...args)
+        output(job, `🐞${module}`, ...args)
       }
     }),
 
@@ -251,7 +277,7 @@ function build (job) {
       } else {
         method = log
       }
-      const text = `${getElapsed()} │ ${status}`
+      const text = `${getElapsed(job)} │ ${status}`
       method(job, '')
       method(job, text)
       method(job, '──────┴'.padEnd(text.length, '─'))
@@ -268,6 +294,8 @@ function build (job) {
     reportOnJobProgress () {
       if (interactive) {
         this.reportIntervalId = setInterval(progress.bind(null, job), 250)
+      } else if (job.outputInterval) {
+        this.reportIntervalId = setInterval(progress.bind(null, job), job.outputInterval)
       }
     },
 
@@ -311,7 +339,7 @@ function build (job) {
     },
 
     browserStart (url) {
-      const text = p80()`${getElapsed()} >> ${pad.lt(url)} [${filename(url)}]`
+      const text = p80()`${getElapsed(job)} >> ${pad.lt(url)} [${filename(url)}]`
       if (interactive) {
         output(job, text)
       } else {
@@ -325,7 +353,7 @@ function build (job) {
       if (page) {
         duration = ' (' + formatTime(page.end - page.start) + ')'
       }
-      const text = p80()`${getElapsed()} << ${pad.lt(url)} ${duration} [${filename(url)}]`
+      const text = p80()`${getElapsed(job)} << ${pad.lt(url)} ${duration} [${filename(url)}]`
       if (interactive) {
         output(job, text)
       } else {
@@ -418,10 +446,6 @@ function build (job) {
       log(job, p80()`Skipping nyc instrumentation (--url)`)
     }),
 
-    qunitEarlyStart: wrap(url => {
-      log(job, p80()`QUnit start without tests in ${pad.lt(url)}`)
-    }),
-
     endpointError: wrap(({ api, url, data, error }) => {
       const p = p80()
       log(job, p`┌──────────${pad.x('─')}┐`)
@@ -514,7 +538,9 @@ function build (job) {
     stop () {
       if (this.reportIntervalId) {
         clearInterval(this.reportIntervalId)
-        clean(job)
+        if (interactive) {
+          clean(job)
+        }
       }
     }
   }