ui5-test-runner 3.3.0 → 3.3.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ui5-test-runner",
-  "version": "3.3.0",
+  "version": "3.3.2",
   "description": "Standalone test runner for UI5",
   "main": "index.js",
   "files": [

package/src/browsers.js

@@ -4,7 +4,7 @@ const { fork } = require('child_process')
 const { join } = require('path')
 const { writeFile, readFile, open, stat, unlink } = require('fs/promises')
 const { recreateDir, filename, allocPromise } = require('./tools')
-const { getPageTimeout } = require('./timeout')
+const { getPageTimeout, pageTimedOut } = require('./timeout')
 const { getOutput } = require('./output')
 const { resolvePackage } = require('./npm')
 const { UTRError } = require('./error')
@@ -174,6 +174,7 @@ async function run (job, pageBrowser) {
   if (timeout) {
     pageBrowser.timeoutId = setTimeout(() => {
       output.browserTimeout(url, dir)
+      pageTimedOut(job, url)
       stop(job, url)
     }, timeout)
   }

package/src/coverage.js

@@ -110,7 +110,20 @@ async function generateCoverageReport (job) {
     await writeFile(coverageFilename, JSON.stringify(coverageData))
   }
   const reporters = job.coverageReporters.map(reporter => `--reporter=${reporter}`)
-  await nyc(job, 'report', ...reporters, '--temp-dir', coverageMergedDir, '--report-dir', job.coverageReportDir, '--nycrc-path', job[$nycSettingsPath])
+  if (!job.coverageReporters.includes('text')) {
+    reporters.push('--reporter=text')
+  }
+  const checks = []
+  if (job.coverageCheckBranches || job.coverageCheckFunctions || job.coverageCheckLines || job.coverageCheckStatements) {
+    checks.push(
+      `--branches=${job.coverageCheckBranches}`,
+      `--functions=${job.coverageCheckFunctions}`,
+      `--lines=${job.coverageCheckLines}`,
+      `--statements=${job.coverageCheckStatements}`,
+      '--check-coverage'
+    )
+  }
+  await nyc(job, 'report', ...reporters, ...checks, '--temp-dir', coverageMergedDir, '--report-dir', job.coverageReportDir, '--nycrc-path', job[$nycSettingsPath])
 }
 
 module.exports = {
@@ -151,6 +164,7 @@ module.exports = {
         coverageGlobalScopeFunc: false
       })
       const instrument = promisify(instrumenter.instrument.bind(instrumenter))
+      const sources = {}
       return [{
         match: /(.*\.js)(\?.*)?$/,
         custom: async (request, response, url) => {
@@ -161,8 +175,15 @@ module.exports = {
           try {
             await access(sourcePath, constants.R_OK)
           } catch (e) {
-            console.log('download', url)
-            await download(origin + url, sourcePath)
+            try {
+              if (sources[url]) {
+                await sources[url]
+              } else {
+                sources[url] = await download(origin + url, sourcePath)
+              }
+            } catch (statusCode) {
+              return statusCode
+            }
           }
           const source = (await readFile(sourcePath)).toString()
           const instrumentedSource = await instrument(source, sourcePath)

package/src/defaults/junit-xml-report.js

@@ -39,6 +39,8 @@ async function main () {
     >`)
         if (test.skip) {
           o('      <skipped></skipped>')
+        } else if (!test.report) {
+          o('      <skipped>(no report found)</skipped>')
         } else if (test.report.failed) {
           test.logs
             .filter(({ result }) => !result)

package/src/defaults/puppeteer.js

@@ -12,10 +12,26 @@ require('./browser')({
       ['-h, --viewport-height <height>', 'Viewport height', 1080],
       ['-l, --language <lang...>', 'Language(s)', ['en-US']],
       ['-u, --unsecure', 'Disable security features', false]
-    ],
-    capabilities: {
+    ] // ,
+    // TODO restore when Node16 is no more supported
+    // capabilities: {
+    //   modules: ['puppeteer'],
+    //   screenshot: '.png',
+    //   scripts: true,
+    //   traces: ['console', 'network']
+    // }
+  },
+
+  // TODO remove when Node16 is no more supported
+  async capabilities () {
+    const version = process.version.match(/^v(\d+\.\d+)/)[1]
+    let screenshot
+    if (!version.startsWith('16')) {
+      screenshot = '.png'
+    }
+    return {
       modules: ['puppeteer'],
-      screenshot: '.png',
+      screenshot,
       scripts: true,
       traces: ['console', 'network']
     }

package/src/defaults/report/default.html

@@ -12,6 +12,9 @@
       <h1>{{ status || 'Test report' }}</h1>
       <div {{if}}="end === undefined" class="elapsed">In progress since {{ elapsed(start) }}</div>
       <div {{else}} class="elapsed">Duration : {{ elapsed(start, end) }} <a href="#" id="download">&#128230;</a></div>
+      <div {{if}}="timedOut">
+        &#9202;&#65039; Timed out
+      </div>
       <table style="visibility: {{ testPageUrls.length > 0 ? 'visible' : 'hidden' }};">
         <tr>
           <th>&nbsp;</th>
@@ -30,7 +33,7 @@
           <td>
             <span {{if}}="!qunitPages[url]">-</span>
             <span {{elseif}}="qunitPages[url].isOpa" title="OPA test">&#129404;</span>
-            <span {{else}} title="Unit test">&#129514;</span>
+            <span {{elseif}}="qunitPages[url].start" title="Unit test">&#129514;</span>
           </td>
           <td>
             <span {{if}}="!qunitPages[url]">-</span>
@@ -56,6 +59,9 @@
       <a href="#">&#9204; back to report</a>
       <div {{if}}="qunitPage.end === undefined" class="elapsed">In progress since {{ elapsed(qunitPage.start) }}</div>
       <div {{else}} class="elapsed">Duration : {{ elapsed(qunitPage.start, qunitPage.end) }}</div>
+      <div {{if}}="timedOut">
+        &#9202;&#65039; Timed out
+      </div>
       <div {{for}}="module of qunitPage.modules">
         <h2>{{ module.name }}</h2>
         <div {{for}}="test of module.tests">

package/src/defaults/text-report.js

@@ -1,19 +1,29 @@
 'use strict'
 
 const { join, isAbsolute } = require('path')
-const [, , reportDir] = process.argv
+const [, , reportDir, expectedWidth] = process.argv
 const { pad } = require('../tools')
 
-const p = pad(process.stdout.columns || 80)
+const p = pad(process.stdout.columns || parseInt(expectedWidth || '80', 10))
 const log = console.log.bind(console)
 
 function collectErrors (page) {
   const errors = []
   page.modules.forEach(module => {
-    module.tests.forEach(test => {
-      if (test.report.failed) {
+    module.tests.every(test => {
+      if (!test.report) {
+        if (!test.skip) {
+          test.logs ??= []
+          test.logs.push({
+            message: '(no report found)'
+          })
+          errors.push({ module: module.name, ...test })
+          return false
+        }
+      } else if (test.report.failed) {
         errors.push({ module: module.name, ...test })
       }
+      return true
     })
   })
   return errors
@@ -28,8 +38,11 @@ async function main () {
   }
   const job = require(jobPath)
   const failedUrls = []
-  log(p`┌─${pad.x('─')}───────────────────┐`)
+  let rendered = 0
   function render (url) {
+    if (++rendered === 1) {
+      log(p`┌─${pad.x('─')}───────────────────┐`)
+    }
     const page = job.qunitPages && job.qunitPages[url]
     if (!page || !page.report) {
       log(p`│${pad.lt(url)} 🧨           │`)
@@ -49,7 +62,9 @@ async function main () {
       render(url)
     }
   })
-  log(p`└─${pad.x('─')}────────────────────┘`)
+  if (rendered > 0) {
+    log(p`└─${pad.x('─')}────────────────────┘`)
+  }
   failedUrls.forEach(url => {
     log()
     log(p`[${pad.lt(url)}]`)

package/src/job.js

@@ -7,7 +7,7 @@ const { name, description, version } = require(join(__dirname, '../package.json'
 const { getOutput } = require('./output')
 const { $valueSources } = require('./symbols')
 const { buildAndCheckMode } = require('./job-mode')
-const { boolean, integer, timeout, url, arrayOf, regex } = require('./options')
+const { boolean, integer, timeout, url, arrayOf, regex, percent } = require('./options')
 
 const $status = Symbol('status')
 
@@ -106,7 +106,7 @@ function getCommand (cwd) {
     .option('-bt, --browser-close-timeout <timeout>', '[💻🔗🧪] Maximum waiting time for browser close', timeout, 2000)
     .option('-br, --browser-retry <count>', '[💻🔗🧪] Browser instantiation retries : if the command fails unexpectedly, it is re-executed (0 means no retry)', 1)
 
-    // Common to legacy and testing
+    // Common to legacy and url
     .option('-pf, --page-filter <regexp>', '[💻🔗] Filter out pages not matching the regexp')
     .option('-pp, --page-params <params>', '[💻🔗] Add parameters to page URL')
     .option('-t, --global-timeout <timeout>', '[💻🔗] Limit the pages execution time, fail the page if it takes longer than the timeout (0 means no timeout)', timeout, 0)
@@ -119,13 +119,18 @@ function getCommand (cwd) {
     .option('--coverage [flag]', '[💻🔗] Enable or disable code coverage', boolean)
     .option('--no-coverage', '[💻🔗] Disable code coverage')
     .option('-cs, --coverage-settings <path>', '[💻🔗] Path to a custom nyc.json file providing settings for instrumentation (relative to cwd or use $/ for provided ones)', '$/nyc.json')
-    .option('-ct, --coverage-temp-dir <path>', '[💻🔗] Directory to output raw coverage information to (relative to cwd)', '.nyc_output')
-    .option('-cr, --coverage-report-dir <path>', '[💻🔗] Directory to store the coverage report files (relative to cwd)', 'coverage')
-    .option('-cr, --coverage-reporters <reporter...>', '[💻🔗] List of nyc reporters to use', ['lcov', 'cobertura'])
+    .option('-ctd, --coverage-temp-dir <path>', '[💻🔗] Directory to output raw coverage information to (relative to cwd)', '.nyc_output')
+    .option('-crd, --coverage-report-dir <path>', '[💻🔗] Directory to store the coverage report files (relative to cwd)', 'coverage')
+    .option('-cr, --coverage-reporters <reporter...>', '[💻🔗] List of nyc reporters to use (text is always used)', ['lcov', 'cobertura'])
+    .option('-ccb, --coverage-check-branches <percent>', '[💻🔗] What % of branches must be covered', percent, 0)
+    .option('-ccf, --coverage-check-functions <percent>', '[💻🔗] What % of functions must be covered', percent, 0)
+    .option('-ccl, --coverage-check-lines <percent>', '[💻🔗] What % of lines must be covered', percent, 0)
+    .option('-ccs, --coverage-check-statements <percent>', '[💻🔗] What % of statements must be covered', percent, 0)
     .option('-s, --serve-only [flag]', '[💻🔗] Serve only', boolean, false)
 
-    // Specific to legacy
+    // Specific to legacy (and might be used with url if pointing to local project)
     .option('--ui5 <url>', '[💻] UI5 url', url, 'https://ui5.sap.com')
+    .option('--no-ui5', '[💻] Disable UI5 mapping (also disable libs)')
     .option('--libs <lib...>', '[💻] Library mapping (<relative>=<path> or <path>)', arrayOf(lib))
     .option('--mappings <mapping...>', '[💻] Custom mapping (<match>=<file|url>(<config>))', arrayOf(mapping))
     .option('--cache <path>', '[💻] Cache UI5 resources locally in the given folder (empty to disable)')
@@ -135,8 +140,8 @@ function getCommand (cwd) {
 
     // Specific to coverage in url mode (experimental)
     .option('-cp, --coverage-proxy [flag]', `[🔗] ${EXPERIMENTAL_OPTION} use internal proxy to instrument remote files`, boolean, false)
-    .option('-cpi, --coverage-proxy-include <regexp>', `[🔗] ${EXPERIMENTAL_OPTION} urls to instrument for coverage`, regex, regex('webapp/.*'))
-    .option('-cpe, --coverage-proxy-exclude <regexp>', `[🔗] ${EXPERIMENTAL_OPTION} urls to ignore for coverage`, regex, regex('(test-)?resources/.*'))
+    .option('-cpi, --coverage-proxy-include <regexp>', `[🔗] ${EXPERIMENTAL_OPTION} urls to instrument for coverage`, regex, regex('.*'))
+    .option('-cpe, --coverage-proxy-exclude <regexp>', `[🔗] ${EXPERIMENTAL_OPTION} urls to ignore for coverage`, regex, regex('/((test-)?resources|tests?)/.*'))
 
     .addOption(new Option('--debug-probe-only', DEBUG_OPTION, boolean).hideHelp())
     .addOption(new Option('--debug-keep-browser-open', DEBUG_OPTION, boolean).hideHelp())

package/src/options.js

@@ -77,5 +77,13 @@ module.exports = {
       result.push(typeValidator(value))
       return result
     }
+  },
+
+  percent (value) {
+    const int = integer(value)
+    if (int > 100) {
+      throw new InvalidArgumentError('Invalid percent')
+    }
+    return int
   }
 }

package/src/options.spec.js

@@ -1,4 +1,4 @@
-const { any, boolean, integer, timeout, url, arrayOf } = require('./options')
+const { any, boolean, integer, timeout, url, arrayOf, percent } = require('./options')
 const { InvalidArgumentError } = require('commander')
 
 function checkType ({ method, validValues, invalidValues }) {
@@ -115,6 +115,22 @@ describe('src/options', () => {
     ]
   })
 
+  checkType({
+    method: percent,
+    validValues: {
+      0: 0,
+      1: 1,
+      10: 10,
+      100: 100
+    },
+    invalidValues: [
+      '',
+      '-1',
+      'abc',
+      '101'
+    ]
+  })
+
   describe('arrayOf', () => {
     it('builds a type validator that aggregates validated values in an array', () => {
       const validator = arrayOf(integer)

package/src/report.js

@@ -22,7 +22,7 @@ async function save (job) {
 
 function generateTextReport (job) {
   const { promise, resolve } = allocPromise()
-  const childProcess = fork(join(__dirname, 'defaults/text-report.js'), [job.reportDir], { stdio: 'pipe' })
+  const childProcess = fork(join(__dirname, 'defaults/text-report.js'), [job.reportDir, process.stdout.columns || ''], { stdio: 'pipe' })
   getOutput(job).monitor(childProcess, true)
   childProcess.on('close', resolve)
   return promise

package/src/simulate.spec.js

@@ -301,7 +301,7 @@ describe('simulate', () => {
 
     describe('global timeout', () => {
       beforeAll(async () => {
-        await setup('timeout', {
+        await setup('global-timeout', {
           parallel: 1,
           globalTimeout: 10000
         })
@@ -325,6 +325,62 @@ describe('simulate', () => {
 
       it('failed', () => {
         expect(job.failed).toStrictEqual(true)
+        expect(job.timedOut).toStrictEqual(true)
+      })
+    })
+
+    describe('page timeout', () => {
+      beforeAll(async () => {
+        await setup('page-timeout', {
+          parallel: 1,
+          pageTimeout: 250
+        })
+        pages = {
+          'testsuite.qunit.html': async referer => {
+            await post('/_/addTestPages', referer, [
+              '/page1.html'
+            ])
+          },
+          'page1.html': async headers => {
+            await post('/_/QUnit/begin', headers, {
+              totalTests: 3,
+              modules: [{
+                name: '0',
+                tests: [{
+                  name: '1',
+                  testId: '1'
+                }, {
+                  name: '2',
+                  testId: '2'
+                }, {
+                  name: '3',
+                  testId: '3'
+                }]
+              }]
+            })
+            await post('/_/QUnit/testStart', headers, { module: '0', name: '1', testId: '1' })
+            await post('/_/QUnit/testDone', headers, { testId: '1', failed: 0, passed: 1 })
+            await post('/_/QUnit/testStart', headers, { module: '0', name: '2', testId: '2' })
+          }
+        }
+        await safeExecute()
+      })
+
+      it('failed', () => {
+        expect(job.failed).toStrictEqual(true)
+        expect(job.timedOut).toStrictEqual(true)
+        const page = job.qunitPages['http://localhost:0/page1.html']
+        expect(page.failed).toStrictEqual(2)
+        expect(page.end).not.toBeUndefined()
+        expect(page.timedOut).toStrictEqual(true)
+        const { tests } = page.modules[0]
+        expect(tests[1].report.failed).toStrictEqual(1)
+        expect(tests[1].report.passed).toStrictEqual(0)
+        expect(tests[1].end).not.toBeUndefined()
+        expect(tests[2].report.failed).toStrictEqual(1)
+        expect(tests[2].report.passed).toStrictEqual(0)
+        expect(tests[2].start).toBeUndefined()
+        expect(tests[2].end).toBeUndefined()
       })
     })
 

package/src/tests.js

@@ -35,6 +35,7 @@ async function run (task, job) {
   if (globallyTimedOut(job)) {
     output.globalTimeout(url)
     job.failed = true
+    job.timedOut = true
   } else if (job.failFast && job.failed) {
     output.failFast(url)
   } else {

package/src/timeout.js

@@ -16,5 +16,38 @@ module.exports = {
       return new Date() - job.start > job.globalTimeout
     }
     return false
+  },
+
+  pageTimedOut (job, url) {
+    const page = job.qunitPages && job.qunitPages[url]
+    if (page) {
+      const now = new Date()
+      page.end = now
+      page.timedOut = true
+      page.modules.forEach(module => {
+        module.tests.forEach(test => {
+          if (!test.report) {
+            ++page.failed
+            if (test.start && !test.end) {
+              test.end = now
+            }
+            test.logs ??= []
+            test.logs.push({
+              result: false,
+              message: 'Page timed out'
+            })
+            test.report = {
+              skipped: false,
+              todo: false,
+              failed: 1,
+              passed: 0,
+              total: 1
+            }
+          }
+        })
+      })
+      job.failed = true
+      job.timedOut = true
+    }
   }
 }

package/src/tools.js

@@ -6,6 +6,7 @@ const { createHash } = require('crypto')
 const { createWriteStream } = require('fs')
 const http = require('http')
 const https = require('https')
+const { unlink } = require('fs/promises')
 
 const recursive = { recursive: true }
 
@@ -136,7 +137,13 @@ async function download (url, filename) {
   await mkdir(dirname(filename), recursive)
   const output = createWriteStream(filename)
   const { promise, resolve, reject } = allocPromise()
-  const request = protocol.request(options, response => {
+  const request = protocol.request(options, async response => {
+    if (response.statusCode !== 200) {
+      reject(response.statusCode)
+      output.end()
+      await unlink(filename)
+      return
+    }
     response.on('error', reject)
     response.on('end', resolve)
     response.pipe(output)

package/src/ui5.js

@@ -7,7 +7,7 @@ const { capture } = require('reserve')
 const { getOutput } = require('./output')
 
 module.exports = job => {
-  if (job.mode === 'url') {
+  if (job.noUi5) {
     return []
   }
 

package/src/unhandled.js

@@ -23,7 +23,7 @@ module.exports = job => {
         getOutput(job).unhandled()
         outputUnhandled = false
       }
-      writeFile(unhandled, `${extractPageUrl(headers)} ${status} ${method} ${url}\n`, {
+      writeFile(unhandled, `${extractPageUrl(headers) || headers.referer} ${status} ${method} ${url}\n`, {
         flag: 'a'
       }, noop)
       return status