ui5-test-runner 3.3.4 → 4.0.0

package/README.md

@@ -20,7 +20,7 @@ A self-sufficient test runner for UI5 applications enabling parallel execution o
 
 ## 💿 How to install
 
-* Works with [Node.js](https://nodejs.org/en/download/) >= 16
+* Works with [Node.js](https://nodejs.org/en/download/) >= 18
 * Local installation
   * `npm install --save-dev ui5-test-runner`
   * Trigger either with `npx ui5-test-runner` or through an npm script invoking `ui5-test-runner`
@@ -49,6 +49,9 @@ A self-sufficient test runner for UI5 applications enabling parallel execution o
 
 ## ⚠️ Breaking changes
 
+### v4
+* Dropping support of Node.js 16
+
 ### v3
 * Dropping support of Node.js 14
 
@@ -58,3 +61,8 @@ A self-sufficient test runner for UI5 applications enabling parallel execution o
 * Dependencies are installed **on demand**
 * Browser instantiation command evolved in an **incompatible way** (see [documentation](https://arnaudbuchholz.github.io/ui5-test-runner/browser.html)).
 * Output is different (report, traces)
+
+## ✒ Contributors
+
+* [Marian Zeis](https://github.com/marianfoo): Documentation page revamp [PR #54](https://github.com/ArnaudBuchholz/ui5-test-runner/pull/54)
+* [Raj Singh](https://github.com/rajxsingh): Basic HTTP Authentication in Puppeteer [PR #71](https://github.com/ArnaudBuchholz/ui5-test-runner/pull/71)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ui5-test-runner",
-  "version": "3.3.4",
+  "version": "4.0.0",
   "description": "Standalone test runner for UI5",
   "main": "index.js",
   "files": [
@@ -12,11 +12,16 @@
     "ui5-test-runner": "./index.js"
   },
   "engines": {
-    "node": ">=16"
+    "node": ">=18"
   },
   "scripts": {
     "lint": "standard --fix",
-    "test": "npm run test:unit && npm run test:integration:jsdom && npm run test:integration:puppeteer && npm run test:integration:selenium-webdriver-chrome && npm run test:integration:playwright",
+    "test": "npm run test:unit && npm run test:browsers && npm run test:samples",
+    "test:browsers": "npm run test:integration:jsdom && npm run test:integration:puppeteer && npm run test:integration:selenium-webdriver-chrome && npm run test:integration:playwright",
+    "test:samples": "npm run test:samples:js && npm run test:samples:ts && npm run test:auth-sample",
+    "test:samples:js": "npm run test:sample:js:legacy && npm run test:sample:js:coverage:legacy && npm run test:sample:js:legacy-remote && npm run test:sample:js:coverage:legacy-remote && npm run test:sample:js:remote && npm run test:sample:js:coverage:remote",
+    "test:samples:ts": "npm run test:sample:ts:remote && npm run test:sample:ts:coverage:remote",
+    "test:auth-sample": "npm run test:auth-sample:remote",
     "test:coverall": "rimraf .nyc_output && jest --coverageDirectory .nyc_output --coverageReporters json && nyc --silent --no-clean npm run test:integration:jsdom && nyc --silent --no-clean npm run test:integration:puppeteer && nyc --silent --no-clean npm run test:integration:selenium-webdriver-chrome && nyc --silent --no-clean npm run test:integration:playwright && nyc merge .nyc_output .nyc_output/final/coverage.json && nyc report --temp-dir .nyc_output/final/ --report-dir coverage --branches 80 --functions 80 --lines 80 --statements 80",
     "test:unit": "jest",
     "test:unit:debug": "jest --runInBand",
@@ -26,7 +31,21 @@
     "test:integration:playwright": "node . --capabilities --browser $/playwright.js",
     "test:report": "node ./src/defaults/report.js ./test/report && reserve --config ./test/report/reserve.json",
     "test:text-report": "node ./src/defaults/text-report.js ./test/report",
-    "build:doc": "node build/doc"
+    "test:sample:js:legacy": "node . --cwd ./test/sample.js",
+    "test:sample:js:coverage:legacy": "node . --cwd ./test/sample.js --coverage --coverage-settings nyc.json --coverage-check-statements 67",
+    "test:sample:js:legacy-remote": "node . --port 8081 --cwd ./test/sample.js --url http://localhost:8081/test/testsuite.qunit.html",
+    "test:sample:js:coverage:legacy-remote": "node . --port 8081 --cwd ./test/sample.js --url http://localhost:8081/test/testsuite.qunit.html --coverage --coverage-settings nyc.json --coverage-check-statements 67",
+    "test:sample:js:remote": "start-server-and-test 'npm run serve:sample:js' http://localhost:8080 'node . --url http://localhost:8080/test/testsuite.qunit.html'",
+    "test:sample:js:coverage:remote": "start-server-and-test 'npm run serve:sample:js' http://localhost:8080 'node . --url http://localhost:8080/test/testsuite.qunit.html --coverage --coverage-check-statements 67'",
+    "serve:sample:js": "ui5 serve --config ./test/sample.js/ui5.yaml",
+    "test:sample:ts:remote": "start-server-and-test 'npm run serve:sample:ts' http://localhost:8080 'node . --url http://localhost:8080/test/testsuite.qunit.html'",
+    "serve:sample:ts": "cd ./test/sample.ts && node ui5.cjs serve",
+    "test:sample:ts:coverage:remote": "start-server-and-test 'npm run serve:sample:ts:coverage' http://localhost:8080 'node . --url http://localhost:8080/test/testsuite.qunit.html --coverage --coverage-check-statements 67'",
+    "serve:sample:ts:coverage": "cd ./test/sample.ts && node ui5.cjs serve --config ui5-coverage.yaml",
+    "test:auth-sample:remote": "start-server-and-test 'npm run serve:auth-sample' http://localhost:8080 'node . --url http://localhost:8080/test/testsuite.qunit.html --browser $/puppeteer.js --browser-args --basic-auth-username testUsername --browser-args --basic-auth-password testPassword'",
+    "serve:auth-sample": "cd ./test/auth_sample.js && reserve --config ./reserve.json",
+    "build:doc": "node build/doc",
+    "clean": "npm uninstall -g ui5-test-runner puppeteer nyc selenium-webdriver playwright webdriverio"
   },
   "repository": {
     "type": "git",
@@ -48,17 +67,23 @@
   },
   "homepage": "https://github.com/ArnaudBuchholz/ui5-test-runner#readme",
   "dependencies": {
-    "commander": "^11.0.0",
+    "commander": "^12.0.0",
     "mime": "^3.0.0",
     "punybind": "^1.2.1",
     "punyexpr": "^1.0.4",
-    "reserve": "^1.15.3"
+    "reserve": "^1.15.6"
   },
   "devDependencies": {
+    "@openui5/types": "^1.120.6",
+    "@ui5/cli": "^3.9.0",
+    "@ui5/middleware-code-coverage": "^1.1.1",
     "jest": "^29.7.0",
-    "nock": "^13.3.3",
+    "nock": "^13.5.1",
     "nyc": "^15.1.0",
-    "standard": "^17.1.0"
+    "standard": "^17.1.0",
+    "start-server-and-test": "^2.0.3",
+    "typescript": "^5.3.3",
+    "ui5-tooling-transpile": "^3.3.3"
   },
   "optionalDependencies": {
     "fsevents": "^2.3.3"
@@ -69,6 +94,10 @@
       "qunit",
       "node",
       "jest"
+    ],
+    "globals": [
+      "sap",
+      "opaTest"
     ]
   },
   "jest": {
@@ -98,4 +127,4 @@
       }
     }
   }
-}
+}

package/src/browsers.js

@@ -214,13 +214,16 @@ async function screenshot (job, url, filename) {
   if (!job.browserCapabilities.screenshot) {
     throw UTRError.BROWSER_SCREENSHOT_NOT_SUPPORTED()
   }
+  const output = getOutput(job)
+  const id = ++lastScreenshotId
   try {
     const { childProcess, reportDir } = job[$browsers][url]
     const absoluteFilename = join(reportDir, filename + job.browserCapabilities.screenshot)
     if (childProcess.connected) {
-      const id = ++lastScreenshotId
+      output.debug('screenshot', id, url, absoluteFilename)
       const { promise, resolve, reject } = allocPromise()
       screenshots[id] = resolve
+      output.debug('screenshot', id, 'sending command')
       childProcess.send({
         id,
         command: 'screenshot',
@@ -229,15 +232,20 @@ async function screenshot (job, url, filename) {
       const timeoutId = setTimeout(() => {
         reject(UTRError.BROWSER_SCREENSHOT_TIMEOUT())
       }, job.screenshotTimeout)
+      output.debug('screenshot', id, 'command sent, waiting for answer')
       await promise
+      output.debug('screenshot', id, 'answer received')
       clearTimeout(timeoutId)
       const result = await stat(absoluteFilename)
+      output.debug('screenshot', id, 'file size :', result.size)
       if (!result.isFile() || result.size === 0) {
         throw new Error('File expected')
       }
+      output.debug('screenshot', id, 'done')
       return absoluteFilename
     }
   } catch (e) {
+    output.debug('screenshot', id, e.message)
     if (e.code === UTRError.BROWSER_SCREENSHOT_TIMEOUT_CODE) {
       throw e
     }

package/src/capabilities/index.js

@@ -4,7 +4,6 @@ const { check, serve, body } = require('reserve')
 const { probe, start, stop } = require('../browsers')
 const { join } = require('path')
 const { getOutput } = require('../output')
-const EventEmitter = require('events')
 const { performance } = require('perf_hooks')
 const { cleanDir, allocPromise, filename } = require('../tools')
 const { $browsers } = require('../symbols')
@@ -60,13 +59,16 @@ async function capabilities (job) {
             const { referer, 'x-page-url': xPageUrl } = request.headers
             const listenerIndex = (xPageUrl || referer).match(/\blistener=(\d+)/)[1]
             const listener = listeners[listenerIndex]
-            listener.emit('endpoint', {
+            await listener({
               endpoint,
               body: JSON.parse(await body(request))
             })
             response.writeHead(200)
             response.end()
           }
+        }, {
+          match: '^/inject/(.*)',
+          file: join(__dirname, '../inject/$1')
         }, {
           match: '^/(.*)',
           file: join(__dirname, '$1')
@@ -105,8 +107,6 @@ async function capabilities (job) {
       const { label, url, scripts, endpoint = () => { } } = filteredTests.shift()
 
       const listenerIndex = listeners.length
-      const listener = new EventEmitter()
-      listeners.push(listener)
       let pageUrl
       if (url.startsWith('http')) {
         pageUrl = url
@@ -151,7 +151,7 @@ async function capabilities (job) {
       const context = {
         job
       }
-      listener.on('endpoint', async data => {
+      listeners[listenerIndex] = async data => {
         try {
           if (await endpoint.call(context, data, pageUrl) !== false) {
             done()
@@ -159,7 +159,7 @@ async function capabilities (job) {
         } catch (e) {
           done(e)
         }
-      })
+      }
 
       start(job, pageUrl, scripts)
         .catch(reason => done(reason))

package/src/capabilities/tests/screenshot/index.html

@@ -3,11 +3,21 @@
   <body>
     <H1>screenshot</H1>
     <p>Checks if the browser supports screenshot</p>
+    <input type="text" id="first" value="first">
+    <input type="text" id="second" value="second">
     <span style="font-size: 32rem;">&#128521;</span>
+    <script src="/inject/post.js"></script>
     <script>
-      const xhr = new XMLHttpRequest()
-      xhr.open('POST', '/_/log')
-      xhr.send('{}')
+      document.getElementById('first').focus()
+      const post = window['ui5-test-runner/post']
+      post('/_/log', { step:'screenshot' })
+        .then(() => {
+          const current = document.activeElement
+          return post('/_/log', {
+            step: 'focus',
+            current: current ? current.id : ''
+          })
+        })
     </script>
   </body>
 </html>

package/src/capabilities/tests/screenshot/index.js

@@ -8,11 +8,17 @@ module.exports = {
   label: 'Screenshot',
   for: capabilities => !!capabilities.screenshot,
   url: 'screenshot/index.html',
-  endpoint: async function (_, url) {
+  endpoint: async function ({ body: { step, current } }, url) {
     const { job } = this
-    const fileName = await screenshot(job, url, 'screenshot')
-    const fileInfo = await stat(fileName)
-    assert.ok(fileInfo.isFile(), 'The file was generated')
-    assert.ok(fileInfo.size > 1024, 'The file contains something')
+    if (step === 'screenshot') {
+      const fileName = await screenshot(job, url, 'screenshot')
+      const fileInfo = await stat(fileName)
+      assert.ok(fileInfo.isFile(), 'The file was generated')
+      assert.ok(fileInfo.size > 1024, 'The file contains something')
+      return false
+    } else {
+      assert.strictEqual(current, 'first')
+      return true
+    }
   }
 }

package/src/capabilities/tests/traces/index.js

@@ -17,7 +17,7 @@ const expectedLogs = [{
   type: 'log',
   text: /^"?complex parameters"? 1 true / // Not sure how objects are handled
 }, {
-  type: 'warning',
+  type: /warning|warn/,
   text: 'A warning'
 }, {
   type: 'error',

package/src/coverage.js

@@ -2,12 +2,14 @@
 
 const { join, dirname, isAbsolute } = require('path')
 const { fork } = require('child_process')
-const { cleanDir, createDir, filename, download } = require('./tools')
+const { cleanDir, createDir, filename, download, allocPromise } = require('./tools')
 const { readdir, readFile, stat, writeFile, access, constants } = require('fs').promises
 const { Readable } = require('stream')
 const { getOutput } = require('./output')
 const { resolvePackage } = require('./npm')
 const { promisify } = require('util')
+const { UTRError } = require('./error')
+const { $remoteOnLegacy } = require('./symbols')
 
 const $nycSettingsPath = Symbol('nycSettingsPath')
 const $coverageFileIndex = Symbol('coverageFileIndex')
@@ -28,9 +30,13 @@ async function nyc (job, ...args) {
   output.nyc(...args)
   const childProcess = fork(nycScript, args, { stdio: 'pipe' })
   output.monitor(childProcess)
-  let done
-  const promise = new Promise(resolve => { done = resolve })
-  childProcess.on('close', done)
+  const { promise, resolve, reject } = allocPromise()
+  childProcess.on('close', async code => {
+    if (code !== 0) {
+      reject(UTRError.NYC_FAILED(`Return code ${code}`))
+    }
+    resolve()
+  })
   return promise
 }
 
@@ -70,15 +76,9 @@ async function instrument (job) {
   settings.exclude.push(join(job.coverageReportDir, '**'))
   await writeFile(job[$nycSettingsPath], JSON.stringify(settings))
   if (job.mode === 'url') {
-    const port = job.port.toString()
-    const useLocal = job.url.some(url => {
-      // ignore host name since the machine might be exposed with any name
-      const parsedUrl = new URL(url)
-      return parsedUrl.port === port
-    })
-    if (!useLocal) {
-      getOutput(job).instrumentationSkipped()
+    if (!job[$remoteOnLegacy]) {
       job[$coverageRemote] = true
+      getOutput(job).instrumentationSkipped()
       return
     }
   }
@@ -125,6 +125,9 @@ async function generateCoverageReport (job) {
   }
   const checks = []
   if (job.coverageCheckBranches || job.coverageCheckFunctions || job.coverageCheckLines || job.coverageCheckStatements) {
+    if (!job.coverageReporters.includes('lcov')) {
+      reporters.push('--reporter=lcov')
+    }
     checks.push(
       `--branches=${job.coverageCheckBranches}`,
       `--functions=${job.coverageCheckFunctions}`,
@@ -134,6 +137,13 @@ async function generateCoverageReport (job) {
     )
   }
   await nyc(job, 'report', ...reporters, ...checks, '--temp-dir', coverageMergedDir, '--report-dir', job.coverageReportDir, '--nycrc-path', job[$nycSettingsPath])
+  if (checks.length) {
+    // The checks are not triggered if the coverage is empty
+    const lcov = await stat(join(job.coverageReportDir, 'lcov.info'))
+    if (lcov.size === 0) {
+      throw UTRError.NYC_FAILED('No coverage information extracted')
+    }
+  }
 }
 
 module.exports = {
@@ -146,18 +156,18 @@ module.exports = {
     }
     await writeFile(coverageFileName, JSON.stringify(coverageData))
   },
-  generateCoverageReport: job => job.coverage && generateCoverageReport(job),
+  generateCoverageReport: job => job.coverage ? generateCoverageReport(job) : Promise.resolve(),
   mappings: async job => {
     if (!job.coverage) {
       return []
     }
     const instrumentedBasePath = join(job.coverageTempDir, 'instrumented')
     const instrumentedMapping = {
-      match: /^\/(.*\.js)$/,
+      match: /(.*\.js)(\?.*)?$/,
       file: join(instrumentedBasePath, '$1'),
       'ignore-if-not-found': true
     }
-    if (job.mode === 'legacy') {
+    if (job.mode === 'legacy' || job[$remoteOnLegacy]) {
       return [{
         ...instrumentedMapping,
         'custom-file-system': job.debugCoverageNoCustomFs ? undefined : customFileSystem

package/src/coverage.spec.js

@@ -4,6 +4,7 @@ const { instrument, generateCoverageReport, mappings } = require('./coverage')
 const { stat } = require('fs/promises')
 const { cleanDir, createDir } = require('./tools')
 const { getOutput } = require('./output')
+const { $remoteOnLegacy } = require('./symbols')
 
 describe('src/coverage', () => {
   const cwd = join(__dirname, '../test/project')
@@ -108,7 +109,8 @@ describe('src/coverage', () => {
         Object.assign(job, {
           mode: 'url',
           port: 8080,
-          url: ['http://localhost:8080/whatever/test.html']
+          url: ['http://localhost:8080/whatever/test.html'],
+          [$remoteOnLegacy]: true // added on job finalization
         })
         await instrument(job)
         expect(instrumentationSkipped).not.toHaveBeenCalled()

package/src/defaults/playwright.js

@@ -88,7 +88,9 @@ require('./browser')({
     networkWriter
   }) {
     const browsers = require(modules.playwright)
-    browser = await browsers[options.browser].launch()
+    browser = await browsers[options.browser].launch({
+      headless: !options.visible
+    })
 
     let recordVideo
     if (options.video) {

package/src/defaults/puppeteer.js

@@ -11,27 +11,13 @@ require('./browser')({
       ['-w, --viewport-width <width>', 'Viewport width', 1920],
       ['-h, --viewport-height <height>', 'Viewport height', 1080],
       ['-l, --language <lang...>', 'Language(s)', ['en-US']],
-      ['-u, --unsecure', 'Disable security features', false]
-    ] // ,
-    // TODO restore when Node16 is no more supported
-    // capabilities: {
-    //   modules: ['puppeteer'],
-    //   screenshot: '.png',
-    //   scripts: true,
-    //   traces: ['console', 'network']
-    // }
-  },
-
-  // TODO remove when Node16 is no more supported
-  async capabilities () {
-    const version = process.version.match(/^v(\d+\.\d+)/)[1]
-    let screenshot
-    if (!version.startsWith('16')) {
-      screenshot = '.png'
-    }
-    return {
+      ['-u, --unsecure', 'Disable security features', false],
+      ['--basic-auth-username <username>', 'Username for basic authentication', ''],
+      ['--basic-auth-password <password>', 'Password for basic authentication', '']
+    ],
+    capabilities: {
       modules: ['puppeteer'],
-      screenshot,
+      screenshot: '.png',
       scripts: true,
       traces: ['console', 'network']
     }
@@ -94,6 +80,10 @@ require('./browser')({
       await page.setBypassCSP(true)
     }
 
+    if (options.basicAuthUsername || options.basicAuthPassword) {
+      await page.authenticate({ username: options.basicAuthUsername, password: options.basicAuthPassword })
+    }
+
     page
       .on('console', message => consoleWriter.append({
         type: message.type(),

package/src/error.js

@@ -37,6 +37,8 @@ const errors = [{
   name: 'BROWSER_MISS_SCRIPTS_CAPABILITY'
 }, {
   name: 'NPM_DEPENDENCY_NOT_FOUND'
+}, {
+  name: 'NYC_FAILED'
 }]
 
 errors.forEach((error, index) => {

package/src/job.js

@@ -5,9 +5,9 @@ const { statSync, accessSync, constants } = require('fs')
 const { dirname, join, isAbsolute } = require('path')
 const { name, description, version } = require(join(__dirname, '../package.json'))
 const { getOutput } = require('./output')
-const { $valueSources } = require('./symbols')
+const { $valueSources, $remoteOnLegacy } = require('./symbols')
 const { buildAndCheckMode } = require('./job-mode')
-const { boolean, integer, timeout, url, arrayOf, regex, percent } = require('./options')
+const { boolean, integer, timeout, url, arrayOf, regex, percent, string } = require('./options')
 
 const $status = Symbol('status')
 
@@ -107,6 +107,7 @@ function getCommand (cwd) {
     .option('-br, --browser-retry <count>', '[💻🔗🧪] Browser instantiation retries : if the command fails unexpectedly, it is re-executed (0 means no retry)', 1)
 
     // Common to legacy and url
+    .option('-qs, --qunit-strict', '[💻🔗] Strict mode on qunit execution (fails if no modules declared)', boolean, false)
     .option('-pf, --page-filter <regexp>', '[💻🔗] Filter out pages not matching the regexp')
     .option('-pp, --page-params <params>', '[💻🔗] Add parameters to page URL')
     .option('-t, --global-timeout <timeout>', '[💻🔗] Limit the pages execution time, fail the page if it takes longer than the timeout (0 means no timeout)', timeout, 0)
@@ -114,7 +115,7 @@ function getCommand (cwd) {
     .option('--no-screenshot', '[💻🔗] Disable screenshots')
     .option('-st, --screenshot-timeout <timeout>', '[💻🔗] Maximum waiting time for browser screenshot', timeout, 5000)
     .option('-rg, --report-generator <path...>', '[💻🔗] Report generator paths (relative to cwd or use $/ for provided ones)', ['$/report.js'])
-    .option('-pp, --progress-page <path>', '[💻🔗] progress page path (relative to cwd or use $/ for provided ones)', '$/report/default.html')
+    .option('--progress-page <path>', '[💻🔗] progress page path (relative to cwd or use $/ for provided ones)', '$/report/default.html')
 
     .option('--coverage [flag]', '[💻🔗] Enable or disable code coverage', boolean)
     .option('--no-coverage', '[💻🔗] Disable code coverage')
@@ -135,7 +136,7 @@ function getCommand (cwd) {
     .option('--mappings <mapping...>', '[💻] Custom mapping (<match>=<file|url>(<config>))', arrayOf(mapping))
     .option('--cache <path>', '[💻] Cache UI5 resources locally in the given folder (empty to disable)')
     .option('--webapp <path>', '[💻] Base folder of the web application (relative to cwd)', 'webapp')
-    .option('--testsuite <path>', '[💻] Path of the testsuite file (relative to webapp)', 'test/testsuite.qunit.html')
+    .option('--testsuite <path>', '[💻] Path of the testsuite file (relative to webapp, URL parameters are supported)', 'test/testsuite.qunit.html')
     .option('-w, --watch [flag]', '[💻] Monitor the webapp folder and re-execute tests on change', boolean, false)
 
     // Specific to coverage in url mode (experimental)
@@ -151,6 +152,7 @@ function getCommand (cwd) {
     .addOption(new Option('--debug-capabilities-no-timeout', DEBUG_OPTION, boolean).hideHelp())
     .addOption(new Option('--debug-coverage', DEBUG_OPTION, boolean).hideHelp())
     .addOption(new Option('--debug-coverage-no-custom-fs', DEBUG_OPTION, boolean).hideHelp())
+    .addOption(new Option('--debug-verbose <module...>', DEBUG_OPTION, arrayOf(string), []).hideHelp())
 
   return command
 }
@@ -226,7 +228,9 @@ function finalize (job) {
   job.mode = buildAndCheckMode(job)
   if (job.mode === 'legacy') {
     checkAccess({ path: job.webapp, label: 'webapp folder' })
-    const testsuitePath = toAbsolute(job.testsuite, job.webapp)
+
+    const [, testsuiteFile] = job.testsuite.match(/([^?]*)(\?.*)?$/)
+    const testsuitePath = toAbsolute(testsuiteFile, job.webapp)
     checkAccess({ path: testsuitePath, label: 'testsuite', file: true })
   } else if (job.mode === 'url') {
     if (job[$valueSources].coverage !== 'cli') {
@@ -287,6 +291,15 @@ function finalize (job) {
     overrideIfNotSet('coverageReporters', settings.reporter)
   }
 
+  if (job.mode === 'url') {
+    const port = job.port.toString()
+    job[$remoteOnLegacy] = job.url.every(url => {
+      // ignore host name since the machine might be exposed with any name
+      const parsedUrl = new URL(url)
+      return parsedUrl.port === port
+    })
+  }
+
   job[$status] = 'Starting'
   Object.defineProperty(job, 'status', {
     get () {

package/src/job.spec.js

@@ -211,6 +211,12 @@ describe('job', () => {
       })).toThrow()
     })
 
+    it('supports parameters for testsuite (stripping ?)', () => {
+      expect(() => buildJob({
+        testsuite: 'test/testsuite.qunit.html?a=b'
+      })).not.toThrow()
+    })
+
     it('fails on a missing file (points to a folder)', () => {
       expect(() => buildJob({
         testsuite: 'lib'

package/src/options.js

@@ -41,6 +41,10 @@ module.exports = {
 
   integer,
 
+  string (value) {
+    return value
+  },
+
   timeout (value) {
     const int = integer(value)
     if (value.endsWith('ms')) {

package/src/output.js

@@ -225,6 +225,12 @@ function build (job) {
       log(job, p80()`Server running at ${pad.lt(url)}`)
     },
 
+    debug: wrap((module, ...args) => {
+      if (job.debugVerbose && job.debugVerbose.includes(module)) {
+        console.log(`🐞${module}`, ...args)
+      }
+    }),
+
     redirected: wrap(({ method, url, statusCode, timeSpent }) => {
       if (url.startsWith('/_/progress')) {
         return // avoids pollution
@@ -412,6 +418,10 @@ function build (job) {
       log(job, p80()`Skipping nyc instrumentation (--url)`)
     }),
 
+    qunitEarlyStart: wrap(url => {
+      log(job, p80()`QUnit start without tests in ${pad.lt(url)}`)
+    }),
+
     endpointError: wrap(({ api, url, data, error }) => {
       const p = p80()
       log(job, p`┌──────────${pad.x('─')}┐`)

package/src/qunit-hooks.js

@@ -13,27 +13,40 @@ function error (job, url, details = '') {
   throw UTRError.QUNIT_ERROR(details)
 }
 
+function invalidTestId (job, url, testId) {
+  error(job, url, `No QUnit unit test found with id ${testId}`)
+}
+
 function get (job, urlWithHash, testId) {
   const url = stripUrlHash(urlWithHash)
   const page = job.qunitPages && job.qunitPages[url]
   if (!page) {
     error(job, url, `No QUnit page found for ${urlWithHash}`)
   }
+  let testModule
   let test
   if (testId !== undefined) {
     page.modules.every(module => {
       test = module.tests.find(test => test.testId === testId)
-      return test === undefined
+      if (test === undefined) {
+        return true
+      } else {
+        testModule = module
+        return false
+      }
     })
-    if (!test) {
-      error(job, url, `No QUnit unit test found with id ${testId}`)
+    if (!test && job.qunitStrict) {
+      invalidTestId(job, url, testId)
     }
   }
-  return { url, page, test }
+  return { url, page, testModule, test }
 }
 
 async function done (job, urlWithHash, report) {
   const { url, page } = get(job, urlWithHash)
+  if (page.earlyStart && page.count === 0) {
+    return // wait
+  }
   if (job.browserCapabilities.screenshot) {
     try {
       await screenshot(job, url, 'done')
@@ -41,11 +54,11 @@ async function done (job, urlWithHash, report) {
       getOutput(job).genericError(error, url)
     }
   }
+  page.end = new Date()
   if (report.__coverage__) {
-    collect(job, url, report.__coverage__)
+    await collect(job, url, report.__coverage__)
     delete report.__coverage__
   }
-  page.end = new Date()
   page.report = report
   stop(job, url)
 }
@@ -55,8 +68,12 @@ module.exports = {
 
   async begin (job, urlWithHash, { isOpa, totalTests, modules }) {
     const url = stripUrlHash(urlWithHash)
-    if (!totalTests || !modules) {
-      error(job, url, 'Invalid begin hook details')
+    const earlyStart = !totalTests || !modules
+    if (earlyStart) {
+      getOutput(job).qunitEarlyStart(url)
+      if (job.qunitStrict) {
+        error(job, url, 'Invalid begin hook details')
+      }
     }
     if (!job.qunitPages) {
       job.qunitPages = {}
@@ -70,16 +87,31 @@ module.exports = {
       count: totalTests,
       modules
     }
+    if (earlyStart) {
+      qunitPage.earlyStart = true
+    }
     job.qunitPages[url] = qunitPage
   },
 
   async testStart (job, urlWithHash, { module, name, testId }) {
-    const { test } = get(job, urlWithHash, testId)
+    let { page, testModule, test } = get(job, urlWithHash, testId)
+    if (!testModule) {
+      testModule = { name: module, tests: [] }
+      page.modules.push(testModule)
+    }
+    if (!test) {
+      test = { name, testId }
+      testModule.tests.push(test)
+      ++page.count
+    }
     test.start = new Date()
   },
 
   async log (job, urlWithHash, { module, name, testId, ...log }) {
     const { url, page, test } = get(job, urlWithHash, testId)
+    if (!test) {
+      invalidTestId(job, url, testId)
+    }
     if (!test.logs) {
       test.logs = []
     }
@@ -97,6 +129,9 @@ module.exports = {
   async testDone (job, urlWithHash, { name, module, testId, assertions, ...report }) {
     const { failed } = report
     const { url, page, test } = get(job, urlWithHash, testId)
+    if (!test) {
+      invalidTestId(job, url, testId)
+    }
     if (failed) {
       if (job.browserCapabilities.screenshot) {
         try {

package/src/qunit-hooks.spec.js

@@ -10,10 +10,12 @@ jest.mock('./coverage.js', () => ({
 const { collect } = require('./coverage')
 
 const mockGenericError = jest.fn()
+const mockQunitEarlyStart = jest.fn()
 
 jest.mock('./output.js', () => ({
   getOutput: () => ({
-    genericError: mockGenericError
+    genericError: mockGenericError,
+    qunitEarlyStart: mockQunitEarlyStart
   })
 }))
 
@@ -174,7 +176,11 @@ describe('src/qunit-hooks', () => {
       })
     })
 
-    describe('validation', () => {
+    describe('validation (--qunit-strict)', () => {
+      beforeEach(() => {
+        job.qunitStrict = true
+      })
+
       afterEach(() => {
         expect(stop).toHaveBeenCalledWith(job, url)
         expect(job.failed).toStrictEqual(true)
@@ -185,6 +191,7 @@ describe('src/qunit-hooks', () => {
           isOpa: false,
           modules: getModules()
         })).rejects.toThrow(UTRError.QUNIT_ERROR('Invalid begin hook details'))
+        expect(mockQunitEarlyStart).toHaveBeenCalled()
       })
 
       it('requires modules', async () => {
@@ -192,6 +199,7 @@ describe('src/qunit-hooks', () => {
           isOpa: false,
           totalTests: 1
         })).rejects.toThrow(UTRError.QUNIT_ERROR('Invalid begin hook details'))
+        expect(mockQunitEarlyStart).toHaveBeenCalled()
       })
     })
   })
@@ -594,7 +602,18 @@ describe('src/qunit-hooks', () => {
       expect(job.failed).toStrictEqual(true)
     })
 
-    it('fails on invalid test id', async () => {
+    it('fails on invalid test id (--qunit-strict)', async () => {
+      job.qunitStrict = true
+      await expect(testDone(job, url, {
+        ...getTestDoneFor1a(),
+        testId: '1c'
+      }))
+        .rejects.toThrow(UTRError.QUNIT_ERROR('No QUnit unit test found with id 1c'))
+      expect(stop).toHaveBeenCalledWith(job, url)
+      expect(job.failed).toStrictEqual(true)
+    })
+
+    it('fails on invalid test id (--no-qunit-strict)', async () => {
       await expect(testDone(job, url, {
         ...getTestDoneFor1a(),
         testId: '1c'

package/src/report.js

@@ -51,7 +51,10 @@ module.exports = {
       })
       return promise
     })
-    promises.push(generateCoverageReport(job))
+    promises.push(generateCoverageReport(job).catch(e => {
+      output.genericError(e)
+      job.failed = true
+    }))
     await Promise.all(promises)
     job.status = 'Done'
   }

package/src/simulate.spec.js

@@ -213,6 +213,31 @@ describe('simulate', () => {
       })
     })
 
+    describe('simple test execution (--no-qunit-strict)', () => {
+      beforeAll(async () => {
+        await setup('simple-early')
+        pages = {
+          'testsuite.qunit.html': async referer => {
+            await post('/_/addTestPages', referer, [
+              referer.replace('testsuite.qunit.html', 'page1.html')
+            ])
+          },
+          'page1.html': async referer => {
+            await post('/_/QUnit/begin', referer, { totalTests: 0, modules: [] })
+            await post('/_/QUnit/done', referer, { failed: 0 })
+            await post('/_/QUnit/testStart', referer, { module: 'module', name: 'test', testId: '1' })
+            await post('/_/QUnit/testDone', referer, { testId: '1', failed: 0, passed: 1 })
+            await post('/_/QUnit/done', referer, { failed: 0 })
+          }
+        }
+        await safeExecute()
+      })
+
+      it('succeeded', () => {
+        expect(job.failed).toStrictEqual(false)
+      })
+    })
+
     describe('error', () => {
       describe('one test fail', () => {
         beforeAll(async () => {

package/src/symbols.js

@@ -4,5 +4,6 @@ module.exports = {
   $probeUrlsCompleted: Symbol('probeUrlsCompleted'),
   $testPagesStarted: Symbol('testPagesStarted'),
   $testPagesCompleted: Symbol('testPagesCompleted'),
-  $valueSources: Symbol('valueSources')
+  $valueSources: Symbol('valueSources'),
+  $remoteOnLegacy: Symbol('remoteOnLegacy')
 }