ui5-test-runner 5.4.2 → 5.5.0

package/index.js

@@ -12,7 +12,10 @@ const { preload } = require('./src/ui5')
 const { probe: probeBrowser } = require('./src/browsers')
 const { recreateDir, allocPromise } = require('./src/tools')
 const reserveConfigurationFactory = require('./src/reserve')
-const start = require('./src/start')
+const { start } = require('./src/start')
+const { executeIf } = require('./src/if')
+const { batch } = require('./src/batch')
+const { end } = require('./src/end')
 
 function send (message) {
   if (process.send) {
@@ -48,9 +51,33 @@ async function main () {
   output = getOutput(job)
   await recreateDir(job.reportDir)
   output.version()
+  if (job.batchMode) {
+    output.batchMode()
+  }
+  output.reportOnJobProgress()
   if (job.mode === 'capabilities') {
     return capabilities(job)
   }
+  if (job.if && !executeIf(job)) {
+    output.skipIf()
+    output.stop()
+    return
+  }
+
+  let startedCommand
+  if (job.startCommand) {
+    startedCommand = await start(job)
+  }
+
+  if (job.mode === 'batch') {
+    return await batch(job)
+      .finally(() => {
+        if (startedCommand) {
+          return startedCommand.stop()
+        }
+      })
+  }
+
   const configuration = await reserveConfigurationFactory(job)
   output.debug('reserve', 'configuration', configuration)
   const server = serve(configuration)
@@ -64,7 +91,6 @@ async function main () {
       job.port = port
       send({ msg: 'ready', port: job.port })
       output.serving(url)
-      output.reportOnJobProgress()
       serverReady()
     })
     .on('error', error => {
@@ -73,11 +99,6 @@ async function main () {
       serverError()
     })
   await serverStarted
-  let startedCommand
-  if (job.start) {
-    output.reportOnJobProgress()
-    startedCommand = await start(job)
-  }
   if (job.preload) {
     await preload(job)
   }
@@ -90,8 +111,8 @@ async function main () {
   if (job.watch) {
     delete job.start
     if (!job.watching) {
-      output.watching(job.webapp)
-      watch(job.webapp, { recursive: true }, async (eventType, filename) => {
+      output.watching(job.watchFolder)
+      watch(job.watchFolder, { recursive: true }, async (eventType, filename) => {
         output.changeDetected(eventType, filename)
         if (!job.start) {
           await recreateDir(job.reportDir)
@@ -106,12 +127,14 @@ async function main () {
   } else if (job.failed) {
     process.exitCode = -1
   }
+  if (job.endScript) {
+    await end(job)
+  }
   output.stop()
   await server.close()
   if (startedCommand) {
     await startedCommand.stop()
   }
-  console.log('done ?')
 }
 
 main()

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ui5-test-runner",
-  "version": "5.4.2",
+  "version": "5.5.0",
   "description": "Standalone test runner for UI5",
   "main": "index.js",
   "bin": {
@@ -17,7 +17,7 @@
     "test:coverall": "rimraf .nyc_output && jest --coverageDirectory .nyc_output --coverageReporters json && nyc --silent --no-clean npm run test:e2e && nyc merge .nyc_output .nyc_output/final/coverage.json && nyc report --temp-dir .nyc_output/final/ --report-dir coverage --branches 80 --functions 80 --lines 80 --statements 80",
     "test:unit": "jest",
     "test:unit:debug": "node --inspect node_modules/jest/bin/jest.js --runInBand --no-coverage",
-    "test:e2e": "node test/e2e",
+    "test:e2e": "node . --batch test/e2e/.*\\.json --report-dir e2e --start \"node test/e2e/serve.js\" --start-wait-url http://localhost:8081 --start-wait-method HEAD --start-timeout 30s",
     "test:report": "node ./src/defaults/report.js ./test/report && reserve --config ./test/report/reserve.json",
     "test:text-report": "node ./src/defaults/text-report.js ./test/report",
     "build:doc": "node build/doc",
@@ -47,20 +47,20 @@
     "ps-tree": "^1.2.0",
     "punybind": "^1.2.1",
     "punyexpr": "^1.0.4",
-    "reserve": "2.0.5"
+    "reserve": "2.1.0"
   },
   "devDependencies": {
-    "@openui5/types": "^1.132.1",
+    "@openui5/types": "^1.133.0",
     "@ui5/cli": "^4.0.13",
     "@ui5/middleware-code-coverage": "^2.0.1",
     "dotenv": "^16.4.7",
     "jest": "^29.7.0",
-    "nock": "^13.5.6",
+    "nock": "^14.0.1",
     "nyc": "^17.1.0",
     "rimraf": "^6.0.1",
     "standard": "^17.1.2",
-    "typescript": "^5.7.3",
-    "ui5-tooling-transpile": "^3.5.3"
+    "typescript": "^5.8.2",
+    "ui5-tooling-transpile": "^3.7.1"
   },
   "optionalDependencies": {
     "fsevents": "^2.3.3"
@@ -77,7 +77,8 @@
     ],
     "globals": [
       "sap",
-      "opaTest"
+      "opaTest",
+      "normalizePath"
     ]
   }
 }

package/src/batch.js

@@ -0,0 +1,196 @@
+const { open, readdir, stat, unlink } = require('fs/promises')
+const { extname, isAbsolute, join } = require('path')
+const { allocPromise, filename } = require('./tools')
+const { fork } = require('child_process')
+const { interactive, getOutput, newProgress } = require('./output')
+const { parallelize } = require('./parallelize')
+const { $statusProgressCount } = require('./symbols')
+const { $valueSources } = require('./symbols')
+const { getCommand, toLongName } = require('./job')
+
+const batchParameters = getCommand('.').options
+  .filter(option => option.description.includes('📡'))
+  .reduce((dictionary, option) => {
+    dictionary[option.long.substring(2)] = option
+    return dictionary
+  }, {})
+
+const root = join(__dirname, '..')
+
+const folder = (batchItems, job, folderPath) => {
+  getOutput(job).debug('batch', `adding folder: ${folderPath}`)
+  batchItems.push({
+    job,
+    path: folderPath,
+    id: filename(folderPath),
+    label: folderPath,
+    args: ['--cwd', folderPath]
+  })
+}
+
+const configurationFile = (batchItems, job, configurationFilePath) => {
+  getOutput(job).debug('batch', `adding configuration file: ${configurationFilePath}`)
+  try {
+    const {
+      batchId: id = filename(configurationFilePath),
+      batchLabel: label = configurationFilePath
+    } = require(configurationFilePath)
+    batchItems.push({
+      job,
+      path: configurationFilePath,
+      id,
+      label,
+      args: ['--config', configurationFilePath]
+    })
+  } catch (e) {
+    getOutput(job).batchFailed(configurationFilePath, 'invalid JSON configuration file')
+  }
+}
+
+const task = async ({ job, id, label, args }) => {
+  const output = getOutput(job)
+  const progress = newProgress(job)
+  const reportDir = join(job.reportDir, id)
+  progress.label = `${label} (${id})`
+  progress.count = 1
+  if (!interactive) {
+    output.log(`${label}...`)
+  }
+  const { promise, resolve, reject } = allocPromise()
+  const parameters = [
+    ...args,
+    '--batch-mode'
+  ]
+  if (job[$valueSources]) {
+    if (job[$valueSources].reportDir === 'cli') {
+      parameters.push('--report-dir', reportDir)
+    }
+    Object.keys(job[$valueSources])
+      .filter(name => job[$valueSources][name] === 'cli')
+      .forEach(name => {
+        const longName = toLongName(name)
+        const option = batchParameters[longName]
+        if (option) {
+          parameters.push(`--${longName}`)
+          if (!option.optional && !option.negate) {
+            if (option.variadic) {
+              parameters.push(...job[name].map(value => value.toString()))
+            } else {
+              parameters.push(job[name].toString())
+            }
+          }
+        }
+      })
+  }
+
+  const stdoutFilename = join(job.reportDir, `${id}.stdout.txt`)
+  const stdout = await open(stdoutFilename, 'w')
+  const stderrFilename = join(job.reportDir, `${id}.stderr.txt`)
+  const stderr = await open(stderrFilename, 'w')
+  const childProcess = fork(
+    join(root, 'index.js'),
+    parameters,
+    {
+      stdio: [0, stdout, stderr, 'ipc']
+    }
+  )
+  childProcess.on('message', data => {
+    if (data.type === 'progress') {
+      const { count, total } = data
+      progress.count = count
+      progress.total = total
+    }
+  })
+  childProcess.on('close', async code => {
+    await stdout.close()
+    await stderr.close()
+    if (code !== 0) {
+      reject(code)
+    } else {
+      await unlink(stdoutFilename)
+      await unlink(stderrFilename)
+      resolve()
+    }
+  })
+  return promise
+    .then(() => {
+      output.log('✔️ ', progress.label)
+    }, (reason) => {
+      ++job.errors
+      output.log('❌', progress.label, reason)
+    })
+    .finally(() => {
+      ++job[$statusProgressCount]
+      progress.done()
+    })
+}
+
+async function batch (job) {
+  /**
+   * job
+   * path: full path
+   * id: batchId | hash (using filename)
+   * label: batchLabel | configuration file path | path
+   * args: []
+   *   --report-dir is always passed to aggregate reports under one root folder
+   */
+  const output = getOutput(job)
+  const batchItems = []
+  for (const batch of job.batch) {
+    output.debug('batch', `processing: ${batch}`)
+    // check if path
+    try {
+      let path = batch
+      if (!isAbsolute(path)) {
+        path = join(job.cwd, path)
+      }
+      const pathStat = await stat(path)
+      if (pathStat.isDirectory()) {
+        folder(batchItems, job, path)
+      } else if (pathStat.isFile() && extname(path) === '.json') {
+        configurationFile(batchItems, job, path)
+      } else {
+        output.batchFailed(batch, 'only folders and JSON configuration files are supported')
+      }
+      continue
+    } catch (e) {
+      // ignore
+    }
+    // Try using regular expression match
+    let re
+    try {
+      re = new RegExp(batch)
+    } catch (e) {
+      getOutput(job).batchFailed(batch, 'invalid regular expression')
+      continue
+    }
+    const scan = async (cwd) => {
+      const names = await readdir(cwd)
+      for (const name of names) {
+        const path = join(cwd, name)
+        const pathStat = await stat(path)
+        if (pathStat.isDirectory()) {
+          if (re.test(path) || re.test(path.replaceAll('\\', '/'))) {
+            folder(batchItems, job, path)
+            continue
+          }
+          await scan(path)
+        } else if (pathStat.isFile() && (re.test(path) || re.test(path.replaceAll('\\', '/')))) {
+          configurationFile(batchItems, job, path)
+        }
+      }
+    }
+    await scan(job.cwd)
+  }
+  if (batchItems.length) {
+    job.status = 'Running batch items...'
+    await parallelize(task, batchItems, job.parallel)
+    // TODO: end command ?
+  } else {
+    output.batchFailed(job.batch, 'no match')
+  }
+  output.stop()
+  return 0
+}
+
+module.exports = { batch, task }

package/src/capabilities/index.js

@@ -8,7 +8,7 @@ const { performance } = require('perf_hooks')
 const { cleanDir, allocPromise, filename } = require('../tools')
 const { $statusProgressTotal, $statusProgressCount } = require('../symbols')
 const tests = require('./tests')
-const parallelize = require('../parallelize')
+const { parallelize } = require('../parallelize')
 
 async function capabilities (job) {
   const output = getOutput(job)
@@ -30,16 +30,14 @@ async function capabilities (job) {
   }
 
   try {
-    output.reportOnJobProgress()
-    try {
-      await probe(job)
-    } catch (e) {
-      output.error('Unable to probe')
-      exit(-1)
-    }
+    await probe(job)
+  } catch (e) {
+    output.error('Unable to probe')
+    exit(-1)
+  }
 
+  try {
     const listeners = []
-
     const configuration = await check({
       port: job.port,
       mappings: [

package/src/capabilities/tests/scripts/index.js

@@ -44,8 +44,8 @@ module.exports = [{
   }
 }, {
   label: 'Scripts (External QUnit)',
-  for: capabilities => !!capabilities.scripts,
-  url: 'https://ui5.sap.com/test-resources/sap/m/demokit/orderbrowser/webapp/test/unit/unitTests.qunit.html',
+  for: capabilities => !!capabilities.scripts && !capabilities.modules.includes('jsdom'),
+  url: 'https://ui5.sap.com/test-resources/sap/m/demokit/orderbrowser/webapp/test/Test.qunit.html?testsuite=test-resources/sap/ui/demo/orderbrowser/testsuite.qunit&test=unit/unitTests',
   scripts: ['post.js', 'qunit-hooks.js'],
   endpoint: qUnitEndpoints
 }, {

package/src/defaults/browser.js

@@ -2,7 +2,7 @@ const { readFile, writeFile } = require('fs/promises')
 const { join } = require('path')
 const { Command, InvalidArgumentError } = require('commander')
 const { buildCsvWriter } = require('../csv-writer')
-const { any, boolean, integer } = require('../options')
+const { any, arrayOf, boolean, integer, string } = require('../options')
 
 const noop = () => { }
 
@@ -51,7 +51,7 @@ module.exports = ({
         ]
       }
       if (option === 'language') {
-        return [['-l, --language <lang...>', 'Language(s)', ['en-US']]]
+        return [['-l, --language <lang...>', 'Language(s) (see rfc5646)', arrayOf(string, true), ['en-US']]]
       }
       if (option === 'unsecure') {
         return [['-u, --unsecure', 'Disable security features', false]]
@@ -152,6 +152,7 @@ module.exports = ({
 
     options.chromeArgs = function () {
       const args = [
+        'true', // Not sure why but this changes the behavior of the browser
         '--start-maximized',
         '--no-sandbox',
         '--disable-gpu',

package/src/defaults/junit-xml-report.js

@@ -74,7 +74,7 @@ async function main () {
     }
   }
   o('</testsuites>')
-  await writeFile(join(reportDir, 'junit.xml'), output.join('\n'))
+  await writeFile(join(reportDir, process.env.JUNIT_XML_REPORT_FILENAME || 'junit.xml'), output.join('\n'))
 }
 
 main()

package/src/end.js

@@ -0,0 +1,58 @@
+const { fork } = require('node:child_process')
+const { join } = require('node:path')
+const { getOutput } = require('./output')
+const { allocPromise } = require('./tools')
+
+async function end (job) {
+  const { endScript: end } = job
+  const output = getOutput(job)
+  const [script, ...args] = end.split(' ')
+
+  job.status = 'Executing end script'
+
+  output.debug('end', 'Starting script :', end)
+  const childProcess = fork(
+    script,
+    [...args, join(job.reportDir, 'job.js')],
+    {
+      cwd: job.cwd,
+      stdio: [0, 'pipe', 'pipe', 'ipc'],
+      windowsHide: true
+    }
+  )
+
+  const { promise: childProcessExit, resolve: childProcessExited } = allocPromise()
+  childProcess.on('close', childProcessExited)
+  output.monitor(childProcess)
+
+  job.status = 'Waiting for script to end'
+
+  if (job.endTimeout) {
+    let timedOut = false
+    const { promise: endTimeoutSignal, resolve: endTimeoutReached } = allocPromise()
+    const timeoutId = setTimeout(() => {
+      timedOut = true
+      endTimeoutReached()
+    }, job.endTimeout)
+
+    await Promise.race([
+      childProcessExit,
+      endTimeoutSignal
+    ])
+    clearTimeout(timeoutId)
+
+    if (timedOut) {
+      childProcess.kill()
+      throw new Error('Timeout while waiting for end script')
+    }
+  } else {
+    await childProcessExit
+  }
+
+  output.debug('end', 'Ended with exit code :', childProcess.exitCode)
+
+  // IMPORTANT : the end command CHANGES the exit code
+  process.exitCode = childProcess.exitCode
+}
+
+module.exports = { end }

package/src/if.js

@@ -0,0 +1,10 @@
+const { punyexpr } = require('punyexpr')
+
+function executeIf (job) {
+  return punyexpr(job.if)({
+    ...process.env,
+    NODE_MAJOR_VERSION: parseInt(parseInt(process.version.match(/v(\d+)\./)[1]))
+  })
+}
+
+module.exports = { executeIf }

package/src/job-mode.js

@@ -17,6 +17,9 @@ function check (job, allowed, forbidden) {
 }
 
 function buildAndCheckMode (job) {
+  if (job.batch) {
+    return 'batch'
+  }
   if (job.capabilities) {
     check(job, [
       'capabilities',
@@ -33,8 +36,12 @@ function buildAndCheckMode (job) {
       'keepAlive',
       'alternateNpmPath',
       'outputInterval',
-      'screenshotTimeout'
-    ], ['start'])
+      'screenshotTimeout',
+      'config',
+      'batchMode',
+      'batchId',
+      'batchLabel'
+    ])
     return 'capabilities'
   }
   if (job.url && job.url.length) {
@@ -43,7 +50,6 @@ function buildAndCheckMode (job) {
       'libs',
       'mappings',
       'cache',
-      'watch',
       'testsuite'
     ])
     return 'url'
@@ -51,8 +57,7 @@ function buildAndCheckMode (job) {
   check(job, undefined, [
     'coverageProxy',
     'coverageProxyInclude',
-    'coverageProxyExclude',
-    'start'
+    'coverageProxyExclude'
   ])
   return 'legacy'
 }

package/src/job.js

@@ -93,68 +93,83 @@ function getCommand (cwd) {
       new Option('-c, --cwd <path>', '[💻🔗🧪] Set working directory')
         .default(cwd, 'current working directory')
     )
+    .option('--config <json>', '[💻🔗🧪] Configuration file (relative to cwd)', string, 'ui5-test-runner.json')
     .option('--port <port>', '[💻🔗🧪] Port to use (0 to use any free one)', integer, 0)
     .option('-r, --report-dir <path>', '[💻🔗🧪] Directory to output test reports (relative to cwd)', 'report')
-    .option('-pt, --page-timeout <timeout>', '[💻🔗🧪] Limit the page execution time, fails the page if it takes longer than the timeout (0 means no timeout)', timeout, 0)
-    .option('-f, --fail-fast [flag]', '[💻🔗🧪] Stop the execution after the first failing page', boolean, false)
-    .option('-fo, --fail-opa-fast [flag]', '[💻🔗] Stop the OPA page execution after the first failing test', boolean, false)
+    .option('-pt, --page-timeout <timeout>', '[💻🔗🧪📡] Limit the page execution time, fails the page if it takes longer than the timeout (0 means no timeout)', timeout, 0)
+    .option('-f, --fail-fast [flag]', '[💻🔗🧪📡] Stop the execution after the first failing page', boolean, false)
+    .option('-fo, --fail-opa-fast [flag]', '[💻🔗📡] Stop the OPA page execution after the first failing test', boolean, false)
     .option('-k, --keep-alive [flag]', '[💻🔗🧪] Keep the server alive', boolean, false)
-    .option('-l, --log-server [flag]', '[💻🔗🧪] Log inner server traces', boolean, false)
+    .option('-l, --log-server [flag]', '[💻🔗🧪📡] Log inner server traces', boolean, false)
     .option('-p, --parallel <count>', '[💻🔗🧪] Number of parallel tests executions', integer, 2)
-    .option('-b, --browser <command>', '[💻🔗🧪] Browser instantiation command (relative to cwd or use $/ for provided ones)', '$/puppeteer.js')
-    .option('--browser-args <argument...>', '[💻🔗🧪] Browser instantiation command parameters (use -- instead)')
-    .option('--alternate-npm-path <path>', '[💻🔗] Alternate NPM path to look for packages (priority: local, alternate, global)')
-    .option('--no-npm-install', '[💻🔗🧪] Prevent any NPM install (execution may fail if a dependency is missing)')
-    .option('-bt, --browser-close-timeout <timeout>', '[💻🔗🧪] Maximum waiting time for browser close', timeout, 2000)
-    .option('-br, --browser-retry <count>', '[💻🔗🧪] Browser instantiation retries : if the command fails unexpectedly, it is re-executed (0 means no retry)', 1)
-    .option('-oi, --output-interval <interval>', '[💻🔗🧪] Interval for reporting progress on non interactive output (CI/CD) (0 means no output)', timeout, 30000)
-    .option('--offline [flag]', '[💻🔗🧪] Limit network usage (implies --no-npm-install)', boolean, false)
+    .option('-b, --browser <command>', '[💻🔗🧪📡] Browser instantiation command (relative to cwd or use $/ for provided ones)', '$/puppeteer.js')
+    .option('--browser-args <argument...>', '[💻🔗🧪📡] Browser instantiation command parameters (use -- instead)')
+    .option('--alternate-npm-path <path>', '[💻🔗📡] Alternate NPM path to look for packages (priority: local, alternate, global)')
+    .option('--no-npm-install', '[💻🔗🧪📡] Prevent any NPM install (execution may fail if a dependency is missing)')
+    .option('-bt, --browser-close-timeout <timeout>', '[💻🔗🧪📡] Maximum waiting time for browser close', timeout, 2000)
+    .option('-br, --browser-retry <count>', '[💻🔗🧪📡] Browser instantiation retries : if the command fails unexpectedly, it is re-executed (0 means no retry)', 1)
+    .option('-oi, --output-interval <interval>', '[💻🔗🧪📡] Interval for reporting progress on non interactive output (CI/CD) (0 means no output)', timeout, 30000)
+    .option('--offline [flag]', '[💻🔗🧪📡] Limit network usage (implies --no-npm-install)', boolean, false)
 
     // Common to legacy and url
     .option('--webapp <path>', '[💻🔗] Base folder of the web application (relative to cwd)', 'webapp')
-    .option('-pf, --page-filter <regexp>', '[💻🔗] Filter out pages not matching the regexp')
-    .option('-pp, --page-params <params>', '[💻🔗] Add parameters to page URL')
-    .option('--page-close-timeout <timeout>', '[💻🔗] Maximum waiting time for page close', timeout, 250)
-    .option('-t, --global-timeout <timeout>', '[💻🔗] Limit the pages execution time, fail the page if it takes longer than the timeout (0 means no timeout)', timeout, 0)
-    .option('--screenshot [flag]', '[💻🔗] Take screenshots during the tests execution (if supported by the browser)', boolean, true)
-    .option('--no-screenshot', '[💻🔗] Disable screenshots')
-    .option('-st, --screenshot-timeout <timeout>', '[💻🔗] Maximum waiting time for browser screenshot', timeout, 5000)
-    .option('-so, --split-opa [flag]', '[💻🔗] Split OPA tests using QUnit modules', boolean, false)
-    .option('-rg, --report-generator <path...>', '[💻🔗] Report generator paths (relative to cwd or use $/ for provided ones)', ['$/report.js'])
-    .option('--progress-page <path>', '[💻🔗] Progress page path (relative to cwd or use $/ for provided ones)', '$/report/default.html')
-
-    .option('--coverage [flag]', '[💻🔗] Enable or disable code coverage', boolean)
-    .option('--no-coverage', '[💻🔗] Disable code coverage')
-    .option('-cs, --coverage-settings <path>', '[💻🔗] Path to a custom .nycrc.json file providing settings for instrumentation (relative to cwd or use $/ for provided ones)', '$/.nycrc.json')
+    .option('-pf, --page-filter <regexp>', '[💻🔗📡] Filter out pages not matching the regexp')
+    .option('-pp, --page-params <params>', '[💻🔗📡] Add parameters to page URL')
+    .option('--page-close-timeout <timeout>', '[💻🔗📡] Maximum waiting time for page close', timeout, 250)
+    .option('-t, --global-timeout <timeout>', '[💻🔗📡] Limit the pages execution time, fail the page if it takes longer than the timeout (0 means no timeout)', timeout, 0)
+    .option('--screenshot [flag]', '[💻🔗📡] Take screenshots during the tests execution (if supported by the browser)', boolean, true)
+    .option('--no-screenshot', '[💻🔗📡] Disable screenshots during the tests execution (but not on failure, see --screenshot-on-failure)')
+    .option('--screenshot-on-failure <flag>', '[💻🔗📡] Take a screenshot when a test fails (even if --screenshot is false)', boolean, true)
+    .option('-st, --screenshot-timeout <timeout>', '[💻🔗📡] Maximum waiting time for browser screenshot', timeout, 5000)
+    .option('-so, --split-opa [flag]', '[💻🔗📡] Split OPA tests using QUnit modules', boolean, false)
+    .option('-rg, --report-generator <path...>', '[💻🔗📡] Report generator paths (relative to cwd or use $/ for provided ones)', ['$/report.js'])
+    .option('--progress-page <path>', '[💻🔗📡] Progress page path (relative to cwd or use $/ for provided ones)', '$/report/default.html')
+
+    .option('--coverage [flag]', '[💻🔗📡] Enable or disable code coverage', boolean)
+    .option('--no-coverage', '[💻🔗📡] Disable code coverage')
+    .option('-cs, --coverage-settings <path>', '[💻🔗📡] Path to a custom .nycrc.json file providing settings for instrumentation (relative to cwd or use $/ for provided ones)', '$/.nycrc.json')
     .option('-ctd, --coverage-temp-dir <path>', '[💻🔗] Directory to output raw coverage information to (relative to cwd)', '.nyc_output')
     .option('-crd, --coverage-report-dir <path>', '[💻🔗] Directory to store the coverage report files (relative to cwd)', 'coverage')
-    .option('-cr, --coverage-reporters <reporter...>', '[💻🔗] List of nyc reporters to use (text is always used)', ['lcov', 'cobertura'])
-    .option('-ccb, --coverage-check-branches <percent>', '[💻🔗] What % of branches must be covered', percent, 0)
-    .option('-ccf, --coverage-check-functions <percent>', '[💻🔗] What % of functions must be covered', percent, 0)
-    .option('-ccl, --coverage-check-lines <percent>', '[💻🔗] What % of lines must be covered', percent, 0)
-    .option('-ccs, --coverage-check-statements <percent>', '[💻🔗] What % of statements must be covered', percent, 0)
-    .option('-crs, --coverage-remote-scanner <path>', '[💻🔗] Scan for files when all coverage is requested', '$/scan-ui5.js')
+    .option('-cr, --coverage-reporters <reporter...>', '[💻🔗📡] List of nyc reporters to use (text is always used)', ['lcov', 'cobertura'])
+    .option('-ccb, --coverage-check-branches <percent>', '[💻🔗📡] What % of branches must be covered', percent, 0)
+    .option('-ccf, --coverage-check-functions <percent>', '[💻🔗📡] What % of functions must be covered', percent, 0)
+    .option('-ccl, --coverage-check-lines <percent>', '[💻🔗📡] What % of lines must be covered', percent, 0)
+    .option('-ccs, --coverage-check-statements <percent>', '[💻🔗📡] What % of statements must be covered', percent, 0)
+    .option('-crs, --coverage-remote-scanner <path>', '[💻🔗📡] Scan for files when all coverage is requested', '$/scan-ui5.js')
     .option('-s, --serve-only [flag]', '[💻🔗] Serve only', boolean, false)
 
+    .option('-w, --watch [flag]', '[💻🔗] Monitor the webapp folder (or the one specified with --watch-folder) and re-execute tests on change', boolean, false)
+    .option('--watch-folder <path>', '[💻🔗] Folder to monitor with watch (enables --watch if not specified)', string)
+
+    .option('--start <command>', '[💻🔗] Start command (might be an NPM script or a shell command)', string)
+    .option('--start-wait-url <command>', '[💻🔗] URL to wait for (🔗 defaulted to first url)', url)
+    .option('--start-wait-method <method>', '[💻🔗] HTTP method to check the waited URL', 'GET')
+    .option('--start-timeout <timeout>', '[💻🔗] Maximum waiting time for the start command (based on when the first URL becomes available)', timeout, 5000)
+
+    .option('--end <script>', '[💻🔗] End script (will receive path to `job.js`)', string)
+    .option('--end-timeout <timeout>', '[💻🔗] Maximum waiting time for the end script', timeout, 5000)
+
     // Specific to legacy (and might be used with url if pointing to local project)
-    .option('--ui5 <url>', '[💻] UI5 url', url, 'https://ui5.sap.com')
-    .option('--disable-ui5 [flag]', '[💻] Disable UI5 mapping (also disable libs)', boolean, false)
-    .option('--libs <lib...>', '[💻] Library mapping (<relative>=<path> or <path>)', arrayOf(lib))
-    .option('--mappings <mapping...>', '[💻] Custom mapping (<match>=<file|url>(<config>))', arrayOf(mapping))
-    .option('--cache <path>', '[💻] Cache UI5 resources locally in the given folder (empty to disable)')
-    .option('--preload <library...>', '[💻] Preload UI5 libraries in the cache folder (only if --cache is used)', arrayOf(string))
+    .option('--ui5 <url>', '[💻📡] UI5 url', url, 'https://ui5.sap.com')
+    .option('--disable-ui5 [flag]', '[💻📡] Disable UI5 mapping (also disable libs)', boolean, false)
+    .option('--libs <lib...>', '[💻📡] Library mapping (<relative>=<path> or <path>)', arrayOf(lib))
+    .option('--mappings <mapping...>', '[💻📡] Custom mapping (<match>=<file|url>(<config>))', arrayOf(mapping))
+    .option('--cache <path>', '[💻📡] Cache UI5 resources locally in the given folder (empty to disable)')
+    .option('--preload <library...>', '[💻📡] Preload UI5 libraries in the cache folder (only if --cache is used)', arrayOf(string))
     .option('--testsuite <path>', '[💻] Path of the testsuite file (relative to webapp, URL parameters are supported)', 'test/testsuite.qunit.html')
-    .option('-w, --watch [flag]', '[💻] Monitor the webapp folder and re-execute tests on change', boolean, false)
-
-    // Specific to url
-    .option('--start <command>', '[🔗] Start command (might be an NPM script or a shell command)', string)
-    .option('--start-timeout <timeout>', '[🔗] Maximum waiting time for the start command (based on when the first URL becomes available)', timeout, 5000)
 
     // Specific to coverage in url mode (experimental)
     .option('-cp, --coverage-proxy [flag]', `[🔗] ${EXPERIMENTAL_OPTION} use internal proxy to instrument remote files`, boolean, false)
     .option('-cpi, --coverage-proxy-include <regexp>', `[🔗] ${EXPERIMENTAL_OPTION} urls to instrument for coverage`, regex, '.*')
     .option('-cpe, --coverage-proxy-exclude <regexp>', `[🔗] ${EXPERIMENTAL_OPTION} urls to ignore for coverage`, regex, '/((test-)?resources|tests?)/')
 
+    // Batch mode related
+    .addOption(new Option('--batch-mode', 'Changes the way options are defaulted (in particular coverage temporary folders)', boolean).hideHelp())
+    .option('--batch <specification...>', 'Batch specification', arrayOf(string))
+    .option('--batch-id <id>', 'Batch id (used for naming report folder)', string)
+    .option('--batch-label <label>', 'Batch label (used while reporting on execution)', string)
+    .option('--if <condition>', 'Condition runner execution', string)
+
     .addOption(new Option('--debug-dev-mode', DEBUG_OPTION, boolean).hideHelp())
     .addOption(new Option('--debug-probe-only', DEBUG_OPTION, boolean).hideHelp())
     .addOption(new Option('--debug-keep-browser-open', DEBUG_OPTION, boolean).hideHelp())
@@ -211,6 +226,7 @@ function checkAccess ({ path, label, file /*, write */ }) {
 
 function finalize (job) {
   function toAbsolute (path, from = job.cwd) {
+    path = path.replace(/📂report\b/, job.reportDir)
     if (!isAbsolute(path)) {
       path = join(from, path)
     }
@@ -230,6 +246,7 @@ function finalize (job) {
   function updateToAbsolute (member, from = job.cwd) {
     job[member] = toAbsolute(job[member], from)
   }
+
   'browser,coverageSettings,coverageRemoteScanner,progressPage'
     .split(',')
     .forEach(setting => { job[setting] = checkDefault(job[setting]) })
@@ -282,6 +299,16 @@ function finalize (job) {
     })
   }
 
+  if (job.watchFolder) {
+    job.watch = true
+    job.watchFolder = updateToAbsolute(job.watchFolder)
+  } else if (job.watch) {
+    job.watchFolder = job.webapp
+  }
+  if (job.watchFolder) {
+    checkAccess({ path: job.watchFolder, label: 'Folder to watch' })
+  }
+
   const output = getOutput(job)
 
   if (job.coverage) {
@@ -336,6 +363,25 @@ function finalize (job) {
     configurable: false
   })
 
+  // Because start and end are already used
+  job.startCommand = job.start
+  delete job.start
+  job.endScript = job.end
+  delete job.end
+
+  if (job.startCommand) {
+    if (!job.startWaitUrl) {
+      job.startWaitUrl = job.url[0]
+    }
+    if (!job.startWaitUrl) {
+      throw new Error('Start command defined but no URL to wait for')
+    }
+  }
+
+  if (job.batchMode) {
+    job.outputInterval = 1000
+  }
+
   /* istanbul ignore next */
   if (process.env.DEBUG_ON_FAILED) {
     let failed
@@ -359,19 +405,34 @@ function finalize (job) {
 function fromCmdLine (cwd, args) {
   let job = parse(cwd, args)
 
-  let defaultPath = join(job.cwd, 'ui5-test-runner.json')
-  if (!isAbsolute(defaultPath)) {
-    defaultPath = join(job.initialCwd, defaultPath)
+  let defaultPath
+  const isConfigSet = job[$valueSources].config === 'cli'
+  if (isAbsolute(job.config)) {
+    defaultPath = job.config
+  } else {
+    defaultPath = join(job.cwd, job.config)
+    if (!isAbsolute(defaultPath)) {
+      defaultPath = join(job.initialCwd, defaultPath)
+    }
   }
   let hasDefaultSettings = false
   try {
     checkAccess({ path: defaultPath, file: true })
     hasDefaultSettings = true
   } catch (e) {
+    if (isConfigSet) {
+      throw e
+    }
     // ignore
   }
   if (hasDefaultSettings) {
     const defaults = require(defaultPath)
+    if (defaults.cwd && !isAbsolute(defaults.cwd)) {
+      // make it relative to the configuration file
+      defaults.cwd = join(dirname(defaultPath), defaults.cwd)
+    } else if (isConfigSet) {
+      defaults.cwd = dirname(defaultPath)
+    }
     const { before, after, browser } = buildArgs(defaults)
     const sep = args.indexOf('--')
     if (sep === -1) {
@@ -397,5 +458,6 @@ function fromObject (cwd, parameters) {
 module.exports = {
   getCommand,
   fromCmdLine,
-  fromObject
+  fromObject,
+  toLongName
 }

package/src/options.js

@@ -70,10 +70,12 @@ module.exports = {
     return value
   },
 
-  arrayOf (typeValidator) {
+  arrayOf (typeValidator, overrideDefault) {
+    let count = 0
     return function (value, previousValue) {
+      ++count
       let result
-      if (previousValue === undefined) {
+      if (previousValue === undefined || (overrideDefault && count === 1)) {
         result = []
       } else {
         result = [...previousValue]

package/src/output.js

@@ -427,6 +427,18 @@ function build (job) {
       browserIssue(job, { type: 'failed', url, code, dir })
     }),
 
+    skipIf: wrap(() => {
+      log(job, p80()`⚠️ [SKIPIF] Skipping execution (--if)`)
+    }),
+
+    batchFailed: wrap((batch, reason) => {
+      log(job, p80()`⚠️ [BATCHF] Failed to resolve batch ${batch}: ${reason}`)
+    }),
+
+    batchMode: wrap((batch, reason) => {
+      log(job, p80()`⚠️ [BATCHM] Batch mode item execution`)
+    }),
+
     startFailed: wrap((url, error) => {
       const p = p80()
       log(job, p`┌──────────${pad.x('─')}┐`)
@@ -492,6 +504,10 @@ function build (job) {
       log(job, p80()`⚠️ [SKPNYC] Skipping nyc instrumentation (--url)`)
     }),
 
+    coverageNotFound: wrap(() => {
+      log(job, p80()`⚠️ [COVMIS] Coverage missing`)
+    }),
+
     assumingOneOrigin: wrap(() => {
       log(job, p80()`⚠️ [COVORG] Considering only one origin`)
     }),

package/src/parallelize.js

@@ -34,7 +34,7 @@ async function run (task) {
     task.stop = true
     reject(error)
   }
-  let remaining = list.length - index - 1
+  let remaining = list.length - task.started
   while (task.active < (parallel + 1) && remaining) {
     --remaining
     ++task.active
@@ -43,7 +43,7 @@ async function run (task) {
   complete(task)
 }
 
-module.exports = function parallelize (method, list, parallel) {
+function parallelize (method, list, parallel) {
   const { promise, resolve, reject } = allocPromise()
   const task = {
     method,
@@ -59,3 +59,5 @@ module.exports = function parallelize (method, list, parallel) {
   run(task)
   return promise
 }
+
+module.exports = { parallelize }

package/src/qunit-hooks.js

@@ -102,6 +102,8 @@ async function done (job, urlWithHash, report) {
     if (report.__coverage__) {
       await collect(job, url, report.__coverage__)
       delete report.__coverage__
+    } else if (job.coverage) {
+      getOutput(job).coverageNotFound()
     }
     page.report = report
     stop(job, url)
@@ -177,7 +179,7 @@ module.exports = {
     }
     ++progress.count
     if (failed) {
-      if (job.browserCapabilities.screenshot) {
+      if (job.browserCapabilities.screenshot && job.screenshotOnFailure) {
         try {
           const absoluteName = await screenshot(job, url, testId)
           test.screenshot = basename(absoluteName)

package/src/report.js

@@ -56,6 +56,6 @@ module.exports = {
       job.failed = true
     }))
     await Promise.all(promises)
-    job.status = 'Done'
+    job.status = 'Reports gemerated'
   }
 }

package/src/start.js

@@ -6,29 +6,34 @@ const psTreeNodeCb = require('ps-tree')
 const { promisify } = require('util')
 const psTree = promisify(psTreeNodeCb)
 
-const $startedProcess = Symbol('startedProcess')
-
-module.exports = async function start (job) {
-  let { start } = job
+async function start (job) {
+  const { startWaitUrl: url, startWaitMethod: method } = job
+  let { startCommand: start } = job
   const output = getOutput(job)
-  const [command] = start.split(' ')
+  const [command, ...parameters] = start.split(' ')
 
   job.status = 'Executing start command'
 
-  // check if existing NPM script
-  const packagePath = join(job.cwd, 'package.json')
-  try {
-    const packageStat = await stat(packagePath)
-    if (packageStat.isFile()) {
-      output.debug('start', 'Found package.json in cwd')
-      const packageFile = JSON.parse(await readFile(packagePath, 'utf-8'))
-      if (packageFile.scripts[command]) {
-        output.debug('start', 'Found matching start script in package.json')
-        start = `npm run ${start}`
+  // check if node
+  if (command === 'node') {
+    output.debug('start', `Replacing node with ${process.argv[0]}`)
+    start = [process.argv[0], ...parameters].join(' ')
+  } else {
+    // check if existing NPM script
+    const packagePath = join(job.cwd, 'package.json')
+    try {
+      const packageStat = await stat(packagePath)
+      if (packageStat.isFile()) {
+        output.debug('start', 'Found package.json in cwd')
+        const packageFile = JSON.parse(await readFile(packagePath, 'utf-8'))
+        if (packageFile.scripts[command]) {
+          output.debug('start', 'Found matching start script in package.json')
+          start = `npm run ${start}`
+        }
       }
+    } catch (e) {
+      output.debug('start', 'Missing or invalid package.json in cwd', e)
     }
-  } catch (e) {
-    output.debug('start', 'Missing or invalid package.json in cwd', e)
   }
 
   let childProcessExited = false
@@ -42,17 +47,14 @@ module.exports = async function start (job) {
     childProcessExited = true
   })
   output.monitor(childProcess)
-  job[$startedProcess] = childProcess
 
   job.status = 'Waiting for URL to be reachable'
 
-  const [url] = job.url
-
   const begin = Date.now()
   // eslint-disable-next-line no-unmodified-loop-condition
   while (!childProcessExited && Date.now() - begin <= job.startTimeout) {
     try {
-      const response = await fetch(url)
+      const response = await fetch(url, { method })
       output.debug('start', url, response.status)
       if (response.status === 200) {
         break
@@ -89,3 +91,5 @@ module.exports = async function start (job) {
     stop
   }
 }
+
+module.exports = { start }

package/src/tests.js

@@ -11,7 +11,7 @@ const {
   $proxifiedUrls
 } = require('./symbols')
 const { UTRError } = require('./error')
-const parallelize = require('./parallelize')
+const { parallelize } = require('./parallelize')
 
 function task (job, method) {
   return async (url, index, { length }) => {
@@ -38,6 +38,11 @@ function task (job, method) {
 }
 
 async function probeUrl (job, url) {
+  const parsedUrl = new URL(url)
+  if (parsedUrl.port === '0') {
+    parsedUrl.port = job.port
+    url = parsedUrl.toString()
+  }
   const output = getOutput(job)
   try {
     let scripts
@@ -122,6 +127,8 @@ async function process (job) {
   }
 
   await generate(job)
+
+  // TODO: #105 integrate end, find a way it can change job.failed
 }
 
 module.exports = {

package/src/ui5.js

@@ -7,7 +7,7 @@ const { capture } = require('reserve')
 const { getOutput, newProgress } = require('./output')
 const { download } = require('./tools')
 const { $statusProgressCount, $statusProgressTotal } = require('./symbols')
-const parallelize = require('./parallelize')
+const { parallelize } = require('./parallelize')
 
 const buildCacheBase = job => {
   const [, hostName] = /https?:\/\/([^/]*)/.exec(job.ui5)