ui5-test-runner 3.3.4 → 3.3.5

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ui5-test-runner",
-  "version": "3.3.4",
+  "version": "3.3.5",
   "description": "Standalone test runner for UI5",
   "main": "index.js",
   "files": [
@@ -16,7 +16,11 @@
   },
   "scripts": {
     "lint": "standard --fix",
-    "test": "npm run test:unit && npm run test:integration:jsdom && npm run test:integration:puppeteer && npm run test:integration:selenium-webdriver-chrome && npm run test:integration:playwright",
+    "test": "npm run test:unit && npm run test:browsers && npm run test:samples",
+    "test:browsers": "npm run test:integration:jsdom && npm run test:integration:puppeteer && npm run test:integration:selenium-webdriver-chrome && npm run test:integration:playwright",
+    "test:samples": "npm run test:samples:js && npm run test:samples:ts",
+    "test:samples:js": "npm run test:sample:js:legacy && npm run test:sample:js:coverage:legacy && npm run test:sample:js:legacy-remote && npm run test:sample:js:coverage:legacy-remote && npm run test:sample:js:remote && npm run test:sample:js:coverage:remote",
+    "test:samples:ts": "npm run test:sample:ts:remote && npm run test:sample:ts:coverage:remote",
     "test:coverall": "rimraf .nyc_output && jest --coverageDirectory .nyc_output --coverageReporters json && nyc --silent --no-clean npm run test:integration:jsdom && nyc --silent --no-clean npm run test:integration:puppeteer && nyc --silent --no-clean npm run test:integration:selenium-webdriver-chrome && nyc --silent --no-clean npm run test:integration:playwright && nyc merge .nyc_output .nyc_output/final/coverage.json && nyc report --temp-dir .nyc_output/final/ --report-dir coverage --branches 80 --functions 80 --lines 80 --statements 80",
     "test:unit": "jest",
     "test:unit:debug": "jest --runInBand",
@@ -26,6 +30,17 @@
     "test:integration:playwright": "node . --capabilities --browser $/playwright.js",
     "test:report": "node ./src/defaults/report.js ./test/report && reserve --config ./test/report/reserve.json",
     "test:text-report": "node ./src/defaults/text-report.js ./test/report",
+    "test:sample:js:legacy": "node . --cwd ./test/sample.js",
+    "test:sample:js:coverage:legacy": "node . --cwd ./test/sample.js --coverage --coverage-settings nyc.json --coverage-check-statements 67",
+    "test:sample:js:legacy-remote": "node . --port 8081 --cwd ./test/sample.js --url http://localhost:8081/test/testsuite.qunit.html",
+    "test:sample:js:coverage:legacy-remote": "node . --port 8081 --cwd ./test/sample.js --url http://localhost:8081/test/testsuite.qunit.html --coverage --coverage-settings nyc.json --coverage-check-statements 67",
+    "test:sample:js:remote": "start-server-and-test 'npm run serve:sample:js' http://localhost:8080 'node . --url http://localhost:8080/test/testsuite.qunit.html'",
+    "test:sample:js:coverage:remote": "start-server-and-test 'npm run serve:sample:js' http://localhost:8080 'node . --url http://localhost:8080/test/testsuite.qunit.html --coverage --coverage-check-statements 67'",
+    "serve:sample:js": "ui5 serve --config ./test/sample.js/ui5.yaml",
+    "test:sample:ts:remote": "start-server-and-test 'npm run serve:sample:ts' http://localhost:8080 'node . --url http://localhost:8080/test/testsuite.qunit.html'",
+    "serve:sample:ts": "cd ./test/sample.ts && node ui5.cjs serve",
+    "test:sample:ts:coverage:remote": "start-server-and-test 'npm run serve:sample:ts:coverage' http://localhost:8080 'node . --url http://localhost:8080/test/testsuite.qunit.html --coverage --coverage-check-statements 67'",
+    "serve:sample:ts:coverage": "cd ./test/sample.ts && node ui5.cjs serve --config ui5-coverage.yaml",
     "build:doc": "node build/doc"
   },
   "repository": {
@@ -52,13 +67,19 @@
     "mime": "^3.0.0",
     "punybind": "^1.2.1",
     "punyexpr": "^1.0.4",
-    "reserve": "^1.15.3"
+    "reserve": "^1.15.4"
   },
   "devDependencies": {
+    "@openui5/types": "^1.119.0",
+    "@ui5/cli": "^3.6.1",
+    "@ui5/middleware-code-coverage": "^1.1.0",
     "jest": "^29.7.0",
     "nock": "^13.3.3",
     "nyc": "^15.1.0",
-    "standard": "^17.1.0"
+    "standard": "^17.1.0",
+    "start-server-and-test": "^2.0.1",
+    "typescript": "^5.2.2",
+    "ui5-tooling-transpile": "^3.2.6"
   },
   "optionalDependencies": {
     "fsevents": "^2.3.3"
@@ -69,6 +90,10 @@
       "qunit",
       "node",
       "jest"
+    ],
+    "globals": [
+      "sap",
+      "opaTest"
     ]
   },
   "jest": {
@@ -98,4 +123,4 @@
       }
     }
   }
-}
+}

package/src/coverage.js

@@ -2,12 +2,14 @@
 
 const { join, dirname, isAbsolute } = require('path')
 const { fork } = require('child_process')
-const { cleanDir, createDir, filename, download } = require('./tools')
+const { cleanDir, createDir, filename, download, allocPromise } = require('./tools')
 const { readdir, readFile, stat, writeFile, access, constants } = require('fs').promises
 const { Readable } = require('stream')
 const { getOutput } = require('./output')
 const { resolvePackage } = require('./npm')
 const { promisify } = require('util')
+const { UTRError } = require('./error')
+const { $remoteOnLegacy } = require('./symbols')
 
 const $nycSettingsPath = Symbol('nycSettingsPath')
 const $coverageFileIndex = Symbol('coverageFileIndex')
@@ -28,9 +30,13 @@ async function nyc (job, ...args) {
   output.nyc(...args)
   const childProcess = fork(nycScript, args, { stdio: 'pipe' })
   output.monitor(childProcess)
-  let done
-  const promise = new Promise(resolve => { done = resolve })
-  childProcess.on('close', done)
+  const { promise, resolve, reject } = allocPromise()
+  childProcess.on('close', async code => {
+    if (code !== 0) {
+      reject(UTRError.NYC_FAILED(`Return code ${code}`))
+    }
+    resolve()
+  })
   return promise
 }
 
@@ -70,15 +76,9 @@ async function instrument (job) {
   settings.exclude.push(join(job.coverageReportDir, '**'))
   await writeFile(job[$nycSettingsPath], JSON.stringify(settings))
   if (job.mode === 'url') {
-    const port = job.port.toString()
-    const useLocal = job.url.some(url => {
-      // ignore host name since the machine might be exposed with any name
-      const parsedUrl = new URL(url)
-      return parsedUrl.port === port
-    })
-    if (!useLocal) {
-      getOutput(job).instrumentationSkipped()
+    if (!job[$remoteOnLegacy]) {
       job[$coverageRemote] = true
+      getOutput(job).instrumentationSkipped()
       return
     }
   }
@@ -125,6 +125,9 @@ async function generateCoverageReport (job) {
   }
   const checks = []
   if (job.coverageCheckBranches || job.coverageCheckFunctions || job.coverageCheckLines || job.coverageCheckStatements) {
+    if (!job.coverageReporters.includes('lcov')) {
+      reporters.push('--reporter=lcov')
+    }
     checks.push(
       `--branches=${job.coverageCheckBranches}`,
       `--functions=${job.coverageCheckFunctions}`,
@@ -134,6 +137,13 @@ async function generateCoverageReport (job) {
     )
   }
   await nyc(job, 'report', ...reporters, ...checks, '--temp-dir', coverageMergedDir, '--report-dir', job.coverageReportDir, '--nycrc-path', job[$nycSettingsPath])
+  if (checks.length) {
+    // The checks are not triggered if the coverage is empty
+    const lcov = await stat(join(job.coverageReportDir, 'lcov.info'))
+    if (lcov.size === 0) {
+      throw UTRError.NYC_FAILED('No coverage information extracted')
+    }
+  }
 }
 
 module.exports = {
@@ -146,18 +156,18 @@ module.exports = {
     }
     await writeFile(coverageFileName, JSON.stringify(coverageData))
   },
-  generateCoverageReport: job => job.coverage && generateCoverageReport(job),
+  generateCoverageReport: job => job.coverage ? generateCoverageReport(job) : Promise.resolve(),
   mappings: async job => {
     if (!job.coverage) {
       return []
     }
     const instrumentedBasePath = join(job.coverageTempDir, 'instrumented')
     const instrumentedMapping = {
-      match: /^\/(.*\.js)$/,
+      match: /(.*\.js)(\?.*)?$/,
       file: join(instrumentedBasePath, '$1'),
       'ignore-if-not-found': true
     }
-    if (job.mode === 'legacy') {
+    if (job.mode === 'legacy' || job[$remoteOnLegacy]) {
       return [{
         ...instrumentedMapping,
         'custom-file-system': job.debugCoverageNoCustomFs ? undefined : customFileSystem

package/src/coverage.spec.js

@@ -4,6 +4,7 @@ const { instrument, generateCoverageReport, mappings } = require('./coverage')
 const { stat } = require('fs/promises')
 const { cleanDir, createDir } = require('./tools')
 const { getOutput } = require('./output')
+const { $remoteOnLegacy } = require('./symbols')
 
 describe('src/coverage', () => {
   const cwd = join(__dirname, '../test/project')
@@ -108,7 +109,8 @@ describe('src/coverage', () => {
         Object.assign(job, {
           mode: 'url',
           port: 8080,
-          url: ['http://localhost:8080/whatever/test.html']
+          url: ['http://localhost:8080/whatever/test.html'],
+          [$remoteOnLegacy]: true // added on job finalization
         })
         await instrument(job)
         expect(instrumentationSkipped).not.toHaveBeenCalled()

package/src/defaults/playwright.js

@@ -88,7 +88,9 @@ require('./browser')({
     networkWriter
   }) {
     const browsers = require(modules.playwright)
-    browser = await browsers[options.browser].launch()
+    browser = await browsers[options.browser].launch({
+      headless: !options.visible
+    })
 
     let recordVideo
     if (options.video) {

package/src/error.js

@@ -37,6 +37,8 @@ const errors = [{
   name: 'BROWSER_MISS_SCRIPTS_CAPABILITY'
 }, {
   name: 'NPM_DEPENDENCY_NOT_FOUND'
+}, {
+  name: 'NYC_FAILED'
 }]
 
 errors.forEach((error, index) => {

package/src/job.js

@@ -5,7 +5,7 @@ const { statSync, accessSync, constants } = require('fs')
 const { dirname, join, isAbsolute } = require('path')
 const { name, description, version } = require(join(__dirname, '../package.json'))
 const { getOutput } = require('./output')
-const { $valueSources } = require('./symbols')
+const { $valueSources, $remoteOnLegacy } = require('./symbols')
 const { buildAndCheckMode } = require('./job-mode')
 const { boolean, integer, timeout, url, arrayOf, regex, percent } = require('./options')
 
@@ -107,6 +107,7 @@ function getCommand (cwd) {
     .option('-br, --browser-retry <count>', '[💻🔗🧪] Browser instantiation retries : if the command fails unexpectedly, it is re-executed (0 means no retry)', 1)
 
     // Common to legacy and url
+    .option('-qs, --qunit-strict', '[💻🔗] Strict mode on qunit execution (fails if no modules declared)', boolean, false)
     .option('-pf, --page-filter <regexp>', '[💻🔗] Filter out pages not matching the regexp')
     .option('-pp, --page-params <params>', '[💻🔗] Add parameters to page URL')
     .option('-t, --global-timeout <timeout>', '[💻🔗] Limit the pages execution time, fail the page if it takes longer than the timeout (0 means no timeout)', timeout, 0)
@@ -287,6 +288,15 @@ function finalize (job) {
     overrideIfNotSet('coverageReporters', settings.reporter)
   }
 
+  if (job.mode === 'url') {
+    const port = job.port.toString()
+    job[$remoteOnLegacy] = job.url.every(url => {
+      // ignore host name since the machine might be exposed with any name
+      const parsedUrl = new URL(url)
+      return parsedUrl.port === port
+    })
+  }
+
   job[$status] = 'Starting'
   Object.defineProperty(job, 'status', {
     get () {

package/src/output.js

@@ -412,6 +412,10 @@ function build (job) {
       log(job, p80()`Skipping nyc instrumentation (--url)`)
     }),
 
+    qunitEarlyStart: wrap(url => {
+      log(job, p80()`QUnit start without tests in ${pad.lt(url)}`)
+    }),
+
     endpointError: wrap(({ api, url, data, error }) => {
       const p = p80()
       log(job, p`┌──────────${pad.x('─')}┐`)

package/src/qunit-hooks.js

@@ -13,27 +13,40 @@ function error (job, url, details = '') {
   throw UTRError.QUNIT_ERROR(details)
 }
 
+function invalidTestId (job, url, testId) {
+  error(job, url, `No QUnit unit test found with id ${testId}`)
+}
+
 function get (job, urlWithHash, testId) {
   const url = stripUrlHash(urlWithHash)
   const page = job.qunitPages && job.qunitPages[url]
   if (!page) {
     error(job, url, `No QUnit page found for ${urlWithHash}`)
   }
+  let testModule
   let test
   if (testId !== undefined) {
     page.modules.every(module => {
       test = module.tests.find(test => test.testId === testId)
-      return test === undefined
+      if (test === undefined) {
+        return true
+      } else {
+        testModule = module
+        return false
+      }
     })
-    if (!test) {
-      error(job, url, `No QUnit unit test found with id ${testId}`)
+    if (!test && job.qunitStrict) {
+      invalidTestId(job, url, testId)
     }
   }
-  return { url, page, test }
+  return { url, page, testModule, test }
 }
 
 async function done (job, urlWithHash, report) {
   const { url, page } = get(job, urlWithHash)
+  if (page.earlyStart && page.count === 0) {
+    return // wait
+  }
   if (job.browserCapabilities.screenshot) {
     try {
       await screenshot(job, url, 'done')
@@ -41,11 +54,11 @@ async function done (job, urlWithHash, report) {
       getOutput(job).genericError(error, url)
     }
   }
+  page.end = new Date()
   if (report.__coverage__) {
-    collect(job, url, report.__coverage__)
+    await collect(job, url, report.__coverage__)
     delete report.__coverage__
   }
-  page.end = new Date()
   page.report = report
   stop(job, url)
 }
@@ -55,8 +68,12 @@ module.exports = {
 
   async begin (job, urlWithHash, { isOpa, totalTests, modules }) {
     const url = stripUrlHash(urlWithHash)
-    if (!totalTests || !modules) {
-      error(job, url, 'Invalid begin hook details')
+    const earlyStart = !totalTests || !modules
+    if (earlyStart) {
+      getOutput(job).qunitEarlyStart(url)
+      if (job.qunitStrict) {
+        error(job, url, 'Invalid begin hook details')
+      }
     }
     if (!job.qunitPages) {
       job.qunitPages = {}
@@ -70,16 +87,31 @@ module.exports = {
       count: totalTests,
       modules
     }
+    if (earlyStart) {
+      qunitPage.earlyStart = true
+    }
     job.qunitPages[url] = qunitPage
   },
 
   async testStart (job, urlWithHash, { module, name, testId }) {
-    const { test } = get(job, urlWithHash, testId)
+    let { page, testModule, test } = get(job, urlWithHash, testId)
+    if (!testModule) {
+      testModule = { name: module, tests: [] }
+      page.modules.push(testModule)
+    }
+    if (!test) {
+      test = { name, testId }
+      testModule.tests.push(test)
+      ++page.count
+    }
     test.start = new Date()
   },
 
   async log (job, urlWithHash, { module, name, testId, ...log }) {
     const { url, page, test } = get(job, urlWithHash, testId)
+    if (!test) {
+      invalidTestId(job, url, testId)
+    }
     if (!test.logs) {
       test.logs = []
     }
@@ -97,6 +129,9 @@ module.exports = {
   async testDone (job, urlWithHash, { name, module, testId, assertions, ...report }) {
     const { failed } = report
     const { url, page, test } = get(job, urlWithHash, testId)
+    if (!test) {
+      invalidTestId(job, url, testId)
+    }
     if (failed) {
       if (job.browserCapabilities.screenshot) {
         try {

package/src/qunit-hooks.spec.js

@@ -10,10 +10,12 @@ jest.mock('./coverage.js', () => ({
 const { collect } = require('./coverage')
 
 const mockGenericError = jest.fn()
+const mockQunitEarlyStart = jest.fn()
 
 jest.mock('./output.js', () => ({
   getOutput: () => ({
-    genericError: mockGenericError
+    genericError: mockGenericError,
+    qunitEarlyStart: mockQunitEarlyStart
   })
 }))
 
@@ -174,7 +176,11 @@ describe('src/qunit-hooks', () => {
       })
     })
 
-    describe('validation', () => {
+    describe('validation (--qunit-strict)', () => {
+      beforeEach(() => {
+        job.qunitStrict = true
+      })
+
       afterEach(() => {
         expect(stop).toHaveBeenCalledWith(job, url)
         expect(job.failed).toStrictEqual(true)
@@ -185,6 +191,7 @@ describe('src/qunit-hooks', () => {
           isOpa: false,
           modules: getModules()
         })).rejects.toThrow(UTRError.QUNIT_ERROR('Invalid begin hook details'))
+        expect(mockQunitEarlyStart).toHaveBeenCalled()
       })
 
       it('requires modules', async () => {
@@ -192,6 +199,7 @@ describe('src/qunit-hooks', () => {
           isOpa: false,
           totalTests: 1
         })).rejects.toThrow(UTRError.QUNIT_ERROR('Invalid begin hook details'))
+        expect(mockQunitEarlyStart).toHaveBeenCalled()
       })
     })
   })
@@ -594,7 +602,18 @@ describe('src/qunit-hooks', () => {
       expect(job.failed).toStrictEqual(true)
     })
 
-    it('fails on invalid test id', async () => {
+    it('fails on invalid test id (--qunit-strict)', async () => {
+      job.qunitStrict = true
+      await expect(testDone(job, url, {
+        ...getTestDoneFor1a(),
+        testId: '1c'
+      }))
+        .rejects.toThrow(UTRError.QUNIT_ERROR('No QUnit unit test found with id 1c'))
+      expect(stop).toHaveBeenCalledWith(job, url)
+      expect(job.failed).toStrictEqual(true)
+    })
+
+    it('fails on invalid test id (--no-qunit-strict)', async () => {
       await expect(testDone(job, url, {
         ...getTestDoneFor1a(),
         testId: '1c'

package/src/report.js

@@ -51,7 +51,10 @@ module.exports = {
       })
       return promise
     })
-    promises.push(generateCoverageReport(job))
+    promises.push(generateCoverageReport(job).catch(e => {
+      output.genericError(e)
+      job.failed = true
+    }))
     await Promise.all(promises)
     job.status = 'Done'
   }

package/src/simulate.spec.js

@@ -213,6 +213,31 @@ describe('simulate', () => {
       })
     })
 
+    describe('simple test execution (--no-qunit-strict)', () => {
+      beforeAll(async () => {
+        await setup('simple-early')
+        pages = {
+          'testsuite.qunit.html': async referer => {
+            await post('/_/addTestPages', referer, [
+              referer.replace('testsuite.qunit.html', 'page1.html')
+            ])
+          },
+          'page1.html': async referer => {
+            await post('/_/QUnit/begin', referer, { totalTests: 0, modules: [] })
+            await post('/_/QUnit/done', referer, { failed: 0 })
+            await post('/_/QUnit/testStart', referer, { module: 'module', name: 'test', testId: '1' })
+            await post('/_/QUnit/testDone', referer, { testId: '1', failed: 0, passed: 1 })
+            await post('/_/QUnit/done', referer, { failed: 0 })
+          }
+        }
+        await safeExecute()
+      })
+
+      it('succeeded', () => {
+        expect(job.failed).toStrictEqual(false)
+      })
+    })
+
     describe('error', () => {
       describe('one test fail', () => {
         beforeAll(async () => {

package/src/symbols.js

@@ -4,5 +4,6 @@ module.exports = {
   $probeUrlsCompleted: Symbol('probeUrlsCompleted'),
   $testPagesStarted: Symbol('testPagesStarted'),
   $testPagesCompleted: Symbol('testPagesCompleted'),
-  $valueSources: Symbol('valueSources')
+  $valueSources: Symbol('valueSources'),
+  $remoteOnLegacy: Symbol('remoteOnLegacy')
 }