ui5-test-runner 2.0.5 → 3.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -20,7 +20,7 @@ A self-sufficient test runner for UI5 applications enabling parallel execution o
20
20
 
21
21
  ## 💿 How to install
22
22
 
23
- * Works with [Node.js](https://nodejs.org/en/download/) >= 14
23
+ * Works with [Node.js](https://nodejs.org/en/download/) >= 16
24
24
  * Local installation
25
25
  * `npm install --save-dev ui5-test-runner`
26
26
  * Trigger either with `npx ui5-test-runner` or through an npm script invoking `ui5-test-runner`
@@ -47,7 +47,10 @@ A self-sufficient test runner for UI5 applications enabling parallel execution o
47
47
  ## ⚖️ License
48
48
  [![FOSSA Status](https://app.fossa.com/api/projects/git%2Bgithub.com%2FArnaudBuchholz%2Fui5-test-runner.svg?type=large)](https://app.fossa.com/projects/git%2Bgithub.com%2FArnaudBuchholz%2Fui5-test-runner?ref=badge_large)
49
49
 
50
- ## ⚠️ Breaking change
50
+ ## ⚠️ Breaking changes
51
+
52
+ ### v3
53
+ * Dropping support of Node.js 14
51
54
 
52
55
  ### v2
53
56
 
package/index.js CHANGED
File without changes
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "ui5-test-runner",
3
- "version": "2.0.5",
3
+ "version": "3.1.0",
4
4
  "description": "Standalone test runner for UI5",
5
5
  "main": "index.js",
6
6
  "files": [
@@ -12,7 +12,7 @@
12
12
  "ui5-test-runner": "./index.js"
13
13
  },
14
14
  "engines": {
15
- "node": ">=14.0.0"
15
+ "node": ">=16"
16
16
  },
17
17
  "scripts": {
18
18
  "lint": "standard --fix",
@@ -45,7 +45,7 @@
45
45
  },
46
46
  "homepage": "https://github.com/ArnaudBuchholz/ui5-test-runner#readme",
47
47
  "dependencies": {
48
- "commander": "^10.0.0",
48
+ "commander": "^10.0.1",
49
49
  "mime": "^3.0.0",
50
50
  "punybind": "^1.2.1",
51
51
  "punyexpr": "^1.0.4",
@@ -53,7 +53,7 @@
53
53
  },
54
54
  "devDependencies": {
55
55
  "jest": "^29.5.0",
56
- "nock": "^13.3.0",
56
+ "nock": "^13.3.1",
57
57
  "nyc": "^15.1.0",
58
58
  "standard": "^17.0.0"
59
59
  },
@@ -0,0 +1,32 @@
1
+ <!DOCTYPE html>
2
+ <html>
3
+ <body>
4
+ <h1>Coverage in an iframe</h1>
5
+ <p>Checks if the coverage information can be hooked in the iframe</p>
6
+ <script>
7
+ // As generated by Istanbul
8
+ function cov_1y52ey2l1c() {
9
+ var global = new Function("return this")();
10
+ var gcv = "__coverage__";
11
+ var path = "coverage.html";
12
+ var coverageData = {
13
+ path: path,
14
+ _coverageSchema: "1a1c01bbd47fc00a2c39e90264f33305004495a9",
15
+ hash: "55ef335c1997f49832eb59e4836d886c3923dbfb",
16
+ status: "ko"
17
+ };
18
+ var coverage = global[gcv] || (global[gcv] = {});
19
+ if (!coverage[path] || coverage[path].hash !== hash) {
20
+ coverage[path] = coverageData;
21
+ }
22
+ var actualCoverage = coverage[path];
23
+ {// @ts-ignore
24
+ cov_1y52ey2l1c = function () { return actualCoverage; };
25
+ }
26
+ return actualCoverage;
27
+ }
28
+ cov_1y52ey2l1c();
29
+ cov_1y52ey2l1c().status = "ok";
30
+ </script>
31
+ </body>
32
+ </html>
@@ -0,0 +1,18 @@
1
+ <!DOCTYPE html>
2
+ <html>
3
+ <head>
4
+ <title>IFrame Coverage</title>
5
+ </head>
6
+ <body>
7
+ <h1>IFrame Coverage</h1>
8
+ <p>Checks if the coverage information can be extracted from the iframe</p>
9
+ <iframe src="coverage.html"></iframe>
10
+ <script>
11
+ document.querySelector('iframe').addEventListener('load', () => {
12
+ const xhr = new XMLHttpRequest()
13
+ xhr.open('POST', '/_/log')
14
+ xhr.send(JSON.stringify(window.__coverage__ || {}))
15
+ })
16
+ </script>
17
+ </body>
18
+ </html>
@@ -47,4 +47,12 @@ module.exports = [{
47
47
  url: 'https://ui5.sap.com/test-resources/sap/m/demokit/orderbrowser/webapp/test/unit/unitTests.qunit.html',
48
48
  scripts: ['qunit-intercept.js', 'post.js', 'qunit-hooks.js'],
49
49
  endpoint: qUnitEndpoints
50
+ }, {
51
+ label: 'Scripts (IFrame Coverage)',
52
+ for: capabilities => !!capabilities.scripts,
53
+ url: 'scripts/iframe.html',
54
+ scripts: ['opa-iframe-coverage.js'],
55
+ endpoint: ({ body }) => {
56
+ assert.strictEqual(body['coverage.html'].status, 'ok')
57
+ }
50
58
  }]
package/src/coverage.js CHANGED
@@ -9,6 +9,7 @@ const { getOutput } = require('./output')
9
9
  const { resolvePackage } = require('./npm')
10
10
 
11
11
  const $nycSettingsPath = Symbol('nycSettingsPath')
12
+ const $coverageFileIndex = Symbol('coverageFileIndex')
12
13
 
13
14
  let nycScript
14
15
 
@@ -42,7 +43,6 @@ const customFileSystem = {
42
43
  }
43
44
 
44
45
  async function instrument (job) {
45
- job.status = 'Instrumenting'
46
46
  if (!nycScript) {
47
47
  const nyc = await resolvePackage(job, 'nyc')
48
48
  nycScript = join(nyc, 'bin/nyc.js')
@@ -62,6 +62,19 @@ async function instrument (job) {
62
62
  settings.exclude.push(join(job.reportDir, '**'))
63
63
  settings.exclude.push(join(job.coverageReportDir, '**'))
64
64
  await writeFile(job[$nycSettingsPath], JSON.stringify(settings))
65
+ if (job.mode === 'url') {
66
+ const port = job.port.toString()
67
+ const useLocal = job.url.some(url => {
68
+ // ignore host name since the machine might be exposed with any name
69
+ const parsedUrl = new URL(url)
70
+ return parsedUrl.port === port
71
+ })
72
+ if (!useLocal) {
73
+ getOutput(job).instrumentationSkipped()
74
+ return
75
+ }
76
+ }
77
+ job.status = 'Instrumenting'
65
78
  await nyc(job, 'instrument', job.webapp, join(job.coverageTempDir, 'instrumented'), '--nycrc-path', job[$nycSettingsPath])
66
79
  }
67
80
 
@@ -76,7 +89,11 @@ async function generateCoverageReport (job) {
76
89
  module.exports = {
77
90
  instrument: job => job.coverage && instrument(job),
78
91
  async collect (job, url, coverageData) {
79
- const coverageFileName = join(job.coverageTempDir, `${filename(url)}.json`)
92
+ job[$coverageFileIndex] = (job[$coverageFileIndex] || 0) + 1
93
+ const coverageFileName = join(job.coverageTempDir, `${filename(url)}_${job[$coverageFileIndex]}.json`)
94
+ if (job.debugCoverage) {
95
+ getOutput(job).wrap(() => console.log('coverage', coverageFileName))
96
+ }
80
97
  await writeFile(coverageFileName, JSON.stringify(coverageData))
81
98
  },
82
99
  generateCoverageReport: job => job.coverage && generateCoverageReport(job),
@@ -85,7 +102,7 @@ module.exports = {
85
102
  match: /^\/(.*\.js)$/,
86
103
  file: join(job.coverageTempDir, 'instrumented', '$1'),
87
104
  'ignore-if-not-found': true,
88
- 'custom-file-system': customFileSystem
105
+ 'custom-file-system': job.debugCoverageNoCustomFs ? undefined : customFileSystem
89
106
  }]
90
107
  : []
91
108
  }
@@ -3,6 +3,7 @@ const { fromObject } = require('./job')
3
3
  const { instrument, generateCoverageReport, mappings } = require('./coverage')
4
4
  const { stat } = require('fs/promises')
5
5
  const { cleanDir, createDir } = require('./tools')
6
+ const { getOutput } = require('./output')
6
7
 
7
8
  describe('src/coverage', () => {
8
9
  const cwd = join(__dirname, '../test/project')
@@ -75,5 +76,43 @@ describe('src/coverage', () => {
75
76
  const coverageMappings = mappings(job)
76
77
  expect(coverageMappings.length).toStrictEqual(1)
77
78
  })
79
+
80
+ describe('--url compatibility', () => {
81
+ let output
82
+ let instrumentationSkipped
83
+
84
+ beforeAll(() => {
85
+ output = getOutput(job)
86
+ instrumentationSkipped = jest.spyOn(output, 'instrumentationSkipped')
87
+ })
88
+
89
+ beforeEach(() => {
90
+ instrumentationSkipped.mockReset()
91
+ })
92
+
93
+ afterAll(() => {
94
+ instrumentationSkipped.mockRestore()
95
+ })
96
+
97
+ it('does *not* instrument if the URL does not match current port', async () => {
98
+ Object.assign(job, {
99
+ mode: 'url',
100
+ port: 8080,
101
+ url: ['http://localhost:8081/whatever/test.html']
102
+ })
103
+ await instrument(job)
104
+ expect(instrumentationSkipped).toHaveBeenCalled()
105
+ })
106
+
107
+ it('**does** instrument anyway if the URL matches current port', async () => {
108
+ Object.assign(job, {
109
+ mode: 'url',
110
+ port: 8080,
111
+ url: ['http://localhost:8080/whatever/test.html']
112
+ })
113
+ await instrument(job)
114
+ expect(instrumentationSkipped).not.toHaveBeenCalled()
115
+ })
116
+ })
78
117
  })
79
118
  })
@@ -30,7 +30,17 @@ require('./browser')({
30
30
  virtualConsole.on('info', (...args) => consoleWriter.append({ type: 'info', text: args.join(' ') }))
31
31
  virtualConsole.on('log', (...args) => consoleWriter.append({ type: 'log', text: args.join(' ') }))
32
32
 
33
+ let mainWindow
34
+
33
35
  const beforeParse = (window) => {
36
+ if (mainWindow === undefined) {
37
+ mainWindow = window
38
+ } else {
39
+ Object.defineProperty(window, 'parent', {
40
+ value: mainWindow,
41
+ writable: false
42
+ })
43
+ }
34
44
  require('./jsdom/compatibility')({ window, networkWriter })
35
45
  if (options.debug) {
36
46
  require('./jsdom/debug')(window)
@@ -43,6 +53,7 @@ require('./browser')({
43
53
  const origCreate = Window.createWindow.bind(Window)
44
54
  Window.createWindow = (...args) => {
45
55
  const window = origCreate(...args)
56
+ window._virtualConsole = virtualConsole
46
57
  beforeParse(window)
47
58
  return window
48
59
  }
@@ -67,7 +67,7 @@ require('./browser')({
67
67
  }
68
68
 
69
69
  browser = await puppeteer.launch({
70
- headless: !options.visible,
70
+ headless: options.visible ? false : 'new',
71
71
  defaultViewport: null,
72
72
  args
73
73
  })
@@ -11,7 +11,7 @@
11
11
  <div {{if}}="!(qunitPage || qunitTest)">
12
12
  <h1>{{ status || 'Test report' }}</h1>
13
13
  <div {{if}}="end === undefined" class="elapsed">In progress since {{ elapsed(start) }}</div>
14
- <div {{else}} class="elapsed">Duration : {{ elapsed(start, end) }}</div>
14
+ <div {{else}} class="elapsed">Duration : {{ elapsed(start, end) }} <a href="#" id="download">&#128230;</a></div>
15
15
  <table style="visibility: {{ testPageUrls.length > 0 ? 'visible' : 'hidden' }};">
16
16
  <tr>
17
17
  <th>&nbsp;</th>
@@ -54,4 +54,16 @@ report.ready.then(update => {
54
54
  })
55
55
  }
56
56
  hashChange(location.hash)
57
+
58
+ window.addEventListener('click', (event) => {
59
+ if (event.target.id === 'download') {
60
+ const link = this.document.createElement('a')
61
+ const blob = new Blob([JSON.stringify(job)], {
62
+ type: 'application/json'
63
+ })
64
+ link.setAttribute('href', URL.createObjectURL(blob))
65
+ link.setAttribute('download', 'ui5-test-runner-job.json')
66
+ link.click()
67
+ }
68
+ })
57
69
  })
@@ -0,0 +1,23 @@
1
+ (function () {
2
+ 'use strict'
3
+
4
+ const MODULE = 'ui5-test-runner/opa-iframe-coverage'
5
+
6
+ if (window[MODULE]) {
7
+ return // already installed
8
+ }
9
+ window[MODULE] = true
10
+
11
+ if (window !== window.top || window !== window.parent) {
12
+ // Inside an iframe
13
+ Object.defineProperty(window, '__coverage__', {
14
+ get () {
15
+ return window.top.__coverage__
16
+ },
17
+ set (value) {
18
+ window.top.__coverage__ = value
19
+ return true
20
+ }
21
+ })
22
+ }
23
+ }())
@@ -1,7 +1,8 @@
1
1
  (function () {
2
2
  'use strict'
3
3
 
4
- if (window['ui5-test-runner/post']) {
4
+ const POST = 'ui5-test-runner/post'
5
+ if (window[POST]) {
5
6
  return
6
7
  }
7
8
 
@@ -70,7 +71,7 @@
70
71
 
71
72
  window['ui5-test-runner/stringify'] = stringify
72
73
 
73
- window['ui5-test-runner/post'] = function post (url, data) {
74
+ window[POST] = function post (url, data) {
74
75
  function request () {
75
76
  return new Promise(function (resolve, reject) {
76
77
  const xhr = new XMLHttpRequest()
@@ -2,10 +2,11 @@
2
2
  (function () {
3
3
  'use strict'
4
4
 
5
- if (window['ui5-test-runner/qunit-hooks']) {
5
+ const MODULE = 'ui5-test-runner/qunit-hooks'
6
+ if (window[MODULE]) {
6
7
  return // already installed
7
8
  }
8
- window['ui5-test-runner/qunit-hooks'] = true
9
+ window[MODULE] = true
9
10
 
10
11
  const post = window['ui5-test-runner/post']
11
12
 
@@ -1,10 +1,11 @@
1
1
  (function () {
2
2
  'use strict'
3
3
 
4
- if (window['ui5-test-runner/qunit-redirect']) {
4
+ const MODULE = 'ui5-test-runner/qunit-redirect'
5
+ if (window[MODULE]) {
5
6
  return // already installed
6
7
  }
7
- window['ui5-test-runner/qunit-redirect'] = true
8
+ window[MODULE] = true
8
9
 
9
10
  /* global suite */
10
11
 
package/src/job.js CHANGED
@@ -2,7 +2,7 @@
2
2
 
3
3
  const { Command, Option, InvalidArgumentError } = require('commander')
4
4
  const { statSync, accessSync, constants } = require('fs')
5
- const { join, isAbsolute } = require('path')
5
+ const { dirname, join, isAbsolute } = require('path')
6
6
  const { name, description, version } = require(join(__dirname, '../package.json'))
7
7
  const { getOutput } = require('./output')
8
8
  const { $valueSources } = require('./symbols')
@@ -95,6 +95,7 @@ function getCommand (cwd) {
95
95
  .option('-r, --report-dir <path>', '[💻🔗🧪] Directory to output test reports (relative to cwd)', 'report')
96
96
  .option('-pt, --page-timeout <timeout>', '[💻🔗🧪] Limit the page execution time, fails the page if it takes longer than the timeout (0 means no timeout)', timeout, 0)
97
97
  .option('-f, --fail-fast [flag]', '[💻🔗🧪] Stop the execution after the first failing page', boolean, false)
98
+ .option('-fo, --fail-opa-fast [flag]', '[💻🔗] Stop the OPA page execution after the first failing test', boolean, false)
98
99
  .option('-k, --keep-alive [flag]', '[💻🔗🧪] Keep the server alive', boolean, false)
99
100
  .option('-l, --log-server [flag]', '[💻🔗🧪] Log inner server traces', boolean, false)
100
101
  .option('-p, --parallel <count>', '[💻🔗🧪] Number of parallel tests executions', 2)
@@ -136,6 +137,8 @@ function getCommand (cwd) {
136
137
  .addOption(new Option('--debug-keep-report', DEBUG_OPTION, boolean).hideHelp())
137
138
  .addOption(new Option('--debug-capabilities-test <name>', DEBUG_OPTION).hideHelp())
138
139
  .addOption(new Option('--debug-capabilities-no-timeout', DEBUG_OPTION, boolean).hideHelp())
140
+ .addOption(new Option('--debug-coverage', DEBUG_OPTION, boolean).hideHelp())
141
+ .addOption(new Option('--debug-coverage-no-custom-fs', DEBUG_OPTION, boolean).hideHelp())
139
142
 
140
143
  return command
141
144
  }
@@ -241,6 +244,37 @@ function finalize (job) {
241
244
  }
242
245
 
243
246
  const output = getOutput(job)
247
+
248
+ if (job.coverage) {
249
+ function overrideIfNotSet (option, valueFromSettings) {
250
+ if (valueFromSettings && job[$valueSources][option] !== 'cli') {
251
+ if (job.debugCoverage) {
252
+ output.wrap(() => console.log(`${option} extracted from nyc settings : ${valueFromSettings}`))
253
+ }
254
+ job[option] = valueFromSettings
255
+ }
256
+ }
257
+
258
+ function overrideDirIfNotSet (option, valueFromSettings) {
259
+ if (valueFromSettings && !isAbsolute(valueFromSettings)) {
260
+ valueFromSettings = join(dirname(job.coverageSettings), valueFromSettings)
261
+ }
262
+ overrideIfNotSet(option, valueFromSettings)
263
+ }
264
+
265
+ checkAccess({ path: job.coverageSettings, file: true, label: 'coverage settings' })
266
+
267
+ let settings
268
+ try {
269
+ settings = require(job.coverageSettings)
270
+ } catch (e) {
271
+ throw new Error(`Unable to read ${job.coverageSettings} as JSON`)
272
+ }
273
+ overrideDirIfNotSet('coverageReportDir', settings['report-dir'])
274
+ overrideDirIfNotSet('coverageTempDir', settings['temp-dir'])
275
+ overrideIfNotSet('coverageReporters', settings.reporter)
276
+ }
277
+
244
278
  job[$status] = 'Starting'
245
279
  Object.defineProperty(job, 'status', {
246
280
  get () {
package/src/output.js CHANGED
@@ -408,6 +408,10 @@ function build (job) {
408
408
  log(job, p80()`nyc ${args.map(arg => arg.toString()).join(' ')}`)
409
409
  }),
410
410
 
411
+ instrumentationSkipped: wrap(() => {
412
+ log(job, p80()`Skipping nyc instrumentation (--url)`)
413
+ }),
414
+
411
415
  endpointError: wrap(({ api, url, data, error }) => {
412
416
  const p = p80()
413
417
  log(job, p`┌──────────${pad.x('─')}┐`)
@@ -32,6 +32,24 @@ function get (job, urlWithHash, testId) {
32
32
  return { url, page, test }
33
33
  }
34
34
 
35
+ async function done (job, urlWithHash, report) {
36
+ const { url, page } = get(job, urlWithHash)
37
+ if (job.browserCapabilities.screenshot) {
38
+ try {
39
+ await screenshot(job, url, 'done')
40
+ } catch (error) {
41
+ getOutput(job).genericError(error, url)
42
+ }
43
+ }
44
+ if (report.__coverage__) {
45
+ collect(job, url, report.__coverage__)
46
+ delete report.__coverage__
47
+ }
48
+ page.end = new Date()
49
+ page.report = report
50
+ stop(job, url)
51
+ }
52
+
35
53
  module.exports = {
36
54
  get,
37
55
 
@@ -95,23 +113,23 @@ module.exports = {
95
113
  }
96
114
  test.end = new Date()
97
115
  test.report = report
116
+ if (job.failOpaFast && failed) {
117
+ // skip remaining tests
118
+ page.modules.forEach(module => {
119
+ module.tests.forEach(test => {
120
+ if (!test.report) {
121
+ test.skip = true
122
+ }
123
+ })
124
+ })
125
+ await done(job, urlWithHash, {
126
+ failed: page.failed,
127
+ passed: page.passed,
128
+ total: page.count,
129
+ runtime: 0
130
+ })
131
+ }
98
132
  },
99
133
 
100
- async done (job, urlWithHash, report) {
101
- const { url, page } = get(job, urlWithHash)
102
- if (job.browserCapabilities.screenshot) {
103
- try {
104
- await screenshot(job, url, 'done')
105
- } catch (error) {
106
- getOutput(job).genericError(error, url)
107
- }
108
- }
109
- if (report.__coverage__) {
110
- collect(job, url, report.__coverage__)
111
- delete report.__coverage__
112
- }
113
- page.end = new Date()
114
- page.report = report
115
- stop(job, url)
116
- }
134
+ done
117
135
  }
@@ -603,6 +603,44 @@ describe('src/qunit-hooks', () => {
603
603
  expect(stop).toHaveBeenCalledWith(job, url)
604
604
  expect(job.failed).toStrictEqual(true)
605
605
  })
606
+
607
+ describe('fail OPA fast behavior', () => {
608
+ beforeEach(async () => {
609
+ job.failOpaFast = true
610
+ await testDone(job, url, {
611
+ ...getTestDoneFor1a(),
612
+ passed: 0,
613
+ failed: 1,
614
+ total: 1
615
+ })
616
+ })
617
+
618
+ it('fails the test immediately', () => {
619
+ const { test } = get(job, url, '1a')
620
+ expect(test).toMatchObject({
621
+ report: {
622
+ passed: 0,
623
+ failed: 1,
624
+ total: 1
625
+ }
626
+ })
627
+ })
628
+
629
+ it('flags the remaining tests as skipped', () => {
630
+ const { page } = get(job, url)
631
+ page.modules.forEach(module => {
632
+ module.tests.forEach(test => {
633
+ if (test.testId !== '1a') {
634
+ expect(test.skip).toStrictEqual(true)
635
+ }
636
+ })
637
+ })
638
+ })
639
+
640
+ it('stops the page immediately', () => {
641
+ expect(stop).toHaveBeenCalledWith(job, url)
642
+ })
643
+ })
606
644
  })
607
645
 
608
646
  describe('done', () => {
package/src/reserve.js CHANGED
@@ -13,7 +13,7 @@ module.exports = job => check({
13
13
  ...job.mappings ?? [],
14
14
  ...job.serveOnly ? [] : endpoints(job),
15
15
  ...ui5(job),
16
- ...job.serveOnly ? [] : coverage(job), {
16
+ ...coverage(job), {
17
17
  // Project mapping
18
18
  match: /^\/(.*)/,
19
19
  file: join(job.webapp, '$1'),
package/src/tests.js CHANGED
@@ -73,7 +73,8 @@ async function runTestPage (job, url) {
73
73
  scripts = [
74
74
  'post.js',
75
75
  'qunit-intercept.js',
76
- 'qunit-hooks.js'
76
+ 'qunit-hooks.js',
77
+ 'opa-iframe-coverage.js'
77
78
  ]
78
79
  }
79
80
  await start(job, url, scripts)
package/src/tools.js CHANGED
@@ -1,17 +1,8 @@
1
1
  'use strict'
2
2
 
3
- const fsPromises = require('fs').promises
4
- const { mkdir, stat } = fsPromises
3
+ const { mkdir, rm, stat } = require('fs').promises
5
4
  const { createHash } = require('crypto')
6
5
 
7
- let rm
8
- /* istanbul ignore next */ // Hard to test both in the same run
9
- if (process.version > 'v14.14') {
10
- rm = fsPromises.rm
11
- } else {
12
- rm = fsPromises.rmdir
13
- }
14
-
15
6
  const recursive = { recursive: true }
16
7
 
17
8
  const stripUrlHash = url => url.split('#')[0]