ui5-test-runner 1.1.4 → 2.0.0

package/src/browsers.js

@@ -2,55 +2,179 @@
 
 const { fork } = require('child_process')
 const { join } = require('path')
-const { recreateDir, filename } = require('./tools')
+const { writeFile, readFile, open, stat, unlink } = require('fs/promises')
+const { recreateDir, filename, allocPromise } = require('./tools')
 const { getPageTimeout } = require('./timeout')
-const output = require('./output')
+const { getOutput } = require('./output')
+const { resolvePackage } = require('./npm')
+const { UTRError } = require('./error')
+const { $browsers } = require('./symbols')
 
 let lastScreenshotId = 0
 const screenshots = {}
 
-function start (job, relativeUrl) {
-  if (!job.browsers) {
-    job.browsers = {}
-  }
-  output.browserStart(relativeUrl)
-  const reportDir = join(job.tstReportDir, filename(relativeUrl))
-  const args = job.args.split(' ')
-    .map(arg => arg
-      .replace('__URL__', `http://localhost:${job.port}${relativeUrl}`)
-      .replace('__REPORT__', reportDir)
-    )
+async function instantiate (job, config) {
+  const { dir, url } = config
+  await recreateDir(dir)
+  const browserConfig = {
+    capabilities: job.browserCapabilities,
+    modules: job.browserModules,
+    ...config,
+    args: job.browserArgs
+  }
+  const browserConfigPath = join(dir, 'browser.json')
+  await writeFile(browserConfigPath, JSON.stringify(browserConfig, undefined, 2))
+  const stdoutFilename = join(dir, 'stdout.txt')
+  const stderrFilename = join(dir, 'stderr.txt')
+  const stdout = await open(stdoutFilename, 'w')
+  const stderr = await open(stderrFilename, 'w')
+  const childProcess = fork(job.browser, [browserConfigPath], {
+    stdio: [0, stdout, stderr, 'ipc']
+  })
+  const { promise, resolve } = allocPromise()
+  childProcess.on('close', async code => {
+    await stdout.close()
+    await stderr.close()
+    if (code !== 0) {
+      getOutput(job).browserFailed(url, code, dir)
+    }
+    resolve(code)
+  })
+  childProcess.closed = promise
+  childProcess.stdoutFilename = stdoutFilename
+  childProcess.stderrFilename = stderrFilename
+  return childProcess
+}
+
+async function probe (job) {
+  if (job.browserCapabilities) {
+    return
+  }
+  const output = getOutput(job)
+  job.status = 'Probing browser instantiation command'
+
+  async function execute (folder) {
+    const dir = join(job.reportDir, folder)
+    const capabilities = join(dir, 'capabilities.json')
+    const childProcess = await instantiate(job, {
+      url: 'about:blank',
+      capabilities,
+      dir
+    })
+    const code = await childProcess.closed
+    if (code !== 0) {
+      throw UTRError.BROWSER_PROBE_FAILED(code.toString())
+    }
+    let browserCapabilities
+    try {
+      browserCapabilities = Object.assign({
+        modules: [],
+        screenshot: null,
+        scripts: false,
+        parallel: true,
+        traces: []
+      }, JSON.parse((await readFile(capabilities)).toString()))
+    } catch (e) {
+      throw UTRError.MISSING_OR_INVALID_BROWSER_CAPABILITIES(e.message)
+    }
+    return browserCapabilities
+  }
+
+  const browserCapabilities = await execute('probe')
+  job.browserCapabilities = browserCapabilities
+
+  const { modules } = browserCapabilities
+  const resolvedModules = {}
+  if (modules.length) {
+    for await (const name of browserCapabilities.modules) {
+      resolvedModules[name] = await resolvePackage(job, name)
+    }
+  }
+  job.browserModules = resolvedModules
+  if (browserCapabilities['probe-with-modules']) {
+    job.browserCapabilities = await execute('probe/with-modules')
+  }
+
+  output.browserCapabilities(job.browserCapabilities)
+}
+
+async function start (job, url, scripts = []) {
+  const output = getOutput(job)
+  if (!job[$browsers]) {
+    job[$browsers] = {}
+  }
+  output.browserStart(url)
+  const reportDir = join(job.reportDir, filename(url))
+  const resolvedScripts = []
+  for await (const script of scripts) {
+    if (script.endsWith('.js')) {
+      const scriptFilename = join(__dirname, 'inject', script)
+      const scriptContent = (await readFile(scriptFilename)).toString()
+      resolvedScripts.push(scriptContent)
+    } else {
+      resolvedScripts.push(script)
+    }
+  }
+  if (resolvedScripts.length) {
+    resolvedScripts.unshift(`window['ui5-test-runner/base-host'] = 'http://localhost:${job.port}'
+`)
+  }
   const pageBrowser = {
-    relativeUrl,
+    url,
     reportDir,
-    args,
+    scripts: resolvedScripts,
     retry: 0
   }
-  const promise = new Promise(resolve => {
-    pageBrowser.done = resolve
-  })
-  job.browsers[relativeUrl] = pageBrowser
-  run(job, pageBrowser)
-  return promise.then(() => {
-    output.browserStopped(relativeUrl)
-  })
+  const { promise, resolve, reject } = allocPromise()
+  pageBrowser.done = value => {
+    delete job[$browsers][url]
+    resolve(value)
+  }
+  pageBrowser.failed = reason => {
+    delete job[$browsers][url]
+    reject(reason)
+  }
+  job[$browsers][url] = pageBrowser
+  await run(job, pageBrowser)
+  await promise
+  output.browserStopped(url)
 }
 
 async function run (job, pageBrowser) {
-  const { relativeUrl } = pageBrowser
-  if (pageBrowser.retry) {
-    output.browserRetry(relativeUrl, pageBrowser.retry)
+  const output = getOutput(job)
+  const { url, retry, reportDir, scripts } = pageBrowser
+  let dir = reportDir
+  if (retry) {
+    output.browserRetry(url, retry)
+    dir = join(dir, retry.toString())
+    if (pageBrowser.console.count) {
+      try {
+        await pageBrowser.console.flush
+          .then(() => unlink(join(reportDir, 'console.jsonl')))
+      } catch (e) {
+        // ignore
+      }
+    }
   }
-  await recreateDir(pageBrowser.reportDir)
+  pageBrowser.console = {
+    count: 0,
+    byApi: {},
+    flush: Promise.resolve()
+  }
+  await recreateDir(dir)
   delete pageBrowser.stopped
-  const childProcess = fork(job.browser, pageBrowser.args.map(arg => arg.replace('__RETRY__', pageBrowser.retry)), { stdio: 'pipe' })
-  output.monitor(childProcess)
+  const childProcess = await instantiate(job, {
+    url,
+    retry,
+    scripts,
+    dir
+  })
   pageBrowser.childProcess = childProcess
   const timeout = getPageTimeout(job)
   if (timeout) {
     pageBrowser.timeoutId = setTimeout(() => {
-      output.browserTimeout(relativeUrl)
-      stop(job, relativeUrl)
+      output.browserTimeout(url, dir)
+      stop(job, url)
     }, timeout)
   }
   childProcess.on('message', message => {
@@ -58,63 +182,100 @@ async function run (job, pageBrowser) {
       const { id } = message
       screenshots[id]()
       delete screenshots[id]
-    } else /* istanbul ignore else */ if (message.command === 'capabilities') {
-      job.browserCapabilities = { ...message }
-      delete job.browserCapabilities.command
-      output.browserCapabilities(job.browserCapabilities)
+    } else if (message.command === 'console') {
+      ++pageBrowser.console.count
+      if (!pageBrowser.console.byApi[message.api]) {
+        pageBrowser.console.byApi[message.api] = 1
+      } else {
+        ++pageBrowser.console.byApi[message.api]
+      }
+      pageBrowser.console.flush = pageBrowser.console.flush
+        .then(() => writeFile(join(reportDir, 'console.jsonl'), JSON.stringify({
+          t: message.t,
+          api: message.api,
+          args: message.args
+        }) + '\n', {
+          flag: 'a+'
+        }))
     }
   })
-  childProcess.on('close', () => {
+  childProcess.on('close', async code => {
     if (!pageBrowser.stopped) {
-      output.browserClosed(relativeUrl)
-      stop(job, relativeUrl, true)
+      if (code === 0) {
+        output.browserClosed(url, code, dir)
+      }
+      childProcess.closed.then(() => stop(job, url, true))
     }
   })
-  if (!job.browserCapabilities) {
-    childProcess.send({ command: 'capabilities' })
-  }
 }
 
-async function screenshot (job, relativeUrl, filename) {
-  if (job.noScreenshot || !job.browserCapabilities || !job.browserCapabilities.screenshot) {
-    return
+async function screenshot (job, url, filename) {
+  if (!job.browserCapabilities.screenshot) {
+    throw UTRError.BROWSER_SCREENSHOT_NOT_SUPPORTED()
   }
-  const pageBrowser = job.browsers[relativeUrl]
-  if (pageBrowser) {
-    const { childProcess } = pageBrowser
+  try {
+    const { childProcess, reportDir } = job[$browsers][url]
+    const absoluteFilename = join(reportDir, filename + job.browserCapabilities.screenshot)
     if (childProcess.connected) {
       const id = ++lastScreenshotId
-      const promise = new Promise(resolve => {
-        screenshots[id] = resolve
-      })
+      const { promise, resolve, reject } = allocPromise()
+      screenshots[id] = resolve
       childProcess.send({
         id,
         command: 'screenshot',
-        filename
+        filename: absoluteFilename
       })
+      const timeoutId = setTimeout(() => {
+        reject(UTRError.BROWSER_SCREENSHOT_TIMEOUT())
+      }, job.screenshotTimeout)
       await promise
+      clearTimeout(timeoutId)
+      const result = await stat(absoluteFilename)
+      if (!result.isFile() || result.size === 0) {
+        throw new Error('File expected')
+      }
+      return absoluteFilename
     }
+  } catch (e) {
+    if (e.code === UTRError.BROWSER_SCREENSHOT_TIMEOUT_CODE) {
+      throw e
+    }
+    throw UTRError.BROWSER_SCREENSHOT_FAILED(e.toString())
   }
 }
 
-async function stop (job, relativeUrl, retry = false) {
-  const pageBrowser = job.browsers[relativeUrl]
+async function stop (job, url, retry = false) {
+  const pageBrowser = job[$browsers][url]
   if (pageBrowser) {
     pageBrowser.stopped = true
-    const { childProcess, done, timeoutId } = pageBrowser
+    const { childProcess, done, failed, timeoutId } = pageBrowser
     if (timeoutId) {
       clearTimeout(timeoutId)
     }
     if (childProcess.connected) {
-      childProcess.send({ command: 'stop' })
+      /* istanbul ignore else */
+      if (!job.debugKeepBrowserOpen) {
+        childProcess.send({ command: 'stop' })
+      }
+      const { promise: closeTimeout, resolve } = allocPromise()
+      const timeoutId = setTimeout(resolve, job.browserCloseTimeout)
+      await Promise.race([
+        childProcess.closed,
+        closeTimeout
+      ])
+      clearTimeout(timeoutId)
     }
-    if (retry && ++pageBrowser.retry <= job.browserRetry) {
-      run(job, pageBrowser)
+    await pageBrowser.console.flush
+    if (retry) {
+      if (++pageBrowser.retry <= job.browserRetry) {
+        run(job, pageBrowser)
+      } else {
+        failed(UTRError.BROWSER_FAILED())
+      }
     } else {
-      delete job.browsers[relativeUrl]
       done()
     }
   }
 }
 
-module.exports = { start, screenshot, stop }
+module.exports = { probe, start, screenshot, stop }

package/src/capabilities/index.js

@@ -0,0 +1,194 @@
+'use strict'
+
+const { check, serve, body } = require('reserve')
+const { probe, start, stop } = require('../browsers')
+const { join } = require('path')
+const { getOutput } = require('../output')
+const EventEmitter = require('events')
+const { performance } = require('perf_hooks')
+const { cleanDir, allocPromise, filename } = require('../tools')
+const { $browsers } = require('../symbols')
+const tests = require('./tests')
+
+async function capabilities (job) {
+  const output = getOutput(job)
+
+  async function exit (code) {
+    if (job.keepAlive) {
+      job.status = 'Serving'
+    } else {
+      if (job.debugKeepReport) {
+        output.wrap(() => console.log('Report folder', job.reportDir, 'not cleaned because of --debug-keep-report.'))
+      } else if (code !== 0) {
+        output.wrap(() => console.error('Report folder', job.reportDir, 'not cleaned because of errors.'))
+      } else {
+        await cleanDir(job.reportDir)
+      }
+      output.stop()
+      process.exit(code)
+    }
+  }
+
+  try {
+    output.reportOnJobProgress()
+    try {
+      await probe(job)
+    } catch (e) {
+      output.wrap(() => console.error('Unable to probe'))
+      exit(-1)
+    }
+
+    const listeners = []
+
+    const configuration = await check({
+      port: job.port,
+      mappings: [
+        require('../cors'), {
+          custom: async (request, response) => {
+            response.setHeader('x-ui5-test-runner', 'capabilities')
+          }
+        }, {
+          match: /x-wait-time=(\d+)/,
+          custom: async (request, response, waitTime) => {
+            response.setHeader('x-wait-time', waitTime)
+            await new Promise(resolve => setTimeout(resolve, parseInt(waitTime)))
+          }
+        }, {
+          method: 'POST',
+          match: '^/_/(.*)',
+          custom: async (request, response, endpoint) => {
+            const { referer, 'x-page-url': xPageUrl } = request.headers
+            const listenerIndex = (xPageUrl || referer).match(/\blistener=(\d+)/)[1]
+            const listener = listeners[listenerIndex]
+            listener.emit('endpoint', {
+              endpoint,
+              body: JSON.parse(await body(request))
+            })
+            response.writeHead(200)
+            response.end()
+          }
+        }, {
+          match: '^/(.*)',
+          file: join(__dirname, '$1')
+        }]
+    })
+    const server = serve(configuration)
+    if (job.logServer) {
+      server.on('redirected', output.redirected)
+    }
+    await new Promise(resolve => server
+      .on('ready', ({ port }) => {
+        job.port = port
+        resolve()
+      })
+    )
+
+    job.status = 'Running tests'
+
+    const filteredTests = tests
+      .filter((test) => !test.for || test.for(job.browserCapabilities))
+      .filter(({ name }) => !job.debugCapabilitiesTest || name.startsWith(job.debugCapabilitiesTest))
+    output.wrap(() => console.log('Number of tests :', filteredTests.length))
+
+    let errors = 0
+
+    const { promise: forEver } = allocPromise()
+
+    const next = async () => {
+      if (filteredTests.length === 0) {
+        if (!job[$browsers] || Object.keys(job[$browsers]).length === 0) {
+          output.wrap(() => console.log('Done.'))
+          exit(errors)
+        }
+        return
+      }
+      const { label, url, scripts, endpoint = () => { } } = filteredTests.shift()
+
+      const listenerIndex = listeners.length
+      const listener = new EventEmitter()
+      listeners.push(listener)
+      let pageUrl
+      if (url.startsWith('http')) {
+        pageUrl = url
+      } else {
+        pageUrl = `http://localhost:${job.port}/tests/${url}`
+      }
+      if (url.includes('?')) {
+        pageUrl += `&listener=${listenerIndex}`
+      } else {
+        pageUrl += `?listener=${listenerIndex}`
+      }
+
+      const now = performance.now()
+      let timeoutId
+      if (!job.debugCapabilitiesNoTimeout) {
+        const timeout = job.pageTimeout || 120000
+        timeoutId = setTimeout(() => done(`Timeout (${timeout})`), timeout)
+      }
+
+      async function done (error) {
+        if (done.called) {
+          return
+        }
+        done.called = true
+        if (timeoutId) {
+          clearTimeout(timeoutId)
+        }
+        await stop(job, pageUrl)
+        const timeSpent = Math.floor(performance.now() - now)
+        if (error) {
+          output.wrap(() => console.log('❌', label, `[${filename(pageUrl)}]`, error))
+          if (job.failFast) {
+            exit(1)
+          }
+          ++errors
+        } else {
+          output.wrap(() => console.log('✔️', label, timeSpent, 'ms'))
+        }
+        await next()
+      }
+
+      const context = {
+        job
+      }
+      listener.on('endpoint', async data => {
+        try {
+          if (await endpoint.call(context, data, pageUrl) !== false) {
+            done()
+          }
+        } catch (e) {
+          done(e)
+        }
+      })
+
+      start(job, pageUrl, scripts)
+        .catch(reason => done(reason))
+        .then(() => {
+          if (!context.stopExpected) {
+            done('Failed')
+          }
+        })
+    }
+
+    let parallel
+    if (!job.browserCapabilities.parallel) {
+      parallel = 1
+    } else {
+      parallel = job.parallel
+    }
+
+    const testsCount = filteredTests.length
+    for (let i = 0; i < Math.max(Math.min(parallel, testsCount), 1); ++i) {
+      next()
+    }
+
+    await forEver
+  } catch (error) {
+    output.wrap(() => console.error(error))
+    exit(-1)
+  }
+}
+
+module.exports = {
+  capabilities
+}

package/src/capabilities/tests/basic/iframe.html

@@ -0,0 +1,8 @@
+<!DOCTYPE html>
+<html>
+  <body>
+    <H1>iframe</H1>
+    <p>Checks if the browser can load UI5 in an iframe</p>
+    <iframe src="ui5.html"></iframe>
+  </body>
+</html>

package/src/capabilities/tests/basic/index.html

@@ -0,0 +1,12 @@
+<!DOCTYPE html>
+<html>
+  <body>
+    <H1>basic</H1>
+    <p>Checks if the browser can load a page</p>
+    <script>
+      const xhr = new XMLHttpRequest()
+      xhr.open('POST', '/_/log')
+      xhr.send('{}')
+    </script>
+  </body>
+</html>

package/src/capabilities/tests/basic/index.js

@@ -0,0 +1,20 @@
+'use strict'
+
+const assert = require('assert')
+
+module.exports = [{
+  label: 'Loads a page',
+  url: 'basic/index.html'
+}, {
+  label: 'Loads a UI5 example',
+  url: 'basic/ui5.html',
+  endpoint: ({ body }) => {
+    assert.strictEqual(body['sap.m.Button'], true)
+  }
+}, {
+  label: 'Loads a UI5 inside an iframe',
+  url: 'basic/iframe.html',
+  endpoint: ({ body }) => {
+    assert.strictEqual(body['sap.m.Button'], true)
+  }
+}]

package/src/capabilities/tests/basic/ui5.html

@@ -0,0 +1,24 @@
+<!DOCTYPE html>
+<html>
+  <body>
+    <H1>UI5 example</H1>
+    <p>Checks if the browser can load a page</p>
+    <script
+        src="https://ui5.sap.com/resources/sap-ui-core.js"
+        id="sap-ui-bootstrap"
+        data-sap-ui-libs="sap.m"
+        data-sap-ui-theme="sap_fiori_3"
+        data-sap-ui-compatVersion="edge">
+    </script>
+    <script>
+        sap.ui.getCore().attachInit(function() {
+            const xhr = new XMLHttpRequest()
+            xhr.open('POST', '/_/log')
+            if (parent !== window) {
+              xhr.setRequestHeader('x-page-url', parent.document.location.toString())
+            }
+            xhr.send('{"sap.m.Button":' + !!sap.m.Button + '}')
+        });
+    </script>
+  </body>
+</html>

package/src/capabilities/tests/dynamic-include/index.js

@@ -0,0 +1,21 @@
+'use strict'
+
+const assert = require('assert')
+
+const endpoint = ({ body }) => {
+  assert.strictEqual(body.test, true)
+}
+
+module.exports = [{
+  label: 'Dynamic include',
+  url: 'dynamic-include/one.html',
+  endpoint
+}, {
+  label: 'Dynamic includes',
+  url: 'dynamic-include/two.html',
+  endpoint
+}, {
+  label: 'Dynamic include mixed with a static one',
+  url: 'dynamic-include/mix.html',
+  endpoint
+}]

package/src/capabilities/tests/dynamic-include/mix.html

@@ -0,0 +1,11 @@
+<!DOCTYPE html>
+<html>
+  <body>
+    <H1>basic</H1>
+    <p>Checks if the browser can mix static script with a dynamic script inserted with document.write</p>
+    <script>
+      document.write('<script src="test.js?x-wait-time=1000"><' + '/script>'); // wait 1s before delivering
+    </script>
+    <script src="post.js"></script>
+  </body>
+</html>

package/src/capabilities/tests/dynamic-include/one.html

@@ -0,0 +1,11 @@
+<!DOCTYPE html>
+<html>
+  <body>
+    <H1>basic</H1>
+    <p>Checks if the browser can load a script inserted with document.write</p>
+    <script>
+      window.test = true
+      document.write('<script src="post.js"><' + '/script>');
+    </script>
+  </body>
+</html>

package/src/capabilities/tests/dynamic-include/post.js

@@ -0,0 +1,3 @@
+const xhr = new XMLHttpRequest()
+xhr.open('POST', '/_/log')
+xhr.send('{"test":' + window.test + '}')

package/src/capabilities/tests/dynamic-include/test.js

@@ -0,0 +1 @@
+window.test = true

package/src/capabilities/tests/dynamic-include/two.html

@@ -0,0 +1,11 @@
+<!DOCTYPE html>
+<html>
+  <body>
+    <H1>basic</H1>
+    <p>Checks if the browser can load sequentially scripts inserted with document.write</p>
+    <script>
+      document.write('<script src="test.js?x-wait-time=1000"><' + '/script>'); // wait 1s before delivering
+      document.write('<script src="post.js"><' + '/script>');
+    </script>
+  </body>
+</html>

package/src/capabilities/tests/index.js

@@ -0,0 +1,16 @@
+const { readdirSync } = require('fs')
+
+module.exports = readdirSync(__dirname)
+  .filter(name => !name.endsWith('.js'))
+  .map(name => {
+    const tests = require(`./${name}`)
+    if (Array.isArray(tests)) {
+      tests.forEach((test, index) => {
+        test.name = `${name}:${index}`
+      })
+    } else {
+      tests.name = name
+    }
+    return tests
+  })
+  .flat()