ui5-test-runner 6.0.0-beta.5 → 6.0.0

package/dist/configuration/options.js

@@ -4,6 +4,7 @@ export const options = [
         short: 'c',
         type: 'fs-entry',
         description: 'set working directory',
+        defaultLabel: 'current working directory',
         default: process.cwd()
     },
     {
@@ -19,26 +20,131 @@ export const options = [
         description: 'path of the testsuite file',
         default: 'test/testsuite.qunit.html'
     },
+    {
+        name: 'agentDetectionInterval',
+        type: 'timeout',
+        browserExposed: true,
+        description: 'initial polling interval when detecting a test framework after page load',
+        default: 100
+    },
+    {
+        name: 'agentDetectionMaxInterval',
+        type: 'timeout',
+        browserExposed: true,
+        description: 'maximum polling interval when detecting a test framework after page load',
+        default: 1000
+    },
+    {
+        name: 'agentDetectionTimeout',
+        type: 'timeout',
+        browserExposed: true,
+        description: 'maximum time to wait for a test framework to be detected after page load',
+        default: 5000
+    },
+    {
+        name: 'agentNoTestsTimeout',
+        type: 'timeout',
+        browserExposed: true,
+        description: 'time to wait after QUnit.done fires with no tests before declaring the page done',
+        default: 5000
+    },
+    {
+        name: 'agentScreenshotTimeout',
+        type: 'timeout',
+        browserExposed: true,
+        description: 'maximum time OPA waits for a pending screenshot before continuing',
+        default: 10_000
+    },
     {
         name: 'alternateNpmPath',
         type: 'fs-entry',
-        description: 'alternate NPM package path'
+        batchForwarded: true,
+        description: 'additional path the runner searches for npm packages after the local and global locations'
+    },
+    {
+        name: 'batch',
+        type: 'string',
+        multiple: true,
+        description: 'batch item specification (folder, config file, or regex pattern)'
+    },
+    {
+        name: 'batchId',
+        type: 'string',
+        description: 'identifier for the batch item'
+    },
+    {
+        name: 'batchLabel',
+        type: 'string',
+        description: 'display label for the batch item'
+    },
+    {
+        name: 'batchOutput',
+        type: 'boolean',
+        description: 'print output.txt for every batch item after execution, in alphabetical batch-id order'
+    },
+    {
+        name: 'batchTimeout',
+        short: 'bt',
+        type: 'timeout',
+        description: 'fails a batch item if it takes longer than this timeout'
     },
     {
         name: 'browser',
         short: 'b',
-        type: 'browser',
-        description: 'browser selection',
-        default: 'puppeteer'
+        type: 'enumeration',
+        typeModifiers: new Set(['chrome', 'chromium', 'firefox', 'webkit', 'edge', 'safari']),
+        batchForwarded: true,
+        description: 'browser selection (per driver)'
+    },
+    {
+        name: 'browserBasicAuthPassword',
+        type: 'secret',
+        description: 'password for HTTP basic authentication'
+    },
+    {
+        name: 'browserBasicAuthUsername',
+        type: 'string',
+        description: 'username for HTTP basic authentication'
+    },
+    {
+        name: 'browserDevice',
+        type: 'string',
+        batchForwarded: true,
+        description: 'emulate a Puppeteer-known device — sets the viewport, user agent, and touch support, overriding browser viewport width and height'
+    },
+    {
+        name: 'browserOptions',
+        type: 'json',
+        batchForwarded: true,
+        description: 'browser-specific options (JSON)',
+        default: {}
+    },
+    {
+        name: 'browserViewportHeight',
+        short: 'H',
+        type: 'integer',
+        typeModifiers: new Set(['positive', 'non-zero']),
+        description: 'height of the browser viewport in pixels',
+        default: 1080
     },
     {
-        name: 'capabilities',
+        name: 'browserViewportWidth',
+        short: 'W',
+        type: 'integer',
+        typeModifiers: new Set(['positive', 'non-zero']),
+        description: 'width of the browser viewport in pixels',
+        default: 1920
+    },
+    {
+        name: 'browserVisible',
+        short: 'V',
         type: 'boolean',
-        description: 'run browser tests'
+        description: 'control if the browser should be visible during the tests'
     },
     {
         name: 'ci',
         type: 'boolean',
+        batchForwarded: true,
         description: 'forces CI mode (no interactive output)',
         default: !process.stdout.isTTY
     },
@@ -49,21 +155,137 @@ export const options = [
         description: 'read options from a configuration file',
         default: 'ui5-test-runner.json'
     },
+    {
+        name: 'coverage',
+        type: 'boolean',
+        description: 'enable code coverage'
+    },
+    {
+        name: 'coverageCheckBranches',
+        short: 'ccb',
+        type: 'percent',
+        description: 'minimum branch coverage threshold (0 = no check)',
+        default: 0
+    },
+    {
+        name: 'coverageCheckFunctions',
+        short: 'ccf',
+        type: 'percent',
+        description: 'minimum function coverage threshold (0 = no check)',
+        default: 0
+    },
+    {
+        name: 'coverageCheckLines',
+        short: 'ccl',
+        type: 'percent',
+        description: 'minimum line coverage threshold (0 = no check)',
+        default: 0
+    },
+    {
+        name: 'coverageCheckStatements',
+        short: 'ccs',
+        type: 'percent',
+        description: 'minimum statement coverage threshold (0 = no check)',
+        default: 0
+    },
+    {
+        name: 'coverageRemoteInstrument',
+        type: 'regexp',
+        multiple: true,
+        browserExposed: true,
+        description: 'resources sub-paths to instrument for coverage in remote mode'
+    },
+    {
+        name: 'coverageReportDir',
+        short: 'crd',
+        type: 'fs-entry',
+        typeModifiers: new Set(['overwrite']),
+        description: 'directory for the final coverage report',
+        default: 'coverage'
+    },
+    {
+        name: 'coverageReporters',
+        short: 'cr',
+        type: 'string',
+        multiple: true,
+        description: 'istanbul-lib-report reporters to use',
+        default: ['lcov', 'cobertura']
+    },
+    {
+        name: 'coverageSettings',
+        short: 'cs',
+        type: 'fs-entry',
+        typeModifiers: new Set(['file', 'safe-default']),
+        description: 'path to the Istanbul configuration file (.nycrc.json)',
+        default: '.nycrc.json'
+    },
+    {
+        name: 'coverageSourceDir',
+        type: 'fs-entry',
+        typeModifiers: new Set(['safe-default']),
+        description: 'directory containing the source files used for coverage reporting'
+    },
+    {
+        name: 'coverageTempDir',
+        short: 'ctd',
+        type: 'fs-entry',
+        typeModifiers: new Set(['overwrite']),
+        description: 'temporary directory for coverage data',
+        default: '.nyc_output'
+    },
     {
         name: 'debugKeepBrowserOpen',
         type: 'boolean',
         description: 'keeps the browser open after the tests completed'
     },
+    {
+        name: 'debugLog',
+        type: 'string',
+        multiple: true,
+        description: 'output debug logs for given sources'
+    },
+    {
+        name: 'debugMcpLocalDocs',
+        type: 'boolean',
+        description: 'use local docs/ directory instead of fetching from GitHub (development only)'
+    },
+    {
+        name: 'driver',
+        short: 'd',
+        type: 'enumeration',
+        typeModifiers: new Set(['puppeteer', 'playwright', 'webdriverio', 'selenium-webdriver']),
+        browserExposed: true,
+        batchForwarded: true,
+        description: 'driver selection',
+        default: 'puppeteer'
+    },
+    {
+        name: 'dumpConfig',
+        type: 'boolean',
+        description: 'dump the resolved configuration as JSON and exit'
+    },
+    {
+        name: 'end',
+        type: 'string',
+        description: 'command to be executed after the tests'
+    },
+    {
+        name: 'endTimeout',
+        type: 'timeout',
+        description: 'maximum waiting time for the end command to execute'
+    },
     {
         name: 'failFast',
         short: 'f',
         type: 'boolean',
+        batchForwarded: true,
         description: 'stop the whole execution after the first failing page'
     },
     {
         name: 'failOpaFast',
         short: 'fo',
         type: 'boolean',
+        batchForwarded: true,
         description: 'stop the OPA page execution after the first failing test'
     },
     {
@@ -75,43 +297,84 @@ export const options = [
     {
         name: 'help',
         type: 'boolean',
-        description: 'display help'
+        description: 'display the list of available options and exit'
     },
     {
-        name: 'keepAlive',
-        short: 'k',
-        type: 'boolean',
-        description: 'keep the server alive'
+        name: 'if',
+        type: 'string',
+        description: 'skip execution if the expression evaluates to falsy'
     },
     {
-        name: 'localhost',
+        name: 'junitXmlFilename',
         type: 'string',
-        description: 'hostname for legacy URL',
-        default: 'localhost'
+        description: 'filename for the JUnit XML report saved in the report directory',
+        default: 'junit.xml'
+    },
+    {
+        name: 'lib',
+        type: 'library-mapping',
+        multiple: true,
+        description: 'map a UI5 resources sub-folder to a local source folder, used in legacy mode to serve custom libraries'
+    },
+    {
+        name: 'llm',
+        type: 'boolean',
+        batchForwarded: true,
+        description: 'forces LLM-friendly output (compact, no color, severity-tagged)'
     },
     {
         name: 'log',
         type: 'fs-entry',
         typeModifiers: new Set(['file']),
-        description: 'read and dump log file using jsonl format'
+        description: 'path to a .logz trace file produced by a previous run; activates log viewer mode'
+    },
+    {
+        name: 'logDump',
+        type: 'boolean',
+        description: 'dump all traces to stdout instead of opening a browser (requires --log)'
+    },
+    {
+        name: 'logFilter',
+        short: 'lf',
+        type: 'string',
+        description: 'JavaScript expression (using punyexpr) to filter logs for dumping with --log-dump'
+    },
+    {
+        name: 'mcp',
+        type: 'boolean',
+        description: 'start a documentation-only MCP server that lets an MCP client answer questions about ui5-test-runner'
     },
     {
-        name: 'logServer',
-        short: 'l',
+        name: 'noBanner',
         type: 'boolean',
-        description: 'log inner server traces'
+        description: 'do not display the startup banner (logo) and environment information'
     },
     {
         name: 'noNpmInstall',
         type: 'boolean',
+        batchForwarded: true,
         description: 'prevent any NPM install'
     },
+    {
+        name: 'npmAllowInstallScripts',
+        type: 'boolean',
+        batchForwarded: true,
+        description: 'allow postinstall scripts when installing missing packages'
+    },
     {
         name: 'npmInstall',
         type: 'string',
         description: 'npm install strategy for missing packages',
         default: 'global'
     },
+    {
+        name: 'npmInstallMinReleaseAge',
+        type: 'integer',
+        typeModifiers: new Set(['positive']),
+        batchForwarded: true,
+        description: 'minimum release age (in days) required before installing a package',
+        default: 3
+    },
     {
         name: 'npmInstallPrefix',
         type: 'fs-entry',
@@ -121,6 +384,7 @@ export const options = [
         name: 'outputInterval',
         short: 'oi',
         type: 'timeout',
+        batchForwarded: true,
         description: 'interval for reporting progress on non interactive output (CI/CD)',
         default: 30_000
     },
@@ -128,31 +392,38 @@ export const options = [
         name: 'pageFilter',
         short: 'pf',
         type: 'regexp',
+        batchForwarded: true,
         description: 'filter pages to execute'
     },
     {
         name: 'pageParams',
         short: 'pp',
         type: 'string',
+        batchForwarded: true,
         description: 'add parameters to page URL'
     },
     {
         name: 'pageTimeout',
         short: 'pt',
         type: 'timeout',
+        batchForwarded: true,
         description: 'fails a page if it takes longer than this timeout'
     },
     {
         name: 'parallel',
         short: 'p',
         type: 'integer',
+        typeModifiers: new Set(['positive', 'non-zero']),
+        browserExposed: true,
+        batchForwarded: true,
         description: 'number of parallel executions',
         default: 2
     },
     {
         name: 'port',
         type: 'integer',
-        description: 'port to use'
+        typeModifiers: new Set(['positive']),
+        description: 'Port the local web server listens on; 0 allocates a free port automatically'
     },
     {
         name: 'reportDir',
@@ -162,16 +433,74 @@ export const options = [
         description: 'directory to output test reports',
         default: 'report'
     },
+    {
+        name: 'screenshot',
+        type: 'boolean',
+        browserExposed: true,
+        batchForwarded: true,
+        description: 'take a screenshot after every OPA assertion'
+    },
+    {
+        name: 'screenshotOnFailure',
+        type: 'boolean',
+        batchForwarded: true,
+        description: 'take a screenshot when a test fails',
+        default: true
+    },
+    {
+        name: 'screenshotTimeout',
+        type: 'timeout',
+        batchForwarded: true,
+        description: 'maximum time allowed to take a screenshot',
+        default: 5000
+    },
+    {
+        name: 'seleniumApiTimeout',
+        type: 'timeout',
+        description: 'fails a page if a selenium webdriver api call does not respond within this timeout, indicating a dead session',
+        default: 10_000
+    },
     {
         name: 'serveOnly',
         short: 's',
         type: 'boolean',
-        description: 'serve only'
+        description: 'start the legacy web server and serve the application without running any tests'
+    },
+    {
+        name: 'splitOpa',
+        short: 'so',
+        type: 'boolean',
+        browserExposed: true,
+        batchForwarded: true,
+        description: 'split OPA tests by QUnit module, creating one test page per module'
+    },
+    {
+        name: 'start',
+        type: 'string',
+        description: 'command to be executed before the tests'
+    },
+    {
+        name: 'startTimeout',
+        type: 'timeout',
+        description: 'maximum waiting time for the start command to become ready',
+        default: 30_000
+    },
+    {
+        name: 'startWaitMethod',
+        type: 'string',
+        description: 'HTTP method used when polling the startWaitUrl',
+        default: 'GET'
+    },
+    {
+        name: 'startWaitUrl',
+        type: 'url',
+        description: 'URL to poll after the start command is executed'
     },
     {
         name: 'ui5',
         type: 'url',
-        description: 'UI5 url',
+        batchForwarded: true,
+        description: 'URL of the UI5 CDN used in legacy mode to serve the UI5 framework resources',
         default: 'https://ui5.sap.com'
     },
     {
@@ -185,19 +514,47 @@ export const options = [
         name: 'version',
         type: 'boolean',
         description: 'display version'
+    },
+    {
+        name: 'watch',
+        type: 'boolean',
+        description: 'watch for file changes and automatically reload'
     }
 ];
 export const defaults = {
-    browser: 'puppeteer',
+    agentDetectionInterval: 100,
+    agentDetectionMaxInterval: 1000,
+    agentDetectionTimeout: 5000,
+    agentNoTestsTimeout: 5000,
+    agentScreenshotTimeout: 10_000,
+    browserOptions: {},
+    browserViewportHeight: 1080,
+    browserViewportWidth: 1920,
     ci: !process.stdout.isTTY,
     config: 'ui5-test-runner.json',
+    coverageCheckBranches: 0,
+    coverageCheckFunctions: 0,
+    coverageCheckLines: 0,
+    coverageCheckStatements: 0,
+    coverageReportDir: 'coverage',
+    coverageReporters: ['lcov', 'cobertura'],
+    coverageSettings: '.nycrc.json',
+    coverageTempDir: '.nyc_output',
     cwd: process.cwd(),
-    localhost: 'localhost',
+    driver: 'puppeteer',
+    junitXmlFilename: 'junit.xml',
     npmInstall: 'global',
+    npmInstallMinReleaseAge: 3,
     outputInterval: 30_000,
     parallel: 2,
     reportDir: 'report',
+    screenshotOnFailure: true,
+    screenshotTimeout: 5000,
+    seleniumApiTimeout: 10_000,
+    startTimeout: 30_000,
+    startWaitMethod: 'GET',
     testsuite: 'test/testsuite.qunit.html',
     ui5: 'https://ui5.sap.com',
     webapp: 'webapp'
 };
+export const sensitiveKeys = new Set(['browserBasicAuthPassword']);

package/dist/configuration/validations.js

@@ -0,0 +1,135 @@
+import { punyexpr } from 'punyexpr';
+import { indexedOptions } from './indexedOptions.js';
+import { OptionValidationError } from './OptionValidationError.js';
+export const validations = [
+    (configuration) => {
+        if (Object.hasOwn(configuration, 'browser') &&
+            !punyexpr("(driver !== 'puppeteer' || browser === 'chrome' || browser === 'firefox')")(configuration)) {
+            throw OptionValidationError.createValidationError(indexedOptions.browser, 'browser is not supported by puppeteer (chrome, firefox)');
+        }
+    },
+    (configuration) => {
+        if (Object.hasOwn(configuration, 'browser') &&
+            !punyexpr("(driver !== 'playwright' || browser === 'chromium' || browser === 'firefox' || browser === 'webkit')")(configuration)) {
+            throw OptionValidationError.createValidationError(indexedOptions.browser, 'browser is not supported by playwright (chromium, firefox, webkit)');
+        }
+    },
+    (configuration) => {
+        if (Object.hasOwn(configuration, 'browser') &&
+            !punyexpr("(driver !== 'webdriverio' || browser === 'chrome' || browser === 'firefox' || browser === 'edge' || browser === 'safari')")(configuration)) {
+            throw OptionValidationError.createValidationError(indexedOptions.browser, 'browser is not supported by webdriverio (chrome, firefox, edge, safari)');
+        }
+    },
+    (configuration) => {
+        if (Object.hasOwn(configuration, 'browser') &&
+            !punyexpr("(driver !== 'selenium-webdriver' || browser === 'chrome' || browser === 'firefox' || browser === 'edge' || browser === 'safari')")(configuration)) {
+            throw OptionValidationError.createValidationError(indexedOptions.browser, 'browser is not supported by selenium-webdriver (chrome, firefox, edge, safari)');
+        }
+    },
+    (configuration) => {
+        if (Object.hasOwn(configuration, 'browserBasicAuthPassword') &&
+            !punyexpr("(driver === 'puppeteer' || driver === 'playwright')")(configuration)) {
+            throw OptionValidationError.createValidationError(indexedOptions.browserBasicAuthPassword, 'browserBasicAuthPassword is only supported with puppeteer and playwright drivers');
+        }
+    },
+    (configuration) => {
+        if (Object.hasOwn(configuration, 'browserBasicAuthPassword') &&
+            !punyexpr('(browserBasicAuthUsername !== undefined)')(configuration)) {
+            throw OptionValidationError.createValidationError(indexedOptions.browserBasicAuthPassword, 'browserBasicAuthPassword requires browserBasicAuthUsername');
+        }
+    },
+    (configuration) => {
+        if (Object.hasOwn(configuration, 'browserBasicAuthUsername') &&
+            !punyexpr("(driver === 'puppeteer' || driver === 'playwright')")(configuration)) {
+            throw OptionValidationError.createValidationError(indexedOptions.browserBasicAuthUsername, 'browserBasicAuthUsername is only supported with puppeteer and playwright drivers');
+        }
+    },
+    (configuration) => {
+        if (Object.hasOwn(configuration, 'browserBasicAuthUsername') &&
+            !punyexpr('(browserBasicAuthPassword !== undefined)')(configuration)) {
+            throw OptionValidationError.createValidationError(indexedOptions.browserBasicAuthUsername, 'browserBasicAuthUsername requires browserBasicAuthPassword');
+        }
+    },
+    (configuration) => {
+        if (Object.hasOwn(configuration, 'browserDevice') && !punyexpr("(driver === 'puppeteer')")(configuration)) {
+            throw OptionValidationError.createValidationError(indexedOptions.browserDevice, 'browserDevice is only supported with the puppeteer driver');
+        }
+    },
+    (configuration) => {
+        if (Object.hasOwn(configuration, 'browserDevice') &&
+            !punyexpr("(browser === undefined || browser === 'chrome')")(configuration)) {
+            throw OptionValidationError.createValidationError(indexedOptions.browserDevice, 'browserDevice requires the chrome browser');
+        }
+    },
+    (configuration) => {
+        if (Object.hasOwn(configuration, 'coverage') &&
+            !punyexpr("(!coverage || webapp !== '' || coverageSourceDir !== '')")(configuration)) {
+            throw OptionValidationError.createValidationError(indexedOptions.coverage, 'at least one of webapp or coverageSourceDir must be set');
+        }
+    },
+    (configuration) => {
+        if (Object.hasOwn(configuration, 'debugMcpLocalDocs') &&
+            !punyexpr("(!debugMcpLocalDocs || mode === 'mcp')")(configuration)) {
+            throw OptionValidationError.createValidationError(indexedOptions.debugMcpLocalDocs, 'this option requires --mcp');
+        }
+    },
+    (configuration) => {
+        if (Object.hasOwn(configuration, 'dumpConfig') && !punyexpr("(mode === 'dumpConfig')")(configuration)) {
+            throw OptionValidationError.createValidationError(indexedOptions.dumpConfig, 'this option cannot be combined with other mode options');
+        }
+    },
+    (configuration) => {
+        if (Object.hasOwn(configuration, 'help') && !punyexpr("(mode === 'help')")(configuration)) {
+            throw OptionValidationError.createValidationError(indexedOptions.help, 'this option cannot be combined with other mode options');
+        }
+    },
+    (configuration) => {
+        if (Object.hasOwn(configuration, 'log') && !punyexpr("(mode === 'log')")(configuration)) {
+            throw OptionValidationError.createValidationError(indexedOptions.log, 'this option cannot be combined with other mode options');
+        }
+    },
+    (configuration) => {
+        if (Object.hasOwn(configuration, 'logDump') && !punyexpr('(log !== undefined)')(configuration)) {
+            throw OptionValidationError.createValidationError(indexedOptions.logDump, 'requires log');
+        }
+    },
+    (configuration) => {
+        if (Object.hasOwn(configuration, 'logFilter') &&
+            !punyexpr('(log !== undefined) && (logDump !== undefined)')(configuration)) {
+            throw OptionValidationError.createValidationError(indexedOptions.logFilter, 'requires log and logDump');
+        }
+    },
+    (configuration) => {
+        if (Object.hasOwn(configuration, 'mcp') && !punyexpr("(mode === 'mcp')")(configuration)) {
+            throw OptionValidationError.createValidationError(indexedOptions.mcp, 'this option cannot be combined with other mode options');
+        }
+    },
+    (configuration) => {
+        if (Object.hasOwn(configuration, 'startTimeout') &&
+            !punyexpr('(start !== undefined) && (startWaitUrl !== undefined)')(configuration)) {
+            throw OptionValidationError.createValidationError(indexedOptions.startTimeout, 'requires start and startWaitUrl');
+        }
+    },
+    (configuration) => {
+        if (Object.hasOwn(configuration, 'startWaitMethod') &&
+            !punyexpr('(start !== undefined) && (startWaitUrl !== undefined)')(configuration)) {
+            throw OptionValidationError.createValidationError(indexedOptions.startWaitMethod, 'requires start and startWaitUrl');
+        }
+    },
+    (configuration) => {
+        if (Object.hasOwn(configuration, 'startWaitUrl') && !punyexpr('(start !== undefined)')(configuration)) {
+            throw OptionValidationError.createValidationError(indexedOptions.startWaitUrl, 'requires start');
+        }
+    },
+    (configuration) => {
+        if (Object.hasOwn(configuration, 'version') && !punyexpr("(mode === 'version')")(configuration)) {
+            throw OptionValidationError.createValidationError(indexedOptions.version, 'this option cannot be combined with other mode options');
+        }
+    },
+    (configuration) => {
+        if (Object.hasOwn(configuration, 'watch') &&
+            !punyexpr("(!watch || (mode === 'mcp' && debugMcpLocalDocs))")(configuration)) {
+            throw OptionValidationError.createValidationError(indexedOptions.watch, 'this option is valid only for --mcp with --debug-mcp-local-docs');
+        }
+    }
+];