ui5-test-runner 6.0.0-beta.5 → 6.0.1

package/dist/modes/test/pageTask.js

@@ -1,145 +1,340 @@
-import { assert, logger } from '../../platform/index.js';
+import { assert, Http, logger, Process } from '../../platform/index.js';
 import { getAgentSource } from './agent.js';
 import { getBrowser } from './browser.js';
 import { Exit, ExitShutdownError } from '../../platform/Exit.js';
-import { setTimeout } from 'node:timers/promises';
-import { reportBuilder } from './report.js';
+import { getReportBuilder } from './report.js';
+import { createEmptyTestResults } from '../../types/CommonTestReportFormat.js';
+import { getBrowserConfigScript } from './browserConfig.js';
+import { collect as collectCoverage } from './coverage/index.js';
+import { makeScreenshotHandlers } from './screenshot.js';
+import { getEffectiveTimeout, isGloballyTimedOut } from '../../utils/node/timeout.js';
+const getPageTimeout = (configuration, startTime) => getEffectiveTimeout(configuration.pageTimeout, configuration.globalTimeout, startTime);
 let lastPageId = 0;
-const reportQunitProgress = (context, agentState) => {
-    if (agentState.isOpa) {
+export const agentStateMessage = (agentState) => {
+    if (agentState.type === 'suite')
+        return 'agent state: suite done';
+    if (agentState.type === 'QUnit') {
+        const kind = agentState.isOpa ? 'OPA' : 'QUnit';
+        const progress = `${agentState.executed}/${agentState.total}`;
+        return agentState.done ? `agent state: ${kind} done ${progress}` : `agent state: ${kind} ${progress}`;
+    }
+    return agentState.type === 'unknown' ? 'agent state: unknown' : 'agent state: loading';
+};
+const reportQunitProgress = (context, agentState, isScreenshotEnabled) => {
+    if (!isScreenshotEnabled && agentState.isOpa) {
         context.loopDelay = 1000;
     }
-    if (agentState.total > 0) {
-        const { executed, total, errors } = agentState;
-        if (executed !== context.lastExecuted || total !== context.lastTotal) {
-            const type = agentState.isOpa ? 'opa' : 'qunit';
-            context.lastExecuted = executed;
-            context.lastTotal = total;
-            context.errors = errors;
-            context.type = type;
-            logger.info({
-                source: 'progress',
-                message: context.url,
-                pageId: context.pageId,
-                data: { max: agentState.total, value: agentState.executed, type, errors }
+    if (agentState.total <= 0) {
+        return;
+    }
+    const { executed, total, errors } = agentState;
+    if (executed === context.lastExecuted && total === context.lastTotal) {
+        return;
+    }
+    const type = agentState.isOpa ? 'opa' : 'qunit';
+    context.lastExecuted = executed;
+    context.lastTotal = total;
+    context.errors = errors;
+    context.type = type;
+    logger.info({
+        source: 'progress',
+        message: context.url,
+        pageId: context.pageId,
+        data: { max: agentState.total, value: agentState.executed, type, errors }
+    });
+};
+export const reportError = (url, message, error) => {
+    const result = createEmptyTestResults();
+    result.summary.tests = 1;
+    result.summary.failed = 1;
+    result.tests.push({
+        name: url,
+        status: 'failed',
+        message: `${message}, check the logs`,
+        trace: error?.stack,
+        duration: 0
+    });
+    getReportBuilder().merge(url, result);
+};
+const shouldUncaughtErrorsFail = (context, errors) => {
+    const { lastUncaughtErrorsCount, pageId } = context;
+    let errorsCount = 0;
+    let firstError;
+    if (lastUncaughtErrorsCount < errors.length) {
+        let index = lastUncaughtErrorsCount;
+        context.lastUncaughtErrorsCount = errors.length;
+        for (; index < errors.length; ++index) {
+            const error = errors[index];
+            let logMethod = 'error';
+            let message = 'uncaught error';
+            if (error.message === 'Called start() outside of a test context too many times' &&
+                (error.stack?.includes('/sap/ui/thirdparty/qunit-2.js') || error.stack?.includes('/sap/ui/thirdparty/qunit.js'))) {
+                logMethod = 'warn';
+                message = 'uncaught error (expected)';
+            }
+            logger[logMethod]({
+                source: 'page',
+                message,
+                pageId,
+                error
             });
+            if (logMethod !== 'error') {
+                continue;
+            }
+            if (firstError === undefined) {
+                firstError = error;
+            }
+            ++errorsCount;
         }
     }
+    if (errorsCount !== 0) {
+        const errorDetail = `${firstError.name}: ${firstError.message}`;
+        reportError(context.url, `Uncaught error(s): ${errorsCount} (first: ${errorDetail})`, firstError);
+        return true;
+    }
+    return false;
 };
-const queryAgentState = async (context) => {
-    const agentState = (await context.page.eval("window['ui5-test-runner'].state"));
-    logger.debug({ source: 'page', message: 'agent state', data: { state: agentState }, pageId: context.pageId });
+const shouldStopBasedOnAgentState = (context, isScreenshotEnabled, agentState) => {
+    logger.debug({
+        source: 'page',
+        message: agentStateMessage(agentState),
+        data: { state: agentState },
+        pageId: context.pageId
+    });
     if (agentState.done) {
         if (agentState.type === 'suite') {
-            for (const page of agentState.pages) {
-                const pageUrl = new URL(page, context.url).toString();
+            const resolvedPages = agentState.pages.map((page) => new URL(page, context.url).href);
+            for (const pageUrl of resolvedPages) {
                 context.urls.push(pageUrl);
             }
+            context.isSuite = true;
+            logger.debug({
+                source: 'page',
+                message: `suite: dispatching ${resolvedPages.length} pages`,
+                pageId: context.pageId,
+                data: { pages: resolvedPages }
+            });
         }
         else if (agentState.type === 'unknown') {
-            logger.fatal({
+            logger.error({
                 source: 'page',
-                message: 'Unable to detect page type',
+                message: 'unable to detect page type',
                 pageId: context.pageId,
                 data: { state: agentState }
             });
-            throw new Error('Unable to detect page type');
+            assert(false, 'unable to detect page type');
         }
         else {
             assert(agentState.type === 'QUnit');
-            reportQunitProgress(context, agentState);
+            reportQunitProgress(context, agentState, isScreenshotEnabled);
         }
         return true;
     }
     if (agentState.type === 'QUnit') {
-        reportQunitProgress(context, agentState);
+        reportQunitProgress(context, agentState, isScreenshotEnabled);
         if (agentState.uncaughtErrors?.length) {
-            return true;
+            return shouldUncaughtErrorsFail(context, agentState.uncaughtErrors);
         }
     }
     return false;
 };
-export const pageTask = async function (url, index, urls) {
-    const pageId = ++lastPageId;
-    logger.debug({ source: 'page', message: 'new page task', pageId, data: { url } });
-    logger.info({
-        source: 'progress',
-        message: url,
+const tryToFetchThePageFirst = async (url, pageId, configuration) => {
+    try {
+        const headers = {};
+        const { browserBasicAuthUsername, browserBasicAuthPassword } = configuration;
+        if (browserBasicAuthUsername !== undefined && browserBasicAuthPassword !== undefined) {
+            const credentials = Buffer.from(`${browserBasicAuthUsername}:${browserBasicAuthPassword.reveal()}`).toString('base64');
+            headers['Authorization'] = `Basic ${credentials}`;
+        }
+        const response = await Http.fetch(url, { headers });
+        if (!response.ok) {
+            throw new Error(`Unexpected status code ${response.status}`);
+        }
+        return response;
+    }
+    catch (error) {
+        logger.info({
+            source: 'progress',
+            message: url,
+            pageId,
+            data: {
+                max: 0,
+                value: 1,
+                type: 'unknown',
+                errors: 1,
+                remove: true
+            }
+        });
+        reportError(url, 'An error occurred while fetching the URL');
+        throw new Error('An error occurred while fetching the URL', { cause: error });
+    }
+};
+const mergeTestResults = (url, pageId, context, testResults) => {
+    if (context.isSuite) {
+        return;
+    }
+    const { passed, failed, tests, duration } = testResults.summary;
+    const durationString = duration === undefined ? '' : ` (${duration}ms)`;
+    logger.debug({
+        source: 'page',
+        message: `test results: passed=${passed} failed=${failed} tests=${tests}${durationString}`,
         pageId,
-        data: { max: 0, value: 1, type: 'unknown', errors: 0 }
+        data: { results: testResults }
     });
-    const { promise: taskStopped, resolve: setTaskAsStopped } = Promise.withResolvers();
-    using _ = Exit.registerAsyncTask({
-        name: url,
-        stop: async () => {
-            try {
-                this.stop(new ExitShutdownError());
-            }
-            catch {
+    getReportBuilder().merge(url, testResults, { pageId, type: context.type });
+};
+const runPollingLoop = async (context, handlePendingScreenshot, signal) => {
+    while (!signal.aborted) {
+        try {
+            await Process.sleep(context.loopDelay);
+            const agentState = (await context.page.eval("window['ui5-test-runner'].state"));
+            if (handlePendingScreenshot) {
+                await handlePendingScreenshot(context.page, agentState, context.pageId);
             }
-            finally {
-                await taskStopped;
+            if (shouldStopBasedOnAgentState(context, handlePendingScreenshot !== false, agentState)) {
+                break;
             }
         }
-    });
-    let page;
-    let context;
-    try {
-        const agentSource = await getAgentSource();
-        const browser = getBrowser();
-        page = await browser.newWindow({
+        catch (error) {
+            logger.error({ source: 'page', message: 'an error occurred', error, pageId: context.pageId, data: {} });
+            break;
+        }
+    }
+};
+export const makePageTask = (configuration) => {
+    const { screenshot } = configuration;
+    const { handlePendingScreenshot, handleFailureScreenshot } = makeScreenshotHandlers(configuration);
+    const ui5VersionByOrigin = new Map();
+    const logUi5VersionOnce = async (probeUrl) => {
+        const origin = new URL(probeUrl).origin;
+        if (ui5VersionByOrigin.has(origin)) {
+            return;
+        }
+        ui5VersionByOrigin.set(origin, '');
+        const versionUrl = new URL('/resources/sap-ui-version.json', probeUrl).href;
+        try {
+            const ui5Version = JSON.parse(await Http.getAsText(versionUrl));
+            const { version: coreVersion } = ui5Version.libraries.find(({ name }) => name === 'sap.ui.core') ?? {
+                version: 'unknown'
+            };
+            ui5VersionByOrigin.set(origin, coreVersion);
+            logger.info({ source: 'job', message: `UI5 version ${coreVersion} (from ${origin})` });
+        }
+        catch {
+        }
+    };
+    return async function (url, _index, urls) {
+        const pageId = ++lastPageId;
+        logger.debug({ source: 'page', message: 'new page task', pageId, data: { url } });
+        logger.info({
+            source: 'progress',
+            message: url,
             pageId,
-            scripts: [agentSource],
-            url
+            data: { max: 0, value: 1, type: 'unknown', errors: 0 }
         });
-        context = {
-            pageId,
-            urls,
-            url,
-            page,
-            loopDelay: 250,
-            type: 'unknown',
-            lastExecuted: 0,
-            errors: 0,
-            lastTotal: 0
-        };
-        while (!this.stopRequested) {
-            try {
-                await setTimeout(context.loopDelay);
-                if (await queryAgentState(context)) {
-                    break;
+        const startTime = getReportBuilder().report.results.summary.start;
+        if (isGloballyTimedOut(configuration.globalTimeout, startTime)) {
+            logger.warn({ source: 'page', message: 'global timeout reached, skipping page', pageId, data: { url } });
+            reportError(url, 'Global timeout reached');
+            return;
+        }
+        await tryToFetchThePageFirst(url, pageId, configuration);
+        await logUi5VersionOnce(url);
+        const { promise: taskStopped, resolve: setTaskAsStopped } = Promise.withResolvers();
+        const stopController = new AbortController();
+        using _ = Exit.registerAsyncTask({
+            name: url,
+            stop: async () => {
+                stopController.abort();
+                try {
+                    this.stop(new ExitShutdownError());
+                }
+                catch {
+                }
+                finally {
+                    await taskStopped;
                 }
             }
-            catch (error) {
-                logger.error({ source: 'page', message: 'An error occurred', error, pageId, data: {} });
-                break;
-            }
-        }
-        const testResults = (await page.eval("window['ui5-test-runner'].results"));
-        logger.debug({ source: 'page', message: 'test results', pageId, data: { results: testResults } });
-        reportBuilder.merge(url, testResults);
-    }
-    finally {
-        if (context !== undefined) {
-            logger.info({
-                source: 'progress',
-                message: url,
+        });
+        let page;
+        let context;
+        try {
+            const pageTimeoutMs = getPageTimeout(configuration, startTime);
+            const agentSource = await getAgentSource();
+            const browserConfig = getBrowserConfigScript(pageId);
+            const scripts = [browserConfig, agentSource];
+            const browser = getBrowser();
+            page = await browser.newWindow({
                 pageId,
-                data: {
-                    max: context.lastTotal,
-                    value: context.lastExecuted,
-                    type: context.type,
-                    errors: context.errors,
-                    remove: true
-                }
+                scripts,
+                url
             });
-        }
-        try {
-            await page?.close();
+            context = {
+                pageId,
+                urls,
+                url,
+                page,
+                loopDelay: 250,
+                type: 'unknown',
+                lastExecuted: 0,
+                errors: 0,
+                lastTotal: 0,
+                isSuite: false,
+                lastUncaughtErrorsCount: 0
+            };
+            const pageTimeoutController = new AbortController();
+            let pageTimeoutHandle;
+            if (pageTimeoutMs > 0) {
+                pageTimeoutHandle = setTimeout(() => {
+                    logger.warn({ source: 'page', message: 'page timed out', pageId, data: { url } });
+                    pageTimeoutController.abort();
+                }, pageTimeoutMs);
+            }
+            try {
+                await runPollingLoop(context, !!screenshot && handlePendingScreenshot, AbortSignal.any([pageTimeoutController.signal, stopController.signal]));
+            }
+            finally {
+                if (pageTimeoutHandle !== undefined) {
+                    clearTimeout(pageTimeoutHandle);
+                }
+            }
+            const testResults = (await page.eval("window['ui5-test-runner'].results"));
+            await handleFailureScreenshot(page, pageId, testResults);
+            await collectCoverage(configuration, context);
+            mergeTestResults(url, pageId, context, testResults);
+            if (pageTimeoutController.signal.aborted) {
+                reportError(url, 'page timed out');
+            }
         }
         catch (error) {
-            logger.error({ source: 'page', message: 'page.close failed', error, pageId, data: {} });
+            logger.error({ source: 'page', message: 'unexpected error while running page', error, pageId, data: {} });
+            reportError(url, 'An unexpected error occurred');
         }
-        setTaskAsStopped();
-    }
+        finally {
+            if (context !== undefined) {
+                logger.info({
+                    source: 'progress',
+                    message: url,
+                    pageId,
+                    data: {
+                        max: context.lastTotal,
+                        value: context.lastExecuted,
+                        type: context.type,
+                        errors: context.errors,
+                        remove: true
+                    }
+                });
+            }
+            if (page) {
+                try {
+                    logger.debug({ source: 'page', message: 'closing page', pageId });
+                    await page.close();
+                }
+                catch (error) {
+                    logger.error({ source: 'page', message: 'page.close failed', error, pageId, data: {} });
+                }
+            }
+            setTaskAsStopped();
+        }
+    };
 };

package/dist/modes/test/report.js

@@ -1,3 +1,20 @@
-import { TestReportBuilder } from '../../utils/shared/TestReportBuilder.js';
-import { version } from '../../platform/version.js';
-export const reportBuilder = new TestReportBuilder(crypto.randomUUID(), await version());
+import { initReportBuilder as buildReportBuilder } from '../../reports/initReportBuilder.js';
+import { assert } from '../../platform/index.js';
+let _reportBuilder;
+export async function initReportBuilder(configuration) {
+    _reportBuilder = await buildReportBuilder(configuration);
+}
+export function getReportBuilder() {
+    assert(_reportBuilder !== undefined, 'reportBuilder not initialized');
+    return _reportBuilder;
+}
+export function setReportBrowserInfo(capabilities) {
+    const environment = getReportBuilder().report.results.environment;
+    if (environment) {
+        environment.extra = {
+            ...environment.extra,
+            browserName: capabilities.browserName,
+            browserVersion: capabilities.browserVersion
+        };
+    }
+}

package/dist/modes/test/reserve.js

@@ -0,0 +1,76 @@
+import { FileSystem, logger, Path } from '../../platform/index.js';
+const hasFolder = async (path) => {
+    try {
+        await FileSystem.stat(path);
+        return true;
+    }
+    catch {
+        return false;
+    }
+};
+export const buildREserveConfiguration = async (configuration) => {
+    const resourcesMatch = /\/((?:test-)?resources\/.*)/;
+    let { ui5 } = configuration;
+    if (!ui5.endsWith('/')) {
+        ui5 += '/';
+    }
+    const ui5Mapping = new URL('$1', ui5).href;
+    const webapp = Path.join(configuration.webapp, '.');
+    const coverageMappings = configuration.coverage
+        ? [
+            {
+                match: /(.*\.js)(\?.*)?$/,
+                cwd: Path.join(configuration.coverageTempDir, 'instrumented'),
+                file: '$1'
+            }
+        ]
+        : [];
+    const libMappings = (configuration.lib ?? []).flatMap(({ resourcesSubFolder, sourceFolder }) => {
+        const match = new RegExp(String.raw `^/resources/${resourcesSubFolder}/(.*?)(?:\?.*)?$`);
+        const shouldUseInstrumented = configuration.coverage && sourceFolder.startsWith(configuration.cwd);
+        if (shouldUseInstrumented) {
+            const instrumentedCwd = Path.join(configuration.coverageTempDir, 'instrumented', sourceFolder.slice(configuration.cwd.length));
+            return [
+                { match, cwd: instrumentedCwd, file: '$1' },
+                { match, cwd: sourceFolder, file: '$1' }
+            ];
+        }
+        return [{ match, cwd: sourceFolder, file: '$1' }];
+    });
+    const isWebappHasResources = (await hasFolder(Path.join(webapp, 'resources'))) || (await hasFolder(Path.join(webapp, 'test-resources')));
+    const webappResourcesMappings = isWebappHasResources
+        ? [
+            {
+                method: 'GET,HEAD',
+                match: resourcesMatch,
+                cwd: webapp,
+                file: '$1'
+            }
+        ]
+        : [];
+    return {
+        port: configuration.port ?? 0,
+        mappings: [
+            ...libMappings,
+            ...webappResourcesMappings,
+            {
+                method: 'GET,HEAD',
+                match: resourcesMatch,
+                url: ui5Mapping,
+                'ignore-unverifiable-certificate': true
+            },
+            ...coverageMappings,
+            {
+                match: /^\/(.*)/,
+                cwd: webapp,
+                file: '$1'
+            },
+            {
+                custom: (request) => {
+                    logger.warn({ source: 'server/unhandled', message: request.url });
+                    return 404;
+                }
+            }
+        ]
+    };
+};

package/dist/modes/test/screenshot.js

@@ -0,0 +1,49 @@
+import { logger, Path } from '../../platform/index.js';
+import { Folder } from '../../utils/node/Folder.js';
+import { setTimeout } from 'node:timers/promises';
+const screenshotTimeoutError = async (ms) => {
+    await setTimeout(ms);
+    throw new Error('screenshot timeout');
+};
+export const makeScreenshotHandlers = (configuration) => {
+    const { screenshotOnFailure, screenshotTimeout, reportDir } = configuration;
+    const takeScreenshot = async (page, filename) => {
+        await Folder.create(reportDir);
+        const absolutePath = Path.join(reportDir, filename);
+        await Promise.race([page.screenshot(absolutePath), screenshotTimeoutError(screenshotTimeout)]);
+        return filename;
+    };
+    const handlePendingScreenshot = async (page, agentState, pageId) => {
+        if (agentState.type !== 'QUnit' || !agentState.pendingScreenshot)
+            return;
+        const filename = agentState.pendingScreenshot;
+        try {
+            await takeScreenshot(page, filename);
+            logger.debug({ source: 'page', message: 'screenshot taken', pageId, data: { filename } });
+        }
+        catch (error) {
+            logger.error({ source: 'page', message: 'screenshot failed', error, pageId });
+        }
+        await page.eval("window['ui5-test-runner'].state.pendingScreenshot = false");
+    };
+    const handleFailureScreenshot = async (page, pageId, testResults) => {
+        if (!screenshotOnFailure || testResults.tests.every((t) => t.status !== 'failed'))
+            return;
+        try {
+            const filename = await takeScreenshot(page, `${pageId}-failure.png`);
+            logger.debug({ source: 'page', message: 'failure screenshot taken', pageId, data: { path: filename } });
+            testResults.tests.push({
+                name: 'failure screenshot',
+                status: 'other',
+                duration: 0,
+                attachments: [{ name: 'failure screenshot', contentType: 'image/png', path: filename }]
+            });
+            testResults.summary.tests += 1;
+            testResults.summary.other += 1;
+        }
+        catch (error) {
+            logger.error({ source: 'page', message: 'failure screenshot failed', error, pageId });
+        }
+    };
+    return { handlePendingScreenshot, handleFailureScreenshot };
+};

package/dist/platform/Crypto.js

@@ -0,0 +1,9 @@
+import { createHash, randomUUID } from 'node:crypto';
+export const Crypto = {
+    sha256hex(input) {
+        return createHash('sha256').update(input).digest('hex');
+    },
+    randomUUID() {
+        return randomUUID();
+    }
+};