webship-js 2.0.11 → 2.0.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.circleci/config.yml +10 -1
- package/.drone.yml +7 -7
- package/.github/workflows/github-actions.yml +48 -0
- package/.gitlab-ci.yml +9 -4
- package/.harness/webship-js-pipeline.yml +5 -6
- package/.semaphore/semaphore.yml +6 -9
- package/.teamcity/settings.kts +7 -6
- package/.travis.yml +6 -1
- package/AGENTS.md +1 -1
- package/CLAUDE.md +65 -23
- package/Jenkinsfile +5 -3
- package/azure-pipelines.yml +17 -8
- package/bamboo-specs/bamboo.yml +9 -15
- package/bin/init-webship.js +53 -10
- package/bitbucket-pipelines.yml +5 -1
- package/buildspec.yml +6 -7
- package/cloudbuild.yaml +28 -12
- package/codefresh.yml +6 -6
- package/cucumber.js +17 -10
- package/docs/00-quick-start.md +8 -0
- package/docs/02-bbr-smart-waits.md +15 -10
- package/docs/03-selector-registry.md +24 -0
- package/docs/04-step-reference.md +51 -25
- package/docs/05-web-first-assertions.md +6 -0
- package/docs/07-auth-state.md +1 -1
- package/docs/08-clock-mocking.md +21 -2
- package/docs/09-api-testing.md +40 -2
- package/docs/10-accessibility.md +2 -0
- package/docs/11-debugging.md +3 -1
- package/docs/12-ai-agent-guide.md +5 -0
- package/docs/13-faq.md +72 -1
- package/docs/14-recipes-cookbook.md +2 -0
- package/docs/15-tag-conventions.md +2 -1
- package/docs/16-ci-cd.md +140 -119
- package/docs/README.md +5 -5
- package/docs/advanced-screenshots.md +3 -1
- package/docs/api-step-definitions.md +16 -2
- package/docs/global-settings.md +69 -17
- package/docs/install-webship-js.md +2 -0
- package/docs/step-definitions/then-i-should-see.md +14 -1
- package/docs/step-definitions/then-item-checkbox.md +2 -0
- package/docs/step-definitions/when-i-fill.md +4 -0
- package/docs/step-definitions/when-i-select-option-select-list.md +2 -0
- package/examples/check-switch.html +37 -0
- package/examples/choice-controls.html +29 -0
- package/examples/click-whitespace.html +37 -0
- package/examples/datetime-fields.html +52 -0
- package/examples/element-selector-hint.html +16 -0
- package/examples/field-states.html +27 -0
- package/examples/fill-in-empty.html +26 -0
- package/examples/fill-unreachable.html +27 -0
- package/examples/follow-hidden-first.html +29 -0
- package/examples/iframe-inner.html +20 -0
- package/examples/iframe-nested.html +16 -0
- package/examples/iframe-outer.html +15 -0
- package/examples/multi-value-fields.html +54 -0
- package/examples/registered-names.html +15 -0
- package/examples/select-by-label.html +33 -0
- package/examples/settle-budget.html +21 -0
- package/examples/sorted-tables.html +38 -0
- package/examples/text-matching.html +48 -0
- package/examples/text-order.html +25 -0
- package/examples/touch.html +22 -0
- package/examples/viewport-top.html +36 -0
- package/package.json +17 -1
- package/playwright.config.ts +3 -1
- package/tests/check-api-launch-url.js +106 -0
- package/tests/check-assets.js +103 -0
- package/tests/check-attachments.js +105 -0
- package/tests/check-breakpoints.js +117 -0
- package/tests/check-clock.js +143 -0
- package/tests/check-config.js +90 -0
- package/tests/check-context.js +215 -0
- package/tests/check-dry-run.js +104 -0
- package/tests/check-examples.js +10 -4
- package/tests/check-exit-report.js +141 -0
- package/tests/check-javascript-errors.js +105 -0
- package/tests/check-must-fail.js +5 -4
- package/tests/check-network-changed.js +137 -0
- package/tests/check-scaffold.js +54 -2
- package/tests/check-screenshots.js +117 -0
- package/tests/check-settle-verbs.js +137 -0
- package/tests/check-time-limit.js +104 -0
- package/tests/check-video.js +156 -0
- package/tests/check-world.js +167 -0
- package/tests/features/api-base-url.feature +25 -0
- package/tests/features/api-form-data.feature +18 -0
- package/tests/features/api-json-paths.feature +24 -0
- package/tests/features/api-response-contains.feature +21 -0
- package/tests/features/api-response-header-presence.feature +17 -0
- package/tests/features/api-response-header.feature +17 -0
- package/tests/features/api-saved-placeholders.feature +37 -0
- package/tests/features/api-step-definitions-individual.feature +1 -1
- package/tests/features/api-tables.feature +40 -0
- package/tests/features/breakpoints.feature +25 -0
- package/tests/features/check-switch.feature +28 -0
- package/tests/features/choice-controls.feature +35 -0
- package/tests/features/click-whitespace.feature +29 -0
- package/tests/features/complete-api-testing-examples.feature +1 -1
- package/tests/features/datetime-field-lookup.feature +29 -0
- package/tests/features/element-selector-hint.feature +14 -0
- package/tests/features/field-state.feature +21 -0
- package/tests/features/fill-in-empty-value.feature +41 -0
- package/tests/features/fill-reachable.feature +13 -0
- package/tests/features/follow-hidden-first.feature +9 -0
- package/tests/features/iframe-nested.feature +43 -0
- package/tests/features/javascript-tolerated-errors.feature +20 -0
- package/tests/features/modal-open.feature +13 -0
- package/tests/features/multi-value-field-lookup.feature +30 -0
- package/tests/features/registered-names.feature +14 -0
- package/tests/features/select-by-label.feature +31 -0
- package/tests/features/selector-files.feature +14 -0
- package/tests/features/selectors-table-header.feature +35 -0
- package/tests/features/settle-budget.feature +11 -0
- package/tests/features/sorted-tables.feature +19 -0
- package/tests/features/storage-quoted-values.feature +26 -0
- package/tests/features/text-matching.feature +38 -0
- package/tests/features/text-order.feature +14 -0
- package/tests/features/touch.feature +21 -0
- package/tests/features/viewport-top.feature +13 -0
- package/tests/features/xml-namespaces.feature +34 -0
- package/tests/must-fail/api-json-paths.feature +21 -0
- package/tests/must-fail/api-response-contains.feature +23 -0
- package/tests/must-fail/api-response-header-presence.feature +17 -0
- package/tests/must-fail/api-response-header.feature +21 -0
- package/tests/must-fail/api-without-base-url.feature +11 -0
- package/tests/must-fail/breakpoints.feature +15 -0
- package/tests/must-fail/choice-controls.feature +25 -0
- package/tests/must-fail/clock.feature +11 -0
- package/tests/must-fail/datetime-field.feature +10 -0
- package/tests/must-fail/element-selector-hint.feature +17 -0
- package/tests/must-fail/field-state.feature +20 -0
- package/tests/must-fail/fill-field.feature +30 -0
- package/tests/must-fail/iframe-text.feature +17 -0
- package/tests/must-fail/modal-open.feature +26 -0
- package/tests/must-fail/multi-value-field.feature +18 -0
- package/tests/must-fail/registered-names.feature +25 -0
- package/tests/must-fail/select-list.feature +15 -0
- package/tests/must-fail/selector-files.feature +10 -0
- package/tests/must-fail/sorted-tables.feature +20 -0
- package/tests/must-fail/storage-before-a-page.feature +13 -0
- package/tests/must-fail/text-matching.feature +16 -0
- package/tests/must-fail/text-order.feature +15 -0
- package/tests/must-fail/viewport-top.feature +20 -0
- package/tests/must-fail/yaml-numbers.feature +48 -0
- package/tests/step-definitions/a11y.steps.js +24 -15
- package/tests/step-definitions/action.steps.js +53 -20
- package/tests/step-definitions/api.steps.js +349 -67
- package/tests/step-definitions/assertion.steps.js +100 -10
- package/tests/step-definitions/auth.steps.js +15 -18
- package/tests/step-definitions/clock.steps.js +22 -0
- package/tests/step-definitions/debug.steps.js +8 -0
- package/tests/step-definitions/dialog.steps.js +7 -0
- package/tests/step-definitions/element.steps.js +132 -17
- package/tests/step-definitions/field.steps.js +412 -96
- package/tests/step-definitions/form.steps.js +88 -37
- package/tests/step-definitions/iframe.steps.js +89 -29
- package/tests/step-definitions/input.steps.js +26 -7
- package/tests/step-definitions/javascript.steps.js +64 -22
- package/tests/step-definitions/modal.steps.js +79 -13
- package/tests/step-definitions/navigation.steps.js +14 -0
- package/tests/step-definitions/network.steps.js +15 -5
- package/tests/step-definitions/response.steps.js +6 -4
- package/tests/step-definitions/responsive.steps.js +26 -33
- package/tests/step-definitions/screenshot.steps.js +25 -8
- package/tests/step-definitions/scroll.steps.js +20 -0
- package/tests/step-definitions/selectors.steps.js +95 -48
- package/tests/step-definitions/storage.steps.js +38 -13
- package/tests/step-definitions/table.steps.js +81 -6
- package/tests/step-definitions/video.steps.js +52 -55
- package/tests/step-definitions/wait.steps.js +88 -23
- package/tests/step-definitions/web-first.steps.js +69 -35
- package/tests/step-definitions/webship.js +741 -145
- package/tests/step-definitions/xml.steps.js +49 -15
- package/tests/step-definitions/yaml.steps.js +45 -20
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
'use strict';
|
|
3
|
+
|
|
4
|
+
// Runs three scenarios with the failure screenshots on and checks which of
|
|
5
|
+
// them leave a "failed_" screenshot: the one whose step fails does; the one
|
|
6
|
+
// that a step skips on purpose (the step returns 'skipped', the way a suite
|
|
7
|
+
// leaves out a scenario that does not apply to the site) and the one that
|
|
8
|
+
// stops at a pending step do not. The failure screenshot is also attached to
|
|
9
|
+
// the failed step in the JSON report (an image/png embedding), so the HTML
|
|
10
|
+
// report shows it; the other two steps carry none.
|
|
11
|
+
//
|
|
12
|
+
// The scenarios never navigate, so no server is needed.
|
|
13
|
+
//
|
|
14
|
+
// Usage: node tests/check-screenshots.js (npm run test:screenshots)
|
|
15
|
+
|
|
16
|
+
const fs = require('fs');
|
|
17
|
+
const os = require('os');
|
|
18
|
+
const path = require('path');
|
|
19
|
+
const { spawnSync } = require('child_process');
|
|
20
|
+
|
|
21
|
+
const ROOT = path.resolve(__dirname, '..');
|
|
22
|
+
const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'webship-js-screenshots-'));
|
|
23
|
+
const shots = path.join(tmp, 'screenshots');
|
|
24
|
+
const report = path.join(tmp, 'outcomes.json');
|
|
25
|
+
const config = path.join(tmp, 'cucumber.js');
|
|
26
|
+
const steps = path.join(tmp, 'outcomes.steps.js');
|
|
27
|
+
const feature = path.join(tmp, 'outcomes.feature');
|
|
28
|
+
|
|
29
|
+
const cucumber = path.join(ROOT, 'node_modules', '@cucumber', 'cucumber');
|
|
30
|
+
fs.writeFileSync(steps, [
|
|
31
|
+
`const { Given, Then } = require(${JSON.stringify(cucumber)});`,
|
|
32
|
+
"Given('this scenario does not apply here', function () { return 'skipped'; });",
|
|
33
|
+
"Given('this step is not written yet', function () { return 'pending'; });",
|
|
34
|
+
"Then('this step fails on purpose', function () { throw new Error('Failing on purpose.'); });",
|
|
35
|
+
'',
|
|
36
|
+
].join('\n'));
|
|
37
|
+
fs.writeFileSync(feature, [
|
|
38
|
+
'Feature: Outcomes and failure screenshots',
|
|
39
|
+
'',
|
|
40
|
+
' Scenario: A step skips the scenario on purpose',
|
|
41
|
+
' Given this scenario does not apply here',
|
|
42
|
+
'',
|
|
43
|
+
' Scenario: A step is pending',
|
|
44
|
+
' Given this step is not written yet',
|
|
45
|
+
'',
|
|
46
|
+
' Scenario: A step fails',
|
|
47
|
+
' Then this step fails on purpose',
|
|
48
|
+
'',
|
|
49
|
+
' Scenario: Named screenshots',
|
|
50
|
+
' Then I save screenshot with name "homepage.png"',
|
|
51
|
+
' And I save fullscreen screenshot with name "full-page.JPG"',
|
|
52
|
+
' And I save screenshot with name "plain-name"',
|
|
53
|
+
'',
|
|
54
|
+
].join('\n'));
|
|
55
|
+
const lines = { skipped: 4, pending: 7, failed: 10 };
|
|
56
|
+
|
|
57
|
+
const base = require(path.join(ROOT, 'cucumber.js')).default;
|
|
58
|
+
const world = JSON.parse(JSON.stringify(base.worldParameters));
|
|
59
|
+
world.screenshot = Object.assign({}, world.screenshot, { dir: shots, onFailed: true, onEveryStep: false, purge: false });
|
|
60
|
+
fs.writeFileSync(config, 'module.exports = { default: ' + JSON.stringify({
|
|
61
|
+
requireModule: base.requireModule,
|
|
62
|
+
require: base.require.concat([steps]),
|
|
63
|
+
paths: [feature],
|
|
64
|
+
format: [`json:${report}`],
|
|
65
|
+
worldParameters: world,
|
|
66
|
+
}) + ' };');
|
|
67
|
+
|
|
68
|
+
spawnSync(process.execPath, [path.join(cucumber, 'bin', 'cucumber.js'), '--config', path.relative(ROOT, config)], {
|
|
69
|
+
cwd: ROOT,
|
|
70
|
+
stdio: ['ignore', 'ignore', 'inherit'],
|
|
71
|
+
env: Object.assign({ SLOW_MO: '0' }, process.env, { WEBSHIP_REPORT_DISABLE: '1', WEBSHIP_VIDEO: 'off', WEBSHIP_JS_ERROR_MODE: 'off' }),
|
|
72
|
+
});
|
|
73
|
+
|
|
74
|
+
const problems = [];
|
|
75
|
+
if (!fs.existsSync(report) || !fs.readFileSync(report, 'utf8').trim()) {
|
|
76
|
+
problems.push('The run wrote no report: cucumber-js did not start.');
|
|
77
|
+
} else {
|
|
78
|
+
const statuses = {};
|
|
79
|
+
const images = {};
|
|
80
|
+
for (const f of JSON.parse(fs.readFileSync(report, 'utf8'))) {
|
|
81
|
+
for (const scenario of f.elements || []) {
|
|
82
|
+
for (const step of (scenario.steps || []).filter((s) => !s.hidden)) {
|
|
83
|
+
statuses[step.line] = step.result && step.result.status;
|
|
84
|
+
images[step.line] = (step.embeddings || []).filter((e) => e.mime_type === 'image/png' && e.data).length;
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
for (const [outcome, line] of Object.entries(lines)) {
|
|
89
|
+
if (statuses[line] !== outcome) problems.push(`The step on line ${line} ended "${statuses[line]}", not "${outcome}": the check itself is broken.`);
|
|
90
|
+
}
|
|
91
|
+
const files = fs.existsSync(shots) ? fs.readdirSync(shots).filter((name) => name.startsWith('failed_')) : [];
|
|
92
|
+
const forLine = (line) => files.filter((name) => new RegExp(`\\.feature_${line}\\.(png|html)$`).test(name));
|
|
93
|
+
if (!forLine(lines.failed).some((name) => name.endsWith('.png'))) {
|
|
94
|
+
problems.push(`The failed step (line ${lines.failed}) left no failure screenshot; files: ${files.join(', ') || 'none'}.`);
|
|
95
|
+
}
|
|
96
|
+
for (const outcome of ['skipped', 'pending']) {
|
|
97
|
+
const found = forLine(lines[outcome]);
|
|
98
|
+
if (found.length) problems.push(`The ${outcome} step (line ${lines[outcome]}) left failure screenshots: ${found.join(', ')}.`);
|
|
99
|
+
if (images[lines[outcome]]) problems.push(`The ${outcome} step (line ${lines[outcome]}) carries ${images[lines[outcome]]} screenshot(s) in the JSON report.`);
|
|
100
|
+
}
|
|
101
|
+
// A name that ends with the extension of a file the step writes keeps
|
|
102
|
+
// that one extension: "homepage.png" is homepage.png, not homepage.png.png.
|
|
103
|
+
const saved = fs.existsSync(shots) ? fs.readdirSync(shots) : [];
|
|
104
|
+
for (const name of ['homepage.png', 'homepage.html', 'full-page.png', 'full-page.html', 'plain-name.png', 'plain-name.html']) {
|
|
105
|
+
if (!saved.includes(name)) problems.push(`The named screenshot ${name} was not written; files: ${saved.filter((f) => !f.startsWith('failed_')).join(', ') || 'none'}.`);
|
|
106
|
+
}
|
|
107
|
+
if (images[lines.failed] !== 1) {
|
|
108
|
+
problems.push(`The failed step (line ${lines.failed}) carries ${images[lines.failed] || 0} screenshot(s) in the JSON report, not 1: no report can show it.`);
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
fs.rmSync(tmp, { recursive: true, force: true });
|
|
113
|
+
if (problems.length) {
|
|
114
|
+
console.error('Failure screenshots:\n - ' + problems.join('\n - '));
|
|
115
|
+
process.exit(1);
|
|
116
|
+
}
|
|
117
|
+
console.log('Failure screenshots: the failed step left one, attached to it in the JSON report; the skipped and the pending step left none. Named screenshots keep the one extension their name gives.');
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
'use strict';
|
|
3
|
+
|
|
4
|
+
// Runs a project's own steps (written here, the way a project writes them)
|
|
5
|
+
// whose words carry a page-changing verb — add, log out, enter, open,
|
|
6
|
+
// delete, save — and checks that the settle after the step waited for the
|
|
7
|
+
// page: the next step reads the element once and finds the change that
|
|
8
|
+
// arrives 600 ms after the action. A built-in assertion whose quoted text
|
|
9
|
+
// holds such a word ("Then I should see "Delete"") gets no settle.
|
|
10
|
+
//
|
|
11
|
+
// The check serves its own page on a free local port, so no other server is
|
|
12
|
+
// needed.
|
|
13
|
+
//
|
|
14
|
+
// Usage: node tests/check-settle-verbs.js (npm run test:settle-verbs)
|
|
15
|
+
|
|
16
|
+
const fs = require('fs');
|
|
17
|
+
const os = require('os');
|
|
18
|
+
const path = require('path');
|
|
19
|
+
const http = require('http');
|
|
20
|
+
const { spawn } = require('child_process');
|
|
21
|
+
|
|
22
|
+
const ROOT = path.resolve(__dirname, '..');
|
|
23
|
+
const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'webship-js-settle-verbs-'));
|
|
24
|
+
const report = path.join(tmp, 'settle.json');
|
|
25
|
+
const config = path.join(tmp, 'cucumber.js');
|
|
26
|
+
const steps = path.join(tmp, 'project.steps.js');
|
|
27
|
+
const feature = path.join(tmp, 'settle.feature');
|
|
28
|
+
const cucumber = path.join(ROOT, 'node_modules', '@cucumber', 'cucumber');
|
|
29
|
+
|
|
30
|
+
const PAGE = `<!doctype html><html><head><title>Shop</title></head><body>
|
|
31
|
+
<h1>Shop</h1>
|
|
32
|
+
<button id="act" type="button">Act</button>
|
|
33
|
+
<p id="status">Nothing yet</p>
|
|
34
|
+
<p>Delete</p>
|
|
35
|
+
<script>
|
|
36
|
+
document.getElementById('act').addEventListener('click', function () {
|
|
37
|
+
setTimeout(function () { document.getElementById('status').textContent = 'Done'; }, 600);
|
|
38
|
+
});
|
|
39
|
+
</script></body></html>`;
|
|
40
|
+
|
|
41
|
+
// A project's own steps: each one clicks the button the way a custom step
|
|
42
|
+
// would, without waiting for anything.
|
|
43
|
+
const phrases = [
|
|
44
|
+
'I add "Laptop" to the cart',
|
|
45
|
+
'I log out of the shop',
|
|
46
|
+
'I enter the 2FA code "123456"',
|
|
47
|
+
'I open the account menu',
|
|
48
|
+
'I delete the draft',
|
|
49
|
+
'I save the order',
|
|
50
|
+
];
|
|
51
|
+
fs.writeFileSync(steps, [
|
|
52
|
+
`const { When } = require(${JSON.stringify(cucumber)});`,
|
|
53
|
+
...phrases.map((p) => {
|
|
54
|
+
const args = (p.match(/"[^"]*"/g) || []).map((_, i) => `value${i}`).join(', ');
|
|
55
|
+
return `When(${JSON.stringify(p.replace(/"[^"]*"/g, '{string}'))}, async function (${args}) { await this.page.click('#act'); });`;
|
|
56
|
+
}),
|
|
57
|
+
'',
|
|
58
|
+
].join('\n'));
|
|
59
|
+
fs.writeFileSync(feature, [
|
|
60
|
+
'Feature: A project step that changes the page is followed by the settle',
|
|
61
|
+
'',
|
|
62
|
+
...phrases.flatMap((p) => [
|
|
63
|
+
` Scenario: ${p}`,
|
|
64
|
+
' Given I am on "/"',
|
|
65
|
+
` When ${p}`,
|
|
66
|
+
' Then I should see "Done" in the "#status" element',
|
|
67
|
+
'',
|
|
68
|
+
]),
|
|
69
|
+
' Scenario: A wait for AJAX in lower case',
|
|
70
|
+
' Given I am on "/"',
|
|
71
|
+
' When I wait for ajax to finish',
|
|
72
|
+
' Then I should see "Shop"',
|
|
73
|
+
'',
|
|
74
|
+
].join('\n'));
|
|
75
|
+
|
|
76
|
+
const server = http.createServer((req, res) => {
|
|
77
|
+
res.writeHead(200, { 'Content-Type': 'text/html; charset=utf-8' });
|
|
78
|
+
res.end(PAGE);
|
|
79
|
+
});
|
|
80
|
+
server.listen(0, '127.0.0.1', () => {
|
|
81
|
+
const base = `http://127.0.0.1:${server.address().port}`;
|
|
82
|
+
const defaults = require(path.join(ROOT, 'cucumber.js')).default;
|
|
83
|
+
const world = JSON.parse(JSON.stringify(defaults.worldParameters));
|
|
84
|
+
world.launchUrl = base;
|
|
85
|
+
fs.writeFileSync(config, 'module.exports = { default: ' + JSON.stringify({
|
|
86
|
+
requireModule: defaults.requireModule,
|
|
87
|
+
require: defaults.require.concat([steps]),
|
|
88
|
+
paths: [feature],
|
|
89
|
+
format: [`json:${report}`],
|
|
90
|
+
worldParameters: world,
|
|
91
|
+
}) + ' };');
|
|
92
|
+
const env = Object.assign({ SLOW_MO: '0' }, process.env, {
|
|
93
|
+
LAUNCH_URL: base,
|
|
94
|
+
WEBSHIP_REPORT_DISABLE: '1',
|
|
95
|
+
WEBSHIP_VIDEO: 'off',
|
|
96
|
+
WEBSHIP_JS_ERROR_MODE: 'off',
|
|
97
|
+
WEBSHIP_SCREENSHOT_ON_FAILED: 'false',
|
|
98
|
+
});
|
|
99
|
+
delete env.WEBSHIP_AUTO_SETTLE;
|
|
100
|
+
const run = spawn(process.execPath, [path.join(cucumber, 'bin', 'cucumber.js'), '--config', path.relative(ROOT, config)], {
|
|
101
|
+
cwd: ROOT,
|
|
102
|
+
stdio: ['ignore', 'ignore', 'pipe'],
|
|
103
|
+
env,
|
|
104
|
+
});
|
|
105
|
+
let errors = '';
|
|
106
|
+
run.stderr.on('data', (chunk) => { errors += chunk; });
|
|
107
|
+
const timer = setTimeout(() => run.kill('SIGKILL'), 180000);
|
|
108
|
+
run.on('exit', () => {
|
|
109
|
+
clearTimeout(timer);
|
|
110
|
+
server.close();
|
|
111
|
+
finish(errors);
|
|
112
|
+
});
|
|
113
|
+
});
|
|
114
|
+
|
|
115
|
+
function finish(errors) {
|
|
116
|
+
const strip = (text) => String(text || '').replace(/\u001b\[[0-9;]*m/g, '');
|
|
117
|
+
const problems = [];
|
|
118
|
+
let count = 0;
|
|
119
|
+
if (!fs.existsSync(report) || !fs.readFileSync(report, 'utf8').trim()) {
|
|
120
|
+
problems.push(`The run wrote no report.\n${strip(errors).slice(-1500)}`);
|
|
121
|
+
} else {
|
|
122
|
+
for (const f of JSON.parse(fs.readFileSync(report, 'utf8'))) {
|
|
123
|
+
for (const scenario of f.elements || []) {
|
|
124
|
+
count += 1;
|
|
125
|
+
const failed = (scenario.steps || []).find((step) => step.result && step.result.status !== 'passed');
|
|
126
|
+
if (failed) problems.push(`"${scenario.name}": ${failed.hidden ? 'a hook' : `"${failed.keyword}${failed.name}"`} ${failed.result.status}: ${strip(failed.result.error_message).split('\n')[0]}`);
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
if (count !== phrases.length + 1) problems.push(`${count} scenarios ran, not ${phrases.length + 1}.`);
|
|
130
|
+
}
|
|
131
|
+
fs.rmSync(tmp, { recursive: true, force: true });
|
|
132
|
+
if (problems.length) {
|
|
133
|
+
console.error('The settle after a project step:\n - ' + problems.join('\n - '));
|
|
134
|
+
process.exit(1);
|
|
135
|
+
}
|
|
136
|
+
console.log('The settle after a project step: every page-changing verb waits for the page; the AJAX wait reads in lower case.');
|
|
137
|
+
}
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
'use strict';
|
|
3
|
+
|
|
4
|
+
// Runs the steps whose length the scenario gives under a step timeout
|
|
5
|
+
// shorter than that length (WEBSHIP_STEP_TIMEOUT=4000) and checks that each
|
|
6
|
+
// one runs past it: "I wait 10 seconds" on a page that never settles waits
|
|
7
|
+
// longer than 4 seconds and passes, a web-first assertion "within 9 seconds"
|
|
8
|
+
// waits for an element that shows after 7 seconds, and one that never holds
|
|
9
|
+
// fails with its own message after its 6 seconds — none of them ends with
|
|
10
|
+
// cucumber's "function timed out".
|
|
11
|
+
//
|
|
12
|
+
// The pages are data: URLs: no server is needed.
|
|
13
|
+
//
|
|
14
|
+
// Usage: node tests/check-time-limit.js (npm run test:time-limit)
|
|
15
|
+
|
|
16
|
+
const fs = require('fs');
|
|
17
|
+
const os = require('os');
|
|
18
|
+
const path = require('path');
|
|
19
|
+
const { spawnSync } = require('child_process');
|
|
20
|
+
|
|
21
|
+
const ROOT = path.resolve(__dirname, '..');
|
|
22
|
+
const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'webship-js-time-limit-'));
|
|
23
|
+
const report = path.join(tmp, 'time-limit.json');
|
|
24
|
+
const config = path.join(tmp, 'cucumber.js');
|
|
25
|
+
const steps = path.join(tmp, 'time-limit.steps.js');
|
|
26
|
+
const feature = path.join(tmp, 'time-limit.feature');
|
|
27
|
+
|
|
28
|
+
// A page that never settles: a timer that reschedules itself and changes the
|
|
29
|
+
// page every 100 ms. And one whose "#late" element shows after 7 seconds.
|
|
30
|
+
const NEVER_SETTLES = '<p id="beat">0</p><script>var n = 0; (function tick() { document.getElementById("beat").textContent = String(++n); setTimeout(tick, 100); })();</script>';
|
|
31
|
+
const SHOWS_LATE = '<p id="late" hidden>Late</p><script>setTimeout(function () { document.getElementById("late").hidden = false; }, 7000);</script>';
|
|
32
|
+
|
|
33
|
+
const cucumber = path.join(ROOT, 'node_modules', '@cucumber', 'cucumber');
|
|
34
|
+
fs.writeFileSync(steps, [
|
|
35
|
+
`const { Given } = require(${JSON.stringify(cucumber)});`,
|
|
36
|
+
`Given('a page that never settles', async function () { await this.page.goto('data:text/html,' + encodeURIComponent(${JSON.stringify(NEVER_SETTLES)})); });`,
|
|
37
|
+
`Given('a page whose element shows late', async function () { await this.page.goto('data:text/html,' + encodeURIComponent(${JSON.stringify(SHOWS_LATE)})); });`,
|
|
38
|
+
'',
|
|
39
|
+
].join('\n'));
|
|
40
|
+
|
|
41
|
+
// name, steps, the status of the last step, its shortest duration in seconds, a text of its failure
|
|
42
|
+
const scenarios = [
|
|
43
|
+
['A wait in seconds runs past the step timeout', 'Given a page that never settles\n When I wait 10 seconds', 'passed', 4.5, null],
|
|
44
|
+
['A wait with a maximum runs past the step timeout', 'Given a page that never settles\n When I wait max of 10 seconds', 'passed', 4.5, null],
|
|
45
|
+
['A wait for a number of seconds runs past the step timeout', 'Given a page that never settles\n When I wait for 10 seconds', 'passed', 4.5, null],
|
|
46
|
+
['A web-first assertion waits as long as it is told', 'Given a page whose element shows late\n Then "#late" should be visible within 9 seconds', 'passed', 6.5, null],
|
|
47
|
+
['A web-first assertion that never holds fails with its own message', 'Given a page that never settles\n Then "#never" should be visible within 6 seconds', 'failed', 5.5, 'Expected "#never" to be visible'],
|
|
48
|
+
];
|
|
49
|
+
fs.writeFileSync(feature, ['Feature: Steps that run for the time the scenario gives', ''].concat(
|
|
50
|
+
scenarios.map(([name, body]) => ` Scenario: ${name}\n ${body}\n`)
|
|
51
|
+
).join('\n'));
|
|
52
|
+
|
|
53
|
+
const base = require(path.join(ROOT, 'cucumber.js')).default;
|
|
54
|
+
fs.writeFileSync(config, 'module.exports = { default: ' + JSON.stringify({
|
|
55
|
+
requireModule: base.requireModule,
|
|
56
|
+
require: base.require.concat([steps]),
|
|
57
|
+
paths: [feature],
|
|
58
|
+
format: [`json:${report}`],
|
|
59
|
+
worldParameters: base.worldParameters,
|
|
60
|
+
}) + ' };');
|
|
61
|
+
|
|
62
|
+
spawnSync(process.execPath, [path.join(cucumber, 'bin', 'cucumber.js'), '--config', path.relative(ROOT, config)], {
|
|
63
|
+
cwd: ROOT,
|
|
64
|
+
stdio: ['ignore', 'ignore', 'inherit'],
|
|
65
|
+
env: Object.assign({ SLOW_MO: '0' }, process.env, {
|
|
66
|
+
WEBSHIP_REPORT_DISABLE: '1',
|
|
67
|
+
WEBSHIP_VIDEO: 'off',
|
|
68
|
+
WEBSHIP_JS_ERROR_MODE: 'off',
|
|
69
|
+
WEBSHIP_STEP_TIMEOUT: '4000',
|
|
70
|
+
}),
|
|
71
|
+
});
|
|
72
|
+
|
|
73
|
+
const problems = [];
|
|
74
|
+
if (!fs.existsSync(report) || !fs.readFileSync(report, 'utf8').trim()) {
|
|
75
|
+
problems.push('The run wrote no report: cucumber-js did not start.');
|
|
76
|
+
} else {
|
|
77
|
+
const found = {};
|
|
78
|
+
for (const f of JSON.parse(fs.readFileSync(report, 'utf8'))) {
|
|
79
|
+
for (const scenario of f.elements || []) found[scenario.name] = scenario;
|
|
80
|
+
}
|
|
81
|
+
for (const [name, , status, seconds, text] of scenarios) {
|
|
82
|
+
const scenario = found[name];
|
|
83
|
+
if (!scenario) { problems.push(`"${name}" did not run.`); continue; }
|
|
84
|
+
const visible = (scenario.steps || []).filter((s) => !s.hidden);
|
|
85
|
+
const last = visible[visible.length - 1];
|
|
86
|
+
const result = (last && last.result) || {};
|
|
87
|
+
const message = String(result.error_message || '');
|
|
88
|
+
const took = (result.duration || 0) / 1e9;
|
|
89
|
+
if (result.status !== status) {
|
|
90
|
+
problems.push(`"${name}": the step ${result.status}, it must have ${status}${message ? `: ${message.split('\n')[0]}` : ''}.`);
|
|
91
|
+
continue;
|
|
92
|
+
}
|
|
93
|
+
if (took < seconds) problems.push(`"${name}": the step ended after ${took.toFixed(1)} s; it must run past the 4-second step timeout (${seconds} s or more).`);
|
|
94
|
+
if (/function timed out/.test(message)) problems.push(`"${name}": the step was cut by the step timeout: ${message.split('\n')[0]}`);
|
|
95
|
+
if (text && !message.includes(text)) problems.push(`"${name}": the failure does not say "${text}": ${message.split('\n')[0]}`);
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
fs.rmSync(tmp, { recursive: true, force: true });
|
|
100
|
+
if (problems.length) {
|
|
101
|
+
console.error('Steps with a length of their own:\n - ' + problems.join('\n - '));
|
|
102
|
+
process.exit(1);
|
|
103
|
+
}
|
|
104
|
+
console.log('Steps with a length of their own: each ran for the time the scenario gave it, past the step timeout.');
|
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
'use strict';
|
|
3
|
+
|
|
4
|
+
// Runs video-recording scenarios in three modes and checks the files they
|
|
5
|
+
// leave in the video folder:
|
|
6
|
+
// - mode "on": "I start video recording" on a scenario that already
|
|
7
|
+
// records leaves one video under its final name, no page@<hash>.webm;
|
|
8
|
+
// the status in the name is in lower case; "print video path" prints
|
|
9
|
+
// the path the video is saved under, not a temporary file;
|
|
10
|
+
// - mode "on-failure": a passing scenario whose video has a reserved name
|
|
11
|
+
// ("I save the current video as") keeps it, another passing one keeps none;
|
|
12
|
+
// - mode "off": "I stop video recording" saves under the filename pattern;
|
|
13
|
+
// WEBSHIP_VIDEO_SIZE sets the recording size.
|
|
14
|
+
//
|
|
15
|
+
// The scenarios write their own page, so no server is needed.
|
|
16
|
+
//
|
|
17
|
+
// Usage: node tests/check-video.js (npm run test:video)
|
|
18
|
+
|
|
19
|
+
const fs = require('fs');
|
|
20
|
+
const os = require('os');
|
|
21
|
+
const path = require('path');
|
|
22
|
+
const { spawnSync } = require('child_process');
|
|
23
|
+
|
|
24
|
+
const ROOT = path.resolve(__dirname, '..');
|
|
25
|
+
const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'webship-js-video-'));
|
|
26
|
+
const cucumber = path.join(ROOT, 'node_modules', '@cucumber', 'cucumber');
|
|
27
|
+
const steps = path.join(tmp, 'video.steps.js');
|
|
28
|
+
const defaults = require(path.join(ROOT, 'cucumber.js')).default;
|
|
29
|
+
|
|
30
|
+
fs.writeFileSync(steps, [
|
|
31
|
+
`const { Given, Then } = require(${JSON.stringify(cucumber)});`,
|
|
32
|
+
"Given('a page that reads {string}', async function (text) {",
|
|
33
|
+
' await this.page.setContent(`<h1>${text}</h1>`);',
|
|
34
|
+
'});',
|
|
35
|
+
"Then('the recording size should be {int} by {int}', function (width, height) {",
|
|
36
|
+
' const size = this._videoSettings && this._videoSettings.size;',
|
|
37
|
+
' if (!size || size.width !== width || size.height !== height) throw new Error(`The recording size is ${JSON.stringify(size)}.`);',
|
|
38
|
+
'});',
|
|
39
|
+
'',
|
|
40
|
+
].join('\n'));
|
|
41
|
+
|
|
42
|
+
const problems = [];
|
|
43
|
+
const strip = (text) => String(text || '').replace(/\u001b\[[0-9;]*m/g, '');
|
|
44
|
+
|
|
45
|
+
function run(name, mode, lines, extraEnv) {
|
|
46
|
+
const dir = path.join(tmp, name);
|
|
47
|
+
const videos = path.join(dir, 'videos');
|
|
48
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
49
|
+
const feature = path.join(dir, `${name}.feature`);
|
|
50
|
+
const report = path.join(dir, 'report.json');
|
|
51
|
+
const config = path.join(dir, 'cucumber.js');
|
|
52
|
+
fs.writeFileSync(feature, lines.join('\n') + '\n');
|
|
53
|
+
const world = JSON.parse(JSON.stringify(defaults.worldParameters));
|
|
54
|
+
world.video = Object.assign({}, world.video, { mode, dir: videos });
|
|
55
|
+
fs.writeFileSync(config, 'module.exports = { default: ' + JSON.stringify({
|
|
56
|
+
requireModule: defaults.requireModule,
|
|
57
|
+
require: defaults.require.concat([steps]),
|
|
58
|
+
paths: [feature],
|
|
59
|
+
format: [`json:${report}`],
|
|
60
|
+
worldParameters: world,
|
|
61
|
+
}) + ' };');
|
|
62
|
+
const env = Object.assign({ SLOW_MO: '0' }, process.env, {
|
|
63
|
+
WEBSHIP_REPORT_DISABLE: '1',
|
|
64
|
+
WEBSHIP_JS_ERROR_MODE: 'off',
|
|
65
|
+
WEBSHIP_SCREENSHOT_ON_FAILED: 'false',
|
|
66
|
+
}, extraEnv || {});
|
|
67
|
+
delete env.WEBSHIP_VIDEO;
|
|
68
|
+
delete env.WEBSHIP_VIDEO_DIR;
|
|
69
|
+
if (!extraEnv || !extraEnv.WEBSHIP_VIDEO_SIZE) delete env.WEBSHIP_VIDEO_SIZE;
|
|
70
|
+
const result = spawnSync(process.execPath, [path.join(cucumber, 'bin', 'cucumber.js'), '--config', path.relative(ROOT, config)], {
|
|
71
|
+
cwd: ROOT,
|
|
72
|
+
encoding: 'utf8',
|
|
73
|
+
stdio: ['ignore', 'pipe', 'pipe'],
|
|
74
|
+
timeout: 180000,
|
|
75
|
+
env,
|
|
76
|
+
});
|
|
77
|
+
if (!fs.existsSync(report) || !fs.readFileSync(report, 'utf8').trim()) {
|
|
78
|
+
problems.push(`${name}: the run wrote no report.\n${strip(result.stderr).slice(-1500)}`);
|
|
79
|
+
} else {
|
|
80
|
+
for (const f of JSON.parse(fs.readFileSync(report, 'utf8'))) {
|
|
81
|
+
for (const scenario of f.elements || []) {
|
|
82
|
+
const failed = (scenario.steps || []).find((step) => step.result && step.result.status !== 'passed');
|
|
83
|
+
if (failed) problems.push(`${name}: "${scenario.name}": ${failed.hidden ? 'a hook' : `"${failed.keyword}${failed.name}"`} ${failed.result.status}: ${strip(failed.result.error_message).split('\n')[0]}`);
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
return {
|
|
88
|
+
files: fs.existsSync(videos) ? fs.readdirSync(videos).sort() : [],
|
|
89
|
+
videos,
|
|
90
|
+
stdout: strip(result.stdout),
|
|
91
|
+
};
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
// Mode "on".
|
|
95
|
+
const on = run('on', 'on', [
|
|
96
|
+
'Feature: Recording that is on',
|
|
97
|
+
'',
|
|
98
|
+
' Scenario: Starting a recording that is already on',
|
|
99
|
+
' Given a page that reads "One"',
|
|
100
|
+
' When I start video recording',
|
|
101
|
+
' And print video path',
|
|
102
|
+
' Then I should see "One"',
|
|
103
|
+
'',
|
|
104
|
+
]);
|
|
105
|
+
if (on.files.some((f) => /^page@/.test(f))) problems.push(`on: a video was left under a temporary name: ${on.files.join(', ')}.`);
|
|
106
|
+
if (on.files.length !== 1) problems.push(`on: ${on.files.length} videos were saved, not 1: ${on.files.join(', ') || 'none'}.`);
|
|
107
|
+
else if (!/\.passed\.webm$/.test(on.files[0])) problems.push(`on: the video name does not end in ".passed.webm": ${on.files[0]}.`);
|
|
108
|
+
const printed = (on.stdout.match(/--- Video path: (\S+)/) || [])[1];
|
|
109
|
+
if (!printed) {
|
|
110
|
+
problems.push('on: "print video path" printed no path.');
|
|
111
|
+
} else if (on.files.length === 1 && path.resolve(printed.replace('<status>', 'passed')) !== path.resolve(on.videos, on.files[0])) {
|
|
112
|
+
problems.push(`on: "print video path" printed ${printed}, but the video was saved as ${path.join(on.videos, on.files[0])}.`);
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
// Mode "on-failure".
|
|
116
|
+
const onFailure = run('on-failure', 'on-failure', [
|
|
117
|
+
'Feature: Recording kept on failure',
|
|
118
|
+
'',
|
|
119
|
+
' Scenario: A passing scenario with a reserved video name',
|
|
120
|
+
' Given a page that reads "Kept"',
|
|
121
|
+
' When I save the current video as "kept.webm"',
|
|
122
|
+
' Then I should see "Kept"',
|
|
123
|
+
'',
|
|
124
|
+
' Scenario: A passing scenario without one',
|
|
125
|
+
' Given a page that reads "Dropped"',
|
|
126
|
+
' Then I should see "Dropped"',
|
|
127
|
+
'',
|
|
128
|
+
]);
|
|
129
|
+
if (onFailure.files.join(',') !== 'kept.webm') problems.push(`on-failure: the folder holds ${onFailure.files.join(', ') || 'nothing'}, not only kept.webm.`);
|
|
130
|
+
|
|
131
|
+
// Mode "off", with a recording size from the environment.
|
|
132
|
+
const off = run('off', 'off', [
|
|
133
|
+
'Feature: Recording started and stopped by steps',
|
|
134
|
+
'',
|
|
135
|
+
' Scenario: Stop and save',
|
|
136
|
+
' When I start video recording',
|
|
137
|
+
' And a page that reads "Stopped"',
|
|
138
|
+
' When I stop video recording',
|
|
139
|
+
'',
|
|
140
|
+
' Scenario: The recording size',
|
|
141
|
+
' When I start video recording',
|
|
142
|
+
' Then the recording size should be 640 by 360',
|
|
143
|
+
' When I stop video recording',
|
|
144
|
+
'',
|
|
145
|
+
], { WEBSHIP_VIDEO_SIZE: '640x360' });
|
|
146
|
+
const stopped = off.files.filter((f) => /Stop_and_save/.test(f));
|
|
147
|
+
if (stopped.length !== 1 || !/\.off\.Stop_and_save\.manual\.webm$/.test(stopped[0])) {
|
|
148
|
+
problems.push(`off: the stopped recording was saved as ${off.files.join(', ') || 'nothing'}, not under the filename pattern with the status "manual".`);
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
fs.rmSync(tmp, { recursive: true, force: true });
|
|
152
|
+
if (problems.length) {
|
|
153
|
+
console.error('Video recording:\n - ' + problems.join('\n - '));
|
|
154
|
+
process.exit(1);
|
|
155
|
+
}
|
|
156
|
+
console.log('Video recording: final names in lower case, no temporary file left, the printed path is the saved one, reserved names kept on failure, stopped recordings named by the pattern, WEBSHIP_VIDEO_SIZE read.');
|