webship-js 2.0.10 → 2.0.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/ISSUE_TEMPLATE/issue.md +37 -0
- package/.github/PULL_REQUEST_TEMPLATE.md +24 -0
- package/.github/workflows/github-actions.yml +56 -4
- package/CLAUDE.md +5 -2
- package/README.md +13 -1
- package/bin/generate-reports.js +51 -7
- package/bin/init-webship.js +85 -17
- package/cucumber.js +29 -1
- package/docs/00-quick-start.md +30 -7
- package/docs/02-bbr-smart-waits.md +6 -0
- package/docs/04-step-reference.md +2 -2
- package/docs/11-debugging.md +21 -0
- package/docs/12-ai-agent-guide.md +3 -1
- package/docs/13-faq.md +3 -0
- package/docs/15-tag-conventions.md +4 -1
- package/docs/16-ci-cd.md +29 -13
- package/docs/global-settings.md +2 -2
- package/docs/install-webship-js/ddev-webship-js.md +3 -0
- package/docs/install-webship-js.md +14 -2
- package/examples/a11y-legacy-widget.html +34 -0
- package/examples/auth-state.html +42 -0
- package/examples/clock.html +33 -0
- package/examples/downloads/webship-archive.zip +0 -0
- package/examples/downloads/webship-notes.txt +4 -0
- package/examples/downloads/webship-report.csv +4 -0
- package/examples/downloads.html +16 -0
- package/examples/element-viewport.html +42 -0
- package/examples/field-types.html +98 -0
- package/examples/iframe-newsletter.html +29 -0
- package/examples/iframe-order.html +26 -0
- package/examples/iframe.html +21 -0
- package/examples/javascript-errors.html +35 -0
- package/examples/links.html +1 -0
- package/examples/metatags.html +18 -0
- package/examples/mouse-and-pointer.html +83 -0
- package/examples/native-dialogs.html +8 -0
- package/examples/network.html +83 -0
- package/examples/scroll-horizontal.html +50 -0
- package/examples/selectors-position.html +29 -0
- package/examples/storage.html +57 -0
- package/examples/viewport-size.html +24 -0
- package/examples/wait-conditions.html +69 -0
- package/examples/web-first.html +99 -0
- package/package.json +9 -2
- package/tests/assets/catalog.xml +19 -0
- package/tests/assets/feed-namespaced.xml +13 -0
- package/tests/assets/sitemap.xml +14 -0
- package/tests/assets/webship-site-expected.yaml +21 -0
- package/tests/assets/webship-site.schema.json +46 -0
- package/tests/assets/webship-site.yaml +21 -0
- package/tests/check-examples.js +235 -0
- package/tests/check-must-fail.js +84 -0
- package/tests/check-paths.js +82 -0
- package/tests/check-report.js +119 -0
- package/tests/check-scaffold.js +211 -0
- package/tests/features/a11y.feature +18 -0
- package/tests/features/auth-state.feature +32 -0
- package/tests/features/clock.feature +27 -0
- package/tests/features/cookie.feature +9 -0
- package/tests/features/dialog.feature +27 -0
- package/tests/features/element.feature +47 -0
- package/tests/features/field.feature +104 -0
- package/tests/features/file-download.feature +64 -0
- package/tests/features/iframe.feature +83 -0
- package/tests/features/javascript.feature +47 -0
- package/tests/features/link.feature +11 -0
- package/tests/features/metatag.feature +38 -0
- package/tests/features/mouse-and-pointer.feature +86 -0
- package/tests/features/network-mocking.feature +80 -0
- package/tests/features/responsive.feature +27 -0
- package/tests/features/scroll-horizontal.feature +55 -0
- package/tests/features/selectors-position.feature +41 -0
- package/tests/features/storage.feature +98 -0
- package/tests/features/test--then--i-see-locator.feature +13 -0
- package/tests/features/test--then--i-should--see-text-in-element.feature +9 -1
- package/tests/features/test--then--the-field-should-contain-value.feature +8 -0
- package/tests/features/video.feature +1 -0
- package/tests/features/wait.feature +89 -0
- package/tests/features/web-first.feature +100 -0
- package/tests/features/xml.feature +96 -0
- package/tests/features/yaml.feature +75 -0
- package/tests/must-fail/assertions.feature +96 -0
- package/tests/must-fail/waits.feature +51 -0
- package/tests/reports/README.md +2 -2
- package/tests/step-definitions/a11y.steps.js +12 -8
- package/tests/step-definitions/assertion.steps.js +235 -61
- package/tests/step-definitions/dialog.steps.js +2 -2
- package/tests/step-definitions/element.steps.js +3 -1
- package/tests/step-definitions/field.steps.js +152 -22
- package/tests/step-definitions/file-download.steps.js +74 -10
- package/tests/step-definitions/screenshot.steps.js +1 -1
- package/tests/step-definitions/selectors.steps.js +9 -3
- package/tests/step-definitions/storage.steps.js +3 -3
- package/tests/step-definitions/wait.steps.js +129 -39
- package/tests/step-definitions/webship.js +144 -3
- package/tests/step-definitions/xml.steps.js +76 -14
- package/tests/step-definitions/yaml.steps.js +46 -27
- package/.scratch/commit.txt +0 -14
- package/.scratch/docs.py +0 -42
- package/.scratch/echo-server.js +0 -8
- package/.scratch/echo.js +0 -35
- package/.scratch/echo.pid +0 -1
- package/.scratch/features/cookies.feature +0 -22
- package/.scratch/http.pid +0 -1
- package/.scratch/https.pid +0 -1
- package/.scratch/issue.md +0 -49
- package/.scratch/one.js +0 -2
- package/.scratch/one.json +0 -280
- package/.scratch/patch.py +0 -163
- package/.scratch/pr.md +0 -30
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$schema": "http://json-schema.org/draft-07/schema#",
|
|
3
|
+
"title": "Site description",
|
|
4
|
+
"type": "object",
|
|
5
|
+
"required": ["name", "namespace", "version", "owner", "pages"],
|
|
6
|
+
"properties": {
|
|
7
|
+
"name": { "type": "string", "pattern": "^[a-z][a-z0-9-]*$" },
|
|
8
|
+
"namespace": { "type": "string", "enum": ["production", "staging", "development"] },
|
|
9
|
+
"version": { "type": "integer", "minimum": 1 },
|
|
10
|
+
"owner": {
|
|
11
|
+
"type": "object",
|
|
12
|
+
"required": ["team", "contact"],
|
|
13
|
+
"properties": {
|
|
14
|
+
"team": { "type": "string" },
|
|
15
|
+
"contact": { "type": "string", "format": "email" }
|
|
16
|
+
}
|
|
17
|
+
},
|
|
18
|
+
"settings": {
|
|
19
|
+
"type": "object",
|
|
20
|
+
"properties": {
|
|
21
|
+
"cache": { "type": "boolean" },
|
|
22
|
+
"languages": { "type": "array", "items": { "type": "string", "minLength": 2, "maxLength": 2 } },
|
|
23
|
+
"homepage": { "type": "string" }
|
|
24
|
+
}
|
|
25
|
+
},
|
|
26
|
+
"pages": {
|
|
27
|
+
"type": "array",
|
|
28
|
+
"minItems": 1,
|
|
29
|
+
"items": {
|
|
30
|
+
"type": "object",
|
|
31
|
+
"required": ["title", "path"],
|
|
32
|
+
"properties": {
|
|
33
|
+
"title": { "type": "string" },
|
|
34
|
+
"path": { "type": "string", "pattern": "^/" }
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
},
|
|
38
|
+
"status": {
|
|
39
|
+
"type": "object",
|
|
40
|
+
"properties": {
|
|
41
|
+
"updated": { "type": "string", "format": "date-time" },
|
|
42
|
+
"revision": { "type": "integer" }
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
# A small site description for the YAML steps.
|
|
2
|
+
name: webship-site
|
|
3
|
+
namespace: production
|
|
4
|
+
version: 2
|
|
5
|
+
owner:
|
|
6
|
+
team: platform
|
|
7
|
+
contact: team@example.com
|
|
8
|
+
settings:
|
|
9
|
+
cache: true
|
|
10
|
+
languages:
|
|
11
|
+
- en
|
|
12
|
+
- fr
|
|
13
|
+
homepage: /welcome
|
|
14
|
+
pages:
|
|
15
|
+
- title: Home
|
|
16
|
+
path: /
|
|
17
|
+
- title: About us
|
|
18
|
+
path: /about-us
|
|
19
|
+
status:
|
|
20
|
+
updated: "2026-01-15T10:00:00Z"
|
|
21
|
+
revision: 41
|
|
@@ -0,0 +1,235 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
'use strict';
|
|
3
|
+
|
|
4
|
+
// Checks the step documentation and the features against the step
|
|
5
|
+
// definitions, without a browser:
|
|
6
|
+
//
|
|
7
|
+
// 1. every step definition in tests/step-definitions carries a JSDoc block
|
|
8
|
+
// with a description and at least one "Example #n:" block,
|
|
9
|
+
// 2. every step line of every example resolves to exactly one definition
|
|
10
|
+
// (none undefined, none ambiguous),
|
|
11
|
+
// 3. at least one example of a definition resolves to that definition,
|
|
12
|
+
// 4. every step in tests/features resolves to exactly one definition.
|
|
13
|
+
//
|
|
14
|
+
// The matching is done by cucumber-js itself (`--dry-run`), so regular
|
|
15
|
+
// expressions and Cucumber expressions are read the way a real run reads
|
|
16
|
+
// them. An example may span several lines: the steps that set the scene, a
|
|
17
|
+
// data table or a doc string count as part of it.
|
|
18
|
+
//
|
|
19
|
+
// Usage: node tests/check-examples.js (npm run test:examples)
|
|
20
|
+
|
|
21
|
+
const fs = require('fs');
|
|
22
|
+
const os = require('os');
|
|
23
|
+
const path = require('path');
|
|
24
|
+
const { spawnSync } = require('child_process');
|
|
25
|
+
|
|
26
|
+
const ROOT = path.resolve(__dirname, '..');
|
|
27
|
+
const STEPS_DIR = path.join(ROOT, 'tests', 'step-definitions');
|
|
28
|
+
const KEYWORD = /^(Given|When|Then|And|But|\*)\s/;
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* Read every step definition of a step file with the JSDoc block above it.
|
|
32
|
+
*
|
|
33
|
+
* @param {string} file - File name inside tests/step-definitions.
|
|
34
|
+
* @returns {Array<Object>} One entry per definition.
|
|
35
|
+
*/
|
|
36
|
+
function readDefinitions(file) {
|
|
37
|
+
const source = fs.readFileSync(path.join(STEPS_DIR, file), 'utf8');
|
|
38
|
+
const definitions = [];
|
|
39
|
+
// Some steps are registered through an alias (ThenF, WhenF): they count.
|
|
40
|
+
const re = /(?:^|\n)[ \t]*(Given|When|Then|defineStep)F?\(\s*(\/(?:\\\/|[^/\n])+\/[a-z]*|'(?:\\'|[^'\n])*'|"(?:\\"|[^"\n])*"|`[^`\n]*`)/g;
|
|
41
|
+
let m;
|
|
42
|
+
while ((m = re.exec(source))) {
|
|
43
|
+
const index = m.index + (m[0].startsWith('\n') ? 1 : 0);
|
|
44
|
+
const before = source.slice(0, index);
|
|
45
|
+
const line = before.split('\n').length;
|
|
46
|
+
let doc = null;
|
|
47
|
+
const docEnd = before.lastIndexOf('*/');
|
|
48
|
+
if (docEnd !== -1 && /^\s*$/.test(before.slice(docEnd + 2))) {
|
|
49
|
+
// The block opens on a line of its own; "/**" may also be part of a
|
|
50
|
+
// URL pattern inside an example.
|
|
51
|
+
let docStart = -1;
|
|
52
|
+
const open = /(^|\n)[ \t]*\/\*\*/g;
|
|
53
|
+
let o;
|
|
54
|
+
while ((o = open.exec(before.slice(0, docEnd)))) docStart = o.index + o[0].indexOf('/**');
|
|
55
|
+
if (docStart !== -1) doc = before.slice(docStart + 3, docEnd);
|
|
56
|
+
}
|
|
57
|
+
const examples = [];
|
|
58
|
+
const description = [];
|
|
59
|
+
if (doc !== null) {
|
|
60
|
+
let current = null;
|
|
61
|
+
for (const raw of doc.split('\n').map((l) => l.replace(/^\s*\*\s?/, ''))) {
|
|
62
|
+
const example = raw.match(/^\s*Example\s*#?\d*[^:]*:\s*(.*)$/);
|
|
63
|
+
if (example) {
|
|
64
|
+
current = [];
|
|
65
|
+
examples.push(current);
|
|
66
|
+
// Text after the colon is the first step, or a caption of the block.
|
|
67
|
+
if (KEYWORD.test(example[1].trim())) current.push(example[1].trim());
|
|
68
|
+
continue;
|
|
69
|
+
}
|
|
70
|
+
if (/^\s*@\w+/.test(raw)) { current = null; continue; }
|
|
71
|
+
if (current) { if (raw.trim()) current.push(raw.trim()); continue; }
|
|
72
|
+
if (!examples.length && raw.trim()) description.push(raw.trim());
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
definitions.push({ file, line, pattern: m[2], hasDoc: doc !== null, description: description.join(' '), examples, own: 0 });
|
|
76
|
+
}
|
|
77
|
+
return definitions;
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/**
|
|
81
|
+
* Turn the examples of the definitions of one file into a feature: one
|
|
82
|
+
* scenario per example. Returns the feature text and, per feature line, the
|
|
83
|
+
* definition the step line belongs to.
|
|
84
|
+
*/
|
|
85
|
+
function buildFeature(file, definitions) {
|
|
86
|
+
const lines = [`Feature: examples of ${file}`, ''];
|
|
87
|
+
const owners = {};
|
|
88
|
+
for (const definition of definitions) {
|
|
89
|
+
definition.examples.forEach((example, n) => {
|
|
90
|
+
const body = [];
|
|
91
|
+
let inDocString = false;
|
|
92
|
+
let steps = 0;
|
|
93
|
+
for (const raw of example) {
|
|
94
|
+
if (/^("""|```)/.test(raw)) { inDocString = !inDocString; if (steps) body.push({ text: ' ' + raw }); continue; }
|
|
95
|
+
if (inDocString) { if (steps) body.push({ text: ' ' + raw }); continue; }
|
|
96
|
+
if (/^\|.*\|\s*(#.*)?$/.test(raw)) { if (steps) body.push({ text: ' ' + raw.replace(/\|\s+#[^|]*$/, '|') }); continue; }
|
|
97
|
+
if (!KEYWORD.test(raw)) continue; // a caption or a comment
|
|
98
|
+
let step = raw.replace(/\s{2,}#\s.*$/, '').replace(/^(\S+)\s+/, '$1 ');
|
|
99
|
+
if (!steps) step = step.replace(/^(And|But|\*) /, 'Given ');
|
|
100
|
+
steps++;
|
|
101
|
+
body.push({ text: ' ' + step, step: raw });
|
|
102
|
+
}
|
|
103
|
+
if (!steps) return;
|
|
104
|
+
lines.push(` Scenario: ${file} line ${definition.line} example ${n + 1}`);
|
|
105
|
+
for (const item of body) {
|
|
106
|
+
lines.push(item.text);
|
|
107
|
+
if (item.step) owners[lines.length] = { definition, step: item.step };
|
|
108
|
+
}
|
|
109
|
+
lines.push('');
|
|
110
|
+
});
|
|
111
|
+
}
|
|
112
|
+
return { text: lines.join('\n'), owners };
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
/**
|
|
116
|
+
* Dry-run cucumber-js over some feature paths and return its JSON report and
|
|
117
|
+
* its usage report.
|
|
118
|
+
*/
|
|
119
|
+
function dryRun(tmp, name, featurePaths) {
|
|
120
|
+
const usage = path.join(tmp, `${name}-usage.json`);
|
|
121
|
+
const report = path.join(tmp, `${name}-report.json`);
|
|
122
|
+
const config = path.join(tmp, `${name}-cucumber.js`);
|
|
123
|
+
const base = require(path.join(ROOT, 'cucumber.js')).default;
|
|
124
|
+
fs.writeFileSync(config, 'module.exports = { default: ' + JSON.stringify({
|
|
125
|
+
requireModule: base.requireModule,
|
|
126
|
+
require: base.require,
|
|
127
|
+
paths: featurePaths,
|
|
128
|
+
format: [`usage-json:${usage}`, `json:${report}`],
|
|
129
|
+
worldParameters: base.worldParameters,
|
|
130
|
+
}) + ' };');
|
|
131
|
+
const bin = path.join(ROOT, 'node_modules', '@cucumber', 'cucumber', 'bin', 'cucumber.js');
|
|
132
|
+
const run = spawnSync(process.execPath, [bin, '--config', path.relative(ROOT, config), '--dry-run'], {
|
|
133
|
+
cwd: ROOT,
|
|
134
|
+
encoding: 'utf8',
|
|
135
|
+
env: Object.assign({}, process.env, { WEBSHIP_REPORT_DISABLE: '1' }),
|
|
136
|
+
maxBuffer: 1 << 28,
|
|
137
|
+
});
|
|
138
|
+
if (!fs.existsSync(report) || !fs.existsSync(usage) || !fs.readFileSync(report, 'utf8').trim()) {
|
|
139
|
+
console.error((run.stdout || '') + (run.stderr || ''));
|
|
140
|
+
console.error(`cucumber-js --dry-run did not produce a report for the ${name}.`);
|
|
141
|
+
process.exit(2);
|
|
142
|
+
}
|
|
143
|
+
return {
|
|
144
|
+
report: JSON.parse(fs.readFileSync(report, 'utf8')),
|
|
145
|
+
usage: JSON.parse(fs.readFileSync(usage, 'utf8')),
|
|
146
|
+
};
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
const problems = [];
|
|
150
|
+
const files = fs.readdirSync(STEPS_DIR).filter((f) => f.endsWith('.steps.js')).sort();
|
|
151
|
+
const all = [];
|
|
152
|
+
const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'webship-js-examples-'));
|
|
153
|
+
const featureOwners = {};
|
|
154
|
+
const featurePaths = [];
|
|
155
|
+
|
|
156
|
+
for (const file of files) {
|
|
157
|
+
const definitions = readDefinitions(file);
|
|
158
|
+
all.push(...definitions);
|
|
159
|
+
for (const d of definitions) {
|
|
160
|
+
const where = `${d.file}:${d.line}`;
|
|
161
|
+
if (!d.hasDoc) problems.push(`${where} no JSDoc block above the definition ${d.pattern}`);
|
|
162
|
+
else if (!d.description) problems.push(`${where} the JSDoc block has no description`);
|
|
163
|
+
if (d.hasDoc && !d.examples.length) problems.push(`${where} no "Example #n:" line`);
|
|
164
|
+
}
|
|
165
|
+
const feature = buildFeature(file, definitions);
|
|
166
|
+
const featurePath = path.join(tmp, `${file}.feature`);
|
|
167
|
+
fs.writeFileSync(featurePath, feature.text);
|
|
168
|
+
featurePaths.push(path.relative(ROOT, featurePath));
|
|
169
|
+
featureOwners[`${file}.feature`] = feature.owners;
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
const byPlace = {};
|
|
173
|
+
for (const d of all) byPlace[`${d.file}:${d.line}`] = d;
|
|
174
|
+
|
|
175
|
+
// 1. The examples.
|
|
176
|
+
const examples = dryRun(tmp, 'examples', featurePaths);
|
|
177
|
+
const matched = {};
|
|
178
|
+
for (const entry of examples.usage) {
|
|
179
|
+
for (const match of entry.matches || []) {
|
|
180
|
+
matched[`${path.basename(match.uri)}:${match.line}`] = `${path.basename(entry.uri)}:${entry.line}`;
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
let exampleSteps = 0;
|
|
184
|
+
for (const feature of examples.report) {
|
|
185
|
+
const owners = featureOwners[path.basename(feature.uri)] || {};
|
|
186
|
+
for (const scenario of feature.elements || []) {
|
|
187
|
+
for (const step of scenario.steps || []) {
|
|
188
|
+
const owner = owners[step.line];
|
|
189
|
+
if (!owner) continue;
|
|
190
|
+
exampleSteps++;
|
|
191
|
+
const where = `${owner.definition.file}:${owner.definition.line}`;
|
|
192
|
+
const status = step.result && step.result.status;
|
|
193
|
+
if (status === 'undefined') problems.push(`${where} example step matches no definition: ${owner.step}`);
|
|
194
|
+
else if (status === 'ambiguous') problems.push(`${where} example step matches more than one definition: ${owner.step}`);
|
|
195
|
+
else if (matched[`${path.basename(feature.uri)}:${step.line}`] === where) owner.definition.own++;
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
for (const d of all) {
|
|
200
|
+
if (d.examples.length && !d.own) problems.push(`${d.file}:${d.line} none of its examples uses the step itself: ${d.pattern}`);
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
// 2. The features of the suite.
|
|
204
|
+
const base = require(path.join(ROOT, 'cucumber.js')).default;
|
|
205
|
+
const suite = dryRun(tmp, 'features', base.paths);
|
|
206
|
+
let featureSteps = 0;
|
|
207
|
+
for (const feature of suite.report) {
|
|
208
|
+
for (const scenario of feature.elements || []) {
|
|
209
|
+
for (const step of scenario.steps || []) {
|
|
210
|
+
if (step.hidden) continue;
|
|
211
|
+
featureSteps++;
|
|
212
|
+
const status = step.result && step.result.status;
|
|
213
|
+
if (status === 'undefined') problems.push(`${feature.uri}:${step.line} undefined step: ${step.keyword}${step.name}`);
|
|
214
|
+
if (status === 'ambiguous') problems.push(`${feature.uri}:${step.line} ambiguous step: ${step.keyword}${step.name}`);
|
|
215
|
+
}
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
// A definition cucumber-js loaded that the reader above did not see would
|
|
219
|
+
// escape every check: say so instead of passing.
|
|
220
|
+
for (const entry of suite.usage) {
|
|
221
|
+
if (!byPlace[`${path.basename(entry.uri)}:${entry.line}`]) {
|
|
222
|
+
problems.push(`${entry.uri}:${entry.line} definition not read by this check: ${entry.pattern}`);
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
fs.rmSync(tmp, { recursive: true, force: true });
|
|
227
|
+
|
|
228
|
+
const exampleCount = all.reduce((n, d) => n + d.examples.length, 0);
|
|
229
|
+
console.log(`${all.length} step definitions in ${files.length} files, ${exampleCount} examples (${exampleSteps} step lines), ${featureSteps} feature steps.`);
|
|
230
|
+
if (problems.length) {
|
|
231
|
+
console.error(`\n${problems.length} problem(s):`);
|
|
232
|
+
for (const p of problems) console.error(' ' + p);
|
|
233
|
+
process.exit(1);
|
|
234
|
+
}
|
|
235
|
+
console.log('Every definition is documented, every example step and every feature step resolves to exactly one definition.');
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
'use strict';
|
|
3
|
+
|
|
4
|
+
// Runs the scenarios of tests/must-fail/ and checks that every one of them
|
|
5
|
+
// FAILS, at its last step, with the message its description announces
|
|
6
|
+
// ("Fails with: ..."), and that the message names the page and carries a
|
|
7
|
+
// hint, with no Playwright call log and no stack trace.
|
|
8
|
+
//
|
|
9
|
+
// A step that cannot fail proves nothing, and a failure nobody can read
|
|
10
|
+
// costs a tester an hour. Each scenario here asserts something that is not
|
|
11
|
+
// true on the example pages; the steps before the last one must pass.
|
|
12
|
+
//
|
|
13
|
+
// Usage: node tests/check-must-fail.js (npm run test:must-fail)
|
|
14
|
+
// The examples/ pages must be served on the launch URL of cucumber.js
|
|
15
|
+
// (npm start; LAUNCH_URL to point elsewhere).
|
|
16
|
+
|
|
17
|
+
const fs = require('fs');
|
|
18
|
+
const os = require('os');
|
|
19
|
+
const path = require('path');
|
|
20
|
+
const { spawnSync } = require('child_process');
|
|
21
|
+
|
|
22
|
+
const ROOT = path.resolve(__dirname, '..');
|
|
23
|
+
const DIR = path.join(ROOT, 'tests', 'must-fail');
|
|
24
|
+
const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'webship-js-must-fail-'));
|
|
25
|
+
const report = path.join(tmp, 'must-fail.json');
|
|
26
|
+
const config = path.join(tmp, 'cucumber.js');
|
|
27
|
+
|
|
28
|
+
const base = require(path.join(ROOT, 'cucumber.js')).default;
|
|
29
|
+
const features = fs.readdirSync(DIR).filter((file) => file.endsWith('.feature')).sort().map((file) => `tests/must-fail/${file}`);
|
|
30
|
+
// The same step definitions and world as the suite; the screenshots of these
|
|
31
|
+
// deliberate failures go to the temporary folder, not to screenshots/.
|
|
32
|
+
const world = JSON.parse(JSON.stringify(base.worldParameters));
|
|
33
|
+
world.screenshot = Object.assign({}, world.screenshot, { dir: path.join(tmp, 'screenshots') });
|
|
34
|
+
fs.writeFileSync(config, 'module.exports = { default: ' + JSON.stringify({
|
|
35
|
+
requireModule: base.requireModule,
|
|
36
|
+
require: base.require,
|
|
37
|
+
paths: features,
|
|
38
|
+
format: [`json:${report}`],
|
|
39
|
+
worldParameters: world,
|
|
40
|
+
}) + ' };');
|
|
41
|
+
|
|
42
|
+
spawnSync(process.execPath, [path.join(ROOT, 'node_modules', '@cucumber', 'cucumber', 'bin', 'cucumber.js'), '--config', path.relative(ROOT, config)], {
|
|
43
|
+
cwd: ROOT,
|
|
44
|
+
stdio: ['ignore', 'ignore', 'inherit'],
|
|
45
|
+
env: Object.assign({ SLOW_MO: '0' }, process.env, { WEBSHIP_REPORT_DISABLE: '1', WEBSHIP_VIDEO: 'off', WEBSHIP_JS_ERROR_MODE: 'off' }),
|
|
46
|
+
});
|
|
47
|
+
|
|
48
|
+
const strip = (text) => String(text || '').replace(/\u001b\[[0-9;]*m/g, '');
|
|
49
|
+
const problems = [];
|
|
50
|
+
let count = 0;
|
|
51
|
+
|
|
52
|
+
if (!fs.existsSync(report) || !fs.readFileSync(report, 'utf8').trim()) {
|
|
53
|
+
problems.push('The must-fail run wrote no report: cucumber-js did not start.');
|
|
54
|
+
} else {
|
|
55
|
+
for (const feature of JSON.parse(fs.readFileSync(report, 'utf8'))) {
|
|
56
|
+
for (const scenario of feature.elements || []) {
|
|
57
|
+
count += 1;
|
|
58
|
+
const where = `${feature.uri}:${scenario.line} "${scenario.name}"`;
|
|
59
|
+
const wanted = ((strip(scenario.description).match(/Fails with:\s*(.+)/) || [])[1] || '').trim();
|
|
60
|
+
const steps = (scenario.steps || []).filter((step) => !step.hidden);
|
|
61
|
+
const last = steps[steps.length - 1];
|
|
62
|
+
const earlier = steps.slice(0, -1).filter((step) => step.result.status !== 'passed');
|
|
63
|
+
if (!wanted) { problems.push(`${where}: no "Fails with:" line in its description.`); continue; }
|
|
64
|
+
if (earlier.length) { problems.push(`${where}: an earlier step did not pass (${earlier[0].keyword}${earlier[0].name}: ${earlier[0].result.status}) ${strip(earlier[0].result.error_message).split('\n')[0]}`); continue; }
|
|
65
|
+
if (!last || last.result.status !== 'failed') { problems.push(`${where}: its last step ${last ? last.result.status : 'is missing'}; it must fail.`); continue; }
|
|
66
|
+
const message = strip(last.result.error_message);
|
|
67
|
+
const shown = message.split('\n').slice(0, 4).join('\n ');
|
|
68
|
+
if (!message.includes(wanted)) { problems.push(`${where}: failed, but not with "${wanted}":\n ${shown}`); continue; }
|
|
69
|
+
if (!/^\s*Hint: \S/m.test(message)) { problems.push(`${where}: the failure carries no hint:\n ${shown}`); continue; }
|
|
70
|
+
if (!/^\s*Page: \S/m.test(message)) { problems.push(`${where}: the failure does not name the page:\n ${shown}`); continue; }
|
|
71
|
+
if (/Call log:|Timeout \d+ms exceeded|^\s+at .+:\d+:\d+\)?$/m.test(message)) { problems.push(`${where}: the failure shows a Playwright call log, a raw timeout or a stack trace:\n ${shown}`); continue; }
|
|
72
|
+
console.log(` fails as it must: ${scenario.name}\n ${message.split('\n')[0].trim()}`);
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
if (!count) problems.push('No must-fail scenario ran.');
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
fs.rmSync(tmp, { recursive: true, force: true });
|
|
79
|
+
|
|
80
|
+
if (problems.length) {
|
|
81
|
+
console.error(`\n${problems.length} problem(s) in ${count} must-fail scenario(s):\n ${problems.join('\n ')}`);
|
|
82
|
+
process.exit(1);
|
|
83
|
+
}
|
|
84
|
+
console.log(`\n${count} must-fail scenarios: each failed at its last step with the announced message, the page and a hint.`);
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
'use strict';
|
|
3
|
+
|
|
4
|
+
// Checks that a feature, a scenario (file:line) or several features named on
|
|
5
|
+
// the command line of cucumber-js run alone, and that a run that names none
|
|
6
|
+
// runs the whole of tests/features/ (cucumber.js, commandLinePaths). Dry runs
|
|
7
|
+
// only: no browser, no server.
|
|
8
|
+
//
|
|
9
|
+
// Usage: node tests/check-paths.js (npm run test:paths)
|
|
10
|
+
|
|
11
|
+
const fs = require('fs');
|
|
12
|
+
const os = require('os');
|
|
13
|
+
const path = require('path');
|
|
14
|
+
const { spawnSync } = require('child_process');
|
|
15
|
+
|
|
16
|
+
const ROOT = path.resolve(__dirname, '..');
|
|
17
|
+
const BIN = path.join(ROOT, 'node_modules', '@cucumber', 'cucumber', 'bin', 'cucumber.js');
|
|
18
|
+
const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'webship-js-paths-'));
|
|
19
|
+
const problems = [];
|
|
20
|
+
|
|
21
|
+
// The scenarios a dry run selects, from a JSON report written to the temporary
|
|
22
|
+
// folder; the report and the formats of cucumber.js are left alone.
|
|
23
|
+
function scenarios(args) {
|
|
24
|
+
const report = path.join(tmp, `run-${Date.now()}-${Math.random().toString(36).slice(2)}.json`);
|
|
25
|
+
const run = spawnSync(process.execPath, [BIN, '--config', 'cucumber.js', '--dry-run', ...args, '--format', `json:${report}`], {
|
|
26
|
+
cwd: ROOT,
|
|
27
|
+
encoding: 'utf8',
|
|
28
|
+
env: Object.assign({}, process.env, { WEBSHIP_REPORT_DISABLE: '1', FORCE_COLOR: '0' }),
|
|
29
|
+
});
|
|
30
|
+
if (!fs.existsSync(report)) throw new Error(`cucumber-js ${args.join(' ')} wrote no report:\n${run.stderr}`);
|
|
31
|
+
const found = [];
|
|
32
|
+
for (const feature of JSON.parse(fs.readFileSync(report, 'utf8'))) {
|
|
33
|
+
for (const scenario of feature.elements || []) found.push(`${feature.uri}:${scenario.line}`);
|
|
34
|
+
}
|
|
35
|
+
return { found, warned: /specified paths in both/.test(run.stderr || '') };
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
function featureFiles() {
|
|
39
|
+
return fs.readdirSync(path.join(ROOT, 'tests', 'features')).filter((f) => f.endsWith('.feature')).sort();
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
try {
|
|
43
|
+
const all = scenarios([]);
|
|
44
|
+
const files = featureFiles();
|
|
45
|
+
if (!all.found.length) problems.push('a run that names no feature selected no scenario');
|
|
46
|
+
const inAll = new Set(all.found.map((s) => s.split(':')[0]));
|
|
47
|
+
for (const f of files) {
|
|
48
|
+
if (!inAll.has(`tests/features/${f}`)) problems.push(`a run that names no feature left out tests/features/${f}`);
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
const [first, second] = files;
|
|
52
|
+
const one = scenarios([`tests/features/${first}`]);
|
|
53
|
+
const own = all.found.filter((s) => s.startsWith(`tests/features/${first}:`));
|
|
54
|
+
if (one.found.length !== own.length || one.found.some((s) => !s.startsWith(`tests/features/${first}:`))) {
|
|
55
|
+
problems.push(`naming tests/features/${first} selected ${one.found.length} scenario(s), not its own ${own.length}`);
|
|
56
|
+
}
|
|
57
|
+
if (one.warned) problems.push('naming a feature makes cucumber-js warn that the paths of the configuration and of the command line are merged');
|
|
58
|
+
|
|
59
|
+
const line = own[0].split(':')[1];
|
|
60
|
+
const byLine = scenarios([`tests/features/${first}:${line}`]);
|
|
61
|
+
if (byLine.found.length !== 1 || byLine.found[0] !== own[0]) {
|
|
62
|
+
problems.push(`naming the scenario tests/features/${first}:${line} selected ${byLine.found.length} scenario(s), not 1`);
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
const two = scenarios([`tests/features/${first}`, '--tags', 'not @no-such-tag', `tests/features/${second}`]);
|
|
66
|
+
const ownTwo = all.found.filter((s) => s.startsWith(`tests/features/${first}:`) || s.startsWith(`tests/features/${second}:`));
|
|
67
|
+
if (two.found.length !== ownTwo.length) {
|
|
68
|
+
problems.push(`naming two features around an option selected ${two.found.length} scenario(s), not their ${ownTwo.length}`);
|
|
69
|
+
}
|
|
70
|
+
if (!problems.length) {
|
|
71
|
+
console.log(` no feature named: ${all.found.length} scenarios; ${first}: ${one.found.length}; ${first}:${line}: 1; ${first} and ${second}: ${two.found.length}.`);
|
|
72
|
+
}
|
|
73
|
+
} finally {
|
|
74
|
+
fs.rmSync(tmp, { recursive: true, force: true });
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
if (problems.length) {
|
|
78
|
+
console.error(`\n${problems.length} problem(s) with the paths of the command line:`);
|
|
79
|
+
for (const p of problems) console.error(' ' + p);
|
|
80
|
+
process.exit(1);
|
|
81
|
+
}
|
|
82
|
+
console.log('A feature or a scenario named on the command line runs alone; naming none runs the whole suite.');
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
'use strict';
|
|
3
|
+
|
|
4
|
+
// Checks the HTML report bin/generate-reports.js writes, from a small
|
|
5
|
+
// cucumber JSON, without a browser:
|
|
6
|
+
//
|
|
7
|
+
// 1. it links no icon of a third-party site,
|
|
8
|
+
// 2. "App Version" is the version of the project the report is made for,
|
|
9
|
+
// and is left out when the project has no package.json,
|
|
10
|
+
// 3. "Executed" is "Local" on a tester's machine and "Remote" on a CI
|
|
11
|
+
// runner, unless it is given.
|
|
12
|
+
//
|
|
13
|
+
// Usage: node tests/check-report.js (npm run test:report)
|
|
14
|
+
|
|
15
|
+
const fs = require('fs');
|
|
16
|
+
const os = require('os');
|
|
17
|
+
const path = require('path');
|
|
18
|
+
const { spawnSync } = require('child_process');
|
|
19
|
+
|
|
20
|
+
const GENERATE = path.resolve(__dirname, '..', 'bin', 'generate-reports.js');
|
|
21
|
+
const problems = [];
|
|
22
|
+
|
|
23
|
+
const SAMPLE = [{
|
|
24
|
+
keyword: 'Feature',
|
|
25
|
+
name: 'A sample feature',
|
|
26
|
+
line: 1,
|
|
27
|
+
id: 'a-sample-feature',
|
|
28
|
+
uri: 'tests/features/sample.feature',
|
|
29
|
+
tags: [],
|
|
30
|
+
elements: [{
|
|
31
|
+
keyword: 'Scenario',
|
|
32
|
+
name: 'A sample scenario',
|
|
33
|
+
line: 3,
|
|
34
|
+
id: 'a-sample-feature;a-sample-scenario',
|
|
35
|
+
type: 'scenario',
|
|
36
|
+
tags: [],
|
|
37
|
+
steps: [{
|
|
38
|
+
keyword: 'Given ',
|
|
39
|
+
name: 'I am on the homepage',
|
|
40
|
+
line: 4,
|
|
41
|
+
arguments: [],
|
|
42
|
+
match: { location: 'tests/step-definitions/navigation.steps.js:44' },
|
|
43
|
+
result: { status: 'passed', duration: 1000000 },
|
|
44
|
+
}],
|
|
45
|
+
}],
|
|
46
|
+
}];
|
|
47
|
+
|
|
48
|
+
/**
|
|
49
|
+
* Generate a report in a new project folder and return its HTML.
|
|
50
|
+
*
|
|
51
|
+
* @param {string} tmp - Parent folder.
|
|
52
|
+
* @param {string} name - Name of the project folder.
|
|
53
|
+
* @param {Object|null} pkg - Its package.json, or null for none.
|
|
54
|
+
* @param {Object} env - Variables set (a null value unsets one).
|
|
55
|
+
* @param {string[]} [args] - Extra arguments of generate-reports.
|
|
56
|
+
* @returns {string}
|
|
57
|
+
*/
|
|
58
|
+
function report(tmp, name, pkg, env, args) {
|
|
59
|
+
const project = path.join(tmp, name);
|
|
60
|
+
fs.mkdirSync(path.join(project, 'tests', 'reports'), { recursive: true });
|
|
61
|
+
if (pkg) fs.writeFileSync(path.join(project, 'package.json'), JSON.stringify(pkg));
|
|
62
|
+
fs.writeFileSync(path.join(project, 'tests', 'reports', 'cucumber_report.json'), JSON.stringify(SAMPLE));
|
|
63
|
+
const environment = Object.assign({}, process.env);
|
|
64
|
+
for (const key of ['CI', 'WEBSHIP_REPORT_APP_VERSION', 'WEBSHIP_REPORT_EXECUTED', 'WEBSHIP_REPORT_ARGS']) delete environment[key];
|
|
65
|
+
for (const [key, value] of Object.entries(env)) {
|
|
66
|
+
if (value === null) delete environment[key];
|
|
67
|
+
else environment[key] = value;
|
|
68
|
+
}
|
|
69
|
+
const run = spawnSync(process.execPath, [GENERATE].concat(args || []), { cwd: project, encoding: 'utf8', env: environment });
|
|
70
|
+
const out = path.join(project, 'tests', 'reports', 'cucumber_report.html');
|
|
71
|
+
if (run.status !== 0 || !fs.existsSync(out)) {
|
|
72
|
+
problems.push(`${name}: generate-reports exited with ${run.status} and wrote ${fs.existsSync(out) ? 'a' : 'no'} report\n${run.stderr}`);
|
|
73
|
+
return '';
|
|
74
|
+
}
|
|
75
|
+
return fs.readFileSync(out, 'utf8');
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
/** The value the Metadata block of a report shows for a key, or null. */
|
|
79
|
+
function metadata(html, key) {
|
|
80
|
+
const text = html.replace(/<[^>]+>/g, ' ').replace(/\s+/g, ' ');
|
|
81
|
+
const match = text.match(new RegExp(key + ':\\s*(\\S+)'));
|
|
82
|
+
return match ? match[1] : null;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'webship-js-report-'));
|
|
86
|
+
try {
|
|
87
|
+
const local = report(tmp, 'local', { name: 'local', version: '3.4.5' }, {});
|
|
88
|
+
if (local) {
|
|
89
|
+
const icons = local.match(/<link\b[^>]*\brel=["']?(?:shortcut )?icon["']?[^>]*>/gi) || [];
|
|
90
|
+
for (const icon of icons) {
|
|
91
|
+
if (/href=["']?(https?:)?\/\//i.test(icon)) problems.push(`the report links an icon of another site: ${icon}`);
|
|
92
|
+
}
|
|
93
|
+
if (/lambdatest/i.test(local)) problems.push('the report names a third-party site (lambdatest)');
|
|
94
|
+
if (!/A sample scenario/.test(local)) problems.push('the report does not show the scenario of the JSON');
|
|
95
|
+
if (metadata(local, 'App Version') !== '3.4.5') problems.push(`App Version is "${metadata(local, 'App Version')}", the project's package.json says 3.4.5`);
|
|
96
|
+
if (metadata(local, 'Executed') !== 'Local') problems.push(`Executed is "${metadata(local, 'Executed')}" for a run without CI set; expected Local`);
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
const ci = report(tmp, 'ci', { name: 'ci', version: '1.0.0' }, { CI: 'true' });
|
|
100
|
+
if (ci && metadata(ci, 'Executed') !== 'Remote') problems.push(`Executed is "${metadata(ci, 'Executed')}" with CI set; expected Remote`);
|
|
101
|
+
|
|
102
|
+
const given = report(tmp, 'given', { name: 'given', version: '1.0.0' }, { WEBSHIP_REPORT_APP_VERSION: '9.9.9' }, ['--executed', 'Staging']);
|
|
103
|
+
if (given) {
|
|
104
|
+
if (metadata(given, 'App Version') !== '9.9.9') problems.push(`WEBSHIP_REPORT_APP_VERSION=9.9.9 gave App Version "${metadata(given, 'App Version')}"`);
|
|
105
|
+
if (metadata(given, 'Executed') !== 'Staging') problems.push(`--executed Staging gave Executed "${metadata(given, 'Executed')}"`);
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
const bare = report(tmp, 'bare', null, {});
|
|
109
|
+
if (bare && metadata(bare, 'App Version') !== null) problems.push(`a project without package.json got App Version "${metadata(bare, 'App Version')}"`);
|
|
110
|
+
} finally {
|
|
111
|
+
fs.rmSync(tmp, { recursive: true, force: true });
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
if (problems.length) {
|
|
115
|
+
console.error(`${problems.length} problem(s) with the HTML report:`);
|
|
116
|
+
for (const p of problems) console.error(' ' + p);
|
|
117
|
+
process.exit(1);
|
|
118
|
+
}
|
|
119
|
+
console.log('The HTML report links no third-party icon; App Version and Executed say what is true for the project and the run.');
|