webship-js 2.0.11 → 2.0.13

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (175) hide show
  1. package/.circleci/config.yml +10 -1
  2. package/.drone.yml +7 -7
  3. package/.github/workflows/github-actions.yml +48 -0
  4. package/.gitlab-ci.yml +9 -4
  5. package/.harness/webship-js-pipeline.yml +5 -6
  6. package/.semaphore/semaphore.yml +6 -9
  7. package/.teamcity/settings.kts +7 -6
  8. package/.travis.yml +6 -1
  9. package/AGENTS.md +1 -1
  10. package/CLAUDE.md +65 -23
  11. package/Jenkinsfile +5 -3
  12. package/azure-pipelines.yml +17 -8
  13. package/bamboo-specs/bamboo.yml +9 -15
  14. package/bin/init-webship.js +53 -10
  15. package/bitbucket-pipelines.yml +5 -1
  16. package/buildspec.yml +6 -7
  17. package/cloudbuild.yaml +28 -12
  18. package/codefresh.yml +6 -6
  19. package/cucumber.js +17 -10
  20. package/docs/00-quick-start.md +8 -0
  21. package/docs/02-bbr-smart-waits.md +15 -10
  22. package/docs/03-selector-registry.md +24 -0
  23. package/docs/04-step-reference.md +51 -25
  24. package/docs/05-web-first-assertions.md +6 -0
  25. package/docs/07-auth-state.md +1 -1
  26. package/docs/08-clock-mocking.md +21 -2
  27. package/docs/09-api-testing.md +40 -2
  28. package/docs/10-accessibility.md +2 -0
  29. package/docs/11-debugging.md +3 -1
  30. package/docs/12-ai-agent-guide.md +5 -0
  31. package/docs/13-faq.md +72 -1
  32. package/docs/14-recipes-cookbook.md +2 -0
  33. package/docs/15-tag-conventions.md +2 -1
  34. package/docs/16-ci-cd.md +140 -119
  35. package/docs/README.md +5 -5
  36. package/docs/advanced-screenshots.md +3 -1
  37. package/docs/api-step-definitions.md +16 -2
  38. package/docs/global-settings.md +69 -17
  39. package/docs/install-webship-js.md +2 -0
  40. package/docs/step-definitions/then-i-should-see.md +14 -1
  41. package/docs/step-definitions/then-item-checkbox.md +2 -0
  42. package/docs/step-definitions/when-i-fill.md +4 -0
  43. package/docs/step-definitions/when-i-select-option-select-list.md +2 -0
  44. package/examples/check-switch.html +37 -0
  45. package/examples/choice-controls.html +29 -0
  46. package/examples/click-whitespace.html +37 -0
  47. package/examples/datetime-fields.html +52 -0
  48. package/examples/element-selector-hint.html +16 -0
  49. package/examples/field-states.html +27 -0
  50. package/examples/fill-in-empty.html +26 -0
  51. package/examples/fill-unreachable.html +27 -0
  52. package/examples/follow-hidden-first.html +29 -0
  53. package/examples/iframe-inner.html +20 -0
  54. package/examples/iframe-nested.html +16 -0
  55. package/examples/iframe-outer.html +15 -0
  56. package/examples/multi-value-fields.html +54 -0
  57. package/examples/registered-names.html +15 -0
  58. package/examples/select-by-label.html +33 -0
  59. package/examples/settle-budget.html +21 -0
  60. package/examples/sorted-tables.html +38 -0
  61. package/examples/text-matching.html +48 -0
  62. package/examples/text-order.html +25 -0
  63. package/examples/touch.html +22 -0
  64. package/examples/viewport-top.html +36 -0
  65. package/package.json +17 -1
  66. package/playwright.config.ts +3 -1
  67. package/tests/check-api-launch-url.js +106 -0
  68. package/tests/check-assets.js +103 -0
  69. package/tests/check-attachments.js +105 -0
  70. package/tests/check-breakpoints.js +117 -0
  71. package/tests/check-clock.js +143 -0
  72. package/tests/check-config.js +90 -0
  73. package/tests/check-context.js +215 -0
  74. package/tests/check-dry-run.js +104 -0
  75. package/tests/check-examples.js +10 -4
  76. package/tests/check-exit-report.js +141 -0
  77. package/tests/check-javascript-errors.js +105 -0
  78. package/tests/check-must-fail.js +5 -4
  79. package/tests/check-network-changed.js +137 -0
  80. package/tests/check-scaffold.js +54 -2
  81. package/tests/check-screenshots.js +117 -0
  82. package/tests/check-settle-verbs.js +137 -0
  83. package/tests/check-time-limit.js +104 -0
  84. package/tests/check-video.js +156 -0
  85. package/tests/check-world.js +167 -0
  86. package/tests/features/api-base-url.feature +25 -0
  87. package/tests/features/api-form-data.feature +18 -0
  88. package/tests/features/api-json-paths.feature +24 -0
  89. package/tests/features/api-response-contains.feature +21 -0
  90. package/tests/features/api-response-header-presence.feature +17 -0
  91. package/tests/features/api-response-header.feature +17 -0
  92. package/tests/features/api-saved-placeholders.feature +37 -0
  93. package/tests/features/api-step-definitions-individual.feature +1 -1
  94. package/tests/features/api-tables.feature +40 -0
  95. package/tests/features/breakpoints.feature +25 -0
  96. package/tests/features/check-switch.feature +28 -0
  97. package/tests/features/choice-controls.feature +35 -0
  98. package/tests/features/click-whitespace.feature +29 -0
  99. package/tests/features/complete-api-testing-examples.feature +1 -1
  100. package/tests/features/datetime-field-lookup.feature +29 -0
  101. package/tests/features/element-selector-hint.feature +14 -0
  102. package/tests/features/field-state.feature +21 -0
  103. package/tests/features/fill-in-empty-value.feature +41 -0
  104. package/tests/features/fill-reachable.feature +13 -0
  105. package/tests/features/follow-hidden-first.feature +9 -0
  106. package/tests/features/iframe-nested.feature +43 -0
  107. package/tests/features/javascript-tolerated-errors.feature +20 -0
  108. package/tests/features/modal-open.feature +13 -0
  109. package/tests/features/multi-value-field-lookup.feature +30 -0
  110. package/tests/features/registered-names.feature +14 -0
  111. package/tests/features/select-by-label.feature +31 -0
  112. package/tests/features/selector-files.feature +14 -0
  113. package/tests/features/selectors-table-header.feature +35 -0
  114. package/tests/features/settle-budget.feature +11 -0
  115. package/tests/features/sorted-tables.feature +19 -0
  116. package/tests/features/storage-quoted-values.feature +26 -0
  117. package/tests/features/text-matching.feature +38 -0
  118. package/tests/features/text-order.feature +14 -0
  119. package/tests/features/touch.feature +21 -0
  120. package/tests/features/viewport-top.feature +13 -0
  121. package/tests/features/xml-namespaces.feature +34 -0
  122. package/tests/must-fail/api-json-paths.feature +21 -0
  123. package/tests/must-fail/api-response-contains.feature +23 -0
  124. package/tests/must-fail/api-response-header-presence.feature +17 -0
  125. package/tests/must-fail/api-response-header.feature +21 -0
  126. package/tests/must-fail/api-without-base-url.feature +11 -0
  127. package/tests/must-fail/breakpoints.feature +15 -0
  128. package/tests/must-fail/choice-controls.feature +25 -0
  129. package/tests/must-fail/clock.feature +11 -0
  130. package/tests/must-fail/datetime-field.feature +10 -0
  131. package/tests/must-fail/element-selector-hint.feature +17 -0
  132. package/tests/must-fail/field-state.feature +20 -0
  133. package/tests/must-fail/fill-field.feature +30 -0
  134. package/tests/must-fail/iframe-text.feature +17 -0
  135. package/tests/must-fail/modal-open.feature +26 -0
  136. package/tests/must-fail/multi-value-field.feature +18 -0
  137. package/tests/must-fail/registered-names.feature +25 -0
  138. package/tests/must-fail/select-list.feature +15 -0
  139. package/tests/must-fail/selector-files.feature +10 -0
  140. package/tests/must-fail/sorted-tables.feature +20 -0
  141. package/tests/must-fail/storage-before-a-page.feature +13 -0
  142. package/tests/must-fail/text-matching.feature +16 -0
  143. package/tests/must-fail/text-order.feature +15 -0
  144. package/tests/must-fail/viewport-top.feature +20 -0
  145. package/tests/must-fail/yaml-numbers.feature +48 -0
  146. package/tests/step-definitions/a11y.steps.js +24 -15
  147. package/tests/step-definitions/action.steps.js +53 -20
  148. package/tests/step-definitions/api.steps.js +349 -67
  149. package/tests/step-definitions/assertion.steps.js +100 -10
  150. package/tests/step-definitions/auth.steps.js +15 -18
  151. package/tests/step-definitions/clock.steps.js +22 -0
  152. package/tests/step-definitions/debug.steps.js +8 -0
  153. package/tests/step-definitions/dialog.steps.js +7 -0
  154. package/tests/step-definitions/element.steps.js +132 -17
  155. package/tests/step-definitions/field.steps.js +412 -96
  156. package/tests/step-definitions/form.steps.js +88 -37
  157. package/tests/step-definitions/iframe.steps.js +89 -29
  158. package/tests/step-definitions/input.steps.js +26 -7
  159. package/tests/step-definitions/javascript.steps.js +64 -22
  160. package/tests/step-definitions/modal.steps.js +79 -13
  161. package/tests/step-definitions/navigation.steps.js +14 -0
  162. package/tests/step-definitions/network.steps.js +15 -5
  163. package/tests/step-definitions/response.steps.js +6 -4
  164. package/tests/step-definitions/responsive.steps.js +26 -33
  165. package/tests/step-definitions/screenshot.steps.js +25 -8
  166. package/tests/step-definitions/scroll.steps.js +20 -0
  167. package/tests/step-definitions/selectors.steps.js +95 -48
  168. package/tests/step-definitions/storage.steps.js +38 -13
  169. package/tests/step-definitions/table.steps.js +81 -6
  170. package/tests/step-definitions/video.steps.js +52 -55
  171. package/tests/step-definitions/wait.steps.js +88 -23
  172. package/tests/step-definitions/web-first.steps.js +69 -35
  173. package/tests/step-definitions/webship.js +741 -145
  174. package/tests/step-definitions/xml.steps.js +49 -15
  175. package/tests/step-definitions/yaml.steps.js +45 -20
@@ -0,0 +1,104 @@
1
+ #!/usr/bin/env node
2
+ 'use strict';
3
+
4
+ // Runs a dry run with the pretty formatter, as the scaffold's cucumber.js
5
+ // does, and checks what it prints: the steps, the "? undefined" line of a
6
+ // step no definition matches and the summary, and no "- skipped" line — in a
7
+ // dry run every hook and every step is skipped, and a line for each of them
8
+ // buries the ones the dry run is for. Then a real run, whose step after a
9
+ // failing one must still be shown as skipped.
10
+ //
11
+ // No page is opened by the dry run; the real run never navigates. No server
12
+ // is needed.
13
+ //
14
+ // Usage: node tests/check-dry-run.js (npm run test:dry-run)
15
+
16
+ const fs = require('fs');
17
+ const os = require('os');
18
+ const path = require('path');
19
+ const { spawnSync } = require('child_process');
20
+
21
+ const ROOT = path.resolve(__dirname, '..');
22
+ const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'webship-js-dry-run-'));
23
+ const config = path.join(tmp, 'cucumber.js');
24
+ const steps = path.join(tmp, 'dry-run.steps.js');
25
+ const dryFeature = path.join(tmp, 'dry-run.feature');
26
+ const realFeature = path.join(tmp, 'real-run.feature');
27
+
28
+ const cucumber = path.join(ROOT, 'node_modules', '@cucumber', 'cucumber');
29
+ fs.writeFileSync(steps, [
30
+ `const { Then } = require(${JSON.stringify(cucumber)});`,
31
+ "Then('this step fails on purpose', function () { throw new Error('Failing on purpose.'); });",
32
+ "Then('this step comes after the failure', function () {});",
33
+ '',
34
+ ].join('\n'));
35
+ fs.writeFileSync(dryFeature, [
36
+ 'Feature: A dry run',
37
+ '',
38
+ ' Scenario: Steps that are defined',
39
+ ' Given I am on "/about-us.html"',
40
+ ' When I press "Submit"',
41
+ ' Then I should see "About Us"',
42
+ '',
43
+ ' Scenario: A step that no definition matches',
44
+ ' Given I am on "/about-us.html"',
45
+ ' Then this step is not defined anywhere',
46
+ '',
47
+ ].join('\n'));
48
+ fs.writeFileSync(realFeature, [
49
+ 'Feature: A real run',
50
+ '',
51
+ ' Scenario: A step after a failing step is skipped',
52
+ ' Then this step fails on purpose',
53
+ ' And this step comes after the failure',
54
+ '',
55
+ ].join('\n'));
56
+
57
+ const base = require(path.join(ROOT, 'cucumber.js')).default;
58
+ fs.writeFileSync(config, 'module.exports = { default: ' + JSON.stringify({
59
+ requireModule: base.requireModule,
60
+ require: base.require.concat([steps]),
61
+ format: ['@cucumber/pretty-formatter'],
62
+ worldParameters: base.worldParameters,
63
+ }) + ' };');
64
+
65
+ const env = Object.assign({ SLOW_MO: '0' }, process.env, { WEBSHIP_REPORT_DISABLE: '1', WEBSHIP_VIDEO: 'off', WEBSHIP_JS_ERROR_MODE: 'off', FORCE_COLOR: '0' });
66
+ const plain = (text) => String(text || '').replace(/\u001b\[[0-9;]*m/g, '');
67
+ const run = (args) => {
68
+ const r = spawnSync(process.execPath, [path.join(cucumber, 'bin', 'cucumber.js'), '--config', path.relative(ROOT, config)].concat(args), {
69
+ cwd: ROOT, encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'], env,
70
+ });
71
+ return plain(`${r.stdout || ''}${r.stderr || ''}`);
72
+ };
73
+
74
+ const problems = [];
75
+ const skipped = /^\s*-\s+skipped\s*$/;
76
+
77
+ const dry = run(['--dry-run', path.relative(ROOT, dryFeature)]).split('\n');
78
+ const dryText = dry.join('\n');
79
+ const skippedLines = dry.filter((line) => skipped.test(line)).length;
80
+ if (skippedLines) problems.push(`The dry run printed ${skippedLines} "- skipped" line(s).`);
81
+ for (const step of ['Given I am on "/about-us.html"', 'When I press "Submit"', 'Then I should see "About Us"', 'Then this step is not defined anywhere']) {
82
+ if (!dry.some((line) => line.trim() === step)) problems.push(`The dry run did not print the step: ${step}`);
83
+ }
84
+ if (!dry.some((line) => /^\s*\?\s+undefined\s*$/.test(line))) problems.push('The dry run did not print the "? undefined" line of the undefined step.');
85
+ if (!/2 scenarios \(1 undefined, 1 skipped\)/.test(dryText)) problems.push('The dry run did not print its summary "2 scenarios (1 undefined, 1 skipped)".');
86
+ // The steps of a scenario follow one another: no empty line left where a
87
+ // "- skipped" line was taken out.
88
+ const first = dry.findIndex((line) => line.trim() === 'Given I am on "/about-us.html"');
89
+ if (first >= 0 && (dry[first + 1] || '').trim() !== 'When I press "Submit"') {
90
+ problems.push(`The dry run left a line between two steps: "${dry[first + 1]}".`);
91
+ }
92
+
93
+ const real = run([path.relative(ROOT, realFeature)]).split('\n');
94
+ const after = real.findIndex((line) => line.trim() === 'And this step comes after the failure');
95
+ if (after < 0 || !skipped.test(real[after + 1] || '')) {
96
+ problems.push('A real run no longer shows the step after a failing step as "- skipped".');
97
+ }
98
+
99
+ fs.rmSync(tmp, { recursive: true, force: true });
100
+ if (problems.length) {
101
+ console.error('Dry run output:\n - ' + problems.join('\n - '));
102
+ process.exit(1);
103
+ }
104
+ console.log('Dry run output: the steps, the undefined step and the summary, with no "- skipped" line; a real run still shows its skipped steps.');
@@ -5,7 +5,8 @@
5
5
  // definitions, without a browser:
6
6
  //
7
7
  // 1. every step definition in tests/step-definitions carries a JSDoc block
8
- // with a description and at least one "Example #n:" block,
8
+ // with a description and at least five "Example #n:" blocks that use
9
+ // the step itself (CLAUDE.md §4.2),
9
10
  // 2. every step line of every example resolves to exactly one definition
10
11
  // (none undefined, none ambiguous),
11
12
  // 3. at least one example of a definition resolves to that definition,
@@ -72,7 +73,7 @@ function readDefinitions(file) {
72
73
  if (!examples.length && raw.trim()) description.push(raw.trim());
73
74
  }
74
75
  }
75
- definitions.push({ file, line, pattern: m[2], hasDoc: doc !== null, description: description.join(' '), examples, own: 0 });
76
+ definitions.push({ file, line, pattern: m[2], hasDoc: doc !== null, description: description.join(' '), examples, own: 0, ownExamples: new Set() });
76
77
  }
77
78
  return definitions;
78
79
  }
@@ -104,7 +105,7 @@ function buildFeature(file, definitions) {
104
105
  lines.push(` Scenario: ${file} line ${definition.line} example ${n + 1}`);
105
106
  for (const item of body) {
106
107
  lines.push(item.text);
107
- if (item.step) owners[lines.length] = { definition, step: item.step };
108
+ if (item.step) owners[lines.length] = { definition, step: item.step, example: n };
108
109
  }
109
110
  lines.push('');
110
111
  });
@@ -192,12 +193,17 @@ for (const feature of examples.report) {
192
193
  const status = step.result && step.result.status;
193
194
  if (status === 'undefined') problems.push(`${where} example step matches no definition: ${owner.step}`);
194
195
  else if (status === 'ambiguous') problems.push(`${where} example step matches more than one definition: ${owner.step}`);
195
- else if (matched[`${path.basename(feature.uri)}:${step.line}`] === where) owner.definition.own++;
196
+ else if (matched[`${path.basename(feature.uri)}:${step.line}`] === where) {
197
+ owner.definition.own++;
198
+ owner.definition.ownExamples.add(owner.example);
199
+ }
196
200
  }
197
201
  }
198
202
  }
203
+ const MIN_EXAMPLES = 5;
199
204
  for (const d of all) {
200
205
  if (d.examples.length && !d.own) problems.push(`${d.file}:${d.line} none of its examples uses the step itself: ${d.pattern}`);
206
+ else if (d.hasDoc && d.ownExamples.size < MIN_EXAMPLES) problems.push(`${d.file}:${d.line} ${d.ownExamples.size} of its examples use the step itself, fewer than ${MIN_EXAMPLES}: ${d.pattern}`);
201
207
  }
202
208
 
203
209
  // 2. The features of the suite.
@@ -0,0 +1,141 @@
1
+ #!/usr/bin/env node
2
+ 'use strict';
3
+
4
+ // Checks the HTML report that tests/step-definitions/webship.js builds when
5
+ // a run of cucumber-js ends, in three runs of a small project:
6
+ //
7
+ // 1. `--parallel 2`: the report is built once, by the process that runs
8
+ // cucumber-js, from the whole JSON report; no worker tries to read a
9
+ // JSON report that is still being written.
10
+ // 2. A JSON report written somewhere else than tests/reports/ (the `json:`
11
+ // formatter of the config file): the report is built from that file.
12
+ // 3. No JSON formatter at all: one line says that there is no report and
13
+ // why, with no stack trace.
14
+ //
15
+ // The scenarios never navigate, so no server is needed; each one opens a
16
+ // browser, as every scenario does.
17
+ //
18
+ // Usage: node tests/check-exit-report.js (npm run test:exit-report)
19
+
20
+ const fs = require('fs');
21
+ const os = require('os');
22
+ const path = require('path');
23
+ const { spawnSync } = require('child_process');
24
+
25
+ const ROOT = path.resolve(__dirname, '..');
26
+ const CUCUMBER = path.join(ROOT, 'node_modules', '@cucumber', 'cucumber');
27
+ const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'webship-js-exit-report-'));
28
+ const problems = [];
29
+
30
+ // A project of its own, so that no tests/reports/ of this repository is read.
31
+ const project = path.join(tmp, 'project');
32
+ fs.mkdirSync(project, { recursive: true });
33
+ fs.writeFileSync(path.join(project, 'playwright.config.js'), 'module.exports = ' + JSON.stringify({
34
+ browser: 'chromium',
35
+ launchOptions: { headless: true },
36
+ contextOptions: { viewport: null },
37
+ }) + ';\n');
38
+ const steps = path.join(project, 'plain.steps.js');
39
+ fs.writeFileSync(steps, [
40
+ `const { Given } = require(${JSON.stringify(CUCUMBER)});`,
41
+ "Given('this step passes', function () {});",
42
+ '',
43
+ ].join('\n'));
44
+ const features = [];
45
+ for (const name of ['first', 'second']) {
46
+ const file = path.join(project, `${name}.feature`);
47
+ fs.writeFileSync(file, [
48
+ `Feature: The ${name} feature`,
49
+ '',
50
+ ` Scenario: The ${name} scenario`,
51
+ ' Given this step passes',
52
+ '',
53
+ ].join('\n'));
54
+ features.push(file);
55
+ }
56
+ const base = require(path.join(ROOT, 'cucumber.js')).default;
57
+ const world = JSON.parse(JSON.stringify(base.worldParameters));
58
+
59
+ /**
60
+ * Run cucumber-js in the project with a config of its own and return what it
61
+ * printed.
62
+ *
63
+ * @param {string} name - Name of the run (and of its config file).
64
+ * @param {string[]} format - Formatters of the config file.
65
+ * @param {string[]} [args] - Extra arguments of cucumber-js.
66
+ * @param {Object} [vars] - Extra environment variables.
67
+ * @returns {{output: string, html: string, status: number}}
68
+ */
69
+ function run(name, format, args, vars) {
70
+ const html = path.join(tmp, `${name}.html`);
71
+ const config = path.join(project, `${name}.cucumber.js`);
72
+ fs.writeFileSync(config, 'module.exports = { default: ' + JSON.stringify({
73
+ require: [path.join(ROOT, 'tests', 'step-definitions', '**', '*.js'), steps],
74
+ paths: features,
75
+ format,
76
+ worldParameters: world,
77
+ }) + ' };');
78
+ const env = Object.assign({ SLOW_MO: '0' }, process.env, {
79
+ WEBSHIP_VIDEO: 'off',
80
+ WEBSHIP_JS_ERROR_MODE: 'off',
81
+ WEBSHIP_REPORT_OUT: html,
82
+ FORCE_COLOR: '0',
83
+ });
84
+ delete env.WEBSHIP_REPORT_DISABLE;
85
+ delete env.WEBSHIP_REPORT_JSON;
86
+ delete env.WEBSHIP_REPORT_ARGS;
87
+ Object.assign(env, vars || {});
88
+ const result = spawnSync(process.execPath, [path.join(CUCUMBER, 'bin', 'cucumber.js'), '--config', path.basename(config)].concat(args || []), {
89
+ cwd: project,
90
+ stdio: ['ignore', 'pipe', 'pipe'],
91
+ env,
92
+ encoding: 'utf8',
93
+ });
94
+ return { output: `${result.stdout}\n${result.stderr}`, html, status: result.status };
95
+ }
96
+
97
+ const count = (text, re) => (text.match(re) || []).length;
98
+ const shown = (text) => text.trim().split('\n').slice(0, 12).map((line) => ` ${line}`).join('\n');
99
+ const STACK = /^\s+at .+:\d+:\d+\)?$/m;
100
+
101
+ // 1. Two workers, the JSON report named by WEBSHIP_REPORT_JSON.
102
+ {
103
+ const json = path.join(tmp, 'parallel.json');
104
+ const { output, html, status } = run('parallel', ['progress', `json:${json}`], ['--parallel', '2'], { WEBSHIP_REPORT_JSON: json });
105
+ if (status !== 0) problems.push(`The --parallel run failed (exit ${status}):\n${shown(output)}`);
106
+ if (/Unable to parse|Unexpected end of JSON input/.test(output)) {
107
+ problems.push(`With --parallel 2, a process read the JSON report before it was complete:\n${shown(output)}`);
108
+ }
109
+ const built = count(output, /\[webship-js\] HTML report/g);
110
+ if (built !== 1) problems.push(`With --parallel 2, the HTML report was built ${built} times, not once.`);
111
+ if (!fs.existsSync(html)) problems.push('With --parallel 2, no HTML report was written.');
112
+ }
113
+
114
+ // 2. The JSON report written somewhere else, by the config file.
115
+ {
116
+ const json = path.join(tmp, 'elsewhere', 'run.json');
117
+ const { output, html } = run('elsewhere', ['progress', `json:${json}`]);
118
+ if (/ENOENT/.test(output) || STACK.test(output)) {
119
+ problems.push(`With the JSON report written elsewhere, the end of the run printed an error:\n${shown(output)}`);
120
+ }
121
+ if (!fs.existsSync(html)) problems.push(`With the JSON report written to ${path.relative(tmp, json)}, no HTML report was built from it.`);
122
+ }
123
+
124
+ // 3. No JSON formatter.
125
+ {
126
+ const { output, html } = run('none', ['progress']);
127
+ if (/ENOENT/.test(output) || STACK.test(output)) {
128
+ problems.push(`With no JSON report, the end of the run printed an error:\n${shown(output)}`);
129
+ }
130
+ if (count(output, /\[webship-js\] No HTML report/g) !== 1) {
131
+ problems.push(`With no JSON report, the end of the run did not say once that there is no HTML report:\n${shown(output)}`);
132
+ }
133
+ if (fs.existsSync(html)) problems.push('With no JSON report, an HTML report was written anyway.');
134
+ }
135
+
136
+ fs.rmSync(tmp, { recursive: true, force: true });
137
+ if (problems.length) {
138
+ console.error('HTML report at the end of a run:\n - ' + problems.join('\n - '));
139
+ process.exit(1);
140
+ }
141
+ console.log('HTML report at the end of a run: built once with --parallel, built from a JSON report written elsewhere, and one plain line when there is none.');
@@ -0,0 +1,105 @@
1
+ #!/usr/bin/env node
2
+ 'use strict';
3
+
4
+ // Runs the JavaScript-error checks with console warnings captured
5
+ // (WEBSHIP_JS_ERROR_LEVELS=error,warning) and checks what each one does:
6
+ // "there should be no JavaScript errors" counts errors only (an uncaught
7
+ // error or console.error), "there should be no JavaScript warnings" counts
8
+ // warnings only, and what a check did not read is still reported at the end
9
+ // of the scenario: a warning after the errors check, an error after the
10
+ // warnings check.
11
+ //
12
+ // The page is the blank page of the browser: no server is needed.
13
+ //
14
+ // Usage: node tests/check-javascript-errors.js (npm run test:js-errors)
15
+
16
+ const fs = require('fs');
17
+ const os = require('os');
18
+ const path = require('path');
19
+ const { spawnSync } = require('child_process');
20
+
21
+ const ROOT = path.resolve(__dirname, '..');
22
+ const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'webship-js-js-errors-'));
23
+ const report = path.join(tmp, 'js-errors.json');
24
+ const config = path.join(tmp, 'cucumber.js');
25
+ const steps = path.join(tmp, 'js-errors.steps.js');
26
+ const feature = path.join(tmp, 'js-errors.feature');
27
+
28
+ const cucumber = path.join(ROOT, 'node_modules', '@cucumber', 'cucumber');
29
+ fs.writeFileSync(steps, [
30
+ `const { Given } = require(${JSON.stringify(cucumber)});`,
31
+ "Given('the page logs a warning', async function () { await this.page.evaluate(() => console.warn('Check warning: this option is deprecated')); await this.page.waitForTimeout(100); });",
32
+ "Given('the page logs an error', async function () { await this.page.evaluate(() => console.error('Check error: the request was refused')); await this.page.waitForTimeout(100); });",
33
+ '',
34
+ ].join('\n'));
35
+
36
+ // name → [the status of its last step, the status of the scenario's After hooks, a text the run must print for it]
37
+ const scenarios = [
38
+ ['A warning is not an error', 'Given the page logs a warning\n Then there should be no JavaScript errors', 'passed', 'passed', 'Check warning'],
39
+ ['An error is an error', 'Given the page logs an error\n Then there should be no JavaScript errors', 'failed', 'passed', null],
40
+ ['A warning is a warning', 'Given the page logs a warning\n Then there should be no JavaScript warnings', 'failed', 'passed', null],
41
+ ['An error is not a warning', 'Given the page logs an error\n Then there should be no JavaScript warnings', 'passed', 'passed', 'Check error'],
42
+ ['@js-fail A warning still fails a scenario in fail mode', 'Given the page logs a warning\n Then there should be no JavaScript errors', 'passed', 'failed', null],
43
+ ['@js-fail An error still fails a scenario in fail mode', 'Given the page logs an error\n Then there should be no JavaScript warnings', 'passed', 'failed', null],
44
+ ];
45
+ fs.writeFileSync(feature, ['Feature: JavaScript errors and warnings', ''].concat(scenarios.map(([name, body]) => {
46
+ const tag = name.startsWith('@') ? name.split(' ')[0] : '';
47
+ const title = tag ? name.slice(tag.length + 1) : name;
48
+ return (tag ? ` ${tag}\n` : '') + ` Scenario: ${title}\n ${body}\n`;
49
+ })).join('\n'));
50
+
51
+ const base = require(path.join(ROOT, 'cucumber.js')).default;
52
+ fs.writeFileSync(config, 'module.exports = { default: ' + JSON.stringify({
53
+ requireModule: base.requireModule,
54
+ require: base.require.concat([steps]),
55
+ paths: [feature],
56
+ format: [`json:${report}`],
57
+ worldParameters: base.worldParameters,
58
+ }) + ' };');
59
+
60
+ const run = spawnSync(process.execPath, [path.join(cucumber, 'bin', 'cucumber.js'), '--config', path.relative(ROOT, config)], {
61
+ cwd: ROOT,
62
+ encoding: 'utf8',
63
+ stdio: ['ignore', 'pipe', 'pipe'],
64
+ env: Object.assign({ SLOW_MO: '0' }, process.env, {
65
+ WEBSHIP_REPORT_DISABLE: '1',
66
+ WEBSHIP_VIDEO: 'off',
67
+ WEBSHIP_JS_ERROR_MODE: 'warn',
68
+ WEBSHIP_JS_ERROR_LEVELS: 'error,warning',
69
+ }),
70
+ });
71
+ const printed = `${run.stdout || ''}\n${run.stderr || ''}`;
72
+
73
+ const problems = [];
74
+ if (!fs.existsSync(report) || !fs.readFileSync(report, 'utf8').trim()) {
75
+ problems.push(`The run wrote no report: cucumber-js did not start.\n${printed}`);
76
+ } else {
77
+ const found = {};
78
+ for (const f of JSON.parse(fs.readFileSync(report, 'utf8'))) {
79
+ for (const scenario of f.elements || []) found[scenario.name] = scenario;
80
+ }
81
+ for (const [name, , lastStatus, afterStatus, text] of scenarios) {
82
+ const title = name.startsWith('@') ? name.slice(name.indexOf(' ') + 1) : name;
83
+ const scenario = found[title];
84
+ if (!scenario) { problems.push(`"${title}" did not run.`); continue; }
85
+ const visible = (scenario.steps || []).filter((s) => !s.hidden);
86
+ const last = visible[visible.length - 1];
87
+ // cucumber-js writes the hooks into the steps of the JSON report, hidden.
88
+ const after = (scenario.steps || []).filter((s) => s.hidden && s.keyword === 'After').map((h) => h.result && h.result.status);
89
+ const afterOutcome = after.includes('failed') ? 'failed' : 'passed';
90
+ if (!last || last.result.status !== lastStatus) {
91
+ problems.push(`"${title}": its check ${last ? last.result.status : 'is missing'}, it must have ${lastStatus}${last && last.result.error_message ? `: ${last.result.error_message.split('\n')[0]}` : ''}.`);
92
+ }
93
+ if (afterOutcome !== afterStatus) problems.push(`"${title}": the end-of-scenario report ${afterOutcome}, it must have ${afterStatus}.`);
94
+ if (text && !new RegExp(`JavaScript errors during "${title}"[\\s\\S]*${text}`).test(printed)) {
95
+ problems.push(`"${title}": the end-of-scenario report did not print "${text}".`);
96
+ }
97
+ }
98
+ }
99
+
100
+ fs.rmSync(tmp, { recursive: true, force: true });
101
+ if (problems.length) {
102
+ console.error('JavaScript errors and warnings:\n - ' + problems.join('\n - '));
103
+ process.exit(1);
104
+ }
105
+ console.log('JavaScript errors and warnings: each check counts its own kind, and what it did not read is reported at the end of the scenario.');
@@ -3,8 +3,9 @@
3
3
 
4
4
  // Runs the scenarios of tests/must-fail/ and checks that every one of them
5
5
  // FAILS, at its last step, with the message its description announces
6
- // ("Fails with: ..."), and that the message names the page and carries a
7
- // hint, with no Playwright call log and no stack trace.
6
+ // ("Fails with: ..."), and that the message names the page (for an API step,
7
+ // the request the response answers) and carries a hint, with no Playwright
8
+ // call log and no stack trace.
8
9
  //
9
10
  // A step that cannot fail proves nothing, and a failure nobody can read
10
11
  // costs a tester an hour. Each scenario here asserts something that is not
@@ -67,7 +68,7 @@ if (!fs.existsSync(report) || !fs.readFileSync(report, 'utf8').trim()) {
67
68
  const shown = message.split('\n').slice(0, 4).join('\n ');
68
69
  if (!message.includes(wanted)) { problems.push(`${where}: failed, but not with "${wanted}":\n ${shown}`); continue; }
69
70
  if (!/^\s*Hint: \S/m.test(message)) { problems.push(`${where}: the failure carries no hint:\n ${shown}`); continue; }
70
- if (!/^\s*Page: \S/m.test(message)) { problems.push(`${where}: the failure does not name the page:\n ${shown}`); continue; }
71
+ if (!/^\s*(?:Page|Request): \S/m.test(message)) { problems.push(`${where}: the failure names neither the page nor the request:\n ${shown}`); continue; }
71
72
  if (/Call log:|Timeout \d+ms exceeded|^\s+at .+:\d+:\d+\)?$/m.test(message)) { problems.push(`${where}: the failure shows a Playwright call log, a raw timeout or a stack trace:\n ${shown}`); continue; }
72
73
  console.log(` fails as it must: ${scenario.name}\n ${message.split('\n')[0].trim()}`);
73
74
  }
@@ -81,4 +82,4 @@ if (problems.length) {
81
82
  console.error(`\n${problems.length} problem(s) in ${count} must-fail scenario(s):\n ${problems.join('\n ')}`);
82
83
  process.exit(1);
83
84
  }
84
- console.log(`\n${count} must-fail scenarios: each failed at its last step with the announced message, the page and a hint.`);
85
+ console.log(`\n${count} must-fail scenarios: each failed at its last step with the announced message, the page (or the request) and a hint.`);
@@ -0,0 +1,137 @@
1
+ #!/usr/bin/env node
2
+ 'use strict';
3
+
4
+ // Checks that a page load which Chromium aborts with net::ERR_NETWORK_CHANGED
5
+ // (the network of the machine changed under it: a Docker network created or
6
+ // removed, as when another DDEV project starts or stops) is loaded once more,
7
+ // with a line in the log, and that a second abort fails the step as before.
8
+ //
9
+ // A project's Before hook (written here, the way a project writes one) makes
10
+ // the page's goto() throw that error once, or twice, before it loads the
11
+ // page: the real navigation step then runs against a real page.
12
+ //
13
+ // The check serves its own page on a free local port, so no other server is
14
+ // needed.
15
+ //
16
+ // Usage: node tests/check-network-changed.js (npm run test:network-changed)
17
+
18
+ const fs = require('fs');
19
+ const os = require('os');
20
+ const path = require('path');
21
+ const http = require('http');
22
+ const { spawn } = require('child_process');
23
+
24
+ const ROOT = path.resolve(__dirname, '..');
25
+ const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'webship-js-network-changed-'));
26
+ const report = path.join(tmp, 'network.json');
27
+ const config = path.join(tmp, 'cucumber.js');
28
+ const steps = path.join(tmp, 'project.steps.js');
29
+ const feature = path.join(tmp, 'network.feature');
30
+ const cucumber = path.join(ROOT, 'node_modules', '@cucumber', 'cucumber');
31
+
32
+ const PAGE = '<!doctype html><html><head><title>Shop</title></head><body><h1>Shop</h1></body></html>';
33
+
34
+ fs.writeFileSync(steps, [
35
+ `const { Before } = require(${JSON.stringify(cucumber)});`,
36
+ "Before({ tags: '@aborted-once or @aborted-twice' }, function (scope) {",
37
+ " let left = scope.pickle.tags.some((tag) => tag.name === '@aborted-twice') ? 2 : 1;",
38
+ ' const page = this.page;',
39
+ ' const goto = page.goto.bind(page);',
40
+ ' page.goto = async (url, options) => {',
41
+ ' if (left > 0) {',
42
+ ' left -= 1;',
43
+ ' throw new Error(`page.goto: net::ERR_NETWORK_CHANGED at ${url}`);',
44
+ ' }',
45
+ ' return goto(url, options);',
46
+ ' };',
47
+ '});',
48
+ '',
49
+ ].join('\n'));
50
+ fs.writeFileSync(feature, [
51
+ 'Feature: A page load aborted by a change of the network of the machine',
52
+ '',
53
+ ' @aborted-once',
54
+ ' Scenario: The page is loaded once more',
55
+ ' Given I am on "/"',
56
+ ' Then I should see "Shop"',
57
+ '',
58
+ ' @aborted-twice',
59
+ ' Scenario: A second abort fails the step',
60
+ ' Given I am on "/"',
61
+ '',
62
+ ' Scenario: A page load that nothing aborts',
63
+ ' Given I am on "/"',
64
+ ' Then I should see "Shop"',
65
+ '',
66
+ ].join('\n'));
67
+
68
+ const server = http.createServer((req, res) => {
69
+ res.writeHead(200, { 'Content-Type': 'text/html; charset=utf-8' });
70
+ res.end(PAGE);
71
+ });
72
+ server.listen(0, '127.0.0.1', () => {
73
+ const base = `http://127.0.0.1:${server.address().port}`;
74
+ const defaults = require(path.join(ROOT, 'cucumber.js')).default;
75
+ const world = JSON.parse(JSON.stringify(defaults.worldParameters));
76
+ world.launchUrl = base;
77
+ fs.writeFileSync(config, 'module.exports = { default: ' + JSON.stringify({
78
+ requireModule: defaults.requireModule,
79
+ require: defaults.require.concat([steps]),
80
+ paths: [feature],
81
+ format: [`json:${report}`],
82
+ worldParameters: world,
83
+ }) + ' };');
84
+ const env = Object.assign({ SLOW_MO: '0' }, process.env, {
85
+ LAUNCH_URL: base,
86
+ WEBSHIP_REPORT_DISABLE: '1',
87
+ WEBSHIP_VIDEO: 'off',
88
+ WEBSHIP_JS_ERROR_MODE: 'off',
89
+ WEBSHIP_SCREENSHOT_ON_FAILED: 'false',
90
+ });
91
+ const run = spawn(process.execPath, [path.join(cucumber, 'bin', 'cucumber.js'), '--config', path.relative(ROOT, config)], {
92
+ cwd: ROOT,
93
+ stdio: ['ignore', 'pipe', 'pipe'],
94
+ env,
95
+ });
96
+ let output = '';
97
+ run.stdout.on('data', (chunk) => { output += chunk; });
98
+ run.stderr.on('data', (chunk) => { output += chunk; });
99
+ const timer = setTimeout(() => run.kill('SIGKILL'), 180000);
100
+ run.on('exit', () => {
101
+ clearTimeout(timer);
102
+ server.close();
103
+ finish(output);
104
+ });
105
+ });
106
+
107
+ function finish(output) {
108
+ const strip = (text) => String(text || '').replace(/\u001b\[[0-9;]*m/g, '');
109
+ const problems = [];
110
+ const results = {};
111
+ if (!fs.existsSync(report) || !fs.readFileSync(report, 'utf8').trim()) {
112
+ problems.push(`The run wrote no report.\n${strip(output).slice(-1500)}`);
113
+ } else {
114
+ for (const f of JSON.parse(fs.readFileSync(report, 'utf8'))) {
115
+ for (const scenario of f.elements || []) {
116
+ const steps = (scenario.steps || []).filter((step) => !step.hidden);
117
+ const failed = steps.find((step) => step.result && step.result.status !== 'passed');
118
+ results[scenario.name] = failed ? { status: failed.result.status, message: strip(failed.result.error_message) } : { status: 'passed' };
119
+ }
120
+ }
121
+ const once = results['The page is loaded once more'];
122
+ const twice = results['A second abort fails the step'];
123
+ const plain = results['A page load that nothing aborts'];
124
+ if (!once || once.status !== 'passed') problems.push(`after one abort the page was not loaded once more: ${once ? once.message.split('\n')[0] : 'the scenario did not run'}`);
125
+ if (!twice || twice.status !== 'failed') problems.push(`after two aborts the step did not fail: ${twice ? twice.status : 'the scenario did not run'}`);
126
+ else if (!/Failed to navigate to/.test(twice.message) || !/ERR_NETWORK_CHANGED/.test(twice.message)) problems.push(`after two aborts the step failed with another message: ${twice.message.split('\n')[0]}`);
127
+ if (!plain || plain.status !== 'passed') problems.push('a page load that nothing aborts did not pass');
128
+ const lines = strip(output).split('\n').filter((line) => /net::ERR_NETWORK_CHANGED, as when a Docker network starts or stops\); loading it once more\./.test(line));
129
+ if (lines.length !== 2) problems.push(`the log says ${lines.length} time(s) that a load was tried once more, not 2 (one per aborted scenario)`);
130
+ }
131
+ fs.rmSync(tmp, { recursive: true, force: true });
132
+ if (problems.length) {
133
+ console.error('A page load aborted by a network change:\n - ' + problems.join('\n - '));
134
+ process.exit(1);
135
+ }
136
+ console.log('A page load aborted by a network change is loaded once more and the log says so; a second abort fails the step.');
137
+ }
@@ -10,8 +10,13 @@
10
10
  // 2. `npm test` in that project is green against a site (the examples/
11
11
  // pages of this repository, served on a free port),
12
12
  // 3. a feature named on the command line runs alone,
13
- // 4. a second scaffold run changes nothing, and a `test` script the project
14
- // chose itself is kept.
13
+ // 4. a second scaffold run changes nothing, a `test` script the project
14
+ // chose itself is kept, and webship-js declared as a dev or peer
15
+ // dependency is not listed a second time,
16
+ // 5. a `test` script that runs Nightwatch (a project from webship-js 1.x)
17
+ // is replaced by the suite and kept as `test:nightwatch`,
18
+ // 6. `--force` rewrites the scaffold's files but keeps the project's own
19
+ // steps in tests/step-definitions/custom.js.
15
20
  //
16
21
  // The browser install is skipped (--skip-browsers): Chromium must be in the
17
22
  // Playwright cache already, as it is after `npx playwright install chromium`.
@@ -184,6 +189,53 @@ async function main() {
184
189
  if (ownPkg.scripts.test !== 'node my-own-tests.js') problems.push(`the scaffold replaced a test script of the project: ${ownPkg.scripts.test}`);
185
190
  if (!ownPkg.scripts['test:chromium']) problems.push('the scaffold did not add test:chromium beside an existing test script');
186
191
 
192
+ // webship-js already declared as a dev dependency (or a peer): the scaffold leaves it there, with
193
+ // its range, and does not list it a second time under dependencies.
194
+ for (const field of ['devDependencies', 'peerDependencies']) {
195
+ const declared = path.join(tmp, `declared-in-${field}`);
196
+ fs.mkdirSync(declared);
197
+ fs.writeFileSync(path.join(declared, 'package.json'), JSON.stringify({ name: `declared-in-${field.toLowerCase()}`, version: '1.0.0', [field]: { 'webship-js': '^2.0.12' } }, null, 2));
198
+ scaffold(declared);
199
+ const declaredPkg = JSON.parse(fs.readFileSync(path.join(declared, 'package.json'), 'utf8'));
200
+ if (declaredPkg.dependencies && declaredPkg.dependencies['webship-js']) {
201
+ problems.push(`with webship-js in ${field}, the scaffold listed it again under dependencies (${declaredPkg.dependencies['webship-js']})`);
202
+ }
203
+ if ((declaredPkg[field] || {})['webship-js'] !== '^2.0.12') {
204
+ problems.push(`the scaffold changed the range of webship-js in ${field}: ${(declaredPkg[field] || {})['webship-js']}`);
205
+ }
206
+ }
207
+
208
+ // A project from webship-js 1.x: its test script runs Nightwatch.
209
+ const legacy = path.join(tmp, 'from-1x');
210
+ fs.mkdirSync(legacy);
211
+ const nightwatch = 'nightwatch --format @cucumber/pretty-formatter --format json:./tests/reports/cucumber-report.json';
212
+ fs.writeFileSync(path.join(legacy, 'package.json'), JSON.stringify({ name: 'from-1x', version: '1.0.0', scripts: { test: nightwatch } }, null, 2));
213
+ const legacyRun = scaffold(legacy);
214
+ const legacyPkg = JSON.parse(fs.readFileSync(path.join(legacy, 'package.json'), 'utf8'));
215
+ if (legacyPkg.scripts.test !== 'cucumber-js --config cucumber.js') {
216
+ problems.push(`the scaffold kept a Nightwatch test script from 1.x: "npm test" is still ${legacyPkg.scripts.test}`);
217
+ }
218
+ if (legacyPkg.scripts['test:nightwatch'] !== nightwatch) {
219
+ problems.push(`the scaffold did not keep the old Nightwatch command as test:nightwatch: ${legacyPkg.scripts['test:nightwatch']}`);
220
+ }
221
+ if (legacyPkg.scripts.test !== nightwatch && !/nightwatch/i.test(legacyRun.stdout || '')) {
222
+ problems.push('the scaffold replaced the Nightwatch test script without saying so');
223
+ }
224
+
225
+ // --force keeps the project's own steps.
226
+ const customFile = path.join(project, 'tests', 'step-definitions', 'custom.js');
227
+ const ownSteps = fs.readFileSync(customFile, 'utf8') + "\nGiven('a step of the project', function () {});\n";
228
+ fs.writeFileSync(customFile, ownSteps);
229
+ fs.writeFileSync(path.join(project, 'playwright.config.ts'), '// changed by the project\n');
230
+ const forced = spawnSync(process.execPath, [INIT, '--cwd', project, '--skip-browsers', '--force'], { encoding: 'utf8' });
231
+ if (forced.status !== 0) problems.push(`the scaffold with --force exited with ${forced.status}`);
232
+ if (fs.readFileSync(customFile, 'utf8') !== ownSteps) {
233
+ problems.push('the scaffold with --force overwrote tests/step-definitions/custom.js, the steps of the project');
234
+ }
235
+ if (fs.readFileSync(path.join(project, 'playwright.config.ts'), 'utf8') === '// changed by the project\n') {
236
+ problems.push('the scaffold with --force did not rewrite playwright.config.ts');
237
+ }
238
+
187
239
  // No package.json at all: the scaffold writes one.
188
240
  const bare = path.join(tmp, 'no-package-json');
189
241
  fs.mkdirSync(bare);