reffy 20.0.13 → 20.0.15

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (78) hide show
  1. package/LICENSE +21 -21
  2. package/README.md +151 -151
  3. package/index.js +29 -29
  4. package/package.json +5 -5
  5. package/reffy.js +324 -324
  6. package/schemas/browserlib/extract-algorithms.json +52 -52
  7. package/schemas/browserlib/extract-cssdfn.json +108 -108
  8. package/schemas/browserlib/extract-dfns.json +90 -90
  9. package/schemas/browserlib/extract-elements.json +17 -17
  10. package/schemas/browserlib/extract-events.json +31 -31
  11. package/schemas/browserlib/extract-headings.json +19 -19
  12. package/schemas/browserlib/extract-ids.json +7 -7
  13. package/schemas/browserlib/extract-links.json +12 -12
  14. package/schemas/browserlib/extract-refs.json +12 -12
  15. package/schemas/common.json +876 -876
  16. package/schemas/files/extracts/algorithms.json +12 -12
  17. package/schemas/files/extracts/css.json +16 -16
  18. package/schemas/files/extracts/dfns.json +12 -12
  19. package/schemas/files/extracts/elements.json +12 -12
  20. package/schemas/files/extracts/events.json +12 -12
  21. package/schemas/files/extracts/headings.json +12 -12
  22. package/schemas/files/extracts/ids.json +12 -12
  23. package/schemas/files/extracts/links.json +12 -12
  24. package/schemas/files/extracts/refs.json +12 -12
  25. package/schemas/files/index.json +59 -59
  26. package/schemas/postprocessing/events.json +50 -50
  27. package/schemas/postprocessing/idlnames-parsed.json +27 -27
  28. package/schemas/postprocessing/idlnames.json +17 -17
  29. package/schemas/postprocessing/idlparsed.json +67 -67
  30. package/src/browserlib/clone-and-clean.mjs +24 -24
  31. package/src/browserlib/create-outline.mjs +353 -353
  32. package/src/browserlib/extract-algorithms.mjs +723 -723
  33. package/src/browserlib/extract-cddl.mjs +125 -125
  34. package/src/browserlib/extract-dfns.mjs +1093 -1093
  35. package/src/browserlib/extract-headings.mjs +76 -76
  36. package/src/browserlib/extract-ids.mjs +28 -28
  37. package/src/browserlib/extract-links.mjs +45 -45
  38. package/src/browserlib/extract-references.mjs +308 -308
  39. package/src/browserlib/extract-webidl.mjs +89 -89
  40. package/src/browserlib/get-absolute-url.mjs +29 -29
  41. package/src/browserlib/get-code-elements.mjs +20 -20
  42. package/src/browserlib/get-generator.mjs +26 -26
  43. package/src/browserlib/get-lastmodified-date.mjs +13 -13
  44. package/src/browserlib/get-revision.mjs +12 -12
  45. package/src/browserlib/get-title.mjs +14 -14
  46. package/src/browserlib/informative-selector.mjs +24 -24
  47. package/src/browserlib/map-ids-to-headings.mjs +173 -173
  48. package/src/browserlib/reffy.json +85 -85
  49. package/src/browserlib/trim-spaces.mjs +35 -35
  50. package/src/cli/check-missing-dfns.js +587 -587
  51. package/src/cli/merge-crawl-results.js +132 -132
  52. package/src/cli/parse-webidl.js +447 -447
  53. package/src/lib/css-grammar-parse-tree.schema.json +109 -109
  54. package/src/lib/css-grammar-parser.js +440 -440
  55. package/src/lib/fetch.js +51 -51
  56. package/src/lib/markdown-report.js +360 -360
  57. package/src/lib/mock-server.js +218 -218
  58. package/src/lib/post-processor.js +322 -322
  59. package/src/lib/throttled-queue.js +129 -129
  60. package/src/postprocessing/annotate-links.js +41 -41
  61. package/src/postprocessing/csscomplete.js +48 -48
  62. package/src/postprocessing/idlnames.js +391 -391
  63. package/src/postprocessing/idlparsed.js +179 -179
  64. package/src/postprocessing/patch-dfns.js +51 -51
  65. package/src/specs/missing-css-rules.json +197 -197
  66. package/src/specs/spec-equivalents.json +149 -149
  67. package/src/browserlib/extract-editors.mjs~ +0 -14
  68. package/src/browserlib/extract-events.mjs~ +0 -3
  69. package/src/browserlib/generate-es-dfn-report.sh~ +0 -4
  70. package/src/browserlib/get-revision.mjs~ +0 -7
  71. package/src/cli/csstree-grammar-check.js +0 -28
  72. package/src/cli/csstree-grammar-check.js~ +0 -10
  73. package/src/cli/csstree-grammar-parser.js +0 -11
  74. package/src/cli/csstree-grammar-parser.js~ +0 -1
  75. package/src/cli/extract-editors.js~ +0 -38
  76. package/src/cli/process-specs.js~ +0 -28
  77. package/src/postprocessing/annotate-links.js~ +0 -8
  78. package/src/postprocessing/events.js~ +0 -245
@@ -1,587 +1,587 @@
1
- #!/usr/bin/env node
2
- /**
3
- * The definitions checker compares CSS, dfns, and IDL extracts created by Reffy
4
- * to detect CSS/IDL terms that do not have a corresponding dfn in the
5
- * specification.
6
- *
7
- * The definitions checker can be called directly through:
8
- *
9
- * `node check-missing-dfns.js [crawl report] [spec] [format]`
10
- *
11
- * where:
12
- * - `crawl report` is the local path to the root folder that contains the
13
- * `index.json` and the extracts (e.g. `reports/ed`)
14
- * - `spec` is the optional shortname of the specification on which to focus or
15
- * `all` (default) to check all specs
16
- * - `format` is the optional output format. Either `json` or `markdown` with
17
- * `markdown` being the default.
18
- *
19
- * Note: CSS extraction already relies on dfns and reports missing dfns in a
20
- * "warnings" property. This checker simply looks at that list.
21
- *
22
- * @module checker
23
- */
24
-
25
- import path from 'node:path';
26
- import { fileURLToPath } from 'node:url';
27
- import process from 'node:process';
28
- import { loadJSON } from '../lib/util.js';
29
-
30
- /**
31
- * List of spec shortnames that, so far, don't follow the dfns data model
32
- */
33
- const specsWithObsoleteDfnsModel = [
34
- 'svg-animations', 'svg-markers', 'svg-strokes', 'SVG2',
35
- 'webgl1', 'webgl2',
36
- 'webrtc-identity'
37
- ];
38
-
39
-
40
- /**
41
- * Return true when provided arrays are "equal", meaning that they contain the
42
- * same items
43
- *
44
- * @function
45
- * @private
46
- * @param {Array} a First array to compare
47
- * @param {Array} b Second array to compare
48
- * @return {boolean} True when arrays are equal
49
- */
50
- function arraysEqual(a, b) {
51
- return Array.isArray(a) &&
52
- Array.isArray(b) &&
53
- a.length === b.length &&
54
- a.every((val, index) => val === b[index]);
55
- }
56
-
57
-
58
- /**
59
- * Return the list of expected definitions from the CSS extract
60
- *
61
- * @function
62
- * @private
63
- * @param {Object} css The root of the object that describes CSS terms in the
64
- * CSS extract
65
- * @return {Array} An array of expected definitions
66
- */
67
- function getExpectedDfnsFromCSS(css) {
68
- const expected = (css.warnings ?? [])
69
- .filter(warning => warning.msg === 'Missing definition')
70
- .map(warning => {
71
- return {
72
- linkingText: [warning.name],
73
- type: warning.type,
74
- 'for': warning.for
75
- };
76
- });
77
-
78
- return expected;
79
- }
80
-
81
-
82
- /**
83
- * Return true when the given CSS definition matches the expected definition
84
- *
85
- * @function
86
- * @private
87
- * @param {Object} expected Expected definition
88
- * @param {Object} actual Actual definition to check
89
- * @return {Boolean} true when actual definition matches the expected one
90
- */
91
- function matchCSSDfn(expected, actual) {
92
- return arraysEqual(expected.linkingText, actual.linkingText) &&
93
- (!expected.for || arraysEqual(expected.for, actual.for)) &&
94
- (!expected.type || (expected.type === actual.type));
95
- }
96
-
97
-
98
- /**
99
- * Return the list of expected definitions from the IDL extract
100
- *
101
- * @function
102
- * @private
103
- * @param {Object} css The root of the object that describes IDL terms in the
104
- * `idlparsed` extract.
105
- * @return {Array} An array of expected definitions
106
- */
107
- function getExpectedDfnsFromIdl(idl = {}) {
108
- // Parse IDL names that the spec defines
109
- const idlNames = Object.values(idl.idlNames || {});
110
- let expected = idlNames.map(name => getExpectedDfnsFromIdlDesc(name)).flat();
111
-
112
- // Parse members of IDL names that the spec extends
113
- const idlExtendedNames = Object.values(idl.idlExtendedNames || {});
114
- expected = expected.concat(idlExtendedNames.map(extended =>
115
- extended.map(name => getExpectedDfnsFromIdlDesc(name, { excludeRoot: true })))
116
- .flat(2));
117
- return expected;
118
- }
119
-
120
-
121
- /**
122
- * Return true if the given parsed IDL object describes a default toJSON
123
- * operation that references:
124
- * https://heycam.github.io/webidl/#default-tojson-steps
125
- *
126
- * @function
127
- * @private
128
- * @param {Object} desc Parsed IDL object to check
129
- * @return {Boolean} true when object describes a default toJSON operation.
130
- */
131
- function isDefaultToJSONOperation(desc) {
132
- return (desc.type === 'operation') &&
133
- (desc.name === 'toJSON') &&
134
- (desc.extAttrs && desc.extAttrs.find(attr => attr.name === "Default"));
135
- }
136
-
137
-
138
- /**
139
- * Return the expected definition for the given parsed IDL structure
140
- *
141
- * @function
142
- * @public
143
- * @param {Object} desc The object that describes the IDL term in the
144
- * `idlparsed` extract.
145
- * @param {Object} parentDesc (optional) The object that describes the parent
146
- * IDL term of the term to parse (used to compute the `for` property).
147
- * @return {Object} The expected definition, or null if no expected definition
148
- * is defined.
149
- */
150
- function getExpectedDfnFromIdlDesc(idl, parentIdl) {
151
- function serializeArgs(args = []) {
152
- return args
153
- .map(arg => arg.variadic ? `...${arg.name}` : arg.name)
154
- .join(', ');
155
- }
156
-
157
- let expected = {
158
- linkingText: [idl.name],
159
- type: idl.type,
160
- 'for': parentIdl && (parentIdl !== idl) ? [parentIdl.name] : []
161
- };
162
-
163
- switch (idl.type) {
164
- case 'attribute':
165
- case 'const':
166
- break;
167
-
168
- case 'constructor':
169
- // Ignore constructors for HTML elements, the spec has a dedicated
170
- // section for them:
171
- // https://html.spec.whatwg.org/multipage/dom.html#html-element-constructors
172
- if (!parentIdl.name.startsWith('HTML')) {
173
- expected.linkingText = [`constructor(${serializeArgs(idl.arguments)})`];
174
- }
175
- else {
176
- expected = null;
177
- }
178
- break;
179
-
180
- case 'enum':
181
- break;
182
-
183
- case 'enum-value':
184
- // The enumeration could include the empty string as a value. There
185
- // cannot be a matching definition in that case.
186
- // Note: look for the quoted value and the unquoted value
187
- const value = idl.value.replace(/^"(.*)"$/, '$1');
188
- expected.linkingText = (value !== '') ? [`"${value}"`, value] : [`"${value}"`];
189
- break;
190
-
191
- case 'field':
192
- expected.type = 'dict-member';
193
- break;
194
-
195
- case 'callback':
196
- case 'callback interface':
197
- case 'dictionary':
198
- case 'interface':
199
- case 'interface mixin':
200
- case 'namespace':
201
- expected.type =
202
- (idl.type === 'callback interface') ? 'callback' :
203
- (idl.type === 'interface mixin') ? 'interface' :
204
- idl.type;
205
- // Ignore partial definition
206
- if (idl.partial) {
207
- expected = null;
208
- }
209
- break;
210
-
211
- case 'includes':
212
- expected = null;
213
- break;
214
-
215
- case 'iterable':
216
- case 'async_iterable':
217
- case 'maplike':
218
- case 'setlike':
219
- // No definition expected for iterable, maplike and setlike members
220
- expected = null;
221
- break;
222
-
223
- case 'operation':
224
- // Stringification behavior is typically defined with a
225
- // "stringification behavior" definition scoped to the interface
226
- if (idl.special === 'stringifier') {
227
- expected.linkingText = ['stringification behavior', 'stringificationbehavior'];
228
- expected.type = 'dfn';
229
- }
230
- // Ignore special "getter", "setter", "deleter" operations when they don't
231
- // have an identifier. They should link to a definition in the prose, but
232
- // the labels seem arbitrary for now.
233
- // Also ignore default toJSON operations. Steps are defined in WebIDL.
234
- else if ((idl.name ||
235
- ((idl.special !== 'getter') &&
236
- (idl.special !== 'setter') &&
237
- (idl.special !== 'deleter'))) &&
238
- !isDefaultToJSONOperation(idl)) {
239
- expected.linkingText = [`${idl.name}(${serializeArgs(idl.arguments)})`];
240
- expected.type = 'method';
241
- }
242
- else {
243
- expected = null;
244
- }
245
- break;
246
-
247
- case 'typedef':
248
- break;
249
-
250
- case 'argument':
251
- expected = null;
252
- break;
253
-
254
- default:
255
- console.warn('Unsupported IDL type', idl.type, idl);
256
- expected = null;
257
- break;
258
- }
259
-
260
- return expected;
261
- }
262
-
263
-
264
- /**
265
- * Return the list of expected definitions from a parsed IDL extract entry.
266
- *
267
- * The function is recursive.
268
- *
269
- * @function
270
- * @private
271
- * @param {Object} idl The object that describes the IDL term in the
272
- * `idlparsed` extract.
273
- * @return {Array} An array of expected definitions
274
- */
275
- function getExpectedDfnsFromIdlDesc(idl, {excludeRoot} = {excludeRoot: false}) {
276
- const res = [];
277
- const parentIdl = idl;
278
- const idlToProcess = excludeRoot ? [] : [idl];
279
-
280
- switch (idl.type) {
281
- case 'enum':
282
- if (idl.values) {
283
- idlToProcess.push(...idl.values);
284
- }
285
- break;
286
-
287
- case 'callback':
288
- case 'callback interface':
289
- case 'dictionary':
290
- case 'interface':
291
- case 'interface mixin':
292
- case 'namespace':
293
- if (idl.members) {
294
- idlToProcess.push(...idl.members);
295
- }
296
- break;
297
- }
298
-
299
- idlToProcess.forEach(idl => {
300
- const expected = getExpectedDfnFromIdlDesc(idl, parentIdl);
301
- if (expected) {
302
- expected.access = 'public';
303
- expected.informative = false;
304
- res.push(expected);
305
- }
306
- });
307
-
308
- return res;
309
- }
310
-
311
-
312
- /**
313
- * Return true when the given IDL definition matches the expected definition.
314
- *
315
- * The function handles overloaded methods, though not properly. That is, it
316
- * will only find the "right" definition for an overloaded method if the number
317
- * and/or the name of the arguments differ between the overloaded definitions.
318
- * Otherwise it will just match the first definition that looks good.
319
- *
320
- * The function works around Respec's issue #3200 for methods and constructors
321
- * that take only optional parameters:
322
- * https://github.com/w3c/respec/issues/3200
323
- *
324
- * @function
325
- * @private
326
- * @param {Object} expected Expected definition
327
- * @param {Object} actual Actual definition to check
328
- * @param {Object} options Comparison options
329
- * @return {Boolean} true when actual definition matches the expected one
330
- */
331
- function matchIdlDfn(expected, actual,
332
- {skipArgs, skipFor, skipType} = {skipArgs: false, skipFor: false, skipType: false}) {
333
- const fixedLt = actual.linkingText
334
- .map(lt => lt.replace(/!overload-\d/, ''))
335
- .map(lt => lt.replace(/\(, /, '('));
336
- let found = expected.linkingText.some(val => fixedLt.includes(val));
337
- if (!found && skipArgs) {
338
- const names = fixedLt.map(lt => lt.replace(/\(.*\)/, ''));
339
- found = expected.linkingText.some(val => {
340
- const valname = val.replace(/\(.*\)/, '');
341
- return names.find(name => name === valname);
342
- });
343
- }
344
- return found &&
345
- (expected.for.every(val => actual.for.includes(val)) || skipFor) &&
346
- (expected.type === actual.type || skipType);
347
- }
348
-
349
-
350
- /**
351
- * Checks the CSS and IDL extracts against the dfns extract for the given spec
352
- *
353
- * @function
354
- * @public
355
- * @param {Object} spec Crawl result for the spec to parse
356
- * @param {String} options Check options. Set the rootFolder property to the
357
- * root folder against which to resolve relative paths to load CSS/IDL
358
- * extracts (only needed if the extracts have not yet been loaded and attached
359
- * to the spec object). Set the includeObsolete property to true to include
360
- * detailed results about specs that use an obsolete dfns data model.
361
- * @return {Object} An object with a css and idl property, each of them holding
362
- * an array of missing CSS or IDL definitions. The function returns null when
363
- * there are no missing definitions.
364
- */
365
- async function checkSpecDefinitions(spec, options = {}) {
366
- if (!options.includeObsolete && specsWithObsoleteDfnsModel.includes(spec.shortname)) {
367
- return { obsoleteDfnsModel: true };
368
- }
369
-
370
- const dfns = (typeof spec.dfns === "string") ?
371
- (await loadJSON(path.resolve(options.rootFolder, spec.dfns))).dfns :
372
- (spec.dfns || []);
373
- const css = (typeof spec.css === "string") ?
374
- (await loadJSON(path.resolve(options.rootFolder, spec.css))) :
375
- (spec.css || {});
376
- const idl = (typeof spec.idlparsed === "string") ?
377
- (await loadJSON(path.resolve(options.rootFolder, spec.idlparsed))).idlparsed :
378
- spec.idlparsed;
379
-
380
- // Make sure that all expected CSS definitions exist in the dfns extract
381
- const expectedCSSDfns = getExpectedDfnsFromCSS(css);
382
- const missingCSSDfns = expectedCSSDfns.map(expected => {
383
- let actual = dfns.find(dfn => matchCSSDfn(expected, dfn));
384
- if (!actual && !expected.type) {
385
- // Right definition is missing. For valuespaces that define functions,
386
- // look for a function definition without the enclosing "<>" instead
387
- const altText = [expected.linkingText[0].replace(/^<(.*)\(\)>$/, '$1()')];
388
- actual = dfns.find(dfn => arraysEqual(altText, dfn.linkingText));
389
- }
390
- if (!actual && expected.value) {
391
- // Still missing? For valuespaces that define functions, this may be
392
- // because there is no definition without parameters, try to find the
393
- // actual value instead
394
- actual = dfns.find(dfn => arraysEqual([expected.value], dfn.linkingText));
395
- }
396
- if (actual) {
397
- // Right definition found
398
- return null;
399
- }
400
- else {
401
- // Right definition is missing, there may be a definition that looks
402
- // like the one we're looking for
403
- const found = dfns.find(dfn =>
404
- arraysEqual(dfn.linkingText, expected.linkingText));
405
- return { expected, found };
406
- }
407
- }).filter(missing => !!missing);
408
-
409
- // Make sure that all expected IDL definitions exist in the dfns extract
410
- const expectedIdlDfns = getExpectedDfnsFromIdl(idl);
411
- const missingIdlDfns = expectedIdlDfns.map(expected => {
412
- let actual = dfns.find(dfn => matchIdlDfn(expected, dfn));
413
- if (actual) {
414
- // Right definition found
415
- return null;
416
- }
417
- else {
418
- // Right definition is missing, include the interface's definitions to
419
- // be able to link to it in the report
420
- let parent = null;
421
- if (expected.for && expected.for[0]) {
422
- parent = dfns.find(dfn =>
423
- (dfn.linkingText[0] === expected.for[0]) &&
424
- ['callback', 'dictionary', 'enum', 'interface', 'namespace'].includes(dfn.type));
425
- }
426
-
427
- // Look for a definition that seems as close as possible to the one
428
- // we're looking for, in the following order:
429
- // 1. For operations, find a definition without taking arguments into
430
- // account and report possible match with a "warning" flag.
431
- // 2. For terms linked to a parent interface-like object, find a match
432
- // scoped to the same parent without taking the type into account.
433
- // 3. Look for a definition with the same name, neither taking the type
434
- // nor the parent into account.
435
- let found = dfns.find(dfn => matchIdlDfn(expected, dfn, { skipArgs: true }));
436
- if (found) {
437
- return { expected, found, for: parent, warning: true };
438
- }
439
- found = dfns.find(dfn => matchIdlDfn(expected, dfn,
440
- { skipArgs: true, skipType: true }));
441
- if (found) {
442
- return { expected, found, for: parent };
443
- }
444
- found = dfns.find(dfn => matchIdlDfn(expected, dfn,
445
- { skipArgs: true, skipType: true, skipFor: true }));
446
- return { expected, found, for: parent };
447
- }
448
- }).filter(missing => !!missing);
449
-
450
- // Report results
451
- return {
452
- css: missingCSSDfns,
453
- idl: missingIdlDfns
454
- };
455
- }
456
-
457
-
458
- /**
459
- * Checks the CSS and IDL extracts against the dfns extract for all specs in
460
- * the report.
461
- *
462
- * @function
463
- * @public
464
- * @param {String} pathToReport Path to the root folder that contains the
465
- * `index.json` report file and the extracts subfolders.
466
- * @param {Object} options Check options. Set the "shortname" property to a
467
- * spec's shortname to only check that spec.
468
- * @return {Array} The list of specifications along with dfn problems that have
469
- * been identified. Each entry has `url`, 'crawled`, `shortname` properties to
470
- * identify the specification, and a `missing` property that is an object that
471
- * may have `css` and `idl` properties which list missing CSS/IDL definitions.
472
- */
473
- async function checkDefinitions(pathToReport, options = {}) {
474
- const rootFolder = path.resolve(process.cwd(), pathToReport);
475
- const index = (await loadJSON(path.resolve(rootFolder, 'index.json'))).results;
476
-
477
- // Check all dfns against CSS and IDL extracts
478
- const checkOptions = {
479
- rootFolder,
480
- includeObsolete: !!options.shortname
481
- };
482
- const missing = index
483
- .filter(spec => !options.shortname || spec.shortname === options.shortname)
484
- .map(async spec => {
485
- const res = {
486
- url: spec.url,
487
- crawled: spec.crawled,
488
- shortname: spec.shortname,
489
- };
490
- if (!spec.dfns) {
491
- return res;
492
- }
493
- res.missing = await checkSpecDefinitions(spec, checkOptions);
494
- return res;
495
- });
496
- return Promise.all(missing);
497
- }
498
-
499
-
500
- /**
501
- * Report missing dfn to the console as Markdown
502
- *
503
- * @function
504
- * @private
505
- * @param {Object} missing Object that describes missing dfn
506
- */
507
- function reportMissing(missing) {
508
- const exp = missing.expected;
509
- const found = missing.found;
510
- const foundFor = (found && found.for && found.for.length > 0) ?
511
- ' for ' + found.for.map(f => `\`${f}\``).join(',') :
512
- '';
513
- console.log(`- \`${exp.linkingText[0]}\` ${exp.type ? `with type \`${exp.type}\`` : ''}` +
514
- (missing.for ? ` for [\`${missing.for.linkingText[0]}\`](${missing.for.href})` : '') +
515
- (found ? `, but found [\`${found.linkingText[0]}\`](${found.href}) with type \`${found.type}\`${foundFor}` : ''));
516
- }
517
-
518
-
519
- /**************************************************
520
- Export methods for use as module
521
- **************************************************/
522
- export {
523
- checkSpecDefinitions,
524
- checkDefinitions,
525
-
526
- // "Inner" functions that the IDL names generator uses to link IDL terms with
527
- // their definition (see generate-idlnames.js)
528
- getExpectedDfnFromIdlDesc,
529
- matchIdlDfn
530
- };
531
-
532
-
533
-
534
- /**************************************************
535
- Code run if the code is run as a stand-alone module
536
- **************************************************/
537
- if (process.argv[1] === fileURLToPath(import.meta.url)) {
538
- const pathToReport = process.argv[2];
539
- const shortname = process.argv[3] || 'all';
540
- const format = process.argv[4] || 'markdown';
541
-
542
- const options = (shortname === 'all') ? undefined : { shortname };
543
- let res = await checkDefinitions(pathToReport, options);
544
- if (shortname === 'all') {
545
- res = res
546
- .filter(result => result.missing &&
547
- !result.missing.obsoleteDfnsModel &&
548
- ((result.missing.css.length > 0) || (result.missing.idl.length > 0)));
549
- }
550
-
551
- if (format === 'json') {
552
- console.log(JSON.stringify(res, null, 2));
553
- }
554
- else {
555
- res.forEach(result => {
556
- const missing = result.missing || {css: [], idl: []};
557
- const errors = ['css', 'idl']
558
- .map(type => result.missing[type].filter(missing => !missing.warning))
559
- .flat();
560
- const warnings = ['css', 'idl']
561
- .map(type => result.missing[type].filter(missing => missing.warning))
562
- .flat();
563
- console.log('<details>');
564
- console.log(`<summary><b><a href="${result.crawled}">${result.shortname}</a></b> (${errors.length} errors, ${warnings.length} warnings)</summary>`);
565
- console.log();
566
- if (errors.length === 0 && warnings.length === 0) {
567
- console.log('All good!');
568
- }
569
- if (errors.length > 0) {
570
- console.log('<details open>');
571
- console.log(`<summary><i>Errors</i> (${errors.length})</summary>`);
572
- console.log();
573
- errors.forEach(reportMissing);
574
- console.log('</details>');
575
- }
576
- if (warnings.length > 0) {
577
- console.log('<details open>');
578
- console.log(`<summary><i>Warnings</i> (${warnings.length})</summary>`);
579
- console.log();
580
- warnings.forEach(reportMissing);
581
- console.log('</details>');
582
- }
583
- console.log('</details>');
584
- console.log();
585
- })
586
- }
587
- }
1
+ #!/usr/bin/env node
2
+ /**
3
+ * The definitions checker compares CSS, dfns, and IDL extracts created by Reffy
4
+ * to detect CSS/IDL terms that do not have a corresponding dfn in the
5
+ * specification.
6
+ *
7
+ * The definitions checker can be called directly through:
8
+ *
9
+ * `node check-missing-dfns.js [crawl report] [spec] [format]`
10
+ *
11
+ * where:
12
+ * - `crawl report` is the local path to the root folder that contains the
13
+ * `index.json` and the extracts (e.g. `reports/ed`)
14
+ * - `spec` is the optional shortname of the specification on which to focus or
15
+ * `all` (default) to check all specs
16
+ * - `format` is the optional output format. Either `json` or `markdown` with
17
+ * `markdown` being the default.
18
+ *
19
+ * Note: CSS extraction already relies on dfns and reports missing dfns in a
20
+ * "warnings" property. This checker simply looks at that list.
21
+ *
22
+ * @module checker
23
+ */
24
+
25
+ import path from 'node:path';
26
+ import { fileURLToPath } from 'node:url';
27
+ import process from 'node:process';
28
+ import { loadJSON } from '../lib/util.js';
29
+
30
+ /**
31
+ * List of spec shortnames that, so far, don't follow the dfns data model
32
+ */
33
+ const specsWithObsoleteDfnsModel = [
34
+ 'svg-animations', 'svg-markers', 'svg-strokes', 'SVG2',
35
+ 'webgl1', 'webgl2',
36
+ 'webrtc-identity'
37
+ ];
38
+
39
+
40
+ /**
41
+ * Return true when provided arrays are "equal", meaning that they contain the
42
+ * same items
43
+ *
44
+ * @function
45
+ * @private
46
+ * @param {Array} a First array to compare
47
+ * @param {Array} b Second array to compare
48
+ * @return {boolean} True when arrays are equal
49
+ */
50
+ function arraysEqual(a, b) {
51
+ return Array.isArray(a) &&
52
+ Array.isArray(b) &&
53
+ a.length === b.length &&
54
+ a.every((val, index) => val === b[index]);
55
+ }
56
+
57
+
58
+ /**
59
+ * Return the list of expected definitions from the CSS extract
60
+ *
61
+ * @function
62
+ * @private
63
+ * @param {Object} css The root of the object that describes CSS terms in the
64
+ * CSS extract
65
+ * @return {Array} An array of expected definitions
66
+ */
67
+ function getExpectedDfnsFromCSS(css) {
68
+ const expected = (css.warnings ?? [])
69
+ .filter(warning => warning.msg === 'Missing definition')
70
+ .map(warning => {
71
+ return {
72
+ linkingText: [warning.name],
73
+ type: warning.type,
74
+ 'for': warning.for
75
+ };
76
+ });
77
+
78
+ return expected;
79
+ }
80
+
81
+
82
+ /**
83
+ * Return true when the given CSS definition matches the expected definition
84
+ *
85
+ * @function
86
+ * @private
87
+ * @param {Object} expected Expected definition
88
+ * @param {Object} actual Actual definition to check
89
+ * @return {Boolean} true when actual definition matches the expected one
90
+ */
91
+ function matchCSSDfn(expected, actual) {
92
+ return arraysEqual(expected.linkingText, actual.linkingText) &&
93
+ (!expected.for || arraysEqual(expected.for, actual.for)) &&
94
+ (!expected.type || (expected.type === actual.type));
95
+ }
96
+
97
+
98
+ /**
99
+ * Return the list of expected definitions from the IDL extract
100
+ *
101
+ * @function
102
+ * @private
103
+ * @param {Object} css The root of the object that describes IDL terms in the
104
+ * `idlparsed` extract.
105
+ * @return {Array} An array of expected definitions
106
+ */
107
+ function getExpectedDfnsFromIdl(idl = {}) {
108
+ // Parse IDL names that the spec defines
109
+ const idlNames = Object.values(idl.idlNames || {});
110
+ let expected = idlNames.map(name => getExpectedDfnsFromIdlDesc(name)).flat();
111
+
112
+ // Parse members of IDL names that the spec extends
113
+ const idlExtendedNames = Object.values(idl.idlExtendedNames || {});
114
+ expected = expected.concat(idlExtendedNames.map(extended =>
115
+ extended.map(name => getExpectedDfnsFromIdlDesc(name, { excludeRoot: true })))
116
+ .flat(2));
117
+ return expected;
118
+ }
119
+
120
+
121
+ /**
122
+ * Return true if the given parsed IDL object describes a default toJSON
123
+ * operation that references:
124
+ * https://heycam.github.io/webidl/#default-tojson-steps
125
+ *
126
+ * @function
127
+ * @private
128
+ * @param {Object} desc Parsed IDL object to check
129
+ * @return {Boolean} true when object describes a default toJSON operation.
130
+ */
131
+ function isDefaultToJSONOperation(desc) {
132
+ return (desc.type === 'operation') &&
133
+ (desc.name === 'toJSON') &&
134
+ (desc.extAttrs && desc.extAttrs.find(attr => attr.name === "Default"));
135
+ }
136
+
137
+
138
+ /**
139
+ * Return the expected definition for the given parsed IDL structure
140
+ *
141
+ * @function
142
+ * @public
143
+ * @param {Object} desc The object that describes the IDL term in the
144
+ * `idlparsed` extract.
145
+ * @param {Object} parentDesc (optional) The object that describes the parent
146
+ * IDL term of the term to parse (used to compute the `for` property).
147
+ * @return {Object} The expected definition, or null if no expected definition
148
+ * is defined.
149
+ */
150
+ function getExpectedDfnFromIdlDesc(idl, parentIdl) {
151
+ function serializeArgs(args = []) {
152
+ return args
153
+ .map(arg => arg.variadic ? `...${arg.name}` : arg.name)
154
+ .join(', ');
155
+ }
156
+
157
+ let expected = {
158
+ linkingText: [idl.name],
159
+ type: idl.type,
160
+ 'for': parentIdl && (parentIdl !== idl) ? [parentIdl.name] : []
161
+ };
162
+
163
+ switch (idl.type) {
164
+ case 'attribute':
165
+ case 'const':
166
+ break;
167
+
168
+ case 'constructor':
169
+ // Ignore constructors for HTML elements, the spec has a dedicated
170
+ // section for them:
171
+ // https://html.spec.whatwg.org/multipage/dom.html#html-element-constructors
172
+ if (!parentIdl.name.startsWith('HTML')) {
173
+ expected.linkingText = [`constructor(${serializeArgs(idl.arguments)})`];
174
+ }
175
+ else {
176
+ expected = null;
177
+ }
178
+ break;
179
+
180
+ case 'enum':
181
+ break;
182
+
183
+ case 'enum-value':
184
+ // The enumeration could include the empty string as a value. There
185
+ // cannot be a matching definition in that case.
186
+ // Note: look for the quoted value and the unquoted value
187
+ const value = idl.value.replace(/^"(.*)"$/, '$1');
188
+ expected.linkingText = (value !== '') ? [`"${value}"`, value] : [`"${value}"`];
189
+ break;
190
+
191
+ case 'field':
192
+ expected.type = 'dict-member';
193
+ break;
194
+
195
+ case 'callback':
196
+ case 'callback interface':
197
+ case 'dictionary':
198
+ case 'interface':
199
+ case 'interface mixin':
200
+ case 'namespace':
201
+ expected.type =
202
+ (idl.type === 'callback interface') ? 'callback' :
203
+ (idl.type === 'interface mixin') ? 'interface' :
204
+ idl.type;
205
+ // Ignore partial definition
206
+ if (idl.partial) {
207
+ expected = null;
208
+ }
209
+ break;
210
+
211
+ case 'includes':
212
+ expected = null;
213
+ break;
214
+
215
+ case 'iterable':
216
+ case 'async_iterable':
217
+ case 'maplike':
218
+ case 'setlike':
219
+ // No definition expected for iterable, maplike and setlike members
220
+ expected = null;
221
+ break;
222
+
223
+ case 'operation':
224
+ // Stringification behavior is typically defined with a
225
+ // "stringification behavior" definition scoped to the interface
226
+ if (idl.special === 'stringifier') {
227
+ expected.linkingText = ['stringification behavior', 'stringificationbehavior'];
228
+ expected.type = 'dfn';
229
+ }
230
+ // Ignore special "getter", "setter", "deleter" operations when they don't
231
+ // have an identifier. They should link to a definition in the prose, but
232
+ // the labels seem arbitrary for now.
233
+ // Also ignore default toJSON operations. Steps are defined in WebIDL.
234
+ else if ((idl.name ||
235
+ ((idl.special !== 'getter') &&
236
+ (idl.special !== 'setter') &&
237
+ (idl.special !== 'deleter'))) &&
238
+ !isDefaultToJSONOperation(idl)) {
239
+ expected.linkingText = [`${idl.name}(${serializeArgs(idl.arguments)})`];
240
+ expected.type = 'method';
241
+ }
242
+ else {
243
+ expected = null;
244
+ }
245
+ break;
246
+
247
+ case 'typedef':
248
+ break;
249
+
250
+ case 'argument':
251
+ expected = null;
252
+ break;
253
+
254
+ default:
255
+ console.warn('Unsupported IDL type', idl.type, idl);
256
+ expected = null;
257
+ break;
258
+ }
259
+
260
+ return expected;
261
+ }
262
+
263
+
264
+ /**
265
+ * Return the list of expected definitions from a parsed IDL extract entry.
266
+ *
267
+ * The function is recursive.
268
+ *
269
+ * @function
270
+ * @private
271
+ * @param {Object} idl The object that describes the IDL term in the
272
+ * `idlparsed` extract.
273
+ * @return {Array} An array of expected definitions
274
+ */
275
+ function getExpectedDfnsFromIdlDesc(idl, {excludeRoot} = {excludeRoot: false}) {
276
+ const res = [];
277
+ const parentIdl = idl;
278
+ const idlToProcess = excludeRoot ? [] : [idl];
279
+
280
+ switch (idl.type) {
281
+ case 'enum':
282
+ if (idl.values) {
283
+ idlToProcess.push(...idl.values);
284
+ }
285
+ break;
286
+
287
+ case 'callback':
288
+ case 'callback interface':
289
+ case 'dictionary':
290
+ case 'interface':
291
+ case 'interface mixin':
292
+ case 'namespace':
293
+ if (idl.members) {
294
+ idlToProcess.push(...idl.members);
295
+ }
296
+ break;
297
+ }
298
+
299
+ idlToProcess.forEach(idl => {
300
+ const expected = getExpectedDfnFromIdlDesc(idl, parentIdl);
301
+ if (expected) {
302
+ expected.access = 'public';
303
+ expected.informative = false;
304
+ res.push(expected);
305
+ }
306
+ });
307
+
308
+ return res;
309
+ }
310
+
311
+
312
+ /**
313
+ * Return true when the given IDL definition matches the expected definition.
314
+ *
315
+ * The function handles overloaded methods, though not properly. That is, it
316
+ * will only find the "right" definition for an overloaded method if the number
317
+ * and/or the name of the arguments differ between the overloaded definitions.
318
+ * Otherwise it will just match the first definition that looks good.
319
+ *
320
+ * The function works around Respec's issue #3200 for methods and constructors
321
+ * that take only optional parameters:
322
+ * https://github.com/w3c/respec/issues/3200
323
+ *
324
+ * @function
325
+ * @private
326
+ * @param {Object} expected Expected definition
327
+ * @param {Object} actual Actual definition to check
328
+ * @param {Object} options Comparison options
329
+ * @return {Boolean} true when actual definition matches the expected one
330
+ */
331
+ function matchIdlDfn(expected, actual,
332
+ {skipArgs, skipFor, skipType} = {skipArgs: false, skipFor: false, skipType: false}) {
333
+ const fixedLt = actual.linkingText
334
+ .map(lt => lt.replace(/!overload-\d/, ''))
335
+ .map(lt => lt.replace(/\(, /, '('));
336
+ let found = expected.linkingText.some(val => fixedLt.includes(val));
337
+ if (!found && skipArgs) {
338
+ const names = fixedLt.map(lt => lt.replace(/\(.*\)/, ''));
339
+ found = expected.linkingText.some(val => {
340
+ const valname = val.replace(/\(.*\)/, '');
341
+ return names.find(name => name === valname);
342
+ });
343
+ }
344
+ return found &&
345
+ (expected.for.every(val => actual.for.includes(val)) || skipFor) &&
346
+ (expected.type === actual.type || skipType);
347
+ }
348
+
349
+
350
+ /**
351
+ * Checks the CSS and IDL extracts against the dfns extract for the given spec
352
+ *
353
+ * @function
354
+ * @public
355
+ * @param {Object} spec Crawl result for the spec to parse
356
+ * @param {String} options Check options. Set the rootFolder property to the
357
+ * root folder against which to resolve relative paths to load CSS/IDL
358
+ * extracts (only needed if the extracts have not yet been loaded and attached
359
+ * to the spec object). Set the includeObsolete property to true to include
360
+ * detailed results about specs that use an obsolete dfns data model.
361
+ * @return {Object} An object with a css and idl property, each of them holding
362
+ * an array of missing CSS or IDL definitions. The function returns null when
363
+ * there are no missing definitions.
364
+ */
365
+ async function checkSpecDefinitions(spec, options = {}) {
366
+ if (!options.includeObsolete && specsWithObsoleteDfnsModel.includes(spec.shortname)) {
367
+ return { obsoleteDfnsModel: true };
368
+ }
369
+
370
+ const dfns = (typeof spec.dfns === "string") ?
371
+ (await loadJSON(path.resolve(options.rootFolder, spec.dfns))).dfns :
372
+ (spec.dfns || []);
373
+ const css = (typeof spec.css === "string") ?
374
+ (await loadJSON(path.resolve(options.rootFolder, spec.css))) :
375
+ (spec.css || {});
376
+ const idl = (typeof spec.idlparsed === "string") ?
377
+ (await loadJSON(path.resolve(options.rootFolder, spec.idlparsed))).idlparsed :
378
+ spec.idlparsed;
379
+
380
+ // Make sure that all expected CSS definitions exist in the dfns extract
381
+ const expectedCSSDfns = getExpectedDfnsFromCSS(css);
382
+ const missingCSSDfns = expectedCSSDfns.map(expected => {
383
+ let actual = dfns.find(dfn => matchCSSDfn(expected, dfn));
384
+ if (!actual && !expected.type) {
385
+ // Right definition is missing. For valuespaces that define functions,
386
+ // look for a function definition without the enclosing "<>" instead
387
+ const altText = [expected.linkingText[0].replace(/^<(.*)\(\)>$/, '$1()')];
388
+ actual = dfns.find(dfn => arraysEqual(altText, dfn.linkingText));
389
+ }
390
+ if (!actual && expected.value) {
391
+ // Still missing? For valuespaces that define functions, this may be
392
+ // because there is no definition without parameters, try to find the
393
+ // actual value instead
394
+ actual = dfns.find(dfn => arraysEqual([expected.value], dfn.linkingText));
395
+ }
396
+ if (actual) {
397
+ // Right definition found
398
+ return null;
399
+ }
400
+ else {
401
+ // Right definition is missing, there may be a definition that looks
402
+ // like the one we're looking for
403
+ const found = dfns.find(dfn =>
404
+ arraysEqual(dfn.linkingText, expected.linkingText));
405
+ return { expected, found };
406
+ }
407
+ }).filter(missing => !!missing);
408
+
409
+ // Make sure that all expected IDL definitions exist in the dfns extract
410
+ const expectedIdlDfns = getExpectedDfnsFromIdl(idl);
411
+ const missingIdlDfns = expectedIdlDfns.map(expected => {
412
+ let actual = dfns.find(dfn => matchIdlDfn(expected, dfn));
413
+ if (actual) {
414
+ // Right definition found
415
+ return null;
416
+ }
417
+ else {
418
+ // Right definition is missing, include the interface's definitions to
419
+ // be able to link to it in the report
420
+ let parent = null;
421
+ if (expected.for && expected.for[0]) {
422
+ parent = dfns.find(dfn =>
423
+ (dfn.linkingText[0] === expected.for[0]) &&
424
+ ['callback', 'dictionary', 'enum', 'interface', 'namespace'].includes(dfn.type));
425
+ }
426
+
427
+ // Look for a definition that seems as close as possible to the one
428
+ // we're looking for, in the following order:
429
+ // 1. For operations, find a definition without taking arguments into
430
+ // account and report possible match with a "warning" flag.
431
+ // 2. For terms linked to a parent interface-like object, find a match
432
+ // scoped to the same parent without taking the type into account.
433
+ // 3. Look for a definition with the same name, neither taking the type
434
+ // nor the parent into account.
435
+ let found = dfns.find(dfn => matchIdlDfn(expected, dfn, { skipArgs: true }));
436
+ if (found) {
437
+ return { expected, found, for: parent, warning: true };
438
+ }
439
+ found = dfns.find(dfn => matchIdlDfn(expected, dfn,
440
+ { skipArgs: true, skipType: true }));
441
+ if (found) {
442
+ return { expected, found, for: parent };
443
+ }
444
+ found = dfns.find(dfn => matchIdlDfn(expected, dfn,
445
+ { skipArgs: true, skipType: true, skipFor: true }));
446
+ return { expected, found, for: parent };
447
+ }
448
+ }).filter(missing => !!missing);
449
+
450
+ // Report results
451
+ return {
452
+ css: missingCSSDfns,
453
+ idl: missingIdlDfns
454
+ };
455
+ }
456
+
457
+
458
+ /**
459
+ * Checks the CSS and IDL extracts against the dfns extract for all specs in
460
+ * the report.
461
+ *
462
+ * @function
463
+ * @public
464
+ * @param {String} pathToReport Path to the root folder that contains the
465
+ * `index.json` report file and the extracts subfolders.
466
+ * @param {Object} options Check options. Set the "shortname" property to a
467
+ * spec's shortname to only check that spec.
468
+ * @return {Array} The list of specifications along with dfn problems that have
469
+ * been identified. Each entry has `url`, 'crawled`, `shortname` properties to
470
+ * identify the specification, and a `missing` property that is an object that
471
+ * may have `css` and `idl` properties which list missing CSS/IDL definitions.
472
+ */
473
+ async function checkDefinitions(pathToReport, options = {}) {
474
+ const rootFolder = path.resolve(process.cwd(), pathToReport);
475
+ const index = (await loadJSON(path.resolve(rootFolder, 'index.json'))).results;
476
+
477
+ // Check all dfns against CSS and IDL extracts
478
+ const checkOptions = {
479
+ rootFolder,
480
+ includeObsolete: !!options.shortname
481
+ };
482
+ const missing = index
483
+ .filter(spec => !options.shortname || spec.shortname === options.shortname)
484
+ .map(async spec => {
485
+ const res = {
486
+ url: spec.url,
487
+ crawled: spec.crawled,
488
+ shortname: spec.shortname,
489
+ };
490
+ if (!spec.dfns) {
491
+ return res;
492
+ }
493
+ res.missing = await checkSpecDefinitions(spec, checkOptions);
494
+ return res;
495
+ });
496
+ return Promise.all(missing);
497
+ }
498
+
499
+
500
+ /**
501
+ * Report missing dfn to the console as Markdown
502
+ *
503
+ * @function
504
+ * @private
505
+ * @param {Object} missing Object that describes missing dfn
506
+ */
507
+ function reportMissing(missing) {
508
+ const exp = missing.expected;
509
+ const found = missing.found;
510
+ const foundFor = (found && found.for && found.for.length > 0) ?
511
+ ' for ' + found.for.map(f => `\`${f}\``).join(',') :
512
+ '';
513
+ console.log(`- \`${exp.linkingText[0]}\` ${exp.type ? `with type \`${exp.type}\`` : ''}` +
514
+ (missing.for ? ` for [\`${missing.for.linkingText[0]}\`](${missing.for.href})` : '') +
515
+ (found ? `, but found [\`${found.linkingText[0]}\`](${found.href}) with type \`${found.type}\`${foundFor}` : ''));
516
+ }
517
+
518
+
519
+ /**************************************************
520
+ Export methods for use as module
521
+ **************************************************/
522
+ export {
523
+ checkSpecDefinitions,
524
+ checkDefinitions,
525
+
526
+ // "Inner" functions that the IDL names generator uses to link IDL terms with
527
+ // their definition (see generate-idlnames.js)
528
+ getExpectedDfnFromIdlDesc,
529
+ matchIdlDfn
530
+ };
531
+
532
+
533
+
534
+ /**************************************************
535
+ Code run if the code is run as a stand-alone module
536
+ **************************************************/
537
+ if (process.argv[1] === fileURLToPath(import.meta.url)) {
538
+ const pathToReport = process.argv[2];
539
+ const shortname = process.argv[3] || 'all';
540
+ const format = process.argv[4] || 'markdown';
541
+
542
+ const options = (shortname === 'all') ? undefined : { shortname };
543
+ let res = await checkDefinitions(pathToReport, options);
544
+ if (shortname === 'all') {
545
+ res = res
546
+ .filter(result => result.missing &&
547
+ !result.missing.obsoleteDfnsModel &&
548
+ ((result.missing.css.length > 0) || (result.missing.idl.length > 0)));
549
+ }
550
+
551
+ if (format === 'json') {
552
+ console.log(JSON.stringify(res, null, 2));
553
+ }
554
+ else {
555
+ res.forEach(result => {
556
+ const missing = result.missing || {css: [], idl: []};
557
+ const errors = ['css', 'idl']
558
+ .map(type => result.missing[type].filter(missing => !missing.warning))
559
+ .flat();
560
+ const warnings = ['css', 'idl']
561
+ .map(type => result.missing[type].filter(missing => missing.warning))
562
+ .flat();
563
+ console.log('<details>');
564
+ console.log(`<summary><b><a href="${result.crawled}">${result.shortname}</a></b> (${errors.length} errors, ${warnings.length} warnings)</summary>`);
565
+ console.log();
566
+ if (errors.length === 0 && warnings.length === 0) {
567
+ console.log('All good!');
568
+ }
569
+ if (errors.length > 0) {
570
+ console.log('<details open>');
571
+ console.log(`<summary><i>Errors</i> (${errors.length})</summary>`);
572
+ console.log();
573
+ errors.forEach(reportMissing);
574
+ console.log('</details>');
575
+ }
576
+ if (warnings.length > 0) {
577
+ console.log('<details open>');
578
+ console.log(`<summary><i>Warnings</i> (${warnings.length})</summary>`);
579
+ console.log();
580
+ warnings.forEach(reportMissing);
581
+ console.log('</details>');
582
+ }
583
+ console.log('</details>');
584
+ console.log();
585
+ })
586
+ }
587
+ }