reffy 7.2.8 → 8.0.0

package/src/lib/specs-crawler.js

@@ -14,16 +14,15 @@ const fs = require('fs');
 const path = require('path');
 const specs = require('web-specs');
 const cssDfnParser = require('./css-grammar-parser');
-const { generateIdlParsed, saveIdlParsed } = require('../cli/generate-idlparsed');
-const { generateIdlNames, saveIdlNames } = require('../cli/generate-idlnames');
+const postProcessor = require('./post-processor');
 const {
     completeWithAlternativeUrls,
     expandBrowserModules,
     expandCrawlResult,
     expandSpecResult,
-    getGeneratedIDLNamesByCSSProperty,
     isLatestLevelThatPasses,
     processSpecification,
+    requireFromWorkingDirectory,
     setupBrowser,
     teardownBrowser,
     createFolderIfNeeded
@@ -72,111 +71,52 @@ async function specOrFallback(spec, fallbackFolder, fallbackData) {
  */
 async function crawlSpec(spec, crawlOptions) {
     crawlOptions = crawlOptions || {};
-    spec.crawled = crawlOptions.publishedVersion ?
+
+    const urlToCrawl = crawlOptions.publishedVersion ?
         (spec.release ? spec.release : spec.nightly) :
         spec.nightly;
     const fallbackFolder = crawlOptions.fallback ?
         path.dirname(crawlOptions.fallback) : '';
 
-    if (spec.error) {
-        return specOrFallback(spec, fallbackFolder, crawlOptions.fallbackData?.results);
-    }
-
     try {
         const fallback = crawlOptions.fallbackData?.results?.find(s => s.url === spec.url);
         let cacheInfo = {};
         if (crawlOptions.fallbackData?.crawler === `reffy-${reffyVersion}`) {
           cacheInfo = Object.assign({}, fallback?.crawlCacheInfo);
         }
-        const result = await processSpecification(
-            spec.crawled,
-            (spec, modules) => {
-                const idToHeading = modules.find(m => m.needsIdToHeadingMap) ?
-                    window.reffy.mapIdsToHeadings() : null;
-                const res = {
-                    crawled: window.location.toString()
-                };
-                modules.forEach(mod => {
-                    res[mod.property] = window.reffy[mod.name](spec, idToHeading);
-                });
-                return res;
-            },
-            [spec, crawlOptions.modules],
-            { quiet: crawlOptions.quiet,
-              forceLocalFetch: crawlOptions.forceLocalFetch,
-              ...cacheInfo}
-        );
-        if (result.status === "notmodified" && fallback) {
-          crawlOptions.quiet ?? console.warn(`skipping ${spec.url}, no change`);
-          const copy = Object.assign({}, fallback);
-          return expandSpecResult(copy, fallbackFolder);
+        let result = null;
+        if (crawlOptions.useCrawl) {
+            result = await expandSpecResult(spec, crawlOptions.useCrawl);
         }
-
-        // Specific rule for IDL extracts:
-        // parse the extracted WebIdl content
-        await generateIdlParsed(result);
-
-        if (result.css) {
-            // Specific rule for CSS properties:
-            // Add CSS property definitions that weren't in a table
-            if (result.dfns) {
-                result.dfns
-                    .filter(dfn => dfn.type == "property" && !dfn.informative)
-                    .forEach(propDfn => {
-                        propDfn.linkingText.forEach(lt => {
-                            if (!result.css.properties.hasOwnProperty(lt)) {
-                                result.css.properties[lt] = {
-                                    name: lt
-                                };
-                            }
-                        });
+        else {
+            result = await processSpecification(
+                urlToCrawl,
+                (spec, modules) => {
+                    const idToHeading = modules.find(m => m.needsIdToHeadingMap) ?
+                        window.reffy.mapIdsToHeadings() : null;
+                    const res = {
+                        crawled: window.location.toString()
+                    };
+                    modules.forEach(mod => {
+                        res[mod.property] = window.reffy[mod.name](spec, idToHeading);
                     });
+                    return res;
+                },
+                [spec, crawlOptions.modules],
+                { quiet: crawlOptions.quiet,
+                  forceLocalFetch: crawlOptions.forceLocalFetch,
+                  ...cacheInfo}
+            );
+            if (result.status === "notmodified" && fallback) {
+              crawlOptions.quiet ?? console.warn(`skipping ${spec.url}, no change`);
+              const copy = Object.assign({}, fallback);
+              return expandSpecResult(copy, fallbackFolder);
             }
+        }
 
-            // Specific rule for CSS properties:
-            // Ideally, the sample definition (property-name) in CSS2 and the custom
-            // property definition (--*) in CSS Variables would not be flagged as
-            // real CSS properties. In practice, they are. Let's remove them from
-            // the extract.
-            ['property-name', '--*'].forEach(prop => {
-                if ((result.css.properties || {})[prop]) {
-                    delete result.css.properties[prop];
-                }
-            });
-
-            // Specific rule for CSS extracts:
-            // Parse extracted CSS definitions and add generated IDL attribute names
-            Object.entries(result.css.properties || {}).forEach(([prop, dfn]) => {
-                if (dfn.value || dfn.newValues) {
-                    try {
-                        dfn.parsedValue = cssDfnParser.parsePropDefValue(
-                            dfn.value || dfn.newValues);
-                    } catch (e) {
-                        dfn.valueParseError = e.message;
-                    }
-                }
-                dfn.styleDeclaration = getGeneratedIDLNamesByCSSProperty(prop);
-            });
-            Object.entries(result.css.descriptors || {}).forEach(([desc, dfn]) => {
-                if (dfn.value) {
-                    try {
-                        dfn.parsedValue = cssDfnParser.parsePropDefValue(
-                            dfn.value);
-                    } catch (e) {
-                        dfn.valueParseError = e.message;
-                    }
-                }
-            });
-            Object.entries(result.css.valuespaces || {}).forEach(([vs, dfn]) => {
-                if (dfn.value) {
-                    try {
-                        dfn.parsedValue = cssDfnParser.parsePropDefValue(
-                            dfn.value);
-                    } catch (e) {
-                        dfn.valueParseError = e.message;
-                    }
-                }
-            });
+        // Run post-processing modules at the spec level
+        for (const mod of (crawlOptions.post ?? [])) {
+            await postProcessor.run(mod, result, crawlOptions);
         }
 
         // Copy results back into initial spec object
@@ -187,9 +127,12 @@ async function crawlSpec(spec, crawlOptions) {
         crawlOptions.modules.forEach(mod => {
             if (result[mod.property]) {
                 spec[mod.property] = result[mod.property];
-                if (mod.property === 'idl') {
-                    spec.idlparsed = result.idlparsed;
-                }
+            }
+        });
+        crawlOptions.post?.forEach(mod => {
+            const prop = mod.property ?? mod.name;
+            if (postProcessor.appliesAtLevel(mod, 'spec') && result[prop]) {
+                spec[prop] = result[prop];
             }
         });
     }
@@ -236,12 +179,6 @@ async function saveSpecResults(spec, settings) {
             continue;
         }
         folders[mod.property] = await getSubfolder(mod.property);
-
-        // Specific rule for IDL:
-        // Raw IDL goes to "idl" subfolder, parsed IDL goes to "idlparsed"
-        if (mod.property === 'idl') {
-            folders.idlparsed = await getSubfolder('idlparsed');
-        }
     }
 
     function getBaseJSON(spec) {
@@ -301,9 +238,6 @@ async function saveSpecResults(spec, settings) {
     if (spec.idl) {
         spec.idl = await saveIdl(spec);
     }
-    if (spec.idlparsed) {
-        spec.idlparsed = await saveIdlParsed(spec, settings.output);
-    }
 
     // Save CSS dumps
     function defineCSSContent(spec) {
@@ -331,14 +265,16 @@ async function saveSpecResults(spec, settings) {
             (typeof thing == 'object') && (Object.keys(thing).length === 0);
     }
 
-    // Save all other extracts
+    // Save all other extracts from crawling modules
     const remainingModules = modules.filter(mod =>
         !mod.metadata && mod.property !== 'css' && mod.property !== 'idl');
     for (const mod of remainingModules) {
         await saveExtract(spec, mod.property, spec => !isEmpty(spec[mod.property]));
-        if (spec[mod.property] && typeof spec[mod.property] !== 'string') {
-            delete spec[mod.property];
-        }
+    }
+
+    // Save extracts from post-processing modules that run at the spec level
+    for (const mod of (settings.post ?? [])) {
+        await postProcessor.save(mod, spec, settings);
     }
 
     return spec;
@@ -372,10 +308,17 @@ async function crawlList(speclist, crawlOptions) {
         }
     }
 
-    // Prepare Puppeteer instance
-    await setupBrowser(crawlOptions.modules);
+    // Prepare Puppeteer instance unless we already have crawl results and
+    // we're only interested in post-processing
+    let list = null;
+    if (crawlOptions.useCrawl) {
+        list = speclist;
+    }
+    else {
+        await setupBrowser(crawlOptions.modules);
+        list = speclist.map(completeWithAlternativeUrls);
+    }
 
-    const list = speclist.map(completeWithAlternativeUrls);
     const listAndPromise = list.map(spec => {
         let resolve = null;
         let reject = null;
@@ -418,7 +361,9 @@ async function crawlList(speclist, crawlOptions) {
     const results = await Promise.all(listAndPromise.map(crawlSpecAndPromise));
 
     // Close Puppeteer instance
-    teardownBrowser();
+    if (!crawlOptions.useCrawl) {
+        teardownBrowser();
+    }
 
     return results;
 }
@@ -564,8 +509,12 @@ function crawlSpecs(options) {
         });
     }
 
-    const requestedList = options?.specs ?
-        prepareListOfSpecs(options.specs) :
+    const crawlIndex = options?.useCrawl ?
+        requireFromWorkingDirectory(options.useCrawl) :
+        null;
+
+    const requestedList = crawlIndex ? crawlIndex.results :
+        options?.specs ? prepareListOfSpecs(options.specs) :
         specs;
 
     // Make a shallow copy of passed options parameter and expand modules
@@ -579,9 +528,11 @@ function crawlSpecs(options) {
             for (const mod of options.modules) {
                 if (mod.extractsPerSeries) {
                     await adjustExtractsPerSeries(results, mod.property, options);
-                    if (mod.property === 'idl') {
-                        await adjustExtractsPerSeries(results, 'idlparsed', options);
-                    }
+                }
+            }
+            for (const mod of options.post ?? []) {
+                if (postProcessor.extractsPerSeries(mod)) {
+                    await adjustExtractsPerSeries(results, mod.property, options);
                 }
             }
             return results;
@@ -612,14 +563,21 @@ function crawlSpecs(options) {
             }
         })
         .then(async crawlIndex => {
-            // Generate IDL names extracts from IDL extracts
-            // (and dfns extracts to create links to definitions)
-            if (!options.output || !crawlIndex?.options?.modules?.find(mod => mod === 'idl')) {
-                return;
+            // Run post-processing modules at the crawl level
+            for (const mod of (options.post ?? [])) {
+                if (!postProcessor.appliesAtLevel(mod, 'crawl')) {
+                    continue;
+                }
+                const crawlResults = await expandCrawlResult(
+                    crawlIndex, options.output, postProcessor.dependsOn(mod));
+                const result = await postProcessor.run(mod, crawlResults, options);
+                await postProcessor.save(mod, result, options);
+
+                if (!options.output) {
+                    console.log();
+                    console.log(JSON.stringify(result, null, 2));
+                }
             }
-            const crawlResults = await expandCrawlResult(crawlIndex, options.output, ['idlparsed', 'dfns']);
-            const idlNames = generateIdlNames(crawlResults.results, options);
-            await saveIdlNames(idlNames, options.output);
         });
 }
 
@@ -627,5 +585,13 @@ function crawlSpecs(options) {
 /**************************************************
 Export methods for use as module
 **************************************************/
-module.exports.crawlList = crawlList;
-module.exports.crawlSpecs = crawlSpecs;
+// TODO: consider more alignment between the two crawl functions or
+// find more explicit names to distinguish between them:
+// - "crawlList" takes an explicit list of specs as input, does not run the
+// post-processor, and returns the results without saving them to files.
+// - "crawlSpecs" takes options as input, runs all steps and saves results
+// to files (or outputs the results to the console). It does not return
+// anything.
+module.exports.crawlSpecs = (...args) => Array.isArray(args[0]) ?
+    crawlList.apply(this, args) :
+    crawlSpecs.apply(this, args);

package/src/lib/util.js

@@ -605,7 +605,7 @@ async function processSpecification(spec, processFunction, args, options) {
                 window.document.head.querySelector("script[src*='respec']");
 
             function sleep(ms) {
-                return new Promise(resolve => setTimeout(resolve, ms));
+                return new Promise(resolve => setTimeout(resolve, ms, 'slept'));
             }
 
             async function isReady(counter) {
@@ -614,7 +614,10 @@ async function processSpecification(spec, processFunction, args, options) {
                     throw new Error('Respec generation took too long');
                 }
                 if (window.document.respec?.ready) {
-                    await window.document.respec.ready;
+                    const res = await Promise.race([window.document.respec.ready, sleep(60000)]);
+                    if (res === 'slept') {
+                        throw new Error('Respec generation took too long');
+                    }
                 }
                 else if (usesRespec) {
                     await sleep(1000);
@@ -741,10 +744,10 @@ function isLatestLevelThatPasses(spec, list, predicate) {
         return true;
     }
     while (spec.seriesNext) {
+        spec = list.find(s => s.shortname === spec.seriesNext);
         if (!spec) {
             break;
         }
-        spec = list.find(s => s.shortname === spec.seriesNext);
         if ((spec.seriesComposition === 'full') && predicate(spec)) {
             return false;
         }
@@ -813,10 +816,19 @@ async function expandSpecResult(spec, baseFolder, properties) {
             // Also drop header that may have been added when extract was
             // serialized.
             if (contents.startsWith('// GENERATED CONTENT - DO NOT EDIT')) {
-                const endOfHeader = contents.indexOf('\n\n');
-                contents = contents.substring(endOfHeader + 2)
-                // remove trailing newline added in saveIdl
-                  .slice(0, -1);
+                const hasWindowsEndings = contents.indexOf('\r\n');
+                if (hasWindowsEndings) {
+                    const endOfHeader = contents.indexOf('\r\n\r\n');
+                    contents = contents.substring(endOfHeader + 4)
+                    // remove trailing newline added in saveIdl
+                      .slice(0, -2);
+                }
+                else {
+                    const endOfHeader = contents.indexOf('\n\n');
+                    contents = contents.substring(endOfHeader + 2)
+                    // remove trailing newline added in saveIdl
+                      .slice(0, -1);
+                }
             }
             spec.idl = contents;
         }
@@ -927,6 +939,66 @@ async function createFolderIfNeeded(folder) {
 }
 
 
+/**
+ * Tree hierarchies on which events may bubble
+ *
+ * First interface is the tree root, further interfaces are deeper levels in
+ * the tree.
+ */
+const trees = {
+  // DOM tree:
+  // https://dom.spec.whatwg.org/#node-trees
+  'dom': ['Window', 'Document', 'Element', 'Node'],
+
+  // IndexedDB tree (defined through "get the parent" algorithms)
+  // https://www.w3.org/TR/IndexedDB/#ref-for-get-the-parent%E2%91%A0
+  // https://www.w3.org/TR/IndexedDB/#ref-for-get-the-parent%E2%91%A1
+  'IndexedDB': ['IDBDatabase', 'IDBTransaction', 'IDBRequest'],
+
+  // Web Bluetooth tree
+  // https://webbluetoothcg.github.io/web-bluetooth/#bluetooth-tree-bluetooth-tree
+  'web-bluetooth': [
+    'Bluetooth', 'BluetoothDevice', 'BluetoothRemoteGATTService',
+    'BluetoothRemoteGATTCharacteristic', 'BluetoothRemoteGATTDescriptor'],
+
+  // Serial tree
+  // https://wicg.github.io/serial/#serialport-interface
+  'serial': ['Serial', 'SerialPort']
+};
+
+
+/**
+ * Return information about the tree hierarchy the IDL interface is linked to.
+ *
+ * @function
+ * @param {String} iface Name of the IDL interface to link to a tree
+ * @param {Array(Object)} interfaces A list of all known IDL interfaces with
+ *   inheritance information in an "inheritance" property.
+ * @return {Object} An object with a "tree" property set to the shortname of the
+ *   spec that defines the tree hierarchy, an "interface" property set to the
+ *   interface name of the closest interface in the inheritance chain of the
+ *   given interface that belongs to the tree, and a "depth" property that gives
+ *   the depth of that interface in the tree hierarchy (where 0 is the tree
+ *   root). The object is null if the interface cannot be associated with a
+ *   tree.
+ */
+function getInterfaceTreeInfo(iface, interfaces) {
+  while (iface) {
+    for (const [tree, nodes] of Object.entries(trees)) {
+      if (nodes.includes(iface)) {
+        return {
+          tree,
+          interface: iface,
+          depth: nodes.findIndex(i => i === iface)
+        };
+      }
+    }
+    iface = interfaces.find(i => i.name === iface)?.inheritance;
+  }
+  return null;
+}
+
+
 module.exports = {
     fetch,
     requireFromWorkingDirectory,
@@ -939,5 +1011,6 @@ module.exports = {
     expandCrawlResult,
     expandSpecResult,
     getGeneratedIDLNamesByCSSProperty,
-    createFolderIfNeeded
+    createFolderIfNeeded,
+    getInterfaceTreeInfo
 };

package/src/postprocessing/csscomplete.js

@@ -0,0 +1,50 @@
+/**
+ * Post-processing module that adds CSS property definitions found in prose
+ * from the dfns extract, clean up property definitions that should never have
+ * been extracted, and adds the generated IDL attribute names in a
+ * styleDeclaration sub-property.
+ *
+ * Module runs at the spec level. It does not create a distinct property but
+ * rather completes the `css` property with additional info.
+ */
+
+const { getGeneratedIDLNamesByCSSProperty } = require('../lib/util');
+
+module.exports = {
+  dependsOn: ['css', 'dfns'],
+  input: 'spec',
+
+  run: async function(spec, options) {
+    if (spec.dfns && spec.css) {
+      spec.dfns
+        .filter(dfn => dfn.type == "property" && !dfn.informative)
+        .forEach(propDfn => {
+          propDfn.linkingText.forEach(lt => {
+            if (!spec.css.properties.hasOwnProperty(lt)) {
+              spec.css.properties[lt] = {
+                name: lt
+              };
+            }
+          });
+        });
+    }
+
+    if (spec.css) {
+      // Add generated IDL attribute names
+      Object.entries(spec.css.properties || {}).forEach(([prop, dfn]) => {
+        dfn.styleDeclaration = getGeneratedIDLNamesByCSSProperty(prop);
+      });
+
+      // Drop the sample definition (property-name) in CSS2 and the custom
+      // property definition (--*) in CSS Variables that specs incorrectly flag
+      // as real CSS properties.
+      ['property-name', '--*'].forEach(prop => {
+          if ((spec.css.properties || {})[prop]) {
+              delete spec.css.properties[prop];
+          }
+      });
+    }
+
+    return spec;
+  }
+};