reffy 7.2.10 → 8.0.2

package/README.md

@@ -32,6 +32,8 @@ npm update -g reffy
 
 Reffy crawls requested specifications and runs a set of processing modules on the content fetched to create relevant extracts from each spec. Which specs get crawled, and which processing modules get run depend on how the crawler gets called. By default, the crawler crawls all specs defined in [browser-specs](https://github.com/w3c/browser-specs/) and runs all core processing modules defined in the [`browserlib`](https://github.com/w3c/reffy/tree/main/src/browserlib) folder.
 
+Reffy can also run post-processing modules on the results of the crawl to create additional views of the data extracted from the spec during the crawl.
+
 Crawl results will either be returned to the console or saved in individual files in a report folder when the `--output` parameter is set.
 
 Examples of information that can be extracted from the specs:
@@ -93,15 +95,6 @@ To create the WebIDL extract in the first place, you will need to run the `idl`
 reffy --spec fetch --module idl > fetch.idl
 ```
 
-### Parsed WebIDL generator
-
-The **Parsed WebIDL generator** takes the results of a crawl as input and applies the WebIDL parser to all specs it contains to create JSON extracts in an `idlparsed` folder. To run the generator: `node src/cli/generate-idlparsed.js [crawl folder] [save folder]`
-
-
-### WebIDL names generator
-
-The **WebIDL names generator** takes the results of a crawl as input and creates a report per referenceable IDL name, that details the complete parsed IDL structure that defines the name across all specs. To run the generator: `node src/cli/generate-idlnames.js [crawl folder] [save folder]`
-
 
 ### Crawl results merger
 

package/index.js

@@ -1,11 +1,9 @@
 module.exports = {
   parseIdl: require("./src/cli/parse-webidl").parse,
-  crawlSpecs: require("./src/lib/specs-crawler").crawlList,
+  crawlSpecs: require("./src/lib/specs-crawler").crawlSpecs,
   expandCrawlResult: require("./src/lib/util").expandCrawlResult,
   mergeCrawlResults: require("./src/lib/util").mergeCrawlResults,
   isLatestLevelThatPasses: require("./src/lib/util").isLatestLevelThatPasses,
-  generateIdlNames: require("./src/cli/generate-idlnames").generateIdlNames,
-  saveIdlNames: require("./src/cli/generate-idlnames").saveIdlNames,
-  generateIdlParsed: require("./src/cli/generate-idlparsed").generateIdlParsed,
-  saveIdlParsed: require("./src/cli/generate-idlparsed").saveIdlParsed
+  getInterfaceTreeInfo: require("./src/lib/util").getInterfaceTreeInfo,
+  postProcessor: require("./src/lib/post-processor")
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "reffy",
-  "version": "7.2.10",
+  "version": "8.0.2",
   "description": "W3C/WHATWG spec dependencies exploration companion. Features a short set of tools to study spec references as well as WebIDL term definitions and references found in W3C specifications.",
   "repository": {
     "type": "git",
@@ -32,7 +32,7 @@
   "bin": "./reffy.js",
   "dependencies": {
     "abortcontroller-polyfill": "1.7.3",
-    "commander": "9.3.0",
+    "commander": "9.4.0",
     "fetch-filecache-for-crawling": "4.1.0",
     "puppeteer": "15.4.0",
     "semver": "^7.3.5",
@@ -42,10 +42,10 @@
   "devDependencies": {
     "chai": "4.3.6",
     "mocha": "10.0.0",
-    "nock": "13.2.8",
+    "nock": "13.2.9",
     "respec": "32.1.10",
     "respec-hljs": "2.1.1",
-    "rollup": "2.76.0"
+    "rollup": "2.77.0"
   },
   "scripts": {
     "test": "mocha --recursive tests/"

package/reffy.js

@@ -27,6 +27,7 @@ const specs = require('web-specs');
 const { version, engines } = require('./package.json');
 const { requireFromWorkingDirectory } = require('./src/lib/util');
 const { crawlSpecs } = require('./src/lib/specs-crawler');
+const { modules } = require('./src/lib/post-processor');
 
 // Warn if version of Node.js does not satisfy requirements
 if (engines && engines.node && !satisfies(process.version, engines.node)) {
@@ -63,6 +64,15 @@ function parseSpecOption(input) {
     }
 }
 
+function parsePostOption(input) {
+    if (input === 'core') {
+      return modules;
+    }
+    else {
+      return input;
+    }
+}
+
 
 const program = new commander.Command();
 program
@@ -73,15 +83,17 @@ program
     .option('-f, --fallback <json>', 'fallback data to use when a spec crawl fails')
     .option('-m, --module <modules...>', 'spec processing modules')
     .option('-o, --output <folder>', 'existing folder/file where crawl results are to be saved')
+    .option('-p, --post <modules...>', 'post-processing modules')
     .option('-q, --quiet', 'do not report progress and other warnings to the console')
     .option('-r, --release', 'crawl release (TR) version of specs')
     .option('-s, --spec <specs...>', 'specs to crawl')
     .option('-t, --terse', 'output crawl results without metadata')
+    .option('-u, --use-crawl <folder>', 'use given crawl result folder as input for post-processing')
     .action(options => {
-        if (!(options.output || options.module || options.spec)) {
+        if (!(options.output || options.module || options.spec || options.useCrawl)) {
           console.error(`
-At least one of the --output, --module or --spec options needs to be specified.
-For usage notes, run:
+At least one of the --output, --module, --spec or --use-crawl options needs to be
+specified. For usage notes, run:
   reffy --help
 
 If you really want to crawl all specs, run all processing modules and report the
@@ -97,7 +109,8 @@ will dump ~100MB of data to the console:
             output: options.output,
             publishedVersion: options.release,
             quiet: options.quiet,
-            terse: options.terse
+            terse: options.terse,
+            useCrawl: options.useCrawl
         };
         if (options.module) {
             crawlOptions.modules = options.module.map(parseModuleOption);
@@ -105,6 +118,9 @@ will dump ~100MB of data to the console:
         if (options.spec) {
             crawlOptions.specs = options.spec.map(parseSpecOption).flat();
         }
+        if (options.post) {
+            crawlOptions.post = options.post.map(parsePostOption).flat();
+        }
 
         if (crawlOptions.terse && crawlOptions.output) {
             console.error('The --terse option cannot be combined with the --output option');
@@ -163,26 +179,22 @@ Usage notes for some of the options:
   Modules must be specified using a relative path to an ".mjs" file that defines
   the processing logic to run on the spec's page in a browser context. For
   instance:
-    $ reffy reports/test --module extract-editors.mjs
+    $ reffy --output reports/test --module extract-editors.mjs
 
   Absolute paths to modules are not properly handled and will likely result in a
   crawling error.
 
   Multiple modules can be specified, repeating the option name or not:
-    $ reffy reports/test -m extract-words.mjs extract-editors.mjs
-    $ reffy reports/test -m extract-words.mjs -m extract-editors.mjs
-
-  The option cannot appear before <folder>, unless you use "--" to flag the end
-  of the list:
-    $ reffy --module extract-editors.mjs -- reports/test
+    $ reffy -o reports/test -m extract-words.mjs extract-editors.mjs
+    $ reffy -o reports/test -m extract-words.mjs -m extract-editors.mjs
 
   Core processing modules may be referenced using the name of the extract folder
   or property that they would create:
-    $ reffy reports/test --module dfns
+    $ reffy --output reports/test --module dfns
 
   To run all core processing modules, use "core". For instance, to apply a
   processing module on top of core processing modules, use:
-    $ reffy reports/test --module core extract-editors.mjs
+    $ reffy --output reports/test --module core extract-editors.mjs
 
   Each module must export a function that takes a spec object as input and
   return a result that can be serialized as JSON. A typical module code looks
@@ -196,7 +208,7 @@ Usage notes for some of the options:
   The name of the folder where extracts get created may be specified for custom
   modules by prefixing the path to the module with the folder name followed by
   ":". For instance, to save extracts to "reports/test/editors", use:
-    $ reffy reports/test --module editors:extract-editors.mjs
+    $ reffy --output reports/test --module editors:extract-editors.mjs
 
 -o, --output <folder>
   By default, crawl results are written to the console as a serialized JSON
@@ -213,6 +225,33 @@ Usage notes for some of the options:
 
   The folder targeted by <folder> must exist.
 
+-p, --post <modules...>
+  Post-processing modules either run after a spec is done crawling or after the
+  entire crawl is over. They allow developers to complete data based on other
+  extracts that were not available when extraction ran.
+
+  To run all core post-processing modules, use "core". Core post-processing
+  modules are defined in:
+    https://github.com/w3c/reffy/blob/main/src/postprocessing.js
+
+  The crawler does not run any post-processing modules by default.
+
+  Custom post-processing modules may be specified using a relative path to a
+  ".js" file that defines the post-processing logic. For instance:
+    $ reffy --output reports/test --post mypostprocessing.js
+
+  Each module must export a "run" function. See the post-processor's code for
+  details:
+    https://github.com/w3c/reffy/blob/main/src/lib/post-processor.js
+
+  Absolute paths to modules are not properly handled and will likely result in a
+  processing error.
+
+  Multiple post-processing modules can be specified, repeating the option name
+  or not:
+    $ reffy -o reports/test -p cssdfns cssidl events
+    $ reffy -o reports/test -p events -p idlparsed -p idlnames
+
 -r, --release
   If the flag is not set, the crawler defaults to crawl nightly versions of the
   specs.
@@ -230,7 +269,7 @@ Usage notes for some of the options:
 
   Use "all" to include all specs in browser-specs in the crawl. For instance, to
   crawl all specs plus one custom spec that does not exist in browser-specs:
-    $ reffy reports/test -s all https://example.org/myspec
+    $ reffy -o reports/test -s all https://example.org/myspec
 
 -t, --terse
   This flag cannot be combined with the --output option and cannot be set if
@@ -243,6 +282,13 @@ Usage notes for some of the options:
   and the processing module results are thus written to the console directly.
   For instance:
     $ reffy --spec fetch --module idl --terse
+
+-u, --use-crawl <folder>
+  Tells Reffy to skip the crawl part and only run requested post-processing
+  modules on the crawl results present in the specified folder.
+
+  If post-processing modules are not specified, Reffy will merely copy the crawl
+  results to the output folder (or to the console).
 `);
 
 program.parse(process.argv);

package/src/browserlib/extract-cssdfn.mjs

@@ -257,10 +257,22 @@ const extractValueSpaces = doc => {
         // https://drafts.csswg.org/css-easing-2/#typedef-step-easing-function
         const prod = text.split(reSplitRules)
             .find(p => p.trim().startsWith(dfn.textContent.trim()));
-        if (!prod) {
-          throw new Error(`Production rule for ${dfn.textContent.trim()} found has unexpected format`);
+        if (prod) {
+          parseProductionRule(prod, { pureSyntax: true });
+        }
+        else {
+          // "=" may appear in another formula in the body of the text, as in:
+          // https://drafts.csswg.org/css-speech-1/#typedef-voice-volume-decibel
+          // It may be worth checking but not an error per se.
+          console.warn('[reffy]', `Found "=" next to definition of ${dfn.textContent.trim()} but no production rule. Did I miss something?`);
+          const name = (dfn.getAttribute('data-lt') ?? dfn.textContent)
+            .trim().replace(/^<?(.*?)>?$/, '<$1>');
+          if (!(name in res)) {
+            res[name] = {
+              prose: parent.textContent.trim().replace(/\s+/g, ' ')
+            };
+          }
         }
-        parseProductionRule(prod, { pureSyntax: true });
       }
       else if (dfn.textContent.trim().match(/^[a-zA-Z_][a-zA-Z0-9_\-]+\([^\)]+\)$/)) {
         // Definition is "prod(foo bar)", create a "prod() = prod(foo bar)" entry

package/src/browserlib/extract-events.mjs

@@ -102,12 +102,12 @@ export default function (spec) {
           if (el.tagName === "DFN" && el.id) {
             event.href = href(el);
           } else if (el.tagName === "A") {
-	    if (!el.getAttribute("href").startsWith("https://")) {
-	      const url = new URL(el.href);
-              event.href = href(document.getElementById(url.hash.slice(1)));
-	    } else {
-	      event.href = el.href;
-	    }
+            if (!el.getAttribute("href").startsWith("https://")) {
+              const url = new URL(el.href);
+                    event.href = href(document.getElementById(url.hash.slice(1)));
+            } else {
+              event.href = el.href;
+            }
           }
           event.src = { format: "summary table", href: href(el.closest('*[id]')) };
           event.type = eventEl.textContent.trim();
@@ -120,9 +120,9 @@ export default function (spec) {
               tr.querySelector(`td:nth-child(${interfaceColumn + 1}) a`)?.textContent ??
               tr.querySelector(`td:nth-child(${interfaceColumn + 1}) code`)?.textContent;
           }
-	  if (targetsColumn >= 0 && !event.targets) {
-	    event.targets = tr.querySelector(`td:nth-child(${targetsColumn + 1})`)?.textContent?.split(',').map(t => t.trim());
-	  }
+          if (targetsColumn >= 0 && !event.targets) {
+            event.targets = tr.querySelector(`td:nth-child(${targetsColumn + 1})`)?.textContent?.split(',').map(t => t.trim());
+          }
           events.push(event);
           eventEl.replaceWith(origEventEl);
         });
@@ -205,8 +205,8 @@ export default function (spec) {
         } else {
           event.type = name;
           // looking at the element following the link
-	  // if its content match the name of the event
-	  const eventEl = a.nextElementSibling?.textContent?.trim() === event.type ? a.nextElementSibling.querySelector("a,dfn") || a.nextElementSibling : null;
+          // if its content match the name of the event
+          const eventEl = a.nextElementSibling?.textContent?.trim() === event.type ? a.nextElementSibling.querySelector("a,dfn") || a.nextElementSibling : null;
           if (eventEl) {
             if (eventEl.tagName === "A" && eventEl.getAttribute("href")) {
               // use the target of the link as our href
@@ -227,7 +227,7 @@ export default function (spec) {
           while ((curEl = curEl.nextElementSibling)) {
             if (curEl.textContent.match(/^([A-Z]+[a-z0-9]*)+Event$/)) {
               iface = curEl.textContent.trim();
-	      break;
+              break;
             }
           }
           if (iface) {
@@ -322,20 +322,20 @@ export default function (spec) {
       // of the section where the definitions are located
       let currentEl = container.parentNode;
       while(currentEl) {
-	if (currentEl.tagName.match(/^H[1-6]$/)) {
-	  break;
-	}
-	currentEl = currentEl.previousElementSibling;
+        if (currentEl.tagName.match(/^H[1-6]$/)) {
+          break;
+        }
+        currentEl = currentEl.previousElementSibling;
       }
       const interfaceEl = currentEl?.querySelector("code");
       if (interfaceEl?.textContent?.match(/^[A-Z][a-z]+Event$/)) {
-	iface = interfaceEl.textContent;
+        iface = interfaceEl.textContent;
       }
     }
     const ev = events.find(e => isSameEvent(event, e));
     if (!ev) {
       if (iface) {
-	event.interface = iface;
+        event.interface = iface;
       }
       event.bubbles = bubbles;
       events.push(event);
@@ -347,12 +347,12 @@ export default function (spec) {
         ev.interface = iface;
       }
       if (!ev.href && event.href) {
-	ev.href = event.href;
+        ev.href = event.href;
       }
       if (bubbles !== undefined) {
         ev.bubbles = bubbles;
       }
     }
   });
-  return events.map(e => e.href && !e.href.startsWith(spec.crawled.url) ? Object.assign(e, {isExtension: true}) : e) ;
+  return events.map(e => e.href && !e.href.startsWith(window.location.toString()) ? Object.assign(e, {isExtension: true}) : e) ;
 }

package/src/browserlib/reffy.json

@@ -24,7 +24,7 @@
   },
   {
     "href": "./extract-events.mjs",
-    "property": "spec-events"
+    "property": "events"
   },
   {
     "href": "./extract-webidl.mjs",

package/src/lib/post-processor.js

@@ -0,0 +1,269 @@
+#!/usr/bin/env node
+/**
+ * The post-processor runs post-processing modules against crawl results.
+ *
+ * There are two types of post-processing modules:
+ * 1. Modules that run against the result of crawling an individual spec. Such
+ * modules take the spec crawl result as input and typically update it in place
+ * 2. Modules that run against an entire crawl result. Such modules take the
+ * entire crawl result as input and return whatever structure they would like
+ * to return.
+ *
+ * The post-processor exposes two main functions:
+ * - run() to run a post-processing module against crawl results or against a
+ * spec crawl result (depending on the module)
+ * - save() to save processing results to files
+ * 
+ * A post-processing module needs to expose the following properties and
+ * functions:
+ * - dependsOn: list of crawl result info that the module depends on. Values
+ * include "css", "dfns", "idl", as well as info that other post-processing
+ * modules may generate such as "idlparsed".
+ * - input: either "crawl" or "spec". Default is "spec". Tells whether the
+ * module operates on a spec crawl result or on the entire crawl result
+ * - property: When "input" is "spec", gives the name of the property that
+ * will be set in the spec crawl result when the post-processing module runs
+ * and of the folder that will contain the spec extracts (unless module has its
+ * "save" logic). For modules that run at the crawl level, gives the name of
+ * the final extract file that gets created (unless module has its own "save"
+ * logic).
+ * - run: Async function to call to apply the post-processing module. The
+ * function is called with either a spec crawl result of the entire crawl result
+ * depending on "input". Second parameter is the crawl options object. The
+ * function should return the created structure when "input" is "crawl" and
+ * the updated spec crawl result when "input" is "spec". Note the function
+ * may update the spec crawl result in place.
+ * - save: Function to call to save the results of the post-processing module.
+ * The function is called with the returned result of running the
+ * post-processing module. Second parameter is the crawl options object. The
+ * function is only needed if "save" needs to do specific things that the
+ * post-processor cannot do on its own. Function must return the relative path
+ * to the file that was saved
+ * - extractsPerSeries: A boolean flag that tells the crawler that it should
+ * clean up extract afterwards to produce extracts per series instead of
+ * extracts per spec. The flag is only meaningful if module runs at the spec
+ * level and if "property" is set.
+ *
+ * @module
+ */
+
+const fs = require('fs');
+const path = require('path');
+const { createFolderIfNeeded, requireFromWorkingDirectory } = require('./util');
+
+
+/**
+ * Core post-processing modules
+ */
+const modules = {
+  csscomplete: require('../postprocessing/csscomplete'),
+  events: require('../postprocessing/events'),
+  idlnames: require('../postprocessing/idlnames'),
+  idlparsed: require('../postprocessing/idlparsed')
+};
+
+
+/**
+ * Returns the post-processing module that match the requested name, or the
+ * given parameter if it is a post-processing module already
+ *
+ * @function
+ * @param {String|Object} mod Module name of known post-processing module, or
+ *   actual post-processing module.
+ * @return {Object} Post-processing module
+ */
+function getModule(mod) {
+  if (typeof mod === 'string') {
+    if (modules[mod]) {
+      return Object.assign({ name: mod }, modules[mod]);
+    }
+    else {
+      const fmod = requireFromWorkingDirectory(mod);
+      if (!fmod) {
+        throw new Error(`Unknown post-processing module "${mod}"`);
+      }
+      if (!isModuleValid(fmod)) {
+        throw new Error(`"${mod}" is not a valid post-processing module`);
+      }
+      return Object.assign({ name: mod }, fmod);
+    }
+  }
+  else if (!isModuleValid(mod)) {
+    throw new Error(`Post-processing module given as parameter does not have a "run" function`);
+  }
+  return mod;
+}
+
+
+/**
+ * Returns true if given module object looks like a valid module, false
+ * otherwise.
+ *
+ * @function
+ * @param {Object} mod Post-processing module object
+ * @return {boolean} True when module looks valid, false otherwise
+ */
+function isModuleValid(mod) {
+  return !!mod && mod.run && (typeof mod.run === 'function');
+}
+
+
+/**
+ * Run a post-processing module against some crawl result
+ *
+ * @function
+ * @param {String|Object} mod Module name for known module or the actual
+ * module implementation.
+ * @param {Object} crawlResult The entire crawl results if module runs at the
+ * "crawl" input level, the result of crawling a spec if module runs at the
+ * "spec" input level.
+ * @param {Object} options Crawl options. See spec crawler for details.
+ * @return {Object} Post-processing structure
+ */
+async function run(mod, crawlResult, options) {
+  mod = getModule(mod);
+
+  if (mod.input === 'crawl') {
+    if (crawlResult.crawled) {
+      // Post-processing module runs at the crawl level and we received
+      // a spec crawl result
+      return;
+    }
+
+    // TODO: make sure that there is at least one spec for which properties
+    // listed in "dependsOn" are set. If not, the module cannot run, which
+    // typically signals that the crawler was called with incompatible settings.
+  }
+  else {
+    if (!crawlResult.crawled) {
+      // Post-processing module runs at the spec level and we received
+      // a full crawl result
+      return;
+    }
+
+    // TODO: check properties listed in "dependsOn". If none is set, no need to
+    // run the module (but not an error per se, it may just be that this
+    // particular spec does not define relevant info)
+  }
+
+  return await mod.run(crawlResult, options); 
+}
+
+
+/**
+ * Save post-processing results
+ *
+ * @function
+ * @param {String|Object} mod Module name for known module or the actual
+ * module implementation.
+ * @param {Object} processResult The post-processing results
+ * @param {Object} options Crawl options. See spec crawler for details.
+ * @return {String} Relative path to the file created
+ */
+async function save(mod, processResult, options) {
+  mod = getModule(mod);
+  processResult = processResult || {};
+  options = options || {};
+
+  if (mod.input === 'crawl') {
+    if (processResult.shortname) {
+      // Post-processing module runs at the crawl level and we received
+      // a spec crawl result
+      return;
+    }
+  }
+  else {
+    if (!processResult.shortname) {
+      // Post-processing module runs at the spec level and we received
+      // a full crawl result
+      return;
+    }
+  }
+
+  if (!options.output) {
+    // Nothing to do if no output folder was given
+    return;
+  }
+
+  if (mod.save) {
+    // For post-processing modules that have some save logic, we'll just let
+    // them do whatever they want
+    return mod.save(processResult, options);
+  }
+  else if (!mod.property) {
+    // For post-processing modules that don't touch any single property, default
+    // save operation is to do nothing.
+    return;
+  }
+  else if (mod.input === 'crawl') {
+    // For post-processing modules that apply at the crawl level, default save
+    // operation is to create a JSON file in the output folder named after the
+    // post-processing module
+    const filename = path.join(options.output, `${mod.property}.json`);
+    await createFolderIfNeeded(options.output);
+    await fs.promises.writeFile(filename, JSON.stringify(processResult, null, 2), 'utf8');
+    return `${mod.property}.json`;
+  }
+  else {
+    // For post-processing modules that apply at the spec level, default save
+    // operation is to create a JSON extract file named after the spec's
+    // shortname under a subfolder named after the post-processing module in the
+    // output folder. Contents of the extract are the contents of the property
+    // that has the same name as the module (or the name of the module's
+    // "property" parameter if defined) in the post-processing result.
+    if (!processResult[mod.property]) {
+      return;
+    }
+    const folder = path.join(options.output, mod.property);
+    const filename = path.join(folder, `${processResult.shortname}.json`);
+    const contents = {
+      spec: {
+        title: processResult.title,
+        url: processResult.crawled
+      }
+    };
+    contents[mod.property] = processResult[mod.property];
+    await createFolderIfNeeded(folder);
+    await fs.promises.writeFile(filename, JSON.stringify(contents, null, 2), 'utf8');
+    processResult[mod.property] = `${mod.property}/${processResult.shortname}.json`;
+    return processResult[mod.property];
+  }
+}
+
+
+/**
+ * Return true if post-processing module generates extracts per spec series
+ */
+function extractsPerSeries(mod) {
+  mod = getModule(mod);
+  return (mod.input !== 'crawl') && !!mod.property && !!mod.extractsPerSeries;
+}
+
+
+/**
+ * Return true if post-processing module generates extracts per spec series
+ */
+function dependsOn(mod) {
+  mod = getModule(mod);
+  return mod.dependsOn;
+}
+
+
+function appliesAtLevel(mod, level) {
+  mod = getModule(mod);
+  const crawlLevel = mod.input === 'crawl';
+  return level === 'crawl' ? crawlLevel : !crawlLevel;
+}
+
+
+
+/**************************************************
+Export post-processing functions
+**************************************************/
+module.exports = {
+  modules: Object.keys(modules),
+  run, save,
+  extractsPerSeries,
+  dependsOn,
+  appliesAtLevel
+};