reffy 4.0.1 → 4.0.5

package/README.md

@@ -11,14 +11,14 @@ The code also currently includes a set of individual tools to study extracts and
 
 ### Pre-requisites
 
-To install Reffy, you need [Node.js](https://nodejs.org/en/).
+To install Reffy, you need [Node.js](https://nodejs.org/en/) 14 or greater.
 
 ### Installation
 
 Reffy is available as an NPM package. To install the package globally, run:
 
 ```bash
-npm install -g reffy`
+npm install -g reffy
 ```
 
 This will install Reffy as a command-line interface tool.
@@ -52,7 +52,7 @@ The crawler can be fully parameterized to crawl a specific list of specs and run
   ```
 - To retrieve the list of specs that the HTML spec references, run (noting that crawling the HTML spec takes some time due to it being a multipage spec):
   ```bash
-  reffy --spec html --module refs`
+  reffy --spec html --module refs
   ```
 - To extract the list of CSS properties defined in CSS Flexible Box Layout Module Level 1, run:
   ```bash

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "reffy",
-  "version": "4.0.1",
+  "version": "4.0.5",
   "description": "W3C/WHATWG spec dependencies exploration companion. Features a short set of tools to study spec references as well as WebIDL term definitions and references found in W3C specifications.",
   "repository": {
     "type": "git",
@@ -32,20 +32,21 @@
   "bin": "./reffy.js",
   "dependencies": {
     "abortcontroller-polyfill": "1.7.3",
-    "browser-specs": "2.14.0",
+    "browser-specs": "2.15.1",
     "commander": "8.2.0",
     "fetch-filecache-for-crawling": "4.0.2",
     "node-pandoc": "0.3.0",
     "puppeteer": "10.4.0",
+    "semver": "^7.3.5",
     "webidl2": "24.1.2"
   },
   "devDependencies": {
     "chai": "4.3.4",
     "mocha": "9.1.2",
     "nock": "13.1.3",
-    "respec": "26.13.4",
+    "respec": "26.14.0",
     "respec-hljs": "2.1.1",
-    "rollup": "2.57.0"
+    "rollup": "2.58.0"
   },
   "scripts": {
     "all": "node src/cli/crawl-and-study.js run ed all && node src/cli/crawl-and-study.js run tr all",

package/reffy.js

@@ -22,11 +22,19 @@
  */
 
 const commander = require('commander');
-const version = require('./package.json').version;
+const satisfies = require('semver/functions/satisfies');
 const specs = require('browser-specs');
+const { version, engines } = require('./package.json');
 const { requireFromWorkingDirectory } = require('./src/lib/util');
 const { crawlSpecs } = require('./src/lib/specs-crawler');
 
+// Warn if version of Node.js does not satisfy requirements
+if (engines && engines.node && !satisfies(process.version, engines.node)) {
+  console.warn(`
+[WARNING] Node.js ${process.version} detected but Reffy needs Node.js ${engines.node}.
+          Please consider upgrading Node.js if the program crashes!`);
+}
+
 
 function parseModuleOption(input) {
     const parts = input.split(':');
@@ -55,6 +63,7 @@ function parseSpecOption(input) {
     }
 }
 
+
 const program = new commander.Command();
 program
     .version(version)
@@ -137,7 +146,7 @@ Usage notes for some of the options:
 -m, --module <modules...>
   If processing modules are not specified, the crawler runs all core processing
   modules defined in:
-    https://github.com/w3c/reffy/tree/main/src/reffy.json
+    https://github.com/w3c/reffy/blob/main/src/browserlib/reffy.json
 
   Modules must be specified using a relative path to an ".mjs" file that defines
   the processing logic to run on the spec's page in a browser context. For
@@ -202,7 +211,10 @@ Usage notes for some of the options:
 
   Valid spec values may be a shortname, a URL, or a relative path to a file that
   contains a list of spec URLs and/or shortnames. All shortnames must exist in
-  browser-specs.
+  browser-specs. Shortname may be the shortname of the spec series, in which
+  case the spec identified as the current specification in the series is used.
+  For instance, as of September 2021, "pointerlock" will map to "pointerlock-2"
+  because Pointer Lock 2.0 is the current level in the series.
 
   Use "all" to include all specs in browser-specs in the crawl. For instance, to
   crawl all specs plus one custom spec that does not exist in browser-specs:

package/src/cli/study-crawl.js

@@ -119,8 +119,9 @@ function studyCrawlResults(results, options = {}) {
             isLatestLevelThatPasses(spec, results, s =>
                 s.idl && s.idl.idlNames && s.idl.idlNames[name])));
 
-    // TODO: we may end up with different variants of the WebIDL spec
-    var WebIDLSpec = results.find(spec => (spec.shortname === 'WebIDL-1')) || {};
+    // WebIDL-1 only kept for historical reasons to process old crawl results
+    var WebIDLSpec = results.find(spec =>
+        spec.shortname === 'webidl' || spec.shortname === 'WebIDL-1') || {};
 
     var sortedResults = results.sort(byTitle);
 

package/src/lib/fetch.js

@@ -5,6 +5,7 @@
  * @module finder
  */
 
+const os = require('os');
 const path = require('path');
 const baseFetch = require('fetch-filecache-for-crawling');
 
@@ -43,6 +44,11 @@ async function fetch(url, options) {
         options.refresh = 'once';
     }
 
+    // Use cache folder in tmp folder by default
+    if (!options.cacheFolder) {
+        options.cacheFolder = path.resolve(os.tmpdir(), 'reffy-cache');
+    }
+
     return baseFetch(url, options);
 }
 

package/src/lib/nock-server.js

@@ -36,7 +36,8 @@ const mockSpecs = {
     }
   },
   "/mediacapture-output/": `<script>respecConfig = { shortName: 'test' };</script><script src='https://www.w3.org/Tools/respec/respec-w3c'></script><div id=abstract></div><pre class='idl'>[Exposed=Window] interface Foo { attribute DOMString bar; };</pre>`,
-  "/accelerometer/": `<html><h2>Normative references</h2><dl><dt>FOO</dt><dd><a href='https://www.w3.org/TR/Foo'>Foo</a></dd></dl>`
+  "/accelerometer/": `<html><h2>Normative references</h2><dl><dt>FOO</dt><dd><a href='https://www.w3.org/TR/Foo'>Foo</a></dd></dl>`,
+  "/pointerlock/": `<html><h1>Pointer Lock 2.0`
 };
 
 nock.disableNetConnect();
@@ -68,7 +69,7 @@ nock("https://respec.org")
 
 nock("https://api.specref.org")
   .persist()
-  .get("/bibrefs?refs=webidl,html").reply(200, {webidl:{href:"https://heycam.github.io/webidl/"}}, {"Access-Control-Allow-Origin": "*"})
+  .get("/bibrefs?refs=webidl,html").reply(200, {webidl:{href:"https://webidl.spec.whatwg.org/"}}, {"Access-Control-Allow-Origin": "*"})
   .get("/bibrefs?refs=HTML").reply(200, {HTML:{href:"https://html.spec.whatwg.org/multipage/"}}, {"Access-Control-Allow-Origin": "*"});
 
 nock("https://www.w3.org")

package/src/lib/specs-crawler.js

@@ -482,7 +482,12 @@ function crawlSpecs(options) {
             if (typeof spec !== 'string') {
                 return spec;
             }
-            const match = specs.find(s => s.url === spec || s.shortname === spec);
+            let match = specs.find(s => s.url === spec || s.shortname === spec);
+            if (!match) {
+                match = specs.find(s => s.series &&
+                    s.series.shortname === spec &&
+                    s.series.currentSpecification === s.shortname);
+            }
             if (match) {
                 return match;
             }
@@ -496,7 +501,8 @@ function crawlSpecs(options) {
                     url = (new URL(spec, `file://${process.cwd()}/`)).href;
                 }
                 else {
-                    throw new Error(`Spec ID "${spec}" can neither be interpreted as a URL, a valid shortname or a relative path to an HTML file`);
+                    const msg = `Spec ID "${spec}" can neither be interpreted as a URL, a valid shortname or a relative path to an HTML file`;
+                    throw new Error(msg);
                 }
             }
             return {

package/src/lib/util.js

@@ -14,6 +14,21 @@ const specEquivalents = require('../specs/spec-equivalents.json');
 const reffyModules = require('../browserlib/reffy.json');
 
 
+/**
+ * Maximum depth difference supported between Reffy's install path and custom
+ * modules that may be provided on the command-line
+ *
+ * TODO: Find a way to get right of that, there should be no limit
+ */
+const maxPathDepth = 20;
+
+
+/**
+ * Returns a range array from 0 to the number provided (not included)
+ */
+const range = n => Array.from(Array(n).keys());
+
+
 /**
  * Shortcut that returns a property extractor iterator
  */
@@ -356,7 +371,7 @@ async function processSpecification(spec, processFunction, args, options) {
                         let depth = requestPath.lastIndexOf('__/') / 3;
                         const filename = requestPath.substring(requestPath.lastIndexOf('__/') + 3);
                         let filePath = path.resolve(__dirname, '..', 'browserlib');
-                        while (depth < 7) {
+                        while (depth < maxPathDepth - 1) {
                             filePath = path.resolve(filePath, '..');
                             depth += 1;
                         }
@@ -547,7 +562,7 @@ async function processSpecification(spec, processFunction, args, options) {
         // "../../node_modules/[...]" and may import other scripts that are
         // higher in the folder tree.
         await page.addScriptTag({
-            url: 'reffy/scripts/__/__/__/__/__/__/__/__/reffy.mjs',
+            url: `reffy/scripts/${range(maxPathDepth).map(n => '__').join('/')}/reffy.mjs`,
             type: 'module'
         });