reffy 3.1.0 → 4.0.3

package/src/cli/crawl-and-study.js

@@ -0,0 +1,212 @@
+#!/usr/bin/env node
+/**
+ * Reffy's command line interface that you can use to crawl and study spec
+ * references. The tool runs the crawler, then the study tools to create the
+ * full reports that typically show up under w3c/webref.
+ *
+ * Tool can be called directly through:
+ *
+ * `node crawl-and-study.js [command]`
+ *
+ * Run `node crawl-and-study.js -h` for help
+ *
+ * @module crawler
+ */
+
+const program = require('commander');
+const version = require('../../package.json').version;
+const fs = require('fs');
+const path = require('path');
+const crawlSpecs = require('../lib/specs-crawler').crawlSpecs;
+const studyCrawl = require('./study-crawl').studyCrawl;
+const generateReport = require('./generate-report').generateReport;
+const pandoc = require('node-pandoc');
+
+
+// List of possible perspectives and associated parameters
+// Note the "ed" perspective produces reports under "whatwg" for backward
+// compatibility reason.
+const perspectives = {
+  'ed': {
+    description: 'Crawls the latest Editor\'s Drafts',
+    refStudy: 'https://w3c.github.io/webref/ed/study.json'
+  },
+  'tr': {
+    description: 'Crawls the latest published versions of specifications in /TR/ space instead of the latest Editor\'s Drafts',
+    publishedVersion: true,
+    refStudy: 'https://w3c.github.io/webref/tr/study.json'
+  }
+};
+
+// List of possible actions for each perspective
+const possibleActions = {
+  'all': 'crawl specs, study report and generate markdown, HTML and diff reports. Default action',
+  'crawl': 'crawl specs and generate a machine-readable report with facts about each spec',
+  'study': 'parse the machine-readable report generated by the crawler, and create a study report of potential anomalies found in the report',
+  'markdown': 'produce a human-readable report in Markdown format out of the anomalies report returned by the study action',
+  'html': 'produce an HTML report out of the Markdown report generated by the markdown action',
+  'diff': 'compare the anomalies report with the latest published anomalies report and generate diff report',
+  'diffnew': 'compare the anomalies report with the latest published anomalies report and generate diff report that only contains new anomalies'
+};
+
+let command = null; 
+program
+  .version(version)
+  .option('-d, --debug', 'run crawl in debug mode (single process, one spec at a time)');
+
+program
+  .command('run <perspective> [action]')
+  .description('run a new crawl and study from the given perspective')
+  .option('-d, --debug', 'run crawl in debug mode (single process, one spec at a time)')
+  .action(async (perspective, action, cmdObj) => {
+    command = 'run';
+    if (!(perspective in perspectives)) {
+      return program.help();
+    }
+    if (action && !(action in possibleActions)) {
+      return program.help();
+    }
+
+    let debug = cmdObj.debug || program.debug;
+    let publishedVersion = perspectives[perspective].publishedVersion;
+    let refStudy = perspectives[perspective].refStudy;
+    let reportFolder = perspectives[perspective].reportFolder ||
+      'reports/' + perspective;
+    let crawlReport = path.join(reportFolder, 'index.json');
+    let studyReport = path.join(reportFolder, 'study.json');
+
+    let promise = Promise.resolve();
+    let actions = (!action || (action === 'all')) ?
+      ['crawl', 'study', 'markdown', 'html', 'diff', 'diffnew'] :
+      [action];
+
+    actions.forEach(action => {
+      switch (action) {
+      case 'crawl':
+        promise = promise
+          .then(_ => crawlSpecs(
+            { publishedVersion, debug, output: reportFolder }));
+        break;
+
+      case 'study':
+        const options = {};
+        if (perspective === 'ed') {
+          const trFolder = perspectives.tr.reportFolder || 'reports/tr';
+          const trReport = path.join(trFolder, 'index.json');
+          if (fs.existsSync(trReport)) {
+            options.trResults = trReport;
+          }
+        }
+        promise = promise
+          .then(_ => studyCrawl(crawlReport, options))
+          .then(results => {
+            fs.writeFileSync(path.join(reportFolder, 'study.json'),
+              JSON.stringify(results, null, 2));
+          });
+        break;
+
+      case 'markdown':
+        promise = promise
+          .then(_ => generateReport(studyReport, { perSpec: true }))
+          .then(report => fs.writeFileSync(path.join(reportFolder, 'index.md'), report))
+          .then(_ => generateReport(studyReport, { perSpec: false }))
+          .then(report => fs.writeFileSync(path.join(reportFolder, 'perissue.md'), report));
+        break;
+
+      case 'html':
+        promise = promise
+          .then(_ => new Promise((resolve, reject) => {
+            let args = [
+              '-f', 'markdown', '-t', 'html5', '--section-divs', '-s',
+              '--template', path.join(__dirname, '..', 'templates', 'report-template.html'),
+              '-o', path.join(reportFolder, 'index.html')
+            ];
+            pandoc(path.join(reportFolder, 'index.md'), args,
+              (err => {
+                if (err) {
+                  return reject(err);
+                }
+                args = [
+                  '-f', 'markdown', '-t', 'html5', '--section-divs', '-s',
+                  '--template', path.join(__dirname, '..', 'templates', 'report-perissue-template.html'),
+                  '-o', path.join(reportFolder, 'perissue.html')];
+                pandoc(path.join(reportFolder, 'perissue.md'), args,
+                  (err => {
+                    if (err) {
+                      return reject(err);
+                    }
+                    return resolve();
+                  }));
+              }));
+            }));
+        break;
+
+      case 'diff':
+        promise = promise
+          .then(_ => generateReport(studyReport, {
+            diffReport: true,
+            refStudyFile: refStudy
+          }))
+          .then(report => fs.writeFileSync(path.join(reportFolder, 'diff.md'), report));
+        break;
+
+      case 'diffnew':
+        promise = promise
+          .then(_ => generateReport(studyReport, {
+            diffReport: true,
+            refStudyFile: refStudy,
+            onlyNew: true
+          }))
+          .then(report => fs.writeFileSync(path.join(reportFolder, 'diffnew.md'), report));
+        break;
+      }
+    });
+
+    return promise;
+  });
+
+program.on('--help', function() {
+  console.log('');
+  console.log('  Possible perspectives:');
+  console.log('');
+  Object.keys(perspectives).forEach(perspective => {
+    console.log('    ' + perspective + ': ' + perspectives[perspective].description);
+  });
+  console.log('');
+
+  console.log('  Possible actions:');
+  console.log('');
+  Object.keys(possibleActions).forEach(action => {
+    console.log('    ' + action + ': ' + possibleActions[action]);
+  });
+  console.log('');
+
+  console.log('  Possible options:');
+  console.log('');
+  console.log('    -d, --debug: run crawl in debug mode (single process, one spec at a time)');
+  console.log('');
+});
+
+program.on('command:*', function () {
+  console.error('Invalid command: %s.\n', program.args.join(' '));
+  program.outputHelp();
+  process.exit(1);
+});
+
+if (!process.argv.slice(2).length) {
+  console.error('Cannot run program without arguments.\n');
+  program.outputHelp();
+  process.exit(1);
+}
+
+program
+  .parseAsync(process.argv)
+  .then(_ => {
+    console.log('-- THE END -- ');
+    process.exit(0);
+  })
+  .catch(err => {
+    console.error('-- ERROR CAUGHT --');
+    console.error(err);
+    process.exit(1);
+  });

package/src/cli/generate-idlnames.js

@@ -9,7 +9,7 @@
  * `node generate-idlnames.js [crawl report] [dfns] [save folder]`
  *
  * where `crawl report` is the path to the folder that contains the
- * `index.json` file and all other crawl results produced by crawl-specs.js,
+ * `index.json` file and all other crawl results produced by specs-crawler.js,
  * `dfns` a param to set to "true" or "dfns" to embed dfns in the generated
  * report, and `save folder` is an optional folder (which must exist) where IDL
  * name extracts are to be saved. In the absence of this parameter, the report
@@ -146,6 +146,7 @@ function generateIdlNames(results, options = {}) {
       }
       names[name] = {
         name: name,
+        type: idl.type,
         defined: desc,
         extended: [],
         inheritance: idl.inheritance,
@@ -437,4 +438,4 @@ if (require.main === module) {
         console.log(JSON.stringify(report, null, 2));
       }
     });
-}
+}

package/src/lib/fetch.js

@@ -5,6 +5,7 @@
  * @module finder
  */
 
+const os = require('os');
 const path = require('path');
 const baseFetch = require('fetch-filecache-for-crawling');
 
@@ -43,6 +44,11 @@ async function fetch(url, options) {
         options.refresh = 'once';
     }
 
+    // Use cache folder in tmp folder by default
+    if (!options.cacheFolder) {
+        options.cacheFolder = path.resolve(os.tmpdir(), 'reffy-cache');
+    }
+
     return baseFetch(url, options);
 }
 

package/src/lib/nock-server.js

@@ -36,7 +36,8 @@ const mockSpecs = {
     }
   },
   "/mediacapture-output/": `<script>respecConfig = { shortName: 'test' };</script><script src='https://www.w3.org/Tools/respec/respec-w3c'></script><div id=abstract></div><pre class='idl'>[Exposed=Window] interface Foo { attribute DOMString bar; };</pre>`,
-  "/accelerometer/": `<html><h2>Normative references</h2><dl><dt>FOO</dt><dd><a href='https://www.w3.org/TR/Foo'>Foo</a></dd></dl>`
+  "/accelerometer/": `<html><h2>Normative references</h2><dl><dt>FOO</dt><dd><a href='https://www.w3.org/TR/Foo'>Foo</a></dd></dl>`,
+  "/pointerlock/": `<html><h1>Pointer Lock 2.0`
 };
 
 nock.disableNetConnect();

package/src/{cli/crawl-specs.js → lib/specs-crawler.js}

@@ -7,42 +7,25 @@
  * reference, and links to external specs), and produces a crawl report with the
  * results of these investigations.
  *
- * The spec crawler can be called directly through:
- *
- * `node crawl-specs.js [options]`
- *
- * Use `--help` option for usage instructions.
- *
- * The JSON file that contains the list of specs to crawl must be an array whose
- * individual items are either:
- * 1. a string that gets interpreted as the URL or the shortname of the spec to
- * crawl. The spec must exist in w3c/browser-specs
- * 2. an object that follows the w3c/browser-specs model:
- * https://github.com/w3c/browser-specs#spec-object
- *
  * @module crawler
  */
 
-const commander = require('commander');
-const version = require('../../package.json').version;
 const fs = require('fs');
 const path = require('path');
 const specs = require('browser-specs');
-const webidlParser = require('./parse-webidl');
-const cssDfnParser = require('../lib/css-grammar-parser');
-const { generateIdlNames, saveIdlNames } = require('./generate-idlnames');
+const webidlParser = require('../cli/parse-webidl');
+const cssDfnParser = require('./css-grammar-parser');
+const { generateIdlNames, saveIdlNames } = require('../cli/generate-idlnames');
 const {
     completeWithAlternativeUrls,
-    fetch,
     expandBrowserModules,
     expandCrawlResult,
     getGeneratedIDLNamesByCSSProperty,
     isLatestLevelThatPasses,
     processSpecification,
-    requireFromWorkingDirectory,
     setupBrowser,
     teardownBrowser
-} = require('../lib/util');
+} = require('./util');
 
 
 /**
@@ -189,9 +172,9 @@ async function crawlSpec(spec, crawlOptions) {
  *   metadata about the spec and the crawl processing results in appropriate
  *   properties.
  * @param {Object} settings Crawl settings. Recognized settings: "modules",
- *   "output" and "quiet". See CLI help (node crawl-specs.js --help) for
- *   details. The "modules" setting is mandatory and note that the function
- *   will not do anything if "output" is not set.
+ *   "output" and "quiet". See CLI help (node reffy.js --help) for details.
+ *   The "modules" setting is mandatory and note that the function will not do
+ *   anything if "output" is not set.
  * @return {Promise<Object>} The promise to get an updated spec object that
  *   contains links to created extracts.
  */
@@ -490,7 +473,7 @@ async function saveResults(data, settings) {
  * @function
  * @param {Object} options Crawl options. Possible options are:
  *   publishedVersion, debug, output, terse, modules and specs.
- *   See CLI help (node crawl-specs.js --help) for details.
+ *   See CLI help (node reffy.js --help) for details.
  * @return {Promise<void>} The promise that the crawl will have been made
  */
 function crawlSpecs(options) {
@@ -499,7 +482,12 @@ function crawlSpecs(options) {
             if (typeof spec !== 'string') {
                 return spec;
             }
-            const match = specs.find(s => s.url === spec || s.shortname === spec);
+            let match = specs.find(s => s.url === spec || s.shortname === spec);
+            if (!match) {
+                match = specs.find(s => s.series &&
+                    s.series.shortname === spec &&
+                    s.series.currentSpecification === s.shortname);
+            }
             if (match) {
                 return match;
             }
@@ -513,7 +501,8 @@ function crawlSpecs(options) {
                     url = (new URL(spec, `file://${process.cwd()}/`)).href;
                 }
                 else {
-                    throw new Error(`Spec ID "${spec}" can neither be interpreted as a URL, a valid shortname or a relative path to an HTML file`);
+                    const msg = `Spec ID "${spec}" can neither be interpreted as a URL, a valid shortname or a relative path to an HTML file`;
+                    throw new Error(msg);
                 }
             }
             return {
@@ -584,188 +573,3 @@ Export methods for use as module
 **************************************************/
 module.exports.crawlList = crawlList;
 module.exports.crawlSpecs = crawlSpecs;
-
-
-/**************************************************
-Code run if the code is run as a stand-alone module
-**************************************************/
-if (require.main === module) {
-
-    function parseModuleOption(input) {
-        const parts = input.split(':');
-        if (parts.length > 2) {
-            console.error('Module input cannot have more than one ":" character');
-            process.exit(2);
-        }
-        if (parts.length === 2) {
-            return {
-                href: parts[1],
-                property: parts[0]
-            };
-        }
-        else {
-            return parts[0];
-        }
-    }
-
-    function parseSpecOption(input) {
-        if (input === 'all') {
-            return specs.map(s => s.shortname);
-        }
-        else {
-            const list = requireFromWorkingDirectory(input);
-            return list ?? input;
-        }
-    }
-
-    const program = new commander.Command();
-    program
-        .version(version)
-        .usage('[options]')
-        .description('Crawls and processes a list of Web specifications')
-        .option('-d, --debug', 'debug mode, crawl one spec at a time')
-        .option('-m, --module <modules...>', 'spec processing modules')
-        .option('-o, --output <folder>', 'existing folder/file where crawl results are to be saved')
-        .option('-q, --quiet', 'do not report progress and other warnings to the console')
-        .option('-r, --release', 'crawl release (TR) version of specs')
-        .option('-s, --spec <specs...>', 'specs to crawl')
-        .option('-t, --terse', 'output crawl results without metadata')
-        .action(options => {
-            const crawlOptions = {
-                debug: options.debug,
-                output: options.output,
-                publishedVersion: options.release,
-                quiet: options.quiet,
-                terse: options.terse
-            };
-            if (options.module) {
-                crawlOptions.modules = options.module.map(parseModuleOption);
-            }
-            if (options.spec) {
-                crawlOptions.specs = options.spec.map(parseSpecOption).flat();
-            }
-
-            if (crawlOptions.terse && crawlOptions.output) {
-                console.error('The --terse option cannot be combined with the --output option');
-                process.exit(2);
-            }
-            if (crawlOptions.terse && (!crawlOptions.modules || crawlOptions.modules.length === 0 || crawlOptions.modules.length > 1)) {
-                console.error('The --terse option can be only be set when only one core processing module runs');
-                process.exit(2);
-            }
-            crawlSpecs(crawlOptions)
-                .then(_ => {
-                    process.exit(0);
-                })
-                .catch(err => {
-                    console.error(err);
-                    process.exit(1);
-                });
-        })
-        .showHelpAfterError('(run with --help for usage information)')
-        .addHelpText('after', `
-Minimal usage example:
-  $ node crawl-specs.js reports/test
-
-Description:
-  Crawls a set of specifications and runs processing modules against each of
-  them to generate extracts.
-
-  Crawl results are written to the console as a serialized JSON array with one
-  entry per spec by default. The order of the specs in the array matches the
-  order of the specs provided as input (or the order of the specs in
-  browser-specs if no explicit spec was provided).
-
-  Resulting array may be large. Crawling all specs with core processing module
-  produces ~100MB of serialized JSON for instance. To avoid janking the console
-  or running into possible memory issues, setting the --output option is
-  strongly recommended.
-
-Usage notes for some of the options:
--m, --module <modules...>
-  If processing modules are not specified, the crawler runs all core processing
-  modules defined in:
-    https://github.com/w3c/reffy/tree/main/src/reffy.json
-
-  Modules must be specified using a relative path to an ".mjs" file that defines
-  the processing logic to run on the spec's page in a browser context. For
-  instance:
-    $ node crawl-specs.js reports/test --module extract-editors.mjs
-
-  Absolute paths to modules are not properly handled and will likely result in a
-  crawling error.
-
-  Multiple modules can be specified, repeating the option name or not:
-    $ node crawl-specs.js reports/test -m extract-words.mjs extract-editors.mjs
-    $ node crawl-specs.js reports/test -m extract-words.mjs -m extract-editors.mjs
-
-  The option cannot appear before <folder>, unless you use "--" to flag the end
-  of the list:
-    $ node crawl-specs.js --module extract-editors.mjs -- reports/test
-
-  Core processing modules may be referenced using the name of the extract folder
-  or property that they would create:
-    $ node crawl-specs.js reports/test --module dfns
-
-  To run all core processing modules, use "core". For instance, to apply a
-  processing module on top of core processing modules, use:
-    $ node crawl-specs.js reports/test --module core extract-editors.mjs
-
-  Each module must export a function that takes a spec object as input and
-  return a result that can be serialized as JSON. A typical module code looks
-  like:
-    https://github.com/w3c/reffy/blob/main/src/browserlib/extract-ids.mjs
-
-  Individual extracts will be created under "<folder>/[camelCaseModule]" where
-  "[camelCaseModule]" is derived from the module's filename. For instance:
-    "extract-editors.mjs" creates extracts under "<folder>/extractEditors"
-
-  The name of the folder where extracts get created may be specified for custom
-  modules by prefixing the path to the module with the folder name followed by
-  ":". For instance, to save extracts to "reports/test/editors", use:
-    $ node crawl-specs.js reports/test --module editors:extract-editors.mjs
-
--o, --output <folder>
-  By default, crawl results are written to the console as a serialized JSON
-  array with one entry per spec, and module processing results attached as
-  property values in each of these entries.
-
-  If an output <folder> is specified, crawl results are rather saved to that
-  folder, with module processing results created under subfolders (see the
-  --module option) and linked from an index.json file created under <folder>.
-
-  Additionally, if an output <folder> is specified and if the IDL processing
-  module is run, the crawler will also creates an index of IDL names named
-  "idlnames.json" that links to relevant extracts in subfolders.
-
--r, --release
-  If the flag is not set, the crawler defaults to crawl nightly versions of the
-  specs.
-
--s, --spec <specs...>
-  If specs to crawl are not specified, all specs in browser-specs get crawled:
-    https://github.com/w3c/browser-specs/
-
-  Valid spec values may be a shortname, a URL, or a relative path to a file that
-  contains a list of spec URLs and/or shortnames. All shortnames must exist in
-  browser-specs.
-
-  Use "all" to include all specs in browser-specs in the crawl. For instance, to
-  crawl all specs plus one custom spec that does not exist in browser-specs:
-    $ node crawl-specs.js reports/test -s all https://example.org/myspec
-
--t, --terse
-  This flag cannot be combined with the --output option and cannot be set if
-  more than one processing module gets run. When set, the crawler writes the
-  processing module results to the console directly without wrapping them with
-  spec metadata. In other words, the spec entry in the crawl results directly
-  contains the outcome of the processing module when the flag is set.
-
-  Additionally, if crawl runs on a single specification, the array is omitted
-  and the processing module results are thus written to the console directly.
-  For instance:
-    $ node crawl-specs.js --spec fetch --module idl --terse
-`);
-
-    program.parse(process.argv);
-}

package/src/lib/util.js

@@ -14,6 +14,21 @@ const specEquivalents = require('../specs/spec-equivalents.json');
 const reffyModules = require('../browserlib/reffy.json');
 
 
+/**
+ * Maximum depth difference supported between Reffy's install path and custom
+ * modules that may be provided on the command-line
+ *
+ * TODO: Find a way to get right of that, there should be no limit
+ */
+const maxPathDepth = 20;
+
+
+/**
+ * Returns a range array from 0 to the number provided (not included)
+ */
+const range = n => Array.from(Array(n).keys());
+
+
 /**
  * Shortcut that returns a property extractor iterator
  */
@@ -356,7 +371,7 @@ async function processSpecification(spec, processFunction, args, options) {
                         let depth = requestPath.lastIndexOf('__/') / 3;
                         const filename = requestPath.substring(requestPath.lastIndexOf('__/') + 3);
                         let filePath = path.resolve(__dirname, '..', 'browserlib');
-                        while (depth < 7) {
+                        while (depth < maxPathDepth - 1) {
                             filePath = path.resolve(filePath, '..');
                             depth += 1;
                         }
@@ -547,7 +562,7 @@ async function processSpecification(spec, processFunction, args, options) {
         // "../../node_modules/[...]" and may import other scripts that are
         // higher in the folder tree.
         await page.addScriptTag({
-            url: 'reffy/scripts/__/__/__/__/__/__/__/__/reffy.mjs',
+            url: `reffy/scripts/${range(maxPathDepth).map(n => '__').join('/')}/reffy.mjs`,
             type: 'module'
         });