reffy 3.1.0 → 4.0.3

package/reffy.js

@@ -1,211 +1,227 @@
 #!/usr/bin/env node
 /**
- * Reffy's main command line interface that you can use to crawl and study spec
- * references.
+ * The spec crawler takes a list of spec URLs as input, gathers some knowledge
+ * about these specs (published versions, URL of the Editor's Draft, etc.),
+ * fetches these specs, parses them, extracts relevant information that they
+ * contain (such as the WebIDL they define, the list of specifications that they
+ * reference, and links to external specs), and produces a crawl report with the
+ * results of these investigations.
  *
- * Reffy can be called directly through:
+ * Provided Reffy was installed as a global package, the spec crawler can be
+ * called directly through:
  *
- * `node reffy.js [command]`
+ * `reffy [options]`
  *
- * Run `node reffy.js -h` for help
+ * Use the `--help` option for usage instructions.
+ *
+ * If Reffy was not installed as a global package, call:
+ *
+ * `node reffy.js [options]`
  *
  * @module crawler
  */
 
-const program = require('commander');
+const commander = require('commander');
 const version = require('./package.json').version;
-const fs = require('fs');
-const path = require('path');
-const crawlSpecs = require('./src/cli/crawl-specs.js').crawlSpecs;
-const studyCrawl = require('./src/cli/study-crawl.js').studyCrawl;
-const generateReport = require('./src/cli/generate-report.js').generateReport;
-const pandoc = require('node-pandoc');
-
-
-// List of possible perspectives and associated parameters
-// Note the "ed" perspective produces reports under "whatwg" for backward
-// compatibility reason.
-const perspectives = {
-  'ed': {
-    description: 'Crawls the latest Editor\'s Drafts',
-    refStudy: 'https://w3c.github.io/webref/ed/study.json'
-  },
-  'tr': {
-    description: 'Crawls the latest published versions of specifications in /TR/ space instead of the latest Editor\'s Drafts',
-    publishedVersion: true,
-    refStudy: 'https://w3c.github.io/webref/tr/study.json'
-  }
-};
-
-// List of possible actions for each perspective
-const possibleActions = {
-  'all': 'crawl specs, study report and generate markdown, HTML and diff reports. Default action',
-  'crawl': 'crawl specs and generate a machine-readable report with facts about each spec',
-  'study': 'parse the machine-readable report generated by the crawler, and create a study report of potential anomalies found in the report',
-  'markdown': 'produce a human-readable report in Markdown format out of the anomalies report returned by the study action',
-  'html': 'produce an HTML report out of the Markdown report generated by the markdown action',
-  'diff': 'compare the anomalies report with the latest published anomalies report and generate diff report',
-  'diffnew': 'compare the anomalies report with the latest published anomalies report and generate diff report that only contains new anomalies'
-};
-
-let command = null; 
-program
-  .version(version)
-  .option('-d, --debug', 'run crawl in debug mode (single process, one spec at a time)');
+const specs = require('browser-specs');
+const { requireFromWorkingDirectory } = require('./src/lib/util');
+const { crawlSpecs } = require('./src/lib/specs-crawler');
 
-program
-  .command('run <perspective> [action]')
-  .description('run a new crawl and study from the given perspective')
-  .option('-d, --debug', 'run crawl in debug mode (single process, one spec at a time)')
-  .action(async (perspective, action, cmdObj) => {
-    command = 'run';
-    if (!(perspective in perspectives)) {
-      return program.help();
+
+function parseModuleOption(input) {
+    const parts = input.split(':');
+    if (parts.length > 2) {
+        console.error('Module input cannot have more than one ":" character');
+        process.exit(2);
+    }
+    if (parts.length === 2) {
+        return {
+            href: parts[1],
+            property: parts[0]
+        };
     }
-    if (action && !(action in possibleActions)) {
-      return program.help();
+    else {
+        return parts[0];
     }
+}
 
-    let debug = cmdObj.debug || program.debug;
-    let publishedVersion = perspectives[perspective].publishedVersion;
-    let refStudy = perspectives[perspective].refStudy;
-    let reportFolder = perspectives[perspective].reportFolder ||
-      'reports/' + perspective;
-    let crawlReport = path.join(reportFolder, 'index.json');
-    let studyReport = path.join(reportFolder, 'study.json');
-
-    let promise = Promise.resolve();
-    let actions = (!action || (action === 'all')) ?
-      ['crawl', 'study', 'markdown', 'html', 'diff', 'diffnew'] :
-      [action];
-
-    actions.forEach(action => {
-      switch (action) {
-      case 'crawl':
-        promise = promise
-          .then(_ => crawlSpecs(
-            { publishedVersion, debug, output: reportFolder }));
-        break;
-
-      case 'study':
-        const options = {};
-        if (perspective === 'ed') {
-          const trFolder = perspectives.tr.reportFolder || 'reports/tr';
-          const trReport = path.join(trFolder, 'index.json');
-          if (fs.existsSync(trReport)) {
-            options.trResults = trReport;
-          }
-        }
-        promise = promise
-          .then(_ => studyCrawl(crawlReport, options))
-          .then(results => {
-            fs.writeFileSync(path.join(reportFolder, 'study.json'),
-              JSON.stringify(results, null, 2));
-          });
-        break;
-
-      case 'markdown':
-        promise = promise
-          .then(_ => generateReport(studyReport, { perSpec: true }))
-          .then(report => fs.writeFileSync(path.join(reportFolder, 'index.md'), report))
-          .then(_ => generateReport(studyReport, { perSpec: false }))
-          .then(report => fs.writeFileSync(path.join(reportFolder, 'perissue.md'), report));
-        break;
-
-      case 'html':
-        promise = promise
-          .then(_ => new Promise((resolve, reject) => {
-            let args = [
-              '-f', 'markdown', '-t', 'html5', '--section-divs', '-s',
-              '--template', path.join(__dirname, 'src', 'templates', 'report-template.html'),
-              '-o', path.join(reportFolder, 'index.html')
-            ];
-            pandoc(path.join(reportFolder, 'index.md'), args,
-              (err => {
-                if (err) {
-                  return reject(err);
-                }
-                args = [
-                  '-f', 'markdown', '-t', 'html5', '--section-divs', '-s',
-                  '--template', path.join(__dirname, 'src', 'templates', 'report-perissue-template.html'),
-                  '-o', path.join(reportFolder, 'perissue.html')];
-                pandoc(path.join(reportFolder, 'perissue.md'), args,
-                  (err => {
-                    if (err) {
-                      return reject(err);
-                    }
-                    return resolve();
-                  }));
-              }));
-            }));
-        break;
-
-      case 'diff':
-        promise = promise
-          .then(_ => generateReport(studyReport, {
-            diffReport: true,
-            refStudyFile: refStudy
-          }))
-          .then(report => fs.writeFileSync(path.join(reportFolder, 'diff.md'), report));
-        break;
-
-      case 'diffnew':
-        promise = promise
-          .then(_ => generateReport(studyReport, {
-            diffReport: true,
-            refStudyFile: refStudy,
-            onlyNew: true
-          }))
-          .then(report => fs.writeFileSync(path.join(reportFolder, 'diffnew.md'), report));
-        break;
-      }
-    });
-
-    return promise;
-  });
-
-program.on('--help', function() {
-  console.log('');
-  console.log('  Possible perspectives:');
-  console.log('');
-  Object.keys(perspectives).forEach(perspective => {
-    console.log('    ' + perspective + ': ' + perspectives[perspective].description);
-  });
-  console.log('');
-
-  console.log('  Possible actions:');
-  console.log('');
-  Object.keys(possibleActions).forEach(action => {
-    console.log('    ' + action + ': ' + possibleActions[action]);
-  });
-  console.log('');
-
-  console.log('  Possible options:');
-  console.log('');
-  console.log('    -d, --debug: run crawl in debug mode (single process, one spec at a time)');
-  console.log('');
-});
-
-program.on('command:*', function () {
-  console.error('Invalid command: %s.\n', program.args.join(' '));
-  program.outputHelp();
-  process.exit(1);
-});
-
-if (!process.argv.slice(2).length) {
-  console.error('Cannot run program without arguments.\n');
-  program.outputHelp();
-  process.exit(1);
+function parseSpecOption(input) {
+    if (input === 'all') {
+        return specs.map(s => s.shortname);
+    }
+    else {
+        const list = requireFromWorkingDirectory(input);
+        return list ?? input;
+    }
 }
 
+const program = new commander.Command();
 program
-  .parseAsync(process.argv)
-  .then(_ => {
-    console.log('-- THE END -- ');
-    process.exit(0);
-  })
-  .catch(err => {
-    console.error('-- ERROR CAUGHT --');
-    console.error(err);
-    process.exit(1);
-  });
+    .version(version)
+    .usage('[options]')
+    .description('Crawls and processes a list of Web specifications')
+    .option('-d, --debug', 'debug mode, crawl one spec at a time')
+    .option('-m, --module <modules...>', 'spec processing modules')
+    .option('-o, --output <folder>', 'existing folder/file where crawl results are to be saved')
+    .option('-q, --quiet', 'do not report progress and other warnings to the console')
+    .option('-r, --release', 'crawl release (TR) version of specs')
+    .option('-s, --spec <specs...>', 'specs to crawl')
+    .option('-t, --terse', 'output crawl results without metadata')
+    .action(options => {
+        if (!(options.output || options.module || options.spec)) {
+          console.error(`
+At least one of the --output, --module or --spec options needs to be specified.
+For usage notes, run:
+  reffy --help
+
+If you really want to crawl all specs, run all processing modules and report the
+JSON outcome to the console, you may run the following command but note that it
+will dump ~100MB of data to the console:
+  reffy --spec all
+`);
+          process.exit(2);
+        }
+        const crawlOptions = {
+            debug: options.debug,
+            output: options.output,
+            publishedVersion: options.release,
+            quiet: options.quiet,
+            terse: options.terse
+        };
+        if (options.module) {
+            crawlOptions.modules = options.module.map(parseModuleOption);
+        }
+        if (options.spec) {
+            crawlOptions.specs = options.spec.map(parseSpecOption).flat();
+        }
+
+        if (crawlOptions.terse && crawlOptions.output) {
+            console.error('The --terse option cannot be combined with the --output option');
+            process.exit(2);
+        }
+        if (crawlOptions.terse && (!crawlOptions.modules || crawlOptions.modules.length === 0 || crawlOptions.modules.length > 1)) {
+            console.error('The --terse option can be only be set when only one core processing module runs');
+            process.exit(2);
+        }
+        crawlSpecs(crawlOptions)
+            .then(_ => {
+                process.exit(0);
+            })
+            .catch(err => {
+                console.error(err);
+                process.exit(1);
+            });
+    })
+    .showHelpAfterError('(run with --help for usage information)')
+    .addHelpText('after', `
+Minimal usage example:
+  To crawl all known specs, run all processing modules, and save generated
+  extracts to the current folder, run:
+    $ reffy -o .
+
+Description:
+  Crawls a set of specifications and runs processing modules against each of
+  them to generate extracts.
+
+  Crawl results are written to the console as a serialized JSON array with one
+  entry per spec by default. The order of the specs in the array matches the
+  order of the specs provided as input (or the order of the specs in
+  browser-specs if no explicit spec was provided).
+
+  Resulting array may be large. Crawling all specs with core processing module
+  produces ~100MB of serialized JSON for instance. To avoid janking the console
+  or running into possible memory issues, setting the --output option is
+  strongly recommended.
+
+Usage notes for some of the options:
+-m, --module <modules...>
+  If processing modules are not specified, the crawler runs all core processing
+  modules defined in:
+    https://github.com/w3c/reffy/tree/main/src/reffy.json
+
+  Modules must be specified using a relative path to an ".mjs" file that defines
+  the processing logic to run on the spec's page in a browser context. For
+  instance:
+    $ reffy reports/test --module extract-editors.mjs
+
+  Absolute paths to modules are not properly handled and will likely result in a
+  crawling error.
+
+  Multiple modules can be specified, repeating the option name or not:
+    $ reffy reports/test -m extract-words.mjs extract-editors.mjs
+    $ reffy reports/test -m extract-words.mjs -m extract-editors.mjs
+
+  The option cannot appear before <folder>, unless you use "--" to flag the end
+  of the list:
+    $ reffy --module extract-editors.mjs -- reports/test
+
+  Core processing modules may be referenced using the name of the extract folder
+  or property that they would create:
+    $ reffy reports/test --module dfns
+
+  To run all core processing modules, use "core". For instance, to apply a
+  processing module on top of core processing modules, use:
+    $ reffy reports/test --module core extract-editors.mjs
+
+  Each module must export a function that takes a spec object as input and
+  return a result that can be serialized as JSON. A typical module code looks
+  like:
+    https://github.com/w3c/reffy/blob/main/src/browserlib/extract-ids.mjs
+
+  Individual extracts will be created under "<folder>/[camelCaseModule]" where
+  "[camelCaseModule]" is derived from the module's filename. For instance:
+    "extract-editors.mjs" creates extracts under "<folder>/extractEditors"
+
+  The name of the folder where extracts get created may be specified for custom
+  modules by prefixing the path to the module with the folder name followed by
+  ":". For instance, to save extracts to "reports/test/editors", use:
+    $ reffy reports/test --module editors:extract-editors.mjs
+
+-o, --output <folder>
+  By default, crawl results are written to the console as a serialized JSON
+  array with one entry per spec, and module processing results attached as
+  property values in each of these entries.
+
+  If an output <folder> is specified, crawl results are rather saved to that
+  folder, with module processing results created under subfolders (see the
+  --module option) and linked from an index.json file created under <folder>.
+
+  Additionally, if an output <folder> is specified and if the IDL processing
+  module is run, the crawler will also creates an index of IDL names named
+  "idlnames.json" that links to relevant extracts in subfolders.
+
+  The folder targeted by <folder> must exist.
+
+-r, --release
+  If the flag is not set, the crawler defaults to crawl nightly versions of the
+  specs.
+
+-s, --spec <specs...>
+  If specs to crawl are not specified, all specs in browser-specs get crawled:
+    https://github.com/w3c/browser-specs/
+
+  Valid spec values may be a shortname, a URL, or a relative path to a file that
+  contains a list of spec URLs and/or shortnames. All shortnames must exist in
+  browser-specs. Shortname may be the shortname of the spec series, in which
+  case the spec identified as the current specification in the series is used.
+  For instance, as of September 2021, "pointerlock" will map to "pointerlock-2"
+  because Pointer Lock 2.0 is the current level in the series.
+
+  Use "all" to include all specs in browser-specs in the crawl. For instance, to
+  crawl all specs plus one custom spec that does not exist in browser-specs:
+    $ reffy reports/test -s all https://example.org/myspec
+
+-t, --terse
+  This flag cannot be combined with the --output option and cannot be set if
+  more than one processing module gets run. When set, the crawler writes the
+  processing module results to the console directly without wrapping them with
+  spec metadata. In other words, the spec entry in the crawl results directly
+  contains the outcome of the processing module when the flag is set.
+
+  Additionally, if crawl runs on a single specification, the array is omitted
+  and the processing module results are thus written to the console directly.
+  For instance:
+    $ reffy --spec fetch --module idl --terse
+`);
+
+program.parse(process.argv);