reffy 20.0.13 → 20.0.15

package/reffy.js

@@ -1,324 +1,324 @@
-#!/usr/bin/env node
-/**
- * The spec crawler takes a list of spec URLs as input, gathers some knowledge
- * about these specs (published versions, URL of the Editor's Draft, etc.),
- * fetches these specs, parses them, extracts relevant information that they
- * contain (such as the WebIDL they define, the list of specifications that they
- * reference, and links to external specs), and produces a crawl report with the
- * results of these investigations.
- *
- * Provided Reffy was installed as a global package, the spec crawler can be
- * called directly through:
- *
- * `reffy [options]`
- *
- * Use the `--help` option for usage instructions.
- *
- * If Reffy was not installed as a global package, call:
- *
- * `node reffy.js [options]`
- *
- * @module crawler
- */
-
-import { Command } from 'commander';
-import satisfies from 'semver/functions/satisfies.js';
-import specs from 'web-specs' with { type: 'json' };
-import packageConfig from './package.json' with { type: 'json' };
-import { crawlSpecs } from './src/lib/specs-crawler.js';
-import postProcessor from './src/lib/post-processor.js';
-import { loadJSON } from './src/lib/util.js';
-
-// Warn if version of Node.js does not satisfy requirements
-if (packageConfig.engines && packageConfig.engines.node &&
-    !satisfies(process.version, packageConfig.engines.node)) {
-  console.warn(`
-[WARNING] Node.js ${process.version} detected but Reffy needs Node.js ${packageConfig.engines.node}.
-          Please consider upgrading Node.js if the program crashes!`);
-}
-
-
-function parseModuleOption(input) {
-    const parts = input.split(':');
-    if (parts.length > 2) {
-        console.error('Module input cannot have more than one ":" character');
-        process.exit(2);
-    }
-    if (parts.length === 2) {
-        return {
-            href: parts[1],
-            property: parts[0]
-        };
-    }
-    else {
-        return parts[0];
-    }
-}
-
-async function parseSpecOption(input) {
-    if (input === 'all') {
-        return specs
-            .filter(s => s.standing !== 'discontinued')
-            .map(s => s.shortname)
-    }
-    else {
-        const list = await loadJSON(input);
-        return list ?? input;
-    }
-}
-
-function parsePostOption(input) {
-    if (input === 'core') {
-      return postProcessor.modules;
-    }
-    else {
-      return input;
-    }
-}
-
-
-const program = new Command();
-program
-    .version(packageConfig.version)
-    .usage('[options]')
-    .description('Crawls and processes a list of Web specifications')
-    .option('-d, --debug', 'debug mode, crawl one spec at a time')
-    .option('-f, --fallback <json>', 'fallback data to use when a spec crawl fails')
-    .option('--md, --markdown', 'output a Markdown report')
-    .option('-m, --module <modules...>', 'spec processing modules')
-    .option('-o, --output <folder>', 'existing folder/file where crawl results are to be saved')
-    .option('-p, --post <modules...>', 'post-processing modules')
-    .option('-q, --quiet', 'do not report progress and other warnings to the console')
-    .option('-r, --release', 'crawl release (TR) version of specs')
-    .option('-s, --spec <specs...>', 'specs to crawl')
-    .option('--summary', 'include a crawl summary in Markdown for each spec')
-    .option('-t, --terse', 'output crawl results without metadata')
-    .option('-u, --use-crawl <folder>', 'use given crawl result folder as input for post-processing')
-    .action(async options => {
-        if (!(options.output || options.module || options.spec || options.useCrawl)) {
-          console.error(`
-At least one of the --output, --module, --spec or --use-crawl options needs to be
-specified. For usage notes, run:
-  reffy --help
-
-If you really want to crawl all specs, run all processing modules and report the
-JSON outcome to the console, you may run the following command but note that it
-will dump ~100MB of data to the console:
-  reffy --spec all
-`);
-          process.exit(2);
-        }
-        const crawlOptions = {
-            debug: options.debug,
-            fallback: options.fallback,
-            markdown: options.markdown,
-            output: options.output,
-            publishedVersion: options.release,
-            quiet: options.quiet,
-            summary: options.summary,
-            terse: options.terse,
-            useCrawl: options.useCrawl
-        };
-        if (options.module) {
-            crawlOptions.modules = options.module.map(parseModuleOption);
-        }
-        if (options.spec) {
-            crawlOptions.specs = (await Promise.all(options.spec.map(parseSpecOption))).flat();
-        }
-        else {
-            crawlOptions.specs = await parseSpecOption('all');
-        }
-        if (options.post) {
-            crawlOptions.post = options.post.map(parsePostOption).flat();
-        }
-
-        if (crawlOptions.terse && crawlOptions.output) {
-            console.error('The --terse option cannot be combined with the --output option');
-            process.exit(2);
-        }
-        if (crawlOptions.terse && (!crawlOptions.modules || crawlOptions.modules.length === 0 || crawlOptions.modules.length > 1)) {
-            console.error('The --terse option can only be set when only one core processing module runs');
-            process.exit(2);
-        }
-        crawlSpecs(crawlOptions)
-            .then(_ => {
-                process.exit(0);
-            })
-            .catch(err => {
-                console.error(err);
-                process.exit(1);
-            });
-    })
-    .showHelpAfterError('(run with --help for usage information)')
-    .addHelpText('after', `
-Minimal usage example:
-  To crawl all known specs, run all processing modules, and save generated
-  extracts to the current folder, run:
-    $ reffy -o .
-
-Description:
-  Crawls a set of specifications and runs processing modules against each of
-  them to generate extracts.
-
-  Crawl results are written to the console as a serialized JSON array with one
-  entry per spec by default. The order of the specs in the array matches the
-  order of the specs provided as input (or the order of the specs in
-  browser-specs if no explicit spec was provided).
-
-  Resulting array may be large. Crawling all specs with core processing module
-  produces ~100MB of serialized JSON for instance. To avoid janking the console
-  or running into possible memory issues, setting the --output option is
-  strongly recommended.
-
-Usage notes for some of the options:
--f, --fallback <jsondata>
-  Provides an existing JSON crawl data file to use as a source of fallback data
-  for specs that fail to be crawled.
-
-  The fallback data gets copied as-is. It is the responsibility of the caller
-  to make sure that extracts it may link to actually exist and match the ones
-  that the crawl would produce in the absence of errors (e.g. same modules).
-
-  The "error" property is set on specs for which fallback data was used.
-
---md, --markdown
-  Output a crawl summary in Markdown instead of a JSON report. The option takes
-  precedence over the \`--output\` option.
-
--m, --module <modules...>
-  If processing modules are not specified, the crawler runs all core processing
-  modules defined in:
-    https://github.com/w3c/reffy/blob/main/src/browserlib/reffy.json
-
-  Modules must be specified using a relative path to an ".mjs" file that defines
-  the processing logic to run on the spec's page in a browser context. For
-  instance:
-    $ reffy --output reports/test --module extract-editors.mjs
-
-  Absolute paths to modules are not properly handled and will likely result in a
-  crawling error.
-
-  Multiple modules can be specified, repeating the option name or not:
-    $ reffy -o reports/test -m extract-words.mjs extract-editors.mjs
-    $ reffy -o reports/test -m extract-words.mjs -m extract-editors.mjs
-
-  Core processing modules may be referenced using the name of the extract folder
-  or property that they would create:
-    $ reffy --output reports/test --module dfns
-
-  To run all core processing modules, use "core". For instance, to apply a
-  processing module on top of core processing modules, use:
-    $ reffy --output reports/test --module core extract-editors.mjs
-
-  Each module must export a function that takes a spec object as input and
-  return a result that can be serialized as JSON. A typical module code looks
-  like:
-    https://github.com/w3c/reffy/blob/main/src/browserlib/extract-ids.mjs
-
-  Individual extracts will be created under "<folder>/[camelCaseModule]" where
-  "[camelCaseModule]" is derived from the module's filename. For instance:
-    "extract-editors.mjs" creates extracts under "<folder>/extractEditors"
-
-  The name of the folder where extracts get created may be specified for custom
-  modules by prefixing the path to the module with the folder name followed by
-  ":". For instance, to save extracts to "reports/test/editors", use:
-    $ reffy --output reports/test --module editors:extract-editors.mjs
-
--o, --output <folder>
-  By default, crawl results are written to the console as a serialized JSON
-  array with one entry per spec, and module processing results attached as
-  property values in each of these entries.
-
-  If an output <folder> is specified, crawl results are rather saved to that
-  folder, with module processing results created under subfolders (see the
-  --module option) and linked from an index.json file created under <folder>.
-
-  Additionally, if an output <folder> is specified and if the IDL processing
-  module is run, the crawler will also creates an index of IDL names named
-  "idlnames.json" that links to relevant extracts in subfolders.
-
-  The folder targeted by <folder> must exist.
-
--p, --post <modules...>
-  Post-processing modules either run after a spec is done crawling or after the
-  entire crawl is over. They allow developers to complete data based on other
-  extracts that were not available when extraction ran.
-
-  To run all core post-processing modules, use "core". Core post-processing
-  modules are defined in:
-    https://github.com/w3c/reffy/blob/main/src/postprocessing.js
-
-  The crawler does not run any post-processing modules by default.
-
-  Custom post-processing modules may be specified using a relative path to a
-  ".js" file that defines the post-processing logic. For instance:
-    $ reffy --output reports/test --post mypostprocessing.js
-
-  Each module must export a "run" function. See the post-processor's code for
-  details:
-    https://github.com/w3c/reffy/blob/main/src/lib/post-processor.js
-
-  Absolute paths to modules are not properly handled and will likely result in a
-  processing error.
-
-  Multiple post-processing modules can be specified, repeating the option name
-  or not:
-    $ reffy -o reports/test -p cssdfns cssidl events
-    $ reffy -o reports/test -p events -p idlparsed -p idlnames
-
--r, --release
-  The crawler defaults to crawling the nightly version of requested specs.
-  Set this flag to tell the crawler to crawl the published version of the specs
-  instead. When the flag is set, the crawler will ignore specs that do not have
-  a published version.
-
--s, --spec <specs...>
-  If specs to crawl are not specified, all specs in browser-specs that are not
-  identified as being discontinued get crawled:
-    https://github.com/w3c/browser-specs/
-
-  Valid spec values may be a shortname, a URL, or a relative path to a file that
-  contains a list of spec URLs and/or shortnames. All shortnames must exist in
-  browser-specs. Shortname may be the shortname of the spec series, in which
-  case the spec identified as the current specification in the series is used.
-  For instance, as of September 2021, "pointerlock" will map to "pointerlock-2"
-  because Pointer Lock 2.0 is the current level in the series.
-
-  Use "all" to include all specs in browser-specs in the crawl. For instance, to
-  crawl all specs plus one custom spec that does not exist in browser-specs:
-    $ reffy -o reports/test -s all https://example.org/myspec
-
-  When "all" is used, to force a crawl on some of the discontinued specs too,
-  include their shortname explicitly (or point to a JSON file that lists their
-  shortnames). For instance, to also crawl the discontinued DOM Level 2 Style
-  spec, run:
-    $ reffy -o reports/test -s all DOM-Level-2-Style
-
---summary
-  Tells Reffy to attach a Markdown summary of the crawl per spec to the JSON
-  report, in a \`crawlSummary\` property. The Markdown report is suitable for
-  inclusion in a GitHub issue or similar. It starts with a summary, and then
-  details a few noteworthy extracts (CSS, dfns, Web IDL) in expandable
-  sections, with links to the online xref database search where appropriate.
-
--t, --terse
-  This flag cannot be combined with the --output option and cannot be set if
-  more than one processing module gets run. When set, the crawler writes the
-  processing module results to the console directly without wrapping them with
-  spec metadata. In other words, the spec entry in the crawl results directly
-  contains the outcome of the processing module when the flag is set.
-
-  Additionally, if crawl runs on a single specification, the array is omitted
-  and the processing module results are thus written to the console directly.
-  For instance:
-    $ reffy --spec fetch --module idl --terse
-
--u, --use-crawl <folder>
-  Tells Reffy to skip the crawl part and only run requested post-processing
-  modules on the crawl results present in the specified folder.
-
-  If post-processing modules are not specified, Reffy will merely copy the crawl
-  results to the output folder (or to the console).
-`);
-
-program.parse(process.argv);
+#!/usr/bin/env node
+/**
+ * The spec crawler takes a list of spec URLs as input, gathers some knowledge
+ * about these specs (published versions, URL of the Editor's Draft, etc.),
+ * fetches these specs, parses them, extracts relevant information that they
+ * contain (such as the WebIDL they define, the list of specifications that they
+ * reference, and links to external specs), and produces a crawl report with the
+ * results of these investigations.
+ *
+ * Provided Reffy was installed as a global package, the spec crawler can be
+ * called directly through:
+ *
+ * `reffy [options]`
+ *
+ * Use the `--help` option for usage instructions.
+ *
+ * If Reffy was not installed as a global package, call:
+ *
+ * `node reffy.js [options]`
+ *
+ * @module crawler
+ */
+
+import { Command } from 'commander';
+import satisfies from 'semver/functions/satisfies.js';
+import specs from 'web-specs' with { type: 'json' };
+import packageConfig from './package.json' with { type: 'json' };
+import { crawlSpecs } from './src/lib/specs-crawler.js';
+import postProcessor from './src/lib/post-processor.js';
+import { loadJSON } from './src/lib/util.js';
+
+// Warn if version of Node.js does not satisfy requirements
+if (packageConfig.engines && packageConfig.engines.node &&
+    !satisfies(process.version, packageConfig.engines.node)) {
+  console.warn(`
+[WARNING] Node.js ${process.version} detected but Reffy needs Node.js ${packageConfig.engines.node}.
+          Please consider upgrading Node.js if the program crashes!`);
+}
+
+
+function parseModuleOption(input) {
+    const parts = input.split(':');
+    if (parts.length > 2) {
+        console.error('Module input cannot have more than one ":" character');
+        process.exit(2);
+    }
+    if (parts.length === 2) {
+        return {
+            href: parts[1],
+            property: parts[0]
+        };
+    }
+    else {
+        return parts[0];
+    }
+}
+
+async function parseSpecOption(input) {
+    if (input === 'all') {
+        return specs
+            .filter(s => s.standing !== 'discontinued')
+            .map(s => s.shortname)
+    }
+    else {
+        const list = await loadJSON(input);
+        return list ?? input;
+    }
+}
+
+function parsePostOption(input) {
+    if (input === 'core') {
+      return postProcessor.modules;
+    }
+    else {
+      return input;
+    }
+}
+
+
+const program = new Command();
+program
+    .version(packageConfig.version)
+    .usage('[options]')
+    .description('Crawls and processes a list of Web specifications')
+    .option('-d, --debug', 'debug mode, crawl one spec at a time')
+    .option('-f, --fallback <json>', 'fallback data to use when a spec crawl fails')
+    .option('--md, --markdown', 'output a Markdown report')
+    .option('-m, --module <modules...>', 'spec processing modules')
+    .option('-o, --output <folder>', 'existing folder/file where crawl results are to be saved')
+    .option('-p, --post <modules...>', 'post-processing modules')
+    .option('-q, --quiet', 'do not report progress and other warnings to the console')
+    .option('-r, --release', 'crawl release (TR) version of specs')
+    .option('-s, --spec <specs...>', 'specs to crawl')
+    .option('--summary', 'include a crawl summary in Markdown for each spec')
+    .option('-t, --terse', 'output crawl results without metadata')
+    .option('-u, --use-crawl <folder>', 'use given crawl result folder as input for post-processing')
+    .action(async options => {
+        if (!(options.output || options.module || options.spec || options.useCrawl)) {
+          console.error(`
+At least one of the --output, --module, --spec or --use-crawl options needs to be
+specified. For usage notes, run:
+  reffy --help
+
+If you really want to crawl all specs, run all processing modules and report the
+JSON outcome to the console, you may run the following command but note that it
+will dump ~100MB of data to the console:
+  reffy --spec all
+`);
+          process.exit(2);
+        }
+        const crawlOptions = {
+            debug: options.debug,
+            fallback: options.fallback,
+            markdown: options.markdown,
+            output: options.output,
+            publishedVersion: options.release,
+            quiet: options.quiet,
+            summary: options.summary,
+            terse: options.terse,
+            useCrawl: options.useCrawl
+        };
+        if (options.module) {
+            crawlOptions.modules = options.module.map(parseModuleOption);
+        }
+        if (options.spec) {
+            crawlOptions.specs = (await Promise.all(options.spec.map(parseSpecOption))).flat();
+        }
+        else {
+            crawlOptions.specs = await parseSpecOption('all');
+        }
+        if (options.post) {
+            crawlOptions.post = options.post.map(parsePostOption).flat();
+        }
+
+        if (crawlOptions.terse && crawlOptions.output) {
+            console.error('The --terse option cannot be combined with the --output option');
+            process.exit(2);
+        }
+        if (crawlOptions.terse && (!crawlOptions.modules || crawlOptions.modules.length === 0 || crawlOptions.modules.length > 1)) {
+            console.error('The --terse option can only be set when only one core processing module runs');
+            process.exit(2);
+        }
+        crawlSpecs(crawlOptions)
+            .then(_ => {
+                process.exit(0);
+            })
+            .catch(err => {
+                console.error(err);
+                process.exit(1);
+            });
+    })
+    .showHelpAfterError('(run with --help for usage information)')
+    .addHelpText('after', `
+Minimal usage example:
+  To crawl all known specs, run all processing modules, and save generated
+  extracts to the current folder, run:
+    $ reffy -o .
+
+Description:
+  Crawls a set of specifications and runs processing modules against each of
+  them to generate extracts.
+
+  Crawl results are written to the console as a serialized JSON array with one
+  entry per spec by default. The order of the specs in the array matches the
+  order of the specs provided as input (or the order of the specs in
+  browser-specs if no explicit spec was provided).
+
+  Resulting array may be large. Crawling all specs with core processing module
+  produces ~100MB of serialized JSON for instance. To avoid janking the console
+  or running into possible memory issues, setting the --output option is
+  strongly recommended.
+
+Usage notes for some of the options:
+-f, --fallback <jsondata>
+  Provides an existing JSON crawl data file to use as a source of fallback data
+  for specs that fail to be crawled.
+
+  The fallback data gets copied as-is. It is the responsibility of the caller
+  to make sure that extracts it may link to actually exist and match the ones
+  that the crawl would produce in the absence of errors (e.g. same modules).
+
+  The "error" property is set on specs for which fallback data was used.
+
+--md, --markdown
+  Output a crawl summary in Markdown instead of a JSON report. The option takes
+  precedence over the \`--output\` option.
+
+-m, --module <modules...>
+  If processing modules are not specified, the crawler runs all core processing
+  modules defined in:
+    https://github.com/w3c/reffy/blob/main/src/browserlib/reffy.json
+
+  Modules must be specified using a relative path to an ".mjs" file that defines
+  the processing logic to run on the spec's page in a browser context. For
+  instance:
+    $ reffy --output reports/test --module extract-editors.mjs
+
+  Absolute paths to modules are not properly handled and will likely result in a
+  crawling error.
+
+  Multiple modules can be specified, repeating the option name or not:
+    $ reffy -o reports/test -m extract-words.mjs extract-editors.mjs
+    $ reffy -o reports/test -m extract-words.mjs -m extract-editors.mjs
+
+  Core processing modules may be referenced using the name of the extract folder
+  or property that they would create:
+    $ reffy --output reports/test --module dfns
+
+  To run all core processing modules, use "core". For instance, to apply a
+  processing module on top of core processing modules, use:
+    $ reffy --output reports/test --module core extract-editors.mjs
+
+  Each module must export a function that takes a spec object as input and
+  return a result that can be serialized as JSON. A typical module code looks
+  like:
+    https://github.com/w3c/reffy/blob/main/src/browserlib/extract-ids.mjs
+
+  Individual extracts will be created under "<folder>/[camelCaseModule]" where
+  "[camelCaseModule]" is derived from the module's filename. For instance:
+    "extract-editors.mjs" creates extracts under "<folder>/extractEditors"
+
+  The name of the folder where extracts get created may be specified for custom
+  modules by prefixing the path to the module with the folder name followed by
+  ":". For instance, to save extracts to "reports/test/editors", use:
+    $ reffy --output reports/test --module editors:extract-editors.mjs
+
+-o, --output <folder>
+  By default, crawl results are written to the console as a serialized JSON
+  array with one entry per spec, and module processing results attached as
+  property values in each of these entries.
+
+  If an output <folder> is specified, crawl results are rather saved to that
+  folder, with module processing results created under subfolders (see the
+  --module option) and linked from an index.json file created under <folder>.
+
+  Additionally, if an output <folder> is specified and if the IDL processing
+  module is run, the crawler will also creates an index of IDL names named
+  "idlnames.json" that links to relevant extracts in subfolders.
+
+  The folder targeted by <folder> must exist.
+
+-p, --post <modules...>
+  Post-processing modules either run after a spec is done crawling or after the
+  entire crawl is over. They allow developers to complete data based on other
+  extracts that were not available when extraction ran.
+
+  To run all core post-processing modules, use "core". Core post-processing
+  modules are defined in:
+    https://github.com/w3c/reffy/blob/main/src/postprocessing.js
+
+  The crawler does not run any post-processing modules by default.
+
+  Custom post-processing modules may be specified using a relative path to a
+  ".js" file that defines the post-processing logic. For instance:
+    $ reffy --output reports/test --post mypostprocessing.js
+
+  Each module must export a "run" function. See the post-processor's code for
+  details:
+    https://github.com/w3c/reffy/blob/main/src/lib/post-processor.js
+
+  Absolute paths to modules are not properly handled and will likely result in a
+  processing error.
+
+  Multiple post-processing modules can be specified, repeating the option name
+  or not:
+    $ reffy -o reports/test -p cssdfns cssidl events
+    $ reffy -o reports/test -p events -p idlparsed -p idlnames
+
+-r, --release
+  The crawler defaults to crawling the nightly version of requested specs.
+  Set this flag to tell the crawler to crawl the published version of the specs
+  instead. When the flag is set, the crawler will ignore specs that do not have
+  a published version.
+
+-s, --spec <specs...>
+  If specs to crawl are not specified, all specs in browser-specs that are not
+  identified as being discontinued get crawled:
+    https://github.com/w3c/browser-specs/
+
+  Valid spec values may be a shortname, a URL, or a relative path to a file that
+  contains a list of spec URLs and/or shortnames. All shortnames must exist in
+  browser-specs. Shortname may be the shortname of the spec series, in which
+  case the spec identified as the current specification in the series is used.
+  For instance, as of September 2021, "pointerlock" will map to "pointerlock-2"
+  because Pointer Lock 2.0 is the current level in the series.
+
+  Use "all" to include all specs in browser-specs in the crawl. For instance, to
+  crawl all specs plus one custom spec that does not exist in browser-specs:
+    $ reffy -o reports/test -s all https://example.org/myspec
+
+  When "all" is used, to force a crawl on some of the discontinued specs too,
+  include their shortname explicitly (or point to a JSON file that lists their
+  shortnames). For instance, to also crawl the discontinued DOM Level 2 Style
+  spec, run:
+    $ reffy -o reports/test -s all DOM-Level-2-Style
+
+--summary
+  Tells Reffy to attach a Markdown summary of the crawl per spec to the JSON
+  report, in a \`crawlSummary\` property. The Markdown report is suitable for
+  inclusion in a GitHub issue or similar. It starts with a summary, and then
+  details a few noteworthy extracts (CSS, dfns, Web IDL) in expandable
+  sections, with links to the online xref database search where appropriate.
+
+-t, --terse
+  This flag cannot be combined with the --output option and cannot be set if
+  more than one processing module gets run. When set, the crawler writes the
+  processing module results to the console directly without wrapping them with
+  spec metadata. In other words, the spec entry in the crawl results directly
+  contains the outcome of the processing module when the flag is set.
+
+  Additionally, if crawl runs on a single specification, the array is omitted
+  and the processing module results are thus written to the console directly.
+  For instance:
+    $ reffy --spec fetch --module idl --terse
+
+-u, --use-crawl <folder>
+  Tells Reffy to skip the crawl part and only run requested post-processing
+  modules on the crawl results present in the specified folder.
+
+  If post-processing modules are not specified, Reffy will merely copy the crawl
+  results to the output folder (or to the console).
+`);
+
+program.parse(process.argv);