reffy 7.2.10 → 8.0.2

package/src/{cli/generate-idlnames.js → postprocessing/idlnames.js}

@@ -1,36 +1,31 @@
-#!/usr/bin/env node
 /**
- * The IDL names generator takes a crawl report as input and creates a report
- * per referenceable IDL name, that details the complete parsed IDL structure
- * that defines the name across all specs.
+ * Post-processing module that creates an index of IDL names and a list of
+ * IDL extracts per IDL name.
  *
- * The spec checker can be called directly through:
- *
- * `node generate-idlnames.js [crawl report] [dfns] [save folder]`
- *
- * where `crawl report` is the path to the folder that contains the
- * `index.json` file and all other crawl results produced by specs-crawler.js,
- * `dfns` a param to set to "true" or "dfns" to embed dfns in the generated
- * report, and `save folder` is an optional folder (which must exist) where IDL
- * name extracts are to be saved. In the absence of this parameter, the report
- * is written to the console.
- *
- * When a folder is provided, the IDL name extracts are saved as a JSON
- * structure in an "idlnamesparsed" subfolder, and as IDL fragments in an
- * "idlnames" folder.
- *
- * @module checker
+ * The module runs at the crawl level. It depends on another post-processing
+ * module, namely the "idlparsed" one, which runs at the spec level (so no need
+ * to worry about ordering, "idlparsed" will always run before this one).
  */
 
 const fs = require('fs');
 const path = require('path');
-const { matchIdlDfn, getExpectedDfnFromIdlDesc } = require('./check-missing-dfns');
 const {
-  expandCrawlResult,
+  matchIdlDfn,
+  getExpectedDfnFromIdlDesc } = require('../cli/check-missing-dfns');
+const {
   isLatestLevelThatPasses,
-  requireFromWorkingDirectory,
-  createFolderIfNeeded
-} = require('../lib/util');
+  createFolderIfNeeded } = require('../lib/util');
+
+
+/**
+ * Definition of the post-processing module
+ */
+module.exports = {
+  dependsOn: ['idlparsed', 'dfns'],
+  input: 'crawl',
+  run: generateIdlNames,
+  save: saveIdlNames
+};
 
 
 /**
@@ -92,12 +87,83 @@ function getRelatedDfns(desc, idlNode, results) {
 }
 
 
+/**
+ * Save IDL names to individual JSON files in the given folder
+ *
+ * @function
+ * @private
+ * @param {Object} names Report generated by generateIdlNames
+ * @param {String} folder Path to folder
+ */
+async function saveParsedIdlNames(names, folder) {
+  await createFolderIfNeeded(folder);
+  await Promise.all(Object.entries(names).map(([name, idl]) => {
+    const json = JSON.stringify(idl, null, 2);
+    const filename = path.join(folder, name + '.json');
+    return fs.promises.writeFile(filename, json);
+  }));
+}
+
+
+
+/**
+ * Save IDL fragments to individual text files in the given folder
+ *
+ * @function
+ * @private
+ * @param {Object} names Report generated by generateIdlNames
+ * @param {String} folder Path to folder
+ */
+async function saveIdlNamesFragments(names, folder) {
+  function serializeNode(node) {
+    return `// Source: ${node.spec.title} (${node.spec.url})\n` +
+      node.fragment;
+  }
+
+  await createFolderIfNeeded(folder);
+  await Promise.all(Object.entries(names).map(([name, idl]) => {
+    const res = [];
+    if (idl.defined) {
+      res.push(serializeNode(idl.defined));
+    }
+    if (idl.extended) {
+      idl.extended.map(node => res.push(serializeNode(node)));
+    }
+    const filename = path.join(folder, name + '.idl');
+    return fs.promises.writeFile(filename, res.join('\n\n'));
+  }));
+}
+
+
+/**
+ * Generate an `idlnames.json` index file that contains all IDL names with
+ * pointers to files in idlnames and idlnamesparsed subfolder.
+ *
+ * @function
+ * @private
+ * @param {Object} names Report generated by generateIdlNames
+ * @param {String} folder Path to folder where index file is to be saved
+ */
+async function saveIndex(names, folder) {
+  await createFolderIfNeeded(folder);
+  let index = {};
+  Object.keys(names).sort().forEach(name => {
+    index[name] = {
+      fragment: `idlnames/${name}.idl`,
+      parsed: `idlnamesparsed/${name}.json`
+    };
+  });
+  const filename = path.join(folder, 'idlnames.json');
+  return fs.promises.writeFile(filename, JSON.stringify(index, null, 2));
+}
+
+
 /**
  * Generate a report per referenceable IDL name from the given crawl results.
  *
  * @function
  * @public
- * @param {Array} results The list of spec crawl results to process
+ * @param {Object} crawl Crawl results
  * @param {Object} options Generation options. Set "dfns" to true to embed
  *   definitions in the final export.
  * @return {Object} A list indexed by referenceable IDL name that details, for
@@ -105,7 +171,7 @@ function getRelatedDfns(desc, idlNode, results) {
  *   along with links to the actual definition of the terms in the specs
  *   (when known).
  */
-function generateIdlNames(results, options = {}) {
+async function generateIdlNames(crawl, options) {
   function specInfo(spec) {
     return {
       spec: {
@@ -123,8 +189,8 @@ function generateIdlNames(results, options = {}) {
   }
 
   // Only keep latest version of specs and delta specs that define some IDL
-  results = results.filter(spec =>
-    (spec.seriesComposition !== 'delta' && isLatestLevelThatPasses(spec, results, defineIDLContent)) ||
+  const results = crawl.results.filter(spec =>
+    (spec.seriesComposition !== 'delta' && isLatestLevelThatPasses(spec, crawl.results, defineIDLContent)) ||
     (spec.seriesComposition === 'delta' && defineIDLContent(spec)));
 
   // Add main definitions of all IDL names
@@ -302,84 +368,6 @@ function generateIdlNames(results, options = {}) {
 }
 
 
-async function generateIdlNamesFromPath(crawlPath, options = {}) {
-  const crawlIndex = requireFromWorkingDirectory(path.resolve(crawlPath, 'index.json'));
-  const crawlResults = await expandCrawlResult(crawlIndex, crawlPath, ['idlparsed', 'dfns']);
-  return generateIdlNames(crawlResults.results, options);
-}
-
-
-/**
- * Save IDL names to individual JSON files in the given folder
- *
- * @function
- * @private
- * @param {Object} names Report generated by generateIdlNames
- * @param {String} folder Path to folder
- */
-async function saveParsedIdlNames(names, folder) {
-  await createFolderIfNeeded(folder);
-  await Promise.all(Object.entries(names).map(([name, idl]) => {
-    const json = JSON.stringify(idl, null, 2);
-    const filename = path.join(folder, name + '.json');
-    return fs.promises.writeFile(filename, json);
-  }));
-}
-
-
-
-/**
- * Save IDL fragments to individual text files in the given folder
- *
- * @function
- * @private
- * @param {Object} names Report generated by generateIdlNames
- * @param {String} folder Path to folder
- */
-async function saveIdlNamesFragments(names, folder) {
-  function serializeNode(node) {
-    return `// Source: ${node.spec.title} (${node.spec.url})\n` +
-      node.fragment;
-  }
-
-  await createFolderIfNeeded(folder);
-  await Promise.all(Object.entries(names).map(([name, idl]) => {
-    const res = [];
-    if (idl.defined) {
-      res.push(serializeNode(idl.defined));
-    }
-    if (idl.extended) {
-      idl.extended.map(node => res.push(serializeNode(node)));
-    }
-    const filename = path.join(folder, name + '.idl');
-    return fs.promises.writeFile(filename, res.join('\n\n'));
-  }));
-}
-
-
-/**
- * Generate an `idlnames.json` index file that contains all IDL names with
- * pointers to files in idlnames and idlnamesparsed subfolder.
- *
- * @function
- * @private
- * @param {Object} names Report generated by generateIdlNames
- * @param {String} folder Path to folder where index file is to be saved
- */
-async function saveIndex(names, folder) {
-  await createFolderIfNeeded(folder);
-  let index = {};
-  Object.keys(names).sort().forEach(name => {
-    index[name] = {
-      fragment: `idlnames/${name}.idl`,
-      parsed: `idlnamesparsed/${name}.json`
-    };
-  });
-  const filename = path.join(folder, 'idlnames.json');
-  return fs.promises.writeFile(filename, JSON.stringify(index, null, 2));
-}
-
-
 /**
  * Generate all IDL names exports: IDL fragments in `idlnames`, parsed
  * structures in `idlnamesparsed`, and index file.
@@ -387,44 +375,15 @@ async function saveIndex(names, folder) {
  * @function
  * @public
  * @param {Object} names Report generated by generateIdlNames
- * @param {String} folder Path to folder where index file is to be saved
+ * @param {Object} options Crawl options ("output" will be used)
  */
-async function saveIdlNames(names, folder) {
+async function saveIdlNames(names, options) {
+  if (!options?.output) {
+    return;
+  }
+
+  const folder = options.output;
   await saveParsedIdlNames(names, path.join(folder, 'idlnamesparsed'));
   await saveIdlNamesFragments(names, path.join(folder, 'idlnames'));
   await saveIndex(names, folder);
 }
-
-
-
-
-
-/**************************************************
-Export methods for use as module
-**************************************************/
-module.exports.generateIdlNames = generateIdlNames;
-module.exports.saveIdlNames = saveIdlNames;
-
-
-/**************************************************
-Code run if the code is run as a stand-alone module
-**************************************************/
-if (require.main === module) {
-  const crawlPath = process.argv[2];
-  if (!crawlPath) {
-    console.error('Required path to crawl results folder is missing');
-    process.exit(2);
-  }
-
-  const dfns = process.argv[3] === 'dfns' || process.argv[3] === 'true';
-  const savePath = process.argv[4];
-  generateIdlNamesFromPath(crawlPath, { dfns })
-    .then(report => {
-      if (savePath) {
-        return saveIdlNames(report, savePath);
-      }
-      else {
-        console.log(JSON.stringify(report, null, 2));
-      }
-    });
-}

package/src/postprocessing/idlparsed.js

@@ -0,0 +1,31 @@
+/**
+ * Post-processing module that creates a parsed IDL structure out of the IDL
+ * extract.
+ * 
+ * The module runs at the spec level and generates an `idlparsed` property.
+ */
+
+const webidlParser = require('../cli/parse-webidl');
+
+module.exports = {
+  dependsOn: ['idl'],
+  input: 'spec',
+  property: 'idlparsed',
+
+  run: async function(spec, options) {
+    if (!spec?.idl) {
+      return spec;
+    }
+    try {
+      spec.idlparsed = await webidlParser.parse(spec.idl);
+      spec.idlparsed.hasObsoleteIdl = webidlParser.hasObsoleteIdl(spec.idl);
+    }
+    catch (err) {
+      // IDL content is invalid and cannot be parsed.
+      // Let's return the error, along with the raw IDL
+      // content so that it may be saved to a file.
+      spec.idlparsed = err.toString();
+    }
+    return spec;
+  }
+};

package/src/cli/generate-idlparsed.js

@@ -1,139 +0,0 @@
-#!/usr/bin/env node
-/**
- * The parsed IDL generator takes a crawl report or a single spec as input, and
- * generates (or re-generates if it already exists) a parsed IDL structure from
- * the raw IDL that the spec defines. Result is dumped to the console or saved
- * to the given folder.
- * 
- * The parsed IDL generator is used by the crawler to create and save the parsed
- * IDL structures. It is also useful to re-generated the parsed IDL info when
- * an IDL patch has been applied to the raw IDL.
- *
- * The parsed IDL generator can be called directly through:
- *
- * `node generate-idlparsed.js [crawl report] [save folder]`
- *
- * where `crawl report` is the path to the folder that contains the
- * `index.json` file and all other crawl results produced by specs-crawler.js,
- * and `save folder` is an optional folder (which must exist) where IDL
- * name extracts are to be saved. In the absence of this parameter, the report
- * is written to the console.
- *
- * When a folder is provided, the IDL name extracts are saved as a JSON
- * structure in an `idlparsed` subfolder.
- */
-
-const fs = require('fs');
-const path = require('path');
-const webidlParser = require('../cli/parse-webidl');
-const {
-  expandCrawlResult,
-  requireFromWorkingDirectory,
-  createFolderIfNeeded
-} = require('../lib/util');
-
-
-/**
- * Update the spec object in place with parsed IDL information.
- * 
- * @function
- * @public
- * @param {Object} spec The spec object to update. The function looks for the
- *   raw IDL in the `idl` property.
- * @return {Object} The updated spec with an `idl` property that contains the
- *   parsed version of the IDL, and the raw IDL moved under the `idl.idl`
- *   sub-property. Note the spec object is updated in place.
- */
-async function generateIdlParsed(spec) {
-  if (!spec?.idl) {
-    return spec;
-  }
-  try {
-    spec.idlparsed = await webidlParser.parse(spec.idl);
-    spec.idlparsed.hasObsoleteIdl = webidlParser.hasObsoleteIdl(spec.idl);
-  }
-  catch (err) {
-    // IDL content is invalid and cannot be parsed.
-    // Let's return the error, along with the raw IDL
-    // content so that it may be saved to a file.
-    spec.idlparsed = err.toString();
-  }
-  return spec;
-}
-
-
-async function generateIdlParsedFromPath(crawlPath) {
-  const crawlIndex = requireFromWorkingDirectory(path.resolve(crawlPath, 'index.json'));
-  const crawlResults = await expandCrawlResult(crawlIndex, crawlPath, ['idl']);
-  await Promise.all(crawlResults.results.map(generateIdlParsed));
-  return crawlResults;
-}
-
-
-/**
- * Generate the `idlparsed` export for the spec.
- * 
- * Note that the raw IDL (under `spec.idl.idl`) gets deleted in the process.
- *
- * @function
- * @public
- * @param {Object} spec Spec object with the parsed IDL
- * @param {String} folder Path to root folder where `idlparsed` folder needs to
- *   appear.
- * @return {String} The relative path from the root folder to the generated file
- */
-async function saveIdlParsed(spec, folder) {
-  function specInfo(spec) {
-    return {
-      spec: {
-        title: spec.title,
-        url: spec.crawled
-      }
-    };
-  }
-
-  const subfolder = path.join(folder, 'idlparsed');
-  await createFolderIfNeeded(subfolder);
-
-  if (!spec?.idlparsed) {
-    return;
-  }
-
-  const json = JSON.stringify(
-    Object.assign(specInfo(spec), { idlparsed: spec.idlparsed }),
-    null, 2);
-  const filename = path.join(subfolder, spec.shortname + '.json');
-  await fs.promises.writeFile(filename, json);
-  return `idlparsed/${spec.shortname}.json`;
-}
-
-
-/**************************************************
-Export methods for use as module
-**************************************************/
-module.exports.generateIdlParsed = generateIdlParsed;
-module.exports.saveIdlParsed = saveIdlParsed;
-
-
-/**************************************************
-Code run if the code is run as a stand-alone module
-**************************************************/
-if (require.main === module) {
-  const crawlPath = process.argv[2];
-  if (!crawlPath) {
-    console.error('Required path to crawl results folder is missing');
-    process.exit(2);
-  }
-
-  const savePath = process.argv[3];
-  generateIdlParsedFromPath(crawlPath)
-    .then(report => {
-      if (savePath) {
-        return Promise.all(report.results.map(
-          spec => saveIdlParsed(spec, savePath)));
-      }
-      else {
-        console.log(JSON.stringify(report, null, 2));
-      }
-    });
-}