reffy 6.2.0 → 6.2.1

package/src/cli/generate-idlparsed.js

@@ -1,139 +1,139 @@
-#!/usr/bin/env node
-/**
- * The parsed IDL generator takes a crawl report or a single spec as input, and
- * generates (or re-generates if it already exists) a parsed IDL structure from
- * the raw IDL that the spec defines. Result is dumped to the console or saved
- * to the given folder.
- * 
- * The parsed IDL generator is used by the crawler to create and save the parsed
- * IDL structures. It is also useful to re-generated the parsed IDL info when
- * an IDL patch has been applied to the raw IDL.
- *
- * The parsed IDL generator can be called directly through:
- *
- * `node generate-idlparsed.js [crawl report] [save folder]`
- *
- * where `crawl report` is the path to the folder that contains the
- * `index.json` file and all other crawl results produced by specs-crawler.js,
- * and `save folder` is an optional folder (which must exist) where IDL
- * name extracts are to be saved. In the absence of this parameter, the report
- * is written to the console.
- *
- * When a folder is provided, the IDL name extracts are saved as a JSON
- * structure in an `idlparsed` subfolder.
- */
-
-const fs = require('fs');
-const path = require('path');
-const webidlParser = require('../cli/parse-webidl');
-const {
-  expandCrawlResult,
-  requireFromWorkingDirectory,
-  createFolderIfNeeded
-} = require('../lib/util');
-
-
-/**
- * Update the spec object in place with parsed IDL information.
- * 
- * @function
- * @public
- * @param {Object} spec The spec object to update. The function looks for the
- *   raw IDL in the `idl` property.
- * @return {Object} The updated spec with an `idl` property that contains the
- *   parsed version of the IDL, and the raw IDL moved under the `idl.idl`
- *   sub-property. Note the spec object is updated in place.
- */
-async function generateIdlParsed(spec) {
-  if (!spec?.idl) {
-    return spec;
-  }
-  try {
-    spec.idlparsed = await webidlParser.parse(spec.idl);
-    spec.idlparsed.hasObsoleteIdl = webidlParser.hasObsoleteIdl(spec.idl);
-  }
-  catch (err) {
-    // IDL content is invalid and cannot be parsed.
-    // Let's return the error, along with the raw IDL
-    // content so that it may be saved to a file.
-    spec.idlparsed = err.toString();
-  }
-  return spec;
-}
-
-
-async function generateIdlParsedFromPath(crawlPath) {
-  const crawlIndex = requireFromWorkingDirectory(path.resolve(crawlPath, 'index.json'));
-  const crawlResults = await expandCrawlResult(crawlIndex, crawlPath, ['idl']);
-  await Promise.all(crawlResults.results.map(generateIdlParsed));
-  return crawlResults;
-}
-
-
-/**
- * Generate the `idlparsed` export for the spec.
- * 
- * Note that the raw IDL (under `spec.idl.idl`) gets deleted in the process.
- *
- * @function
- * @public
- * @param {Object} spec Spec object with the parsed IDL
- * @param {String} folder Path to root folder where `idlparsed` folder needs to
- *   appear.
- * @return {String} The relative path from the root folder to the generated file
- */
-async function saveIdlParsed(spec, folder) {
-  function specInfo(spec) {
-    return {
-      spec: {
-        title: spec.title,
-        url: spec.crawled
-      }
-    };
-  }
-
-  const subfolder = path.join(folder, 'idlparsed');
-  await createFolderIfNeeded(subfolder);
-
-  if (!spec?.idlparsed) {
-    return;
-  }
-
-  const json = JSON.stringify(
-    Object.assign(specInfo(spec), { idlparsed: spec.idlparsed }),
-    null, 2);
-  const filename = path.join(subfolder, spec.shortname + '.json');
-  await fs.promises.writeFile(filename, json);
-  return `idlparsed/${spec.shortname}.json`;
-}
-
-
-/**************************************************
-Export methods for use as module
-**************************************************/
-module.exports.generateIdlParsed = generateIdlParsed;
-module.exports.saveIdlParsed = saveIdlParsed;
-
-
-/**************************************************
-Code run if the code is run as a stand-alone module
-**************************************************/
-if (require.main === module) {
-  const crawlPath = process.argv[2];
-  if (!crawlPath) {
-    console.error('Required path to crawl results folder is missing');
-    process.exit(2);
-  }
-
-  const savePath = process.argv[3];
-  generateIdlParsedFromPath(crawlPath)
-    .then(report => {
-      if (savePath) {
-        return Promise.all(report.results.map(
-          spec => saveIdlParsed(spec, savePath)));
-      }
-      else {
-        console.log(JSON.stringify(report, null, 2));
-      }
-    });
-}
+#!/usr/bin/env node
+/**
+ * The parsed IDL generator takes a crawl report or a single spec as input, and
+ * generates (or re-generates if it already exists) a parsed IDL structure from
+ * the raw IDL that the spec defines. Result is dumped to the console or saved
+ * to the given folder.
+ * 
+ * The parsed IDL generator is used by the crawler to create and save the parsed
+ * IDL structures. It is also useful to re-generated the parsed IDL info when
+ * an IDL patch has been applied to the raw IDL.
+ *
+ * The parsed IDL generator can be called directly through:
+ *
+ * `node generate-idlparsed.js [crawl report] [save folder]`
+ *
+ * where `crawl report` is the path to the folder that contains the
+ * `index.json` file and all other crawl results produced by specs-crawler.js,
+ * and `save folder` is an optional folder (which must exist) where IDL
+ * name extracts are to be saved. In the absence of this parameter, the report
+ * is written to the console.
+ *
+ * When a folder is provided, the IDL name extracts are saved as a JSON
+ * structure in an `idlparsed` subfolder.
+ */
+
+const fs = require('fs');
+const path = require('path');
+const webidlParser = require('../cli/parse-webidl');
+const {
+  expandCrawlResult,
+  requireFromWorkingDirectory,
+  createFolderIfNeeded
+} = require('../lib/util');
+
+
+/**
+ * Update the spec object in place with parsed IDL information.
+ * 
+ * @function
+ * @public
+ * @param {Object} spec The spec object to update. The function looks for the
+ *   raw IDL in the `idl` property.
+ * @return {Object} The updated spec with an `idl` property that contains the
+ *   parsed version of the IDL, and the raw IDL moved under the `idl.idl`
+ *   sub-property. Note the spec object is updated in place.
+ */
+async function generateIdlParsed(spec) {
+  if (!spec?.idl) {
+    return spec;
+  }
+  try {
+    spec.idlparsed = await webidlParser.parse(spec.idl);
+    spec.idlparsed.hasObsoleteIdl = webidlParser.hasObsoleteIdl(spec.idl);
+  }
+  catch (err) {
+    // IDL content is invalid and cannot be parsed.
+    // Let's return the error, along with the raw IDL
+    // content so that it may be saved to a file.
+    spec.idlparsed = err.toString();
+  }
+  return spec;
+}
+
+
+async function generateIdlParsedFromPath(crawlPath) {
+  const crawlIndex = requireFromWorkingDirectory(path.resolve(crawlPath, 'index.json'));
+  const crawlResults = await expandCrawlResult(crawlIndex, crawlPath, ['idl']);
+  await Promise.all(crawlResults.results.map(generateIdlParsed));
+  return crawlResults;
+}
+
+
+/**
+ * Generate the `idlparsed` export for the spec.
+ * 
+ * Note that the raw IDL (under `spec.idl.idl`) gets deleted in the process.
+ *
+ * @function
+ * @public
+ * @param {Object} spec Spec object with the parsed IDL
+ * @param {String} folder Path to root folder where `idlparsed` folder needs to
+ *   appear.
+ * @return {String} The relative path from the root folder to the generated file
+ */
+async function saveIdlParsed(spec, folder) {
+  function specInfo(spec) {
+    return {
+      spec: {
+        title: spec.title,
+        url: spec.crawled
+      }
+    };
+  }
+
+  const subfolder = path.join(folder, 'idlparsed');
+  await createFolderIfNeeded(subfolder);
+
+  if (!spec?.idlparsed) {
+    return;
+  }
+
+  const json = JSON.stringify(
+    Object.assign(specInfo(spec), { idlparsed: spec.idlparsed }),
+    null, 2);
+  const filename = path.join(subfolder, spec.shortname + '.json');
+  await fs.promises.writeFile(filename, json);
+  return `idlparsed/${spec.shortname}.json`;
+}
+
+
+/**************************************************
+Export methods for use as module
+**************************************************/
+module.exports.generateIdlParsed = generateIdlParsed;
+module.exports.saveIdlParsed = saveIdlParsed;
+
+
+/**************************************************
+Code run if the code is run as a stand-alone module
+**************************************************/
+if (require.main === module) {
+  const crawlPath = process.argv[2];
+  if (!crawlPath) {
+    console.error('Required path to crawl results folder is missing');
+    process.exit(2);
+  }
+
+  const savePath = process.argv[3];
+  generateIdlParsedFromPath(crawlPath)
+    .then(report => {
+      if (savePath) {
+        return Promise.all(report.results.map(
+          spec => saveIdlParsed(spec, savePath)));
+      }
+      else {
+        console.log(JSON.stringify(report, null, 2));
+      }
+    });
+}

package/src/cli/merge-crawl-results.js

@@ -1,128 +1,128 @@
-#!/usr/bin/env node
-/**
- * The crawl report merger can be used to merge a new crawl report into a
- * reference one. This tool is typically useful to make incremental updates to a
- * reference crawl, used as knowledge database. It replaces the crawl results of
- * a given spec by the new results where appropriate.
- *
- * The crawl report merge can be called directly through:
- *
- * `node merge-crawl-results.js [new report] [ref report] [merged report]`
- *
- * where `new report` is the name of the new report to merge into `ref report`
- * to produce the `merged report` file.
- *
- * @module merger
- */
-
-const fs = require('fs');
-const requireFromWorkingDirectory = require('../lib/util').requireFromWorkingDirectory;
-
-
-/**
- * Compares specs for ordering by URL
- */
-const byURL = (a, b) => a.url.localeCompare(b.url);
-
-
-/**
- * Merge given crawl results with the given reference crawl results and return
- * the new results.
- *
- * @function
- * @param {String} newCrawl The crawl results to merge
- * @param {String} refCrawl The reference crawl results
- * @param {Object} options Merge options. Only "matchTitle" is supported for now
- * @return {Promise} The promise to get a new crawl results that contains the
- *   results of the merge
- */
-function mergeCrawlResults(newCrawl, refCrawl, options) {
-    options = options || {};
-
-    let newResults = newCrawl.results || [];
-    let refResults = refCrawl.results || [];
-
-    let results = refResults.filter(refSpec => !newResults.some(newSpec =>
-        (refSpec.url && newSpec.url && (refSpec.url === newSpec.url)) ||
-        (refSpec.html && newSpec.html && (refSpec.html === newSpec.html)) ||
-        (refSpec.latest && newSpec.latest && (refSpec.latest === newSpec.latest)) ||
-        (refSpec.shortname && newSpec.shortname && (refSpec.shortname === newSpec.shortname)) ||
-        (refSpec.versions && newSpec.versions &&
-            refSpec.versions.some(refVersion => newSpec.versions.some(newVersion => (refVersion === newVersion)))) ||
-        (options.matchTitle && refSpec.title && newSpec.title && (refSpec.title === newSpec.title))
-    )).concat(newResults);
-
-    let crawlData = {};
-    crawlData.title = newCrawl.title || refCrawl.title || 'Reffy crawl';
-    if (newCrawl.description || refCrawl.description) {
-        crawlData.description = newCrawl.description || refCrawl.description;
-    }
-    crawlData.date = (new Date()).toJSON();
-    crawlData.stats = {};
-    crawlData.results = results;
-    crawlData.results.sort(byURL);
-    crawlData.stats = {
-        crawled: crawlData.results.length,
-        errors: crawlData.results.filter(spec => !!spec.error).length
-    };
-
-    return Promise.resolve(crawlData);
-}
-
-
-/**
- * Merge the crawl results in the first JSON file with the crawl results in the
- * second JSON file, and create a third JSON file with the results.
- *
- * @function
- * @param {String} newCrawlPath The JSON file that contains the results to merge
- * @param {String} refCrawlPath The JSON file that contains the reference results
- * @param {String} resPath The JSON file that will contain the result of the merge
- * @param {Object} options Merge options. Only "matchTitle" is supported for now
- * @return {Promise} The promise to have merged the two JSON files into one
- */
-function mergeCrawlFiles(newCrawlPath, refCrawlPath, resPath, options) {
-    options = options || {};
-
-    let newCrawl = requireFromWorkingDirectory(newCrawlPath);
-    let refCrawl = requireFromWorkingDirectory(refCrawlPath);
-    return mergeCrawlResults(newCrawl, refCrawl, options)
-        .then(filedata => new Promise((resolve, reject) =>
-            fs.writeFile(resPath, JSON.stringify(filedata, null, 2),
-                     err => { if (err) return reject(err); resolve(); })))
-}
-
-
-/**************************************************
-Export the methods for use as module
-**************************************************/
-module.exports.mergeCrawlResults = mergeCrawlResults;
-module.exports.mergeCrawlFiles = mergeCrawlFiles;
-
-
-/**************************************************
-Code run if the code is run as a stand-alone module
-**************************************************/
-if (require.main === module) {
-    let newCrawlPath = process.argv[2];
-    let refCrawlPath = process.argv[3];
-    let resPath = process.argv[4];
-    if (!newCrawlPath || !refCrawlPath || !resPath) {
-        console.error('Command needs 3 filename parameters:');
-        console.error(' 1. the crawl results to merge into the reference crawl results');
-        console.error(' 2. the reference crawl results');
-        console.error(' 3. where to save the result of the merge');
-        process.exit(2);
-    }
-    let mergeOptions = {
-        matchTitle: true
-    };
-
-    console.log('Merging crawl files into: ' + resPath);
-    mergeCrawlFiles(newCrawlPath, refCrawlPath, resPath, mergeOptions)
-        .then(_ => console.log('Finished'))
-        .catch(err => {
-            console.error(err);
-            process.exit(64)
-        });
-}
+#!/usr/bin/env node
+/**
+ * The crawl report merger can be used to merge a new crawl report into a
+ * reference one. This tool is typically useful to make incremental updates to a
+ * reference crawl, used as knowledge database. It replaces the crawl results of
+ * a given spec by the new results where appropriate.
+ *
+ * The crawl report merge can be called directly through:
+ *
+ * `node merge-crawl-results.js [new report] [ref report] [merged report]`
+ *
+ * where `new report` is the name of the new report to merge into `ref report`
+ * to produce the `merged report` file.
+ *
+ * @module merger
+ */
+
+const fs = require('fs');
+const requireFromWorkingDirectory = require('../lib/util').requireFromWorkingDirectory;
+
+
+/**
+ * Compares specs for ordering by URL
+ */
+const byURL = (a, b) => a.url.localeCompare(b.url);
+
+
+/**
+ * Merge given crawl results with the given reference crawl results and return
+ * the new results.
+ *
+ * @function
+ * @param {String} newCrawl The crawl results to merge
+ * @param {String} refCrawl The reference crawl results
+ * @param {Object} options Merge options. Only "matchTitle" is supported for now
+ * @return {Promise} The promise to get a new crawl results that contains the
+ *   results of the merge
+ */
+function mergeCrawlResults(newCrawl, refCrawl, options) {
+    options = options || {};
+
+    let newResults = newCrawl.results || [];
+    let refResults = refCrawl.results || [];
+
+    let results = refResults.filter(refSpec => !newResults.some(newSpec =>
+        (refSpec.url && newSpec.url && (refSpec.url === newSpec.url)) ||
+        (refSpec.html && newSpec.html && (refSpec.html === newSpec.html)) ||
+        (refSpec.latest && newSpec.latest && (refSpec.latest === newSpec.latest)) ||
+        (refSpec.shortname && newSpec.shortname && (refSpec.shortname === newSpec.shortname)) ||
+        (refSpec.versions && newSpec.versions &&
+            refSpec.versions.some(refVersion => newSpec.versions.some(newVersion => (refVersion === newVersion)))) ||
+        (options.matchTitle && refSpec.title && newSpec.title && (refSpec.title === newSpec.title))
+    )).concat(newResults);
+
+    let crawlData = {};
+    crawlData.title = newCrawl.title || refCrawl.title || 'Reffy crawl';
+    if (newCrawl.description || refCrawl.description) {
+        crawlData.description = newCrawl.description || refCrawl.description;
+    }
+    crawlData.date = (new Date()).toJSON();
+    crawlData.stats = {};
+    crawlData.results = results;
+    crawlData.results.sort(byURL);
+    crawlData.stats = {
+        crawled: crawlData.results.length,
+        errors: crawlData.results.filter(spec => !!spec.error).length
+    };
+
+    return Promise.resolve(crawlData);
+}
+
+
+/**
+ * Merge the crawl results in the first JSON file with the crawl results in the
+ * second JSON file, and create a third JSON file with the results.
+ *
+ * @function
+ * @param {String} newCrawlPath The JSON file that contains the results to merge
+ * @param {String} refCrawlPath The JSON file that contains the reference results
+ * @param {String} resPath The JSON file that will contain the result of the merge
+ * @param {Object} options Merge options. Only "matchTitle" is supported for now
+ * @return {Promise} The promise to have merged the two JSON files into one
+ */
+function mergeCrawlFiles(newCrawlPath, refCrawlPath, resPath, options) {
+    options = options || {};
+
+    let newCrawl = requireFromWorkingDirectory(newCrawlPath);
+    let refCrawl = requireFromWorkingDirectory(refCrawlPath);
+    return mergeCrawlResults(newCrawl, refCrawl, options)
+        .then(filedata => new Promise((resolve, reject) =>
+            fs.writeFile(resPath, JSON.stringify(filedata, null, 2),
+                     err => { if (err) return reject(err); resolve(); })))
+}
+
+
+/**************************************************
+Export the methods for use as module
+**************************************************/
+module.exports.mergeCrawlResults = mergeCrawlResults;
+module.exports.mergeCrawlFiles = mergeCrawlFiles;
+
+
+/**************************************************
+Code run if the code is run as a stand-alone module
+**************************************************/
+if (require.main === module) {
+    let newCrawlPath = process.argv[2];
+    let refCrawlPath = process.argv[3];
+    let resPath = process.argv[4];
+    if (!newCrawlPath || !refCrawlPath || !resPath) {
+        console.error('Command needs 3 filename parameters:');
+        console.error(' 1. the crawl results to merge into the reference crawl results');
+        console.error(' 2. the reference crawl results');
+        console.error(' 3. where to save the result of the merge');
+        process.exit(2);
+    }
+    let mergeOptions = {
+        matchTitle: true
+    };
+
+    console.log('Merging crawl files into: ' + resPath);
+    mergeCrawlFiles(newCrawlPath, refCrawlPath, resPath, mergeOptions)
+        .then(_ => console.log('Finished'))
+        .catch(err => {
+            console.error(err);
+            process.exit(64)
+        });
+}