reffy 16.1.1 → 17.0.0

package/index.js

@@ -1,10 +1,19 @@
-module.exports = {
-  parseIdl: require("./src/cli/parse-webidl").parse,
-  crawlSpecs: require("./src/lib/specs-crawler").crawlSpecs,
-  expandCrawlResult: require("./src/lib/util").expandCrawlResult,
-  mergeCrawlResults: require("./src/lib/util").mergeCrawlResults,
-  isLatestLevelThatPasses: require("./src/lib/util").isLatestLevelThatPasses,
-  getInterfaceTreeInfo: require("./src/lib/util").getInterfaceTreeInfo,
-  getSchemaValidationFunction: require("./src/lib/util").getSchemaValidationFunction,
-  postProcessor: require("./src/lib/post-processor")
-};
+import { parseIdl } from "./src/cli/parse-webidl.js";
+import { crawlSpecs } from "./src/lib/specs-crawler.js";
+import { expandCrawlResult } from "./src/lib/util.js";
+import { mergeCrawlResults } from "./src/lib/util.js";
+import { isLatestLevelThatPasses } from "./src/lib/util.js";
+import { getInterfaceTreeInfo } from "./src/lib/util.js";
+import { getSchemaValidationFunction } from "./src/lib/util.js";
+import postProcessor from "./src/lib/post-processor.js";
+
+export {
+  parseIdl,
+  crawlSpecs,
+  expandCrawlResult,
+  mergeCrawlResults,
+  isLatestLevelThatPasses,
+  getInterfaceTreeInfo,
+  getSchemaValidationFunction,
+  postProcessor
+};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "reffy",
-  "version": "16.1.1",
+  "version": "17.0.0",
   "description": "W3C/WHATWG spec dependencies exploration companion. Features a short set of tools to study spec references as well as WebIDL term definitions and references found in W3C specifications.",
   "repository": {
     "type": "git",
@@ -29,23 +29,24 @@
   "engines": {
     "node": ">=20.12.1"
   },
+  "type": "module",
   "main": "index.js",
   "bin": "./reffy.js",
   "dependencies": {
-    "ajv": "8.16.0",
+    "ajv": "8.17.1",
     "ajv-formats": "3.0.1",
     "commander": "12.1.0",
     "fetch-filecache-for-crawling": "5.1.1",
-    "puppeteer": "22.13.0",
+    "puppeteer": "22.13.1",
     "semver": "^7.3.5",
     "web-specs": "3.13.1",
     "webidl2": "24.4.1"
   },
   "devDependencies": {
-    "mocha": "10.6.0",
+    "mocha": "10.7.0",
     "respec": "35.1.1",
     "respec-hljs": "2.1.1",
-    "rollup": "4.18.1",
+    "rollup": "4.19.0",
     "undici": "^6.1.0"
   },
   "overrides": {

package/reffy.js

@@ -21,16 +21,17 @@
  * @module crawler
  */
 
-const commander = require('commander');
-const satisfies = require('semver/functions/satisfies');
-const specs = require('web-specs');
-const { version, engines } = require('./package.json');
-const { requireFromWorkingDirectory } = require('./src/lib/util');
-const { crawlSpecs } = require('./src/lib/specs-crawler');
-const { modules } = require('./src/lib/post-processor');
+import { Command } from 'commander';
+import satisfies from 'semver/functions/satisfies.js';
+import specs from 'web-specs' with { type: 'json' };
+import packageConfig from './package.json' with { type: 'json' };
+import { crawlSpecs } from './src/lib/specs-crawler.js';
+import { modules } from './src/lib/post-processor.js';
+import { loadJSON } from './src/lib/util.js';
 
 // Warn if version of Node.js does not satisfy requirements
-if (engines && engines.node && !satisfies(process.version, engines.node)) {
+if (packageConfig.engines && packageConfig.engines.node &&
+    !satisfies(process.version, packageConfig.engines.node)) {
   console.warn(`
 [WARNING] Node.js ${process.version} detected but Reffy needs Node.js ${engines.node}.
           Please consider upgrading Node.js if the program crashes!`);
@@ -54,14 +55,14 @@ function parseModuleOption(input) {
     }
 }
 
-function parseSpecOption(input) {
+async function parseSpecOption(input) {
     if (input === 'all') {
         return specs
             .filter(s => s.standing !== 'discontinued')
             .map(s => s.shortname)
     }
     else {
-        const list = requireFromWorkingDirectory(input);
+        const list = await loadJSON(input);
         return list ?? input;
     }
 }
@@ -76,9 +77,9 @@ function parsePostOption(input) {
 }
 
 
-const program = new commander.Command();
+const program = new Command();
 program
-    .version(version)
+    .version(packageConfig.version)
     .usage('[options]')
     .description('Crawls and processes a list of Web specifications')
     .option('-d, --debug', 'debug mode, crawl one spec at a time')
@@ -91,7 +92,7 @@ program
     .option('-s, --spec <specs...>', 'specs to crawl')
     .option('-t, --terse', 'output crawl results without metadata')
     .option('-u, --use-crawl <folder>', 'use given crawl result folder as input for post-processing')
-    .action(options => {
+    .action(async options => {
         if (!(options.output || options.module || options.spec || options.useCrawl)) {
           console.error(`
 At least one of the --output, --module, --spec or --use-crawl options needs to be
@@ -118,10 +119,10 @@ will dump ~100MB of data to the console:
             crawlOptions.modules = options.module.map(parseModuleOption);
         }
         if (options.spec) {
-            crawlOptions.specs = options.spec.map(parseSpecOption).flat();
+            crawlOptions.specs = (await Promise.all(options.spec.map(parseSpecOption))).flat();
         }
         else {
-            crawlOptions.specs = parseSpecOption('all');
+            crawlOptions.specs = await parseSpecOption('all');
         }
         if (options.post) {
             crawlOptions.post = options.post.map(parsePostOption).flat();

package/src/browserlib/extract-algorithms.mjs

@@ -404,14 +404,17 @@ function getDefinedNameIn(el) {
  */
 function findIntroParagraph(algo) {
   let paragraph;
-  let container = algo.root.closest('.algorithm');
+  let container = algo.root.closest('li,.algorithm');
   while (container) {
     const dfn = container.querySelector('dfn');
-    if (dfn) {
-      paragraph = dfn.closest('p,div');
+    if (dfn && !algo.root.contains(dfn)) {
+      paragraph = dfn.closest('p,div,li');
+      break;
+    }
+    if (container.nodeName === 'LI') {
       break;
     }
-    container = container.parentElement.closest('.algorithm');
+    container = container.parentElement.closest('li,.algorithm');
   }
 
   if (!paragraph) {
@@ -441,9 +444,10 @@ function getAlgorithmInfo(algo, context) {
   // Note some specs add the "algorithm" class to the `<ol>` and to the
   // wrapping container, and define the name in the wrapping container.
   let info = {};
+
   let container = algo.root.closest('.algorithm');
-  while (container) {
-    if (container && !context?.nested) {
+  if (!context?.nested) {
+    while (container) {
       if (container.getAttribute('data-algorithm')) {
         info.name = normalize(container.getAttribute('data-algorithm'));
         if (container.getAttribute('data-algorithm-for')) {
@@ -462,13 +466,15 @@ function getAlgorithmInfo(algo, context) {
             info.href = dfn.href;
           }
         }
-        break;
       }
       else {
         info = getDefinedNameIn(container);
+        if (info.name || info.href) {
+          break;
+        }
       }
+      container = container.parentElement.closest('.algorithm');
     }
-    container = container.parentElement.closest('.algorithm');
   }
 
   // Get the introductory prose from the previous paragraph

package/src/cli/check-missing-dfns.js

@@ -22,7 +22,10 @@
  * @module checker
  */
 
-const path = require('path');
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import process from 'node:process';
+import { loadJSON } from '../lib/util.js';
 
 /**
  * List of spec shortnames that, so far, don't follow the dfns data model
@@ -358,19 +361,19 @@ function matchIdlDfn(expected, actual,
  *   an array of missing CSS or IDL definitions. The function returns null when
  *   there are no missing definitions.
  */
-function checkSpecDefinitions(spec, options = {}) {
+async function checkSpecDefinitions(spec, options = {}) {
   if (!options.includeObsolete && specsWithObsoleteDfnsModel.includes(spec.shortname)) {
     return { obsoleteDfnsModel: true };
   }
 
   const dfns = (typeof spec.dfns === "string") ?
-    require(path.resolve(options.rootFolder, spec.dfns)).dfns :
+    (await loadJSON(path.resolve(options.rootFolder, spec.dfns))).dfns :
     (spec.dfns || []);
   const css = (typeof spec.css === "string") ?
-    require(path.resolve(options.rootFolder, spec.css)) :
+    (await loadJSON(path.resolve(options.rootFolder, spec.css))) :
     (spec.css || {});
   const idl = (typeof spec.idlparsed === "string") ?
-    require(path.resolve(options.rootFolder, spec.idlparsed)).idlparsed :
+    (await loadJSON(path.resolve(options.rootFolder, spec.idlparsed))).idlparsed :
     spec.idlparsed;
 
   // Make sure that all expected CSS definitions exist in the dfns extract
@@ -466,9 +469,9 @@ function checkSpecDefinitions(spec, options = {}) {
  *  identify the specification, and a `missing` property that is an object that
  *  may have `css` and `idl` properties which list missing CSS/IDL definitions.
  */
-function checkDefinitions(pathToReport, options = {}) {
+async function checkDefinitions(pathToReport, options = {}) {
   const rootFolder = path.resolve(process.cwd(), pathToReport);
-  const index = require(path.resolve(rootFolder, 'index.json')).results;
+  const index = (await loadJSON(path.resolve(rootFolder, 'index.json'))).results;
 
   // Check all dfns against CSS and IDL extracts
   const checkOptions = {
@@ -477,7 +480,7 @@ function checkDefinitions(pathToReport, options = {}) {
   };
   const missing = index
     .filter(spec => !options.shortname || spec.shortname === options.shortname)
-    .map(spec => {
+    .map(async spec => {
       const res = {
         url: spec.url,
         crawled: spec.crawled,
@@ -486,11 +489,10 @@ function checkDefinitions(pathToReport, options = {}) {
       if (!spec.dfns) {
         return res;
       }
-      res.missing = checkSpecDefinitions(spec, checkOptions);
+      res.missing = await checkSpecDefinitions(spec, checkOptions);
       return res;
     });
-
-  return missing;
+  return Promise.all(missing);
 }
 
 
@@ -516,26 +518,28 @@ function reportMissing(missing) {
 /**************************************************
 Export methods for use as module
 **************************************************/
-module.exports.checkSpecDefinitions = checkSpecDefinitions;
-module.exports.checkDefinitions = checkDefinitions;
+export {
+  checkSpecDefinitions,
+  checkDefinitions,
 
-// "Inner" functions that the IDL names generator uses to link IDL terms with
-// their definition (see generate-idlnames.js)
-module.exports.getExpectedDfnFromIdlDesc = getExpectedDfnFromIdlDesc;
-module.exports.matchIdlDfn = matchIdlDfn;
+  // "Inner" functions that the IDL names generator uses to link IDL terms with
+  // their definition (see generate-idlnames.js)
+  getExpectedDfnFromIdlDesc,
+  matchIdlDfn
+};
 
 
 
 /**************************************************
 Code run if the code is run as a stand-alone module
 **************************************************/
-if (require.main === module) {
+if (process.argv[1] === fileURLToPath(import.meta.url)) {
     const pathToReport = process.argv[2];
     const shortname = process.argv[3] || 'all';
     const format = process.argv[4] || 'markdown';
 
     const options = (shortname === 'all') ? undefined : { shortname };
-    let res = checkDefinitions(pathToReport, options);
+    let res = await checkDefinitions(pathToReport, options);
     if (shortname === 'all') {
       res = res
         .filter(result => result.missing &&

package/src/cli/merge-crawl-results.js

@@ -15,8 +15,10 @@
  * @module merger
  */
 
-const fs = require('fs');
-const requireFromWorkingDirectory = require('../lib/util').requireFromWorkingDirectory;
+import fs from 'node:fs';
+import { fileURLToPath } from 'node:url';
+import process from 'node:process';
+import { loadJSON } from '../lib/util.js';
 
 
 /**
@@ -81,11 +83,11 @@ function mergeCrawlResults(newCrawl, refCrawl, options) {
  * @param {Object} options Merge options. Only "matchTitle" is supported for now
  * @return {Promise} The promise to have merged the two JSON files into one
  */
-function mergeCrawlFiles(newCrawlPath, refCrawlPath, resPath, options) {
+async function mergeCrawlFiles(newCrawlPath, refCrawlPath, resPath, options) {
     options = options || {};
 
-    let newCrawl = requireFromWorkingDirectory(newCrawlPath);
-    let refCrawl = requireFromWorkingDirectory(refCrawlPath);
+    let newCrawl = await loadJSON(newCrawlPath);
+    let refCrawl = await loadJSON(refCrawlPath);
     return mergeCrawlResults(newCrawl, refCrawl, options)
         .then(filedata => new Promise((resolve, reject) =>
             fs.writeFile(resPath, JSON.stringify(filedata, null, 2),
@@ -96,14 +98,16 @@ function mergeCrawlFiles(newCrawlPath, refCrawlPath, resPath, options) {
 /**************************************************
 Export the methods for use as module
 **************************************************/
-module.exports.mergeCrawlResults = mergeCrawlResults;
-module.exports.mergeCrawlFiles = mergeCrawlFiles;
+export {
+    mergeCrawlResults,
+    mergeCrawlFiles
+};
 
 
 /**************************************************
 Code run if the code is run as a stand-alone module
 **************************************************/
-if (require.main === module) {
+if (process.argv[1] === fileURLToPath(import.meta.url)) {
     let newCrawlPath = process.argv[2];
     let refCrawlPath = process.argv[3];
     let resPath = process.argv[4];

package/src/cli/parse-webidl.js

@@ -18,7 +18,9 @@
  * @module webidlParser
  */
 
-const WebIDL2 = require("webidl2");
+import * as WebIDL2 from 'webidl2';
+import { fileURLToPath } from 'node:url';
+import process from 'node:process';
 
 
 /**
@@ -410,15 +412,17 @@ function addDependency(name, idlNames, externalDependencies) {
 /**************************************************
 Export the parse method for use as module
 **************************************************/
-module.exports.parse = parse;
-module.exports.hasObsoleteIdl = hasObsoleteIdl;
+export {
+    parse,
+    hasObsoleteIdl
+};
 
 
 /**************************************************
 Code run if the code is run as a stand-alone module
 **************************************************/
-if (require.main === module) {
-    const fs = require("fs");
+if (process.argv[1] === fileURLToPath(import.meta.url)) {
+    const fs = await import('node:fs');
     const idlFile = process.argv[2];
     if (!idlFile) {
         console.error("No IDL file to parse");

package/src/lib/css-grammar-parser.js

@@ -437,4 +437,4 @@ const parsePropDefValue = (value) => {
   return res.length === 1 ? res[0] : res;
 };
 
-module.exports.parsePropDefValue =  parsePropDefValue;
+export { parsePropDefValue };

package/src/lib/fetch.js

@@ -5,16 +5,14 @@
  * @module finder
  */
 
-const os = require('os');
-const path = require('path');
-const baseFetch = require('fetch-filecache-for-crawling');
+import os from 'node:os';
+import path from 'node:path';
+import baseFetch from 'fetch-filecache-for-crawling';
+import { loadJSON } from './util.js';
 
 // Read configuration parameters from `config.json` file
-let config = null;
-try {
-    config = require(path.resolve('config.json'));
-}
-catch (err) {
+let config = await loadJSON('config.json');
+if (!config) {
     config = {};
 }
 
@@ -32,7 +30,7 @@ catch (err) {
  *   options for fetch-filecache-for-crawling)
  * @return {Promise(Response)} Promise to get an HTTP response
  */
-async function fetch(url, options) {
+export default async function fetch(url, options) {
     options = Object.assign({headers: {}}, options);
     ['cacheFolder', 'resetCache', 'cacheRefresh', 'logToConsole'].forEach(param => {
         let fetchParam = (param === 'cacheRefresh') ? 'refresh' : param;
@@ -51,6 +49,3 @@ async function fetch(url, options) {
 
     return baseFetch(url, options);
 }
-
-
-module.exports = fetch;

package/src/lib/mock-server.js

@@ -5,9 +5,12 @@
  * @module mock-server
  */
 
-const { MockAgent, setGlobalDispatcher } = require('undici');
-const path = require("path");
-const { existsSync, readFileSync } = require('fs');
+import { MockAgent, setGlobalDispatcher } from 'undici';
+import path from 'node:path';
+import { existsSync, readFileSync } from 'node:fs';
+import { fileURLToPath } from 'node:url';
+
+const scriptPath = path.dirname(fileURLToPath(import.meta.url));
 
 /**
  * Determine the path to the "node_modules" folder. The path depends on whether
@@ -17,7 +20,7 @@ const { existsSync, readFileSync } = require('fs');
  * @return {String} Path to the node_modules folder.
  */
 function getModulesFolder() {
-    const rootFolder = path.resolve(__dirname, '../..');
+    const rootFolder = path.resolve(scriptPath, '../..');
     let folder = path.resolve(rootFolder, 'node_modules');
     if (existsSync(folder)) {
         return folder;
@@ -185,4 +188,4 @@ nock.emitter.on('no match', function(req, options, requestBody) {
   }
 });*/
 
-module.exports = mockAgent;
+export default mockAgent;

package/src/lib/post-processor.js

@@ -47,24 +47,69 @@
  * @module
  */
 
-const fs = require('fs');
-const path = require('path');
-const { createFolderIfNeeded, requireFromWorkingDirectory } = require('./util');
+import fs from 'node:fs';
+import path from 'node:path';
+import { pathToFileURL } from 'node:url';
+import { createFolderIfNeeded } from './util.js';
+import csscomplete from '../postprocessing/csscomplete.js';
+import events from '../postprocessing/events.js';
+import idlnames from '../postprocessing/idlnames.js';
+import idlparsed from '../postprocessing/idlparsed.js';
+import annotatelinks from '../postprocessing/annotate-links.js';
+import patchdfns from '../postprocessing/patch-dfns.js';
 
 
 /**
  * Core post-processing modules
  */
 const modules = {
-  csscomplete: require('../postprocessing/csscomplete'),
-  events: require('../postprocessing/events'),
-  idlnames: require('../postprocessing/idlnames'),
-  idlparsed: require('../postprocessing/idlparsed'),
-  annotatelinks: require('../postprocessing/annotate-links'),
-  patchdfns: require('../postprocessing/patch-dfns')
+  csscomplete,
+  events,
+  idlnames,
+  idlparsed,
+  annotatelinks,
+  patchdfns
 };
 
 
+/**
+ * Custom post-processing modules
+ */
+const customModules = {};
+
+
+/**
+ * Loads post-processing modules once and for all
+ */
+async function loadModules(mods) {
+  for (const mod of mods) {
+    if (typeof mod === 'string') {
+      if (modules[mod]) {
+        // Core post-processing module, already loaded
+        continue;
+      }
+      else {
+        try {
+          customModules[mod] = await import(pathToFileURL(mod));
+        }
+        catch (err) {
+          throw new Error(`Unknown post-processing module "${mod}"`);
+        }
+        if (!isModuleValid(customModules[mod])) {
+          throw new Error(`"${mod}" is not a valid post-processing module`);
+        }
+      }
+    }
+    else {
+      // Given post-processing moduel should already be a good one
+      if (!isModuleValid(customModules[mod])) {
+        throw new Error(`Post-processing module given as parameter does not have a "run" function`);
+      }
+    }
+  }
+}
+
+
 /**
  * Returns the post-processing module that match the requested name, or the
  * given parameter if it is a post-processing module already
@@ -79,20 +124,13 @@ function getModule(mod) {
     if (modules[mod]) {
       return Object.assign({ name: mod }, modules[mod]);
     }
+    else if (customModules[mod]) {
+      return Object.assign({ name: mod }, customModules[mod]);
+    }
     else {
-      const fmod = requireFromWorkingDirectory(mod);
-      if (!fmod) {
-        throw new Error(`Unknown post-processing module "${mod}"`);
-      }
-      if (!isModuleValid(fmod)) {
-        throw new Error(`"${mod}" is not a valid post-processing module`);
-      }
-      return Object.assign({ name: mod }, fmod);
+      throw new Error(`Unknown post-processing module "${mod}"`);
     }
   }
-  else if (!isModuleValid(mod)) {
-    throw new Error(`Post-processing module given as parameter does not have a "run" function`);
-  }
   return mod;
 }
 
@@ -270,11 +308,13 @@ function appliesAtLevel(mod, level) {
 /**************************************************
 Export post-processing functions
 **************************************************/
-module.exports = {
-  modules: Object.keys(modules),
+const moduleNames = Object.keys(modules);
+export {
+  moduleNames as modules,
+  loadModules,
   run, save,
   extractsPerSeries,
   dependsOn,
   getProperty,
   appliesAtLevel
-};
+};

package/src/lib/specs-crawler.js

@@ -10,27 +10,27 @@
  * @module crawler
  */
 
-const fs = require('fs');
-const path = require('path');
-const specs = require('web-specs');
-const inspect = require('util').inspect;
-const cssDfnParser = require('./css-grammar-parser');
-const postProcessor = require('./post-processor');
-const ThrottledQueue = require('./throttled-queue');
-const {
+import fs from 'node:fs';
+import path from 'node:path';
+import { inspect } from 'node:util';
+import specs from 'web-specs' with { type: 'json' };
+import * as postProcessor from './post-processor.js';
+import ThrottledQueue from './throttled-queue.js';
+import {
     completeWithAlternativeUrls,
     expandBrowserModules,
     expandCrawlResult,
     expandSpecResult,
     isLatestLevelThatPasses,
     processSpecification,
-    requireFromWorkingDirectory,
     setupBrowser,
     teardownBrowser,
-    createFolderIfNeeded
-} = require('./util');
+    createFolderIfNeeded,
+    loadJSON
+} from './util.js';
 
-const {version: reffyVersion} = require('../../package.json');
+import packageConfig from '../../package.json' with { type: 'json' };
+const reffyVersion = packageConfig.version;
 
 
 /**
@@ -342,6 +342,9 @@ async function crawlList(speclist, crawlOptions) {
         list = list.filter(spec => !!spec.release);
     }
 
+    // Load post-processing modules as needed
+    await postProcessor.loadModules(crawlOptions.post ?? []);
+
     const nbStr = '' + list.length;
     async function processSpec(spec, idx) {
         const logCounter = ('' + (idx + 1)).padStart(nbStr.length, ' ') + '/' + nbStr;
@@ -465,7 +468,7 @@ async function saveResults(contents, settings) {
  *   See CLI help (node reffy.js --help) for details.
  * @return {Promise<void>} The promise that the crawl will have been made
  */
-function crawlSpecs(options) {
+async function crawlSpecs(options) {
     function prepareListOfSpecs(list) {
         return list.map(spec => {
             if (typeof spec !== 'string') {
@@ -506,7 +509,7 @@ function crawlSpecs(options) {
     }
 
     const crawlIndex = options?.useCrawl ?
-        requireFromWorkingDirectory(options.useCrawl) :
+        await loadJSON(path.join(options.useCrawl, 'index.json')) :
         null;
 
     const requestedList = crawlIndex ? crawlIndex.results :
@@ -606,6 +609,10 @@ Export methods for use as module
 // - "crawlSpecs" takes options as input, runs all steps and saves results
 // to files (or outputs the results to the console). It does not return
 // anything.
-module.exports.crawlSpecs = (...args) => Array.isArray(args[0]) ?
-    crawlList.apply(this, args) :
-    crawlSpecs.apply(this, args);
+function crawl(...args) {
+    return Array.isArray(args[0]) ?
+        crawlList.apply(this, args) :
+        crawlSpecs.apply(this, args);
+}
+
+export { crawl as crawlSpecs };

package/src/lib/throttled-queue.js

@@ -43,7 +43,7 @@ function getOrigin(url) {
  * while guaranteeing that only one request will be sent to a given origin
  * server at a time.
  */
-module.exports = class ThrottledQueue {
+export default class ThrottledQueue {
   originQueue = {};
   maxParallel = 4;
   sleepInterval = 2000;

package/src/lib/util.js

@@ -2,26 +2,27 @@
  * A bunch of utility functions common to multiple scripts
  */
 
-const fs = require('fs').promises;
-const { existsSync, readdirSync } = require('fs');
-const path = require('path');
-const puppeteer = require('puppeteer');
-const crypto = require('crypto');
-const { Buffer } = require('buffer');
-const Ajv = require('ajv');
-const addFormats = require('ajv-formats');
-const commonSchema = require('../../schemas/common.json');
-const fetch = require('./fetch');
-const specEquivalents = require('../specs/spec-equivalents.json');
-
-
-const reffyModules = require('../browserlib/reffy.json');
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import crypto from 'node:crypto';
+import { Buffer } from 'node:buffer';
+import { fileURLToPath } from 'node:url';
+import puppeteer from 'puppeteer';
+import Ajv from 'ajv';
+import addFormats from 'ajv-formats';
+import fetch from './fetch.js';
+
+import commonSchema from '../../schemas/common.json' with { type: 'json' };
+import specEquivalents from '../specs/spec-equivalents.json' with { type: 'json' };
+import reffyModules from '../browserlib/reffy.json' with { type: 'json' };
+
+const scriptPath = path.dirname(fileURLToPath(import.meta.url));
 
 /**
  * Maximum depth difference supported between Reffy's install path and custom
  * modules that may be provided on the command-line
  *
- * TODO: Find a way to get right of that, there should be no limit
+ * TODO: Find a way to get rid of that, there should be no limit
  */
 const maxPathDepth = 20;
 
@@ -38,39 +39,30 @@ const prop = p => x => x[p];
 
 
 /**
- * Wrapper around the "require" function to require files relative to the
- * current working directory (CWD), instead of relative to the current JS
- * file.
- *
- * This is typically needed to be able to use "require" to load JSON config
- * files provided as command-line arguments.
+ * Load a JSON file as JS object.
  *
  * @function
  * @param {String} filename The path to the file to require
- * @return {Object} The result of requiring the file relative to the current
- *   working directory.
+ * @return {Object} The result of loading and parsing the file relative to the
+ *   current working directory.
  */
-function requireFromWorkingDirectory(filename) {
-    try {
-        return require(path.resolve(filename));
-    }
-    catch (err) {
-        return null;
-    }
-}
+async function loadJSON(filename) {
+  try {
+    const json = await fs.readFile(filename, 'utf8');
+    return JSON.parse(json);
+  }
+  catch (err) {
+    return null;
+  }
+};
 
 
 /**
  * Path to the "webidl2" folder to resolve relative links in the ES6 browser
  * lib modules. The path depends on whether Reffy is run directly, or installed
  * as a library.
- *
- * Code relies on the "require.resolve" function, but note that, when given a
- * simple module name, that function returns the path to the file targeted by
- * the "main" property in "package.json" which, in the case of the webidl2
- * module, is "dist/webidl2.js".
  */
-const webidl2Folder = path.resolve(path.dirname(require.resolve('webidl2')), '..');
+const webidl2Folder = path.dirname(fileURLToPath(import.meta.resolve('webidl2')));
 
 
 /**
@@ -128,7 +120,7 @@ function expandBrowserModules(modules) {
         return name;
     }
 
-    const browserlibPath = path.resolve(__dirname, '..', 'browserlib');
+    const browserlibPath = path.resolve(scriptPath, '..', 'browserlib');
     if (!modules) {
         return reffyModules.map(mod => Object.assign({
             name: getCamelCaseName(mod.href),
@@ -391,7 +383,7 @@ async function processSpecification(spec, processFunction, args, options) {
                         const requestPath = request.url.substring(request.url.indexOf(reffyPath) + reffyPath.length);
                         let depth = requestPath.lastIndexOf('__/') / 3;
                         const filename = requestPath.substring(requestPath.lastIndexOf('__/') + 3);
-                        let filePath = path.resolve(__dirname, '..', 'browserlib');
+                        let filePath = path.resolve(scriptPath, '..', 'browserlib');
                         while (depth < maxPathDepth - 1) {
                             filePath = path.resolve(filePath, '..');
                             depth += 1;
@@ -1041,7 +1033,7 @@ function getInterfaceTreeInfo(iface, interfaces) {
  * @return {function} The "validate" function for Ajv. The function returns null
  *   if the requested schema does not exist.
  */
-function getSchemaValidationFunction(schemaName) {
+async function getSchemaValidationFunction(schemaName) {
     // Helper function that selects the right schema file from the given
     // schema name.
     function getSchemaFileFromSchemaName(name) {
@@ -1065,11 +1057,11 @@ function getSchemaValidationFunction(schemaName) {
         }
     }
 
-    const schemasFolder = path.join(__dirname, '..', '..', 'schemas');
+    const schemasFolder = path.resolve(scriptPath, '..', '..', 'schemas');
     const schemaFile = getSchemaFileFromSchemaName(schemaName);
     let schema;
     try {
-        schema = require(path.join(schemasFolder, schemaFile));
+        schema = await loadJSON(path.join(schemasFolder, schemaFile));
     }
     catch (err) {
         return null;
@@ -1082,10 +1074,11 @@ function getSchemaValidationFunction(schemaName) {
         // The files schemas reference the browserlib ones, which need to
         // be explicitly added for Ajv to resolve references
         const folder = path.join(schemasFolder, 'browserlib');
-        const files = readdirSync(folder);
+        const files = await fs.readdir(folder);
         for (const file of files) {
             if (file.endsWith('.json')) {
-                ajvWithSchemas = ajvWithSchemas.addSchema(require(path.join(schemasFolder, 'browserlib', file)));
+                const json = await loadJSON(path.join(schemasFolder, 'browserlib', file));
+                ajvWithSchemas = ajvWithSchemas.addSchema(json);
             }
         }
     }
@@ -1118,10 +1111,8 @@ function getSchemaValidationFunction(schemaName) {
     };
 }
 
-
-module.exports = {
+export {
     fetch,
-    requireFromWorkingDirectory,
     expandBrowserModules,
     setupBrowser,
     teardownBrowser,
@@ -1133,5 +1124,6 @@ module.exports = {
     getGeneratedIDLNamesByCSSProperty,
     createFolderIfNeeded,
     getInterfaceTreeInfo,
-    getSchemaValidationFunction
+    getSchemaValidationFunction,
+    loadJSON
 };

package/src/postprocessing/annotate-links.js

@@ -18,7 +18,7 @@ function canonicalizeUrl(url) {
 
 const needsSaving = {};
 
-module.exports = {
+export default {
   dependsOn: ['links'],
   input: 'spec',
 
@@ -38,3 +38,4 @@ module.exports = {
     return spec;
   }
 };
+

package/src/postprocessing/csscomplete.js

@@ -8,9 +8,9 @@
  * rather completes the `css` property with additional info.
  */
 
-const { getGeneratedIDLNamesByCSSProperty } = require('../lib/util');
+import { getGeneratedIDLNamesByCSSProperty } from '../lib/util.js';
 
-module.exports = {
+export default {
   dependsOn: ['css', 'dfns'],
   input: 'spec',
 

package/src/postprocessing/events.js

@@ -3,9 +3,9 @@
  * per event.
  */
 
-const { isLatestLevelThatPasses, getInterfaceTreeInfo } = require('../lib/util');
+import { isLatestLevelThatPasses, getInterfaceTreeInfo } from '../lib/util.js';
 
-module.exports = {
+export default {
   dependsOn: ['events'],
   input: 'crawl',
   property: 'events',

package/src/postprocessing/idlnames.js

@@ -7,20 +7,20 @@
  * to worry about ordering, "idlparsed" will always run before this one).
  */
 
-const fs = require('fs');
-const path = require('path');
-const {
+import fs from 'node:fs';
+import path from 'node:path';
+import {
   matchIdlDfn,
-  getExpectedDfnFromIdlDesc } = require('../cli/check-missing-dfns');
-const {
+  getExpectedDfnFromIdlDesc } from '../cli/check-missing-dfns.js';
+import {
   isLatestLevelThatPasses,
-  createFolderIfNeeded } = require('../lib/util');
+  createFolderIfNeeded } from '../lib/util.js';
 
 
 /**
  * Definition of the post-processing module
  */
-module.exports = {
+export default {
   dependsOn: ['idlparsed', 'dfns'],
   input: 'crawl',
   run: generateIdlNames,

package/src/postprocessing/idlparsed.js

@@ -5,9 +5,9 @@
  * The module runs at the spec level and generates an `idlparsed` property.
  */
 
-const webidlParser = require('../cli/parse-webidl');
+import * as webidlParser from '../cli/parse-webidl.js';
 
-module.exports = {
+export default {
   dependsOn: ['dfns', 'idl'],
   input: 'spec',
   property: 'idlparsed',

package/src/postprocessing/patch-dfns.js

@@ -10,7 +10,7 @@
  * The module runs at the spec level.
  */
 
-module.exports = {
+export default {
   dependsOn: ['dfns'],
   input: 'spec',