reffy 6.2.2 → 6.3.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "reffy",
-  "version": "6.2.2",
+  "version": "6.3.0",
   "description": "W3C/WHATWG spec dependencies exploration companion. Features a short set of tools to study spec references as well as WebIDL term definitions and references found in W3C specifications.",
   "repository": {
     "type": "git",
@@ -34,7 +34,7 @@
     "abortcontroller-polyfill": "1.7.3",
     "browser-specs": "2.27.0",
     "commander": "9.0.0",
-    "fetch-filecache-for-crawling": "4.0.2",
+    "fetch-filecache-for-crawling": "4.1.0",
     "puppeteer": "13.1.3",
     "semver": "^7.3.5",
     "webidl2": "24.2.0"
@@ -42,10 +42,10 @@
   "devDependencies": {
     "chai": "4.3.6",
     "mocha": "9.2.0",
-    "nock": "13.2.2",
-    "respec": "29.0.5",
+    "nock": "13.2.4",
+    "respec": "29.0.6",
     "respec-hljs": "2.1.1",
-    "rollup": "2.67.0"
+    "rollup": "2.67.1"
   },
   "scripts": {
     "test": "mocha --recursive tests/"

package/src/lib/fetch.js

@@ -33,7 +33,7 @@ catch (err) {
  * @return {Promise(Response)} Promise to get an HTTP response
  */
 async function fetch(url, options) {
-    options = Object.assign({}, options);
+    options = Object.assign({headers: {}}, options);
     ['cacheFolder', 'resetCache', 'cacheRefresh', 'logToConsole'].forEach(param => {
         let fetchParam = (param === 'cacheRefresh') ? 'refresh' : param;
         if (config[param] && !options.hasOwnProperty(fetchParam)) {
@@ -53,4 +53,4 @@ async function fetch(url, options) {
 }
 
 
-module.exports = fetch;
+module.exports = fetch;

package/src/lib/nock-server.js

@@ -98,7 +98,14 @@ nock("https://www.w3.org")
   .get("/Tools/respec/respec-w3c").replyWithFile(200,
     path.join(modulesFolder, "respec", "builds", "respec-w3c.js"),
     { "Content-Type": "application/js" })
-  .get("/TR/idontexist/").reply(404, '');
+  .get("/TR/idontexist/").reply(404, '')
+  .get("/TR/ididnotchange/").reply(function() {
+    if (this.req.headers['if-modified-since'][0] === "Fri, 11 Feb 2022 00:00:42 GMT") {
+      return [304, ''];
+    } else {
+      return [200, 'Unexpected path'];
+    }
+  });
 
 nock("https://drafts.csswg.org")
   .persist()
@@ -117,4 +124,4 @@ nock.emitter.on('no match', function(req, options, requestBody) {
   }
 });
 
-module.exports = nock;
+module.exports = nock;

package/src/lib/specs-crawler.js

@@ -29,6 +29,7 @@ const {
     createFolderIfNeeded
 } = require('./util');
 
+const {version: reffyVersion} = require('../../package.json');
 
 /**
  * Return the spec if crawl succeeded or crawl result from given fallback list
@@ -78,10 +79,15 @@ async function crawlSpec(spec, crawlOptions) {
         path.dirname(crawlOptions.fallback) : '';
 
     if (spec.error) {
-        return specOrFallback(spec, fallbackFolder, crawlOptions.fallbackData);
+        return specOrFallback(spec, fallbackFolder, crawlOptions.fallbackData?.results);
     }
 
     try {
+        const fallback = crawlOptions.fallbackData?.results?.find(s => s.url === spec.url);
+        let cacheInfo = {};
+        if (crawlOptions.fallbackData?.crawler === `reffy-${reffyVersion}`) {
+          cacheInfo = Object.assign({}, fallback?.crawlCacheInfo);
+        }
         const result = await processSpecification(
             spec.crawled,
             (spec, modules) => {
@@ -97,8 +103,14 @@ async function crawlSpec(spec, crawlOptions) {
             },
             [spec, crawlOptions.modules],
             { quiet: crawlOptions.quiet,
-              forceLocalFetch: crawlOptions.forceLocalFetch }
+              forceLocalFetch: crawlOptions.forceLocalFetch,
+              ...cacheInfo}
         );
+        if (result.status === "notmodified" && fallback) {
+          crawlOptions.quiet ?? console.warn(`skipping ${spec.url}, no change`);
+          const copy = Object.assign({}, fallback);
+          return expandSpecResult(copy, fallbackFolder);
+        }
 
         // Specific rule for IDL extracts:
         // parse the extracted WebIdl content
@@ -169,6 +181,9 @@ async function crawlSpec(spec, crawlOptions) {
 
         // Copy results back into initial spec object
         spec.crawled = result.crawled;
+        if (result.crawlCacheInfo) {
+          spec.crawlCacheInfo = result.crawlCacheInfo;
+        }
         crawlOptions.modules.forEach(mod => {
             if (result[mod.property]) {
                 spec[mod.property] = result[mod.property];
@@ -183,7 +198,7 @@ async function crawlSpec(spec, crawlOptions) {
         spec.error = err.toString() + (err.stack ? ' ' + err.stack : '');
     }
 
-    return specOrFallback(spec, fallbackFolder, crawlOptions.fallbackData);
+    return specOrFallback(spec, fallbackFolder, crawlOptions.fallbackData?.results);
 }
 
 
@@ -351,7 +366,7 @@ async function crawlList(speclist, crawlOptions) {
     // Load fallback data if necessary
     if (crawlOptions.fallback) {
         try {
-            crawlOptions.fallbackData = JSON.parse(await fs.promises.readFile(crawlOptions.fallback)).results;
+            crawlOptions.fallbackData = JSON.parse(await fs.promises.readFile(crawlOptions.fallback));
         } catch (e) {
             throw new Error(`Could not parse fallback data file ${crawlOptions.fallback}`);
         }
@@ -469,12 +484,14 @@ async function saveResults(data, settings) {
 
     // Save all results to an index.json file
     const indexFilename = path.join(settings.output, 'index.json');
+
     const contents = {
         type: 'crawl',
         title: 'Reffy crawl',
         date: (new Date()).toJSON(),
         options: settings,
         stats: {},
+        crawler: `reffy-${reffyVersion}`,
         results: data
     };
     contents.options.modules = contents.options.modules.map(mod => mod.property);

package/src/lib/util.js

@@ -13,7 +13,6 @@ const specEquivalents = require('../specs/spec-equivalents.json');
 
 const reffyModules = require('../browserlib/reffy.json');
 
-
 /**
  * Maximum depth difference supported between Reffy's install path and custom
  * modules that may be provided on the command-line
@@ -22,6 +21,7 @@ const reffyModules = require('../browserlib/reffy.json');
  */
 const maxPathDepth = 20;
 
+let prefetchedResponses = {};
 
 /**
  * Returns a range array from 0 to the number provided (not included)
@@ -325,7 +325,8 @@ async function teardownBrowser() {
  *   flag tells the function that all network requests need to be only handled
  *   by Node.js's "fetch" function (as opposed to falling back to Puppeteer's
  *   network and caching logic), which is useful to keep full control of network
- *   requests in tests.
+ *   requests in tests. The "etag" and "lastModified" options give input
+ *   to the conditional fetch request sent for the primary crawled URL
  * @return {Promise} The promise to get the results of the processing function
  */
 async function processSpecification(spec, processFunction, args, options) {
@@ -409,17 +410,18 @@ async function processSpecification(spec, processFunction, args, options) {
                         await cdp.send('Fetch.continueRequest', { requestId });
                         return;
                     }
+                    const response = prefetchedResponses[request.url] ?? await fetch(request.url, { signal: controller.signal, headers: request.headers });
 
-                    const response = await fetch(request.url, { signal: controller.signal });
                     const body = await response.buffer();
+
                     await cdp.send('Fetch.fulfillRequest', {
                         requestId,
                         responseCode: response.status,
                         responseHeaders: Object.keys(response.headers.raw()).map(header => {
-                            return {
-                                name: header,
-                                value: response.headers.raw()[header].join(',')
-                            };
+                          return {
+                            name: header,
+                            value: response.headers.raw()[header].join(',')
+                          };
                         }),
                         body: body.toString('base64')
                     });
@@ -442,8 +444,8 @@ async function processSpecification(spec, processFunction, args, options) {
                     await cdp.send('Fetch.failRequest', { requestId, errorReason: 'Failed' });
                 }
                 else {
-                    options.quiet ?? console.warn(`[warn] Fall back to regular network request for ${request.url}`, err);
                     try {
+                        options.quiet ?? console.warn(`[warn] Fall back to regular network request for ${request.url}`, err);
                         await cdp.send('Fetch.continueRequest', { requestId });
                     }
                     catch (err) {
@@ -457,6 +459,34 @@ async function processSpecification(spec, processFunction, args, options) {
     }
 
     try {
+        // Fetch the spec URL if using https
+        // This allow to skip launching a browser
+        // if we have a fallback data source
+        // with a defined cache target for the spec
+        if (!spec.url.startsWith('file://')) {
+          let response;
+          // We set a conditional request header
+          // Use If-Modified-Since in preference as it is in practice
+          // more reliable for conditional requests
+          let headers = {'Accept-Encoding': 'gzip, deflate, br', 'Upgrade-Insecure-Requests': 1, 'User-Agent': browser.userAgent()};
+          if (options.lastModified) {
+            headers["If-Modified-Since"] = options.lastModified;
+          } else if (options.etag) {
+            headers["If-None-Match"] = options.etag;
+          }
+          try {
+            response = await fetch(spec.url, {headers});
+            if (response.status === 304) {
+              return {status: "notmodified"};
+            }
+            prefetchedResponses[spec.url] = response;
+          } catch (err) {
+            throw new Error(`Loading ${spec.url} triggered network error ${err}`);
+          }
+          if (response.status !== 200) {
+            throw new Error(`Loading ${spec.url} triggered HTTP status ${response.status}`);
+          }
+        }
         const page = await browser.newPage();
 
         // Disable cache if caller wants to handle all network requests
@@ -497,13 +527,27 @@ async function processSpecification(spec, processFunction, args, options) {
 
         // Load the page
         // (note HTTP status is 0 when `file://` URLs are loaded)
+        let cacheInfo;
         if (spec.html) {
             await page.setContent(spec.html, loadOptions);
         }
         else {
-            const result = await page.goto(spec.url, loadOptions);
+            let result;
+            try {
+              result = await page.goto(spec.url, loadOptions);
+            } catch (err) {
+              throw new Error(`Loading ${spec.url} triggered network error ${err}`);
+            }
             if ((result.status() !== 200) && (!spec.url.startsWith('file://') || (result.status() !== 0))) {
-                throw new Error(`Loading ${spec.url} triggered HTTP status ${result.status()}`);
+              throw new Error(`Loading ${spec.url} triggered HTTP status ${result.status()}`);
+            }
+            const responseHeaders = result.headers();
+            // Use Last-Modified in preference as it is in practice
+            // more reliable for conditional requests
+            if (responseHeaders['last-modified']) {
+              cacheInfo = {lastModified: responseHeaders['last-modified']};
+            } else if (responseHeaders.etag) {
+              cacheInfo = {etag: responseHeaders.etag};
             }
         }
 
@@ -613,7 +657,7 @@ async function processSpecification(spec, processFunction, args, options) {
 
         // Run the processFunction method in the browser context
         const results = await page.evaluate(processFunction, ...args);
-
+        results.crawlCacheInfo = cacheInfo;
         // Pending network requests may still be in the queue, flag the page
         // as closed not to send commands on a CDP session that's no longer
         // attached to anything