reffy 12.0.0 → 12.1.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "reffy",
-  "version": "12.0.0",
+  "version": "12.1.0",
   "description": "W3C/WHATWG spec dependencies exploration companion. Features a short set of tools to study spec references as well as WebIDL term definitions and references found in W3C specifications.",
   "repository": {
     "type": "git",
@@ -37,9 +37,9 @@
     "ajv-formats": "2.1.1",
     "commander": "10.0.0",
     "fetch-filecache-for-crawling": "4.1.0",
-    "puppeteer": "19.7.2",
+    "puppeteer": "19.7.3",
     "semver": "^7.3.5",
-    "web-specs": "2.50.0",
+    "web-specs": "2.51.0",
     "webidl2": "24.2.2"
   },
   "devDependencies": {
@@ -48,7 +48,7 @@
     "nock": "13.3.0",
     "respec": "32.7.1",
     "respec-hljs": "2.1.1",
-    "rollup": "3.17.2"
+    "rollup": "3.18.0"
   },
   "scripts": {
     "test": "mocha --recursive tests/"

package/src/browserlib/create-outline.mjs

@@ -8,7 +8,7 @@
  * As a by-product of generating the outline, the function also generates a
  * mapping between elements and the (conceptual) section that contains them in
  * the outline. To save memory, this mapping is only done for elements that have
- * an ID.
+ * an ID (or a "name" attribute).
  *
  * Both the outline and the mapping are returned.
  */
@@ -307,8 +307,9 @@ export default function (root) {
     // In addition, whenever the walk exits a node, after doing the steps above,
     // if the node is not associated with a section yet, associate the node with
     // the section current section.
-    // (we will only do that for elements that have an ID)
-    if (node.getAttribute('id') && !nodeToSection.has(node)) {
+    // (we will only do that for elements that have an ID or a "name" attribute)
+    if ((node.getAttribute('id') || node.getAttribute('name')) &&
+        !nodeToSection.has(node)) {
       nodeToSection.set(node, currentSection);
     }
   }

package/src/browserlib/extract-cssdfn.mjs

@@ -333,6 +333,14 @@ const dfnLabel2Property = label => label.trim()
   .replace(/^newValue$/, 'newValues');
 
 
+/**
+ * Selector to use to exclude inner blocks that list tests, references and/or
+ * link to implementation statuses, which would provide too much detailed info
+ * in prose content.
+ */
+const asideSelector = 'aside, .mdn-anno, .wpt-tests-block';
+
+
 /**
  * Extract a CSS definition from a table
  *
@@ -343,7 +351,7 @@ const extractTableDfn = table => {
   const lines = [...table.querySelectorAll('tr')]
     .map(line => {
       const cleanedLine = line.cloneNode(true);
-      const annotations = cleanedLine.querySelectorAll("aside, .mdn-anno");
+      const annotations = cleanedLine.querySelectorAll(asideSelector);
       annotations.forEach(n => n.remove());
       return {
         name: dfnLabel2Property(cleanedLine.querySelector(':first-child').textContent),
@@ -580,7 +588,7 @@ const extractTypedDfn = dfn => {
   // and remove MDN annotations as well
   [...parent.querySelectorAll('sup')]
     .map(sup => sup.parentNode.removeChild(sup));
-  [...parent.querySelectorAll('aside, .mdn-anno')]
+  [...parent.querySelectorAll(asideSelector)]
     .map(annotation => annotation.parentNode.removeChild(annotation));
 
   const text = parent.textContent.trim();
@@ -655,7 +663,7 @@ const extractTypedDfn = dfn => {
       });
       [...dd.querySelectorAll('sup')]
         .map(sup => sup.parentNode.removeChild(sup));
-      [...dd.querySelectorAll('aside, .mdn-anno')]
+      [...dd.querySelectorAll(asideSelector)]
         .map(annotation => annotation.parentNode.removeChild(annotation));
 
       res = {
@@ -711,7 +719,7 @@ const extractProductionRules = root => {
     .filter(el => !el.closest(informativeSelector))
     .map(el => el.cloneNode(true))
     .map(el => {
-      [...el.querySelectorAll('aside, .mdn-anno')]
+      [...el.querySelectorAll(asideSelector)]
         .map(aside => aside.parentNode.removeChild(aside));
       return el;
     })

package/src/browserlib/extract-dfns.mjs

@@ -226,6 +226,9 @@ export default function (spec, idToHeading = {}) {
 
   const shortname = (typeof spec === 'string') ? spec : spec.shortname;
   switch (shortname) {
+  case "CSS2":
+    preProcessCSS21();
+    break;
   case "html":
     preProcessHTML();
     break;
@@ -667,6 +670,52 @@ function preProcessHTML() {
     });
 }
 
+function preProcessCSS21() {
+  document.querySelectorAll('span.index-def')
+    .forEach(span => {
+      // Definition ID is to be found in a nearby anchor
+      const anchor = span.querySelector('a[name]') ?? span.closest('a[name]');
+      if (!anchor) {
+        return;
+      }
+
+      // Once in a while, definition has a "<dfn>", and once in a while, that
+      // "<dfn>" already follows the dfn data model.
+      let dfn = span.querySelector('dfn') ?? span.closest('dfn');
+      if (dfn?.id) {
+        return;
+      }
+
+      // No "<dfn>"? Let's create it
+      if (!dfn) {
+        dfn = document.createElement('dfn');
+        for (let child of [...span.childNodes]) {
+          dfn.appendChild(child);
+        }
+        span.appendChild(dfn);
+      }
+
+      // Complete the "<dfn>" with expected attributes
+      dfn.id = anchor.getAttribute('name');
+      dfn.dataset.export = '';
+      if (span.getAttribute('title')) {
+        dfn.dataset.lt = span.getAttribute('title');
+      }
+      let dfnType = null;
+      switch (anchor.getAttribute('class') ?? '') {
+        case 'propdef-title':
+          dfnType = 'property';
+          break;
+        case 'value-def':
+          dfnType = 'value';
+          break;
+      }
+      if (dfnType) {
+        dfn.dataset.dfnType = dfnType;
+      }
+    });
+}
+
 function preProcessSVG2() {
   const idl = extractWebIdl();
   const idlTree = parse(idl);

package/src/browserlib/extract-headings.mjs

@@ -19,7 +19,13 @@ export default function (spec, idToHeading) {
       number: headingNumber
     };
   });
-  return esHeadings.concat([...document.querySelectorAll('h1[id], h2[id], h3[id], h4[id], h5[id] ,h6[id]')].map(n => {
+
+  const headingsSelector = [
+    ':is(h1,h2,h3,h4,h5,h6)[id]',                 // Regular headings
+    ':is(h1,h2,h3,h4,h5,h6):not([id]) > a[name]'  // CSS 2.1 headings
+  ].join(',');
+
+  return esHeadings.concat([...document.querySelectorAll(headingsSelector)].map(n => {
     // Note: In theory, all <hX> heading elements that have an ID are associated
     // with a heading in idToHeading. One exception to the rule: when the
     // heading element appears in a <hgroup> element, the mapping is not
@@ -27,9 +33,11 @@ export default function (spec, idToHeading) {
     // headings not to create a mess in the outline). In practice, this only
     // really happens so far for WHATWG spec titles that (correctly) group the
     // title and subtitle headings in a <hgroup>.
-    const href = getAbsoluteUrl(n, { singlePage });
+    const idAttr = n.hasAttribute('id') ? 'id' : 'name';
+    const headingEl = n.hasAttribute('id') ? n : n.parentNode;
+    const href = getAbsoluteUrl(n, { singlePage, attribute: idAttr });
     const heading = idToHeading[href] || {
-      id: n.id,
+      id: n.getAttribute(idAttr),
       href,
       title: n.textContent.trim()
     };
@@ -37,7 +45,7 @@ export default function (spec, idToHeading) {
     const res = {
       id: heading.id,
       href: heading.href,
-      level: parseInt(n.tagName.slice(1), 10),
+      level: parseInt(headingEl.tagName.slice(1), 10),
       title: heading.title
     };
     if (heading.number) {

package/src/browserlib/map-ids-to-headings.mjs

@@ -2,14 +2,33 @@ import createOutline from './create-outline.mjs';
 import getAbsoluteUrl from './get-absolute-url.mjs';
 
 // Regular expression to capture the numbering of a heading. The expression
-// extracts numbers such as "1.", "A.", "A.3", "13.3.4.". Note: a top-level
-// number always ends with a ".", but there may be no final "." in sublevels
-// (Bikeshed adds one, ReSpec does not).
-const reNumber = /^([A-Z0-9]\.|[A-Z](\.[0-9]+)+\.?|[0-9]+(\.[0-9]+)+\.?)\s/;
+// extracts numbers such as "1.", "A.", "A.3", "13.3.4.". Notes:
+// - A top-level number always ends with a ".", except in CSS 2.1, some IETF RFCs
+// and WebGL specs.
+// - There may be no final "." in sublevels (Bikeshed adds one, not ReSpec)
+// - Top level appendices (e.g. in CSS 2.1, IETF RFCs and Bikeshed specs) start
+// with "Appendix", sometimes followed by ":"
+const reNumber = /^([A-Z\d]\.|[A-Z](\.\d+)+\.?|\d+(\.\d+)+\.?|\d|Appendix [A-Z][\.:])\s/;
 
 /**
- * Generate a mapping between elements that have an ID and the closest heading
- * (that also has an ID) under which these elements appear in the DOM tree.
+ * Retrieve a "cleaned" version of the node's text content, without aside notes
+ * such as links to tests, MDN or references.
+ *
+ * Note that this is mainly intended for CSS Color 3, which has test annotations
+ * within headings.
+ */
+function getCleanTextContent(node) {
+  const asideSelector = 'aside, .mdn-anno, .wpt-tests-block, .annotation';
+  const cleanedNode = node.cloneNode(true);
+  const annotations = cleanedNode.querySelectorAll(asideSelector);
+  annotations.forEach(n => n.remove());
+  return cleanedNode.textContent.trim().replace(/\s+/g, ' ');
+}
+
+/**
+ * Generate a mapping between elements that have an ID (or a "name") and the
+ * closest heading (that also has an ID) under which these elements appear in
+ * the DOM tree.
  *
  * The main difficulty is that the structure of a DOM tree does not necessarily
  * follow the structure of the outline of the document, which means that there
@@ -48,7 +67,7 @@ export default function () {
   const singlePage = !document.querySelector('[data-reffy-page]');
 
   const mappingTable = {};
-  [...document.querySelectorAll('[id]')].forEach(node => {
+  [...document.querySelectorAll('[id],[name]')].forEach(node => {
     let parentSection = nodeToSection.get(node);
     while (parentSection) {
       if (parentSection.heading !== '__implied') {
@@ -62,20 +81,32 @@ export default function () {
     // Compute the absolute URL with fragment
     // (Note the crawler merges pages of a multi-page spec in the first page
     // to ease parsing logic, and we want to get back to the URL of the page)
-    const nodeid = getAbsoluteUrl(node, { singlePage });
+    const idAttr = node.hasAttribute('id') ? 'id' : 'name';
+    const nodeid = getAbsoluteUrl(node, { singlePage, attribute: idAttr });
     let href = nodeid;
 
     if (parentSection) {
+      let id;
+
       const heading = parentSection.heading;
-      let id = heading.id;
-      href = getAbsoluteUrl(heading, { singlePage });
+      if (heading.hasAttribute('id')) {
+        id = heading.id;
+        href = getAbsoluteUrl(heading, { singlePage });
+      }
+      else {
+        const anchor = heading.querySelector('a[name]');
+        if (anchor) {
+          id = anchor.getAttribute('name');
+          href = getAbsoluteUrl(anchor, { singlePage, attribute: 'name' });
+        }
+      }
 
       if (parentSection.root && parentSection.root.hasAttribute('id')) {
         id = parentSection.root.id;
         href = getAbsoluteUrl(parentSection.root, { singlePage });
       }
 
-      const trimmedText = heading.textContent.trim();
+      const trimmedText = getCleanTextContent(heading);
       const match = trimmedText.match(reNumber);
       const number = match ? match[1] : null;
 
@@ -84,12 +115,13 @@ export default function () {
         mapping.id = id;
       }
       mapping.href = href;
-      mapping.title = trimmedText.replace(reNumber, '').trim().replace(/\s+/g, ' ');
+      mapping.title = trimmedText.replace(reNumber, '');
       mappingTable[nodeid] = mapping;
 
       if (number) {
-        // Store the number without the final "."
-        mappingTable[nodeid].number = number.replace(/\.$/, '');
+        // Store the number without the final "." or ":"
+        // (and without the "Appendix" prefix in the CSS 2.1 case)
+        mappingTable[nodeid].number = number.replace(/[\.:]$/, '').replace(/^Appendix /, '');
       }
     }
   });
@@ -115,7 +147,7 @@ function esMapIdToHeadings() {
       if (!section) return;
 
       const heading = section.querySelector("h1");
-      const trimmedText = heading.textContent.trim();
+      const trimmedText = getCleanTextContent(heading);
       const nodeid = getAbsoluteUrl(el, { singlePage });
       const href = getAbsoluteUrl(section, { singlePage });
 
@@ -127,7 +159,7 @@ function esMapIdToHeadings() {
         mapping.id = section.id;
       }
       mapping.href = href;
-      mapping.title = trimmedText.replace(reNumber, '').trim().replace(/\s+/g, ' ');
+      mapping.title = trimmedText.replace(reNumber, '');
       mappingTable[nodeid] = mapping;
 
       if (number) {