reffy 3.1.0 → 4.0.3

package/src/browserlib/extract-dfns.mjs

@@ -201,6 +201,9 @@ export default function (spec, idToHeading = {}) {
   case "html":
     preProcessHTML();
     break;
+  case "ecmascript":
+    preProcessEcmascript();
+    break;
   case "SVG2":
     preProcessSVG2();
     break;
@@ -225,6 +228,339 @@ export default function (spec, idToHeading = {}) {
     .map(node => definitionMapper(node, idToHeading));
 }
 
+function preProcessEcmascript() {
+  // Skip elements in sections marked as legacy
+  const legacySectionFilter = n => !n.closest("[legacy]");
+
+  const wrapWithDfn = (el) => {
+    // wrap with a dfn
+    const dfn = document.createElement("dfn");
+    for (let child of [...el.childNodes]) {
+      dfn.appendChild(child);
+    }
+    el.appendChild(dfn);
+    // set id
+    dfn.setAttribute("id", el.parentNode.getAttribute("id"));
+    dfn.dataset.ltNodefault = true;
+    return dfn;
+  };
+
+  const cleanMethodName = (name) => {
+    return name.replace(/\[/g, '')
+      .replace(/\]/g, '') // removing brackets used to mark optional args
+      .replace(/ \( */, '(')
+      .replace(/ *\)/, ')')
+      .replace(/ *,/g, ','); // trimming internal spaces
+  };
+
+  let definitionNames = new Set();
+  let idlTypes = {};
+
+  // We find the list of abstract methods
+  // to help with scoping abstract operations
+  let abstractMethods = {};
+  const abstractMethodCaptions = [...document.querySelectorAll("figcaption")]
+        .filter(el => el.textContent.match(/(abstract|additional) method/i) && el.parentNode.querySelector("emu-xref"));
+  for (const figcaption of abstractMethodCaptions) {
+    const scope = figcaption.querySelector("emu-xref").textContent;
+    const table = figcaption.parentNode.querySelector("tbody");
+    for (const td of table.querySelectorAll("tr td:first-child")) {
+      // We only consider the name of the method, not the potential parameters
+      // as they're not necessarily consistently named across
+      // the list and the definition
+      const methodName = td.textContent.trim().split('(')[0];
+      abstractMethods[methodName] = scope;
+    }
+  }
+
+  const sectionNumberRegExp = /^([A-Z]\.)?[0-9\.]+ /;
+  [...document.querySelectorAll("h1")]
+    .filter(legacySectionFilter)
+    .forEach(el => {
+      let dfnName = el.textContent.replace(sectionNumberRegExp, '').trim() ;// remove section number
+      const dfnId = el.parentNode.id;
+      if (dfnId.match(/-objects?$/) && dfnName.match(/ Objects?$/)) {
+
+        // Skip headings that look like object definitions, but aren't
+        const notObjectIds = ["sec-global-object", "sec-fundamental-objects", "sec-waiterlist-objects"];
+        if (notObjectIds.includes(dfnId)) return;
+
+        // only keep ids that match a credible pattern for object names
+        // i.e. a single word
+        // there are exceptions to that simple rule
+        // RegExp includes its expansion (regular expansion) in the id
+        // WeakRef is translated into weak-ref in the id
+        const objectsIdsExceptions = ["sec-regexp-regular-expression-objects", "sec-weak-ref-objects", "sec-aggregate-error-objects", "sec-finalization-registry-objects", "sec-async-function-objects"];
+
+        if (!dfnId.match(/sec-[a-z]+-objects?/)
+            && !objectsIdsExceptions.includes(dfnId)
+           ) return;
+        const dfn = wrapWithDfn(el);
+        // set data-lt
+        dfnName = dfnName
+          .replace(/^The /, '')
+          .replace(/ Objects?$/, '')
+        // regexp def includes "(Regular Expression)"
+          .replace(/ \([^\)]*\)/, '') ;
+        dfn.dataset.lt = dfnName;
+
+        // FIXME
+        // These interfaces are also defined in WebIDL, which in general is
+        // the prefered source for these terms
+        // Because bikeshed does not yet support spec-specific imports,
+        // we hide these terms as not exported
+        // cf https://github.com/w3c/reffy/pull/732#issuecomment-925950287
+        const exportExceptions = [ "Promise", "DataView", "ArrayBuffer" ];
+        if (exportExceptions.includes(dfnName)) {
+          dfn.dataset.noexport = "";
+        }
+
+        if (dfnName.match(/^[A-Z]/)) {
+          // set dfn-type
+          if (dfnName.match(/Error$/)) {
+            dfn.dataset.dfnType = "exception";
+          } else if (!el.parentNode.querySelector('[id$="constructor"]')) {
+            // Objects without constructors match to the namespace type
+            dfn.dataset.dfnType = "namespace";
+          } else {
+            dfn.dataset.dfnType = "interface";
+          }
+          // We keep track of types associated with a name
+          // to associate the same type to the relevant intrinsic object
+          // à la %Math%
+          idlTypes[dfnName] = dfn.dataset.dfnType;
+        }
+        definitionNames.add(dfnName);
+      } else if (dfnId.match(/-[a-z]+error$/) && !dfnName.match(/\(/)) {
+        const dfn = wrapWithDfn(el);
+        dfn.dataset.lt = dfnName;
+        dfn.dataset.dfnType = "exception";
+        definitionNames.add(dfnName);
+        idlTypes[dfnName] = dfn.dataset.dfnType;
+      } else if (dfnId.match(/[-\.]prototype[-\.]/)) {
+        // methods and attributes on objects
+
+        // Skip headings with a space and no parenthesis
+        // (they mention prototype but aren't a prototype property def)
+        // with the exception of "set " and "get " headings
+        // (which describe setters and getters)
+        if (!dfnName.match(/\(/) && (dfnName.match(/ /) && !dfnName.match(/^[gs]et /))) return;
+
+        // Skip unscoped internal methods à la [[SetPrototypeOf]](V)
+        if (dfnName.match(/\[\[/)) return;
+
+        // Skip symbol-based property definitions;
+        // not clear they're useful as externally referenceable names
+        if (dfnName.match(/@@/)) return;
+
+        // Skip .constructor as that cannot be considered as an attribute
+        if (dfnName.match(/\.constructor$/)) return;
+
+        const dfn = wrapWithDfn(el);
+        // set definition scope
+        dfn.dataset.dfnFor = dfnName.replace(/\.prototype\..*/, '')
+          .replace(/^[gs]et /, ''); // remove "get"/"set" markers
+
+        // Remove parent object prototype (set as scope)
+        dfnName = dfnName.replace(/.*\.prototype\./, '');
+
+        dfn.dataset.lt = dfnName;
+        // set dfn-type
+        if (dfn.dataset.lt.match(/\(/)) {
+          dfnName = cleanMethodName(dfnName);
+          dfn.dataset.lt = dfnName;
+          dfn.dataset.dfnType = "method";
+        } else {
+          dfn.dataset.dfnType = "attribute";
+        }
+      } else if (el.closest("#sec-value-properties-of-the-global-object")) {
+        // properties of the global object
+        if (el.id !== "#sec-value-properties-of-the-global-object"){
+          const dfn = wrapWithDfn(el);
+          dfn.dataset.lt = dfnName;
+          dfn.dataset.dfnType = "attribute";
+          dfn.dataset.dfnFor = "globalThis";
+        }
+      } else {
+        // We handle other headings that look like a method / property
+        // on an object instance (rather than its prototype)
+        // or an abstract op
+
+        // if there is already a dfn element, we move on
+        if (el.querySelector("dfn")) return;
+
+        // only dealing with well-known patterns
+        if (!dfnName.match(/^[a-z]+\.[a-z]+/i) // à la JSON.parse
+            && !dfnName.match(/^([a-z]+)+ *\(/i) // à la ArrayCreate ( or decodeURI (
+           ) return;
+        // Skip symbol-based property definitions
+        if (dfnName.match(/@@/)) return;
+
+        // Skip .prototype as that cannot be considered
+        // as an attribute
+        if (dfnName.match(/\.prototype$/)) return;
+
+        // Skip headings where foo.bar appears as part of a longer phrase
+        if (!dfnName.match(/\(/) && dfnName.match(/ /)) return;
+
+        // redundant definitions of constructors on the global object
+        // e.g. "Array ( . . . )"
+        if (dfnName.match(/\. \. \./)) return;
+
+        const dfn = wrapWithDfn(el);
+
+        if (dfnName.match(/^[a-z]+\.[a-z]+/i)) {
+          // set definition scope
+          // This assumes that such methods and attributes are only defined
+          // one-level deep from the global scope
+          dfn.dataset.dfnFor = dfnName.replace(/\..*$/, '');
+          dfnName = dfnName.replace(dfn.dataset.dfnFor + ".", '');
+          if (dfnName.match(/\(/)) {
+            dfnName = cleanMethodName(dfnName);
+            dfn.dataset.lt = dfnName;
+            dfn.dataset.dfnType = "method";
+          } else {
+            dfn.dataset.lt = dfnName;
+            if (dfnName.match(/^[A-Z]+$/)) {
+              dfn.dataset.dfnType = "const";
+            } else {
+              dfn.dataset.dfnType = "attribute";
+            }
+          }
+        } else if (dfnName.match(/^([A-Z]+[a-z]*)+ *\(/)) { // Abstract ops à la ArrayCreate or global constructor
+          dfnName = cleanMethodName(dfnName);
+          dfn.dataset.lt = dfnName;
+          const opName = dfnName.split('(')[0];
+
+          // distinguish global constructors from abstract operations
+          if (idlTypes[opName]) {
+            dfn.dataset.dfnType = "constructor";
+            dfn.dataset.dfnFor = opName;
+          } else {
+            // If the name is listed as an Abstract Method
+            // we set the dfn-for accordingly
+            if (abstractMethods[opName]) {
+              dfn.dataset.dfnFor = abstractMethods[opName];
+            }
+
+            dfn.dataset.dfnType = "abstract-op";
+          }
+        } else { // methods of the global object
+          dfnName = cleanMethodName(dfnName);
+          dfn.dataset.lt = dfnName;
+          dfn.dataset.dfnType = "method";
+          dfn.dataset.dfnFor = "globalThis";
+        }
+        definitionNames.add(dfnName);
+      }
+    });
+  // Extract abstract operations from <emu-eqn> with aoid attribute
+  [...document.querySelectorAll("emu-eqn[aoid]")]
+    .filter(legacySectionFilter)
+    .forEach(el => {
+      // Skip definitions of constant values (e.g. msPerDay)
+      if (el.textContent.match(/=/)) return;
+      const dfn = wrapWithDfn(el);
+      dfn.dataset.lt = el.getAttribute("aoid");
+      dfn.dataset.dfnType = "abstract-op";
+      dfn.id = el.id;
+    });
+
+  // Extract State Components from tables
+  [...document.querySelectorAll("figure > table")]
+    .filter(legacySectionFilter)
+    .forEach(el => {
+      const title = el.parentNode.querySelector("figcaption")?.textContent || "";
+      if (!title.match(/state components for/i)) return;
+      const scope = title.replace(/^.*state components for/i, '').trim();
+      for (const td of el.querySelectorAll("tr td:first-child")) {
+        const dfn = wrapWithDfn(td);
+        dfn.dataset.dfnFor = scope;
+        dfn.id = el.closest("emu-table[id],emu-clause[id]").id;
+      }
+    });
+
+  [...document.querySelectorAll("dfn")]
+    .filter(legacySectionFilter)
+    .forEach(el => {
+      // Skip definitions in conformance page and conventions page
+      if (el.closest('section[data-reffy-page$="conformance.html"]') ||
+          el.closest('section[data-reffy-page$="notational-conventions.html"]')) {
+        el.removeAttribute("id");
+        return;
+      }
+
+      // rely on the aoid attribute as a hint we're dealing
+      // with an abstract-op
+      if (el.getAttribute("aoid")) {
+        el.dataset.dfnType = "abstract-op";
+      }
+
+      // Mark well-known intrinsic objects as the same type as their visible object (if set), defaulting to "interface"
+      if (el.textContent.match(/^%[A-Z].*%$/)) {
+        el.dataset.dfnType = idlTypes[el.textContent.replace(/%/g, '')] || "interface";
+        definitionNames.add(el.textContent.trim());
+      }
+
+      // %names% in the global object section are operations of the globalThis object
+      if (el.closest('[data-reffy-page$="global-object.html"]') && el.textContent.match(/^%[a-z]+%/i)) {
+        el.dataset.dfnFor = "globalThis";
+        // TODO: this doesn't capture the arguments
+        el.dataset.dfnType = "method";
+      }
+
+      // Mark well-known symbols as "const"
+      // for lack of a better type, and as the WebIDL spec has been doing
+      if (el.textContent.match(/^@@[a-z]*$/i)) {
+        el.dataset.dfnType = "const";
+      }
+      if (el.getAttribute("variants")) {
+        el.dataset.lt = (el.dataset.lt ?? el.textContent.trim()) + "|" + el.getAttribute("variants");
+      }
+
+      // Skip definitions that have already been identified
+      // with a more specific typing
+      if (!el.dataset.dfnType) {
+        // we already have a matching typed definition
+        if (definitionNames.has(el.textContent.trim())) return;
+      }
+
+      // If the <dfn> has no id, we attach it the one from the closest
+      // <emu-clause> with an id
+      // Note that this means several definitions can share the same id
+      if (!el.getAttribute("id")) {
+        if (el.closest("emu-clause[id]")) {
+          el.setAttribute("id", el.closest("emu-clause").getAttribute("id"));
+        }
+      }
+
+      // Any generic <dfn> not previously filtered out
+      // is deemed to be exported, scoped to ECMAScript
+      if (!el.dataset.dfnType) {
+        if (!el.dataset.dfnFor) {
+          el.dataset.dfnFor = "ECMAScript";
+        }
+        el.dataset.export = "";
+      }
+    });
+  // Another pass of clean up for duplicates
+  // This cannot be done in the first pass
+  // because %Foo.prototype% does not necessarily get identified before
+  // the equivalent " prototype object" dfn
+
+  [...document.querySelectorAll("dfn[id][data-export]")]
+    .filter(legacySectionFilter)
+    .forEach(dfn => {
+      // we have the syntactic equivalent %x.prototype%
+      let m = dfn.textContent.trim().match(/^(.*) prototype( object)?$/);
+      if (m && definitionNames.has(`%${m[1].trim()}.prototype%`)) {
+        dfn.removeAttribute("id");
+        delete dfn.dataset.export;
+        return;
+        }
+    });
+}
+
 function preProcessHTML() {
   const headingSelector = [
     'h2[id]:not([data-dfn-type]) dfn',
@@ -235,7 +571,7 @@ function preProcessHTML() {
   ].join(',');
 
   // we copy the id on the dfn when it is set on the surrounding heading
-  [...document.querySelectorAll(headingSelector)]
+  document.querySelectorAll(headingSelector)
     .forEach(el => {
       const headingId = el.closest("h2, h3, h4, h5, h6").id;
       if (!el.id) {
@@ -246,11 +582,12 @@ function preProcessHTML() {
   // all the definitions in indices.html are non-normative, so we skip them
   // to avoid having to properly type them
   // they're not all that interesting
-  [...document.querySelectorAll('section[data-reffy-page$="indices.html"] dfn[id]')].forEach(el => {
-    el.dataset.dfnSkip = true;
-  });
+  document.querySelectorAll('section[data-reffy-page$="indices.html"] dfn[id]')
+    .forEach(el => {
+      el.dataset.dfnSkip = true;
+    });
 
-  [...document.querySelectorAll("dfn[id]:not([data-dfn-type]):not([data-skip])")]
+  document.querySelectorAll("dfn[id]:not([data-dfn-type]):not([data-skip])")
     .forEach(el => {
       // Hard coded rules for special ids
       // dom-style is defined elsewhere
@@ -281,7 +618,7 @@ function preProcessSVG2() {
     linkHeading.dataset.lt = "link";
   }
 
-  [...document.querySelectorAll(".attrdef dfn[id]:not([data-dfn-type]):not([data-skip])")]
+  document.querySelectorAll(".attrdef dfn[id]:not([data-dfn-type]):not([data-skip])")
     .forEach(el => {
       el.dataset.dfnType = "element-attr";
       const attrDesc = document.querySelector('[data-reffy-page$="attindex.html"] th span.attr-name a[href$="#' + el.id + '"]');
@@ -291,7 +628,8 @@ function preProcessSVG2() {
         console.error("Could not find description for " + el.textContent);
       }
     });
-  [...document.querySelectorAll("dt[id] > .adef, dt[id] > .property")].forEach(el => {
+  document.querySelectorAll("dt[id] > .adef, dt[id] > .property")
+    .forEach(el => {
     const dt = el.parentNode;
     const newDt = document.createElement("dt");
     const dfn = document.createElement("dfn");
@@ -309,7 +647,7 @@ function preProcessSVG2() {
     newDt.appendChild(dfn);
     dt.replaceWith(newDt);
   });
-  [...document.querySelectorAll('b[id^="__svg__"]')].forEach(el => {
+  document.querySelectorAll('b[id^="__svg__"]').forEach(el => {
     const [,, containername, membername] = el.id.split('__');
     if (containername && membername) {
       let container = idlTree.find(i => i.name === containername);
@@ -326,14 +664,14 @@ function preProcessSVG2() {
       }
     }
   });
-  [...document.querySelectorAll('h3[id^="Interface"]:not([data-dfn-type])')].forEach(el => {
+  document.querySelectorAll('h3[id^="Interface"]:not([data-dfn-type])').forEach(el => {
     const name = el.id.slice("Interface".length);
     if (idlTree.find(i => i.name === name && i.type === "interface")) {
       el.dataset.dfnType = "interface";
       el.dataset.lt = name;
     }
   });
-  [...document.querySelectorAll('b[id]:not([data-dfn-type])')].forEach(el => {
+  document.querySelectorAll('b[id]:not([data-dfn-type])').forEach(el => {
     const name = el.textContent;
     const idlItem = idlTree.find(i => i.name === name) ;
     if (idlItem) {

package/src/browserlib/map-ids-to-headings.mjs

@@ -1,6 +1,12 @@
 import createOutline from './create-outline.mjs';
 import getAbsoluteUrl from './get-absolute-url.mjs';
 
+// Regular expression to capture the numbering of a heading. The expression
+// extracts numbers such as "1.", "A.", "A.3", "13.3.4.". Note: a top-level
+// number always ends with a ".", but there may be no final "." in sublevels
+// (Bikeshed adds one, ReSpec does not).
+const reNumber = /^([A-Z0-9]\.|[A-Z](\.[0-9]+)+\.?|[0-9]+(\.[0-9]+)+\.?)\s/;
+
 /**
  * Generate a mapping between elements that have an ID and the closest heading
  * (that also has an ID) under which these elements appear in the DOM tree.
@@ -21,11 +27,12 @@ import getAbsoluteUrl from './get-absolute-url.mjs';
  *   such a heading.
  */
 export default function () {
-  // Regular expression to capture the numbering of a heading. The expression
-  // extracts numbers such as "1.", "A.", "A.3", "13.3.4.". Note: a top-level
-  // number always ends with a ".", but there may be no final "." in sublevels
-  // (Bikeshed adds one, ReSpec does not).
-  const reNumber = /^([A-Z0-9]\.|[A-Z](\.[0-9]+)+\.?|[0-9]+(\.[0-9]+)+\.?)\s/;
+  // Special-casing ecmascript specs which use special markup for sections
+  // <emu-clause>
+  if (document.querySelector("emu-clause")) {
+    return esMapIdToHeadings();
+  }
+
 
   // Get a flat list of all conceptual sections
   function flattenSections(outline) {
@@ -86,4 +93,44 @@ export default function () {
   });
 
   return mappingTable;
-}
+}
+
+function esMapIdToHeadings() {
+  // Based on https://tc39.es/ecmarkup/
+  // and actual emu-* tags used in the ecmascript spec with ids
+  const ignoreTags = ["emu-xref"];
+  const sectionTags = ["emu-intro", "emu-clause", "emu-annex"];
+
+  // Compute once whether we created a single page version out of multiple pages
+  const singlePage = !document.querySelector('[data-reffy-page]');
+
+  let mappingTable = {};
+  [...document.querySelectorAll(`[id]:not(${ignoreTags.join(',')}`)]
+    .forEach(el => {
+      const section = el.closest(`${sectionTags.map(t => `${t}[id]`).join(',')}`);
+
+      // These are spec UI-related ids, so not a loss
+      if (!section) return;
+
+      const heading = section.querySelector("h1");
+      const trimmedText = heading.textContent.trim();
+      const nodeid = getAbsoluteUrl(el, { singlePage });
+      const href = getAbsoluteUrl(section, { singlePage });
+
+      const match = trimmedText.match(reNumber);
+      const number = match ? match[1] : null;
+
+      mappingTable[nodeid] = {
+        id: section.id,
+        href,
+        title: trimmedText.replace(reNumber, '').trim().replace(/\s+/g, ' ')
+      };
+
+      if (number) {
+        // Store the number without the final "."
+        mappingTable[nodeid].number = number.replace(/\.$/, '');
+      }
+
+    });
+  return mappingTable;
+}

package/src/cli/check-specs.js

@@ -28,7 +28,7 @@ const path = require('path');
 const browserSpecs = require('browser-specs');
 const requireFromWorkingDirectory = require('../lib/util').requireFromWorkingDirectory;
 const expandCrawlResult = require('../lib/util').expandCrawlResult;
-const crawlList = require('./crawl-specs').crawlList;
+const crawlList = require('../lib/specs-crawler').crawlList;
 const mergeCrawlResults = require('./merge-crawl-results').mergeCrawlResults;
 const studyCrawl = require('./study-crawl').studyCrawl;