reffy 10.2.3 → 11.0.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "reffy",
-  "version": "10.2.3",
+  "version": "11.0.0",
   "description": "W3C/WHATWG spec dependencies exploration companion. Features a short set of tools to study spec references as well as WebIDL term definitions and references found in W3C specifications.",
   "repository": {
     "type": "git",
@@ -33,13 +33,13 @@
   "bin": "./reffy.js",
   "dependencies": {
     "abortcontroller-polyfill": "1.7.5",
-    "ajv": "8.11.0",
+    "ajv": "8.11.2",
     "ajv-formats": "2.1.1",
     "commander": "9.4.1",
     "fetch-filecache-for-crawling": "4.1.0",
-    "puppeteer": "19.2.2",
+    "puppeteer": "19.3.0",
     "semver": "^7.3.5",
-    "web-specs": "2.35.0",
+    "web-specs": "2.36.0",
     "webidl2": "24.2.2"
   },
   "devDependencies": {
@@ -48,7 +48,7 @@
     "nock": "13.2.9",
     "respec": "32.3.0",
     "respec-hljs": "2.1.1",
-    "rollup": "3.2.5"
+    "rollup": "3.5.0"
   },
   "scripts": {
     "test": "mocha --recursive tests/"

package/schemas/browserlib/extract-cssdfn.json

@@ -4,17 +4,18 @@
 
   "type": "object",
   "additionalProperties": false,
-  "required": ["properties", "atrules", "valuespaces"],
+  "required": ["properties", "atrules", "selectors", "values"],
   "properties": {
     "properties": {
-      "type": "object",
-      "propertyNames": { "$ref": "../common.json#/$defs/cssPropertyName" },
-      "additionalProperties": {
+      "type": "array",
+      "items": {
         "type": "object",
         "additionalProperties": true,
+        "required": ["name"],
         "properties": {
           "name": { "$ref": "../common.json#/$defs/cssPropertyName" },
           "value": { "$ref": "../common.json#/$defs/cssValue" },
+          "values": { "$ref": "../common.json#/$defs/cssValues" },
           "styleDeclaration": {
             "type": "array",
             "items": { "type": "string" },
@@ -25,25 +26,26 @@
     },
 
     "atrules": {
-      "type": "object",
-      "propertyNames": {
-        "type": "string",
-        "pattern": "^@"
-      },
-      "additionalProperties": {
+      "type": "array",
+      "items": {
         "type": "object",
+        "required": ["name", "descriptors"],
         "additionalProperties": false,
         "properties": {
+          "name": { "type": "string", "pattern": "^@" },
           "value": { "$ref": "../common.json#/$defs/cssValue" },
+          "prose": { "type": "string" },
           "descriptors": {
             "type": "array",
             "items": {
               "type": "object",
+              "required": ["name", "for"],
               "additionalProperties": true,
               "properties": {
                 "name": { "type": "string" },
                 "for": { "type": "string" },
-                "value": { "$ref": "../common.json#/$defs/cssValue" }
+                "value": { "$ref": "../common.json#/$defs/cssValue" },
+                "values": { "$ref": "../common.json#/$defs/cssValues" }
               }
             }
           }
@@ -51,21 +53,49 @@
       }
     },
 
-    "valuespaces": {
-      "type": "object",
-      "propertyNames": {
-        "type": "string",
-        "pattern": "^<[^>]+>$"
-      },
-      "additionalProperties": {
+    "selectors": {
+      "type": "array",
+      "items": {
         "type": "object",
+        "required": ["name"],
         "additionalProperties": false,
         "properties": {
+          "name": { "$ref": "../common.json#/$defs/cssPropertyName" },
           "prose": { "type": "string" },
           "value": { "$ref": "../common.json#/$defs/cssValue" },
-          "legacyValue": { "$ref": "../common.json#/$defs/cssValue" }
+          "values": { "$ref": "../common.json#/$defs/cssValues" }
         }
       }
+    },
+
+    "values": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "required": ["name", "type"],
+        "additionalProperties": false,
+        "properties": {
+          "name": { "type": "string", "pattern": "^<[^>]+>$|^.*()$" },
+          "type": { "type": "string", "enum": ["type", "function"] },
+          "prose": { "type": "string" },
+          "value": { "$ref": "../common.json#/$defs/cssValue" },
+          "legacyValue": { "$ref": "../common.json#/$defs/cssValue" },
+          "values": { "$ref": "../common.json#/$defs/cssValues" }
+        }
+      }
+    },
+
+    "warnings": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "required": ["msg", "name"],
+        "properties": {
+          "msg": { "type": "string" },
+          "name": { "type": "string" }
+        }
+      },
+      "minItems": 1
     }
   }
 }

package/schemas/common.json

@@ -39,6 +39,23 @@
       "minLength": 1
     },
 
+    "cssValues": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "required": ["name", "type", "value"],
+        "additionalProperties": false,
+        "properties": {
+          "name": { "$ref": "#/$defs/cssValue" },
+          "type": { "type": "string", "enum": ["type", "function", "value"] },
+          "prose": { "type": "string" },
+          "value": { "$ref": "#/$defs/cssValue" },
+          "legacyValue": { "$ref": "#/$defs/cssValue" },
+          "values": { "$ref": "#/$defs/cssValues" }
+        }
+      }
+    },
+
     "interface": {
       "type": "string",
       "pattern": "^[A-Z]([A-Za-z0-9_])*$|^console$",

package/src/browserlib/extract-cssdfn.mjs

@@ -8,44 +8,306 @@ import informativeSelector from './informative-selector.mjs';
  * @return {Promise} The promise to get an extract of the CSS definitions, or
  *   an empty CSS description object if the spec does not contain any CSS
  *   definition. The return object will have properties named "properties",
- *  "descriptors", and "valuespaces".
+ *  "descriptors", "selectors" and "values".
  */
 export default function () {
-  let res = {
-    properties: extractTableDfns(document, 'propdef', { unique: true }),
-    atrules: {},
-    valuespaces: extractValueSpaces(document)
+  // List of inconsistencies and errors found in the spec while trying to make
+  // sense of the CSS definitions and production rules it contains
+  const warnings = [];
+
+  const res = {
+    // Properties are always defined in dedicated tables in modern CSS specs
+    properties: extractDfns({
+      selector: 'table.propdef:not(.attrdef)',
+      extractor: extractTableDfn,
+      duplicates: 'merge',
+      mayReturnMultipleDfns: true,
+      warnings
+    }),
+
+    // At-rules, selectors, functions and types are defined through dfns with
+    // the right "data-dfn-type" attribute
+    // Note some selectors are re-defined locally in HTML and Fullscreen. We
+    // won't import them.
+    atrules: extractDfns({
+      selector: 'dfn[data-dfn-type=at-rule]',
+      extractor: extractTypedDfn,
+      duplicates: 'reject',
+      warnings
+    }),
+    selectors: extractDfns({
+      selector: 'dfn[data-dfn-type=selector][data-export]',
+      extractor: extractTypedDfn,
+      duplicates: 'reject',
+      warnings
+    }),
+    values: extractDfns({
+      selector: ['dfn[data-dfn-type=function]:not([data-dfn-for])',
+                 'dfn[data-dfn-type=function][data-dfn-for=""]',
+                 'dfn[data-dfn-type=type]:not([data-dfn-for])',
+                 'dfn[data-dfn-type=type][data-dfn-for=""]'
+                ].join(','),
+      extractor: extractTypedDfn,
+      duplicates: 'reject',
+      keepDfnType: true,
+      warnings
+    })
   };
-  let descriptors = extractTableDfns(document, 'descdef', { unique: false });
 
-  // Try old recipes if we couldn't extract anything
-  if ((Object.keys(res.properties).length === 0) &&
-      (Object.keys(descriptors).length === 0)) {
-    res.properties = extractDlDfns(document, 'propdef', { unique: true });
-    descriptors = extractDlDfns(document, 'descdef', { unique: false });
+  // At-rules have descriptors, defined in dedicated tables in modern CSS specs
+  // Note some of the descriptors are defined with a "type" property set to
+  // "range". Not sure what the type of a descriptor is supposed to mean, but
+  // let's keep that information around. One such example is the
+  // "-webkit-device-pixel-ratio" descriptor for "@media" at-rule in compat:
+  // https://compat.spec.whatwg.org/#css-media-queries-webkit-device-pixel-ratio
+  let descriptors = extractDfns({
+    selector: 'table.descdef:not(.attrdef)',
+    extractor: extractTableDfn,
+    duplicates: 'push',
+    mayReturnMultipleDfns: true,
+    keepDfnType: true,
+    warnings
+  });
+
+  // Older specs may follow older recipes, let's give them a try if we couldn't
+  // extract properties or descriptors
+  if (res.properties.length === 0 && descriptors.length === 0) {
+    res.properties = extractDfns({
+      selector: 'div.propdef dl',
+      extractor: extractDlDfn,
+      duplicates: 'merge',
+      mayReturnMultipleDfns: true,
+      warnings
+    });
+    descriptors = extractDfns({
+      selector: 'div.descdef dl',
+      extractor: extractDlDfn,
+      duplicates: 'push',
+      mayReturnMultipleDfns: true,
+      warnings
+    });
   }
 
-  // Move at-rules definitions from valuespaces to at-rules structure
-  for (const [name, dfn] of Object.entries(res.valuespaces)) {
-    if (name.startsWith('@')) {
-      if (!res.atrules[name]) {
-        res.atrules[name] = Object.assign(dfn, { descriptors: [] });
+  // Move descriptors to at-rules structure
+  for (const desclist of descriptors) {
+    for (const desc of desclist) {
+      let rule = res.atrules.find(r => r.name === desc.for);
+      if (rule) {
+        if (!rule.descriptors) {
+          rule.descriptors = [];
+        }
+      }
+      else {
+        rule = { name: desc.for, descriptors: [] };
+        res.atrules.push(rule);
       }
-      delete res.valuespaces[name];
+      rule.descriptors.push(desc);
+    }
+  }
+  for (const rule of res.atrules) {
+    if (!rule.descriptors) {
+      rule.descriptors = [];
     }
   }
 
-  // Move descriptors to at-rules structure
-  for (const [name, desclist] of Object.entries(descriptors)) {
+  // Keep an index of "root" (non-namespaced + descriptors) dfns
+  const rootDfns = Object.values(res).flat();
+  for (const desclist of descriptors) {
     for (const desc of desclist) {
-      const rule = desc.for;
-      if (!res.atrules[rule]) {
-        res.atrules[rule] = { descriptors: [] };
+      rootDfns.push(desc);
+    }
+  }
+
+  // Extract value dfns.
+  // Note some of the values can be namespaced "function" or "type" dfns, such
+  // as "<content-replacement>" in css-content-3:
+  // https://drafts.csswg.org/css-content-3/#typedef-content-content-replacement
+  const values = extractDfns({
+    selector: ['dfn[data-dfn-type=value][data-dfn-for]:not([data-dfn-for=""])',
+               'dfn[data-dfn-type=function][data-dfn-for]:not([data-dfn-for=""])',
+               'dfn[data-dfn-type=type][data-dfn-for]:not([data-dfn-for=""])'
+              ].join(','),
+    extractor: extractTypedDfn,
+    duplicates: 'push',
+    keepDfnType: true,
+    warnings
+  }).flat();
+
+  const matchName = (name, { approx = false } = {}) => dfn => {
+    let res = dfn.name === name;
+    if (!res && name.match(/^@.+\/.+$/)) {
+      // Value reference might be for an at-rule descriptor:
+      // https://tabatkins.github.io/bikeshed/#dfn-for
+      const parts = name.split('/');
+      res = dfn.name === parts[1] && dfn.for === parts[0];
+    }
+    if (!res && approx) {
+      res = `<${dfn.name}>` === name;
+    }
+    return res;
+  };
+
+  // Extract production rules from pre.prod contructs
+  // and complete result structure accordingly
+  const rules = extractProductionRules(document);
+  for (const rule of rules) {
+    const dfn = rootDfns.find(matchName(rule.name)) ??
+      rootDfns.find(matchName(rule.name, { approx: true }));
+    if (dfn) {
+      dfn.value = rule.value;
+      if (rule.legacyValue) {
+        dfn.legacyValue = rule.legacyValue;
+      }
+    }
+    else {
+      let matchingValues = values.filter(matchName(rule.name));
+      if (matchingValues.length === 0) {
+        matchingValues = values.filter(matchName(rule.name, { approx: true }));
+      }
+      for (const matchingValue of matchingValues) {
+        matchingValue.value = rule.value;
+        if (rule.legacyValue) {
+          matchingValue.legacyValue = rule.legacyValue;
+        }
+      }
+      if (matchingValues.length === 0) {
+        // Dangling production rule. That should never happen for properties,
+        // at-rules, descriptors and functions, since they should always be
+        // defined somewhere. That happens from time to time for types that are
+        // described in prose and that don't have a dfn. One could perhaps argue
+        // that these constructs ought to have a dfn too.
+        if (!res.warnings) {
+          res.warnings = []
+        }
+        const warning = Object.assign({ msg: 'Missing definition' }, rule);
+        warnings.push(warning);
+        rootDfns.push(warning);
       }
-      res.atrules[rule].descriptors.push(desc);
     }
   }
 
+  // We now need to associate values with dfns. CSS specs tend to list in
+  // "data-dfn-for" attributes of values the construct to which the value
+  // applies directly but also the constructs to which the value indirectly
+  // applies. For instance, "open-quote" in css-content-3 directly applies to
+  // "<quote>" and indirectly applies to "<content-list>" (which has "<quote>"
+  // in its value syntax) and to "content" (which has "<content-list>" in its
+  // value syntax), and all 3 appear in the "data-dfn-for" attribute:
+  // https://drafts.csswg.org/css-content-3/#valdef-content-open-quote
+  //
+  // To make it easier to make sense of the extracted data and avoid duplicates
+  // in the resulting structure, the goal is to only keep the constructs to
+  // which the value applies directly. In the previous example, the goal is to
+  // list "open-quote" under "<quote>" but not under "<content-list>" and
+  // "<content>".
+
+  // Start by looking at values that are "for" something. Their list of parents
+  // appears in the "data-dfn-for" attribute of their definition.
+  const parents = {};
+  for (const value of values) {
+    if (!parents[value.name]) {
+      parents[value.name] = [];
+    }
+    parents[value.name].push(...value.for.split(',').map(ref => ref.trim()));
+  }
+
+  // Then look at non-namespaced types and functions. Their list of parents
+  // are all the definitions whose values reference them (for instance,
+  // "<quote>" has "<content-list>" as parent because "<content-list>" has
+  // "<quote>" in its value syntax).
+  for (const type of res.values) {
+    if (!parents[type.name]) {
+      parents[type.name] = [];
+    }
+    for (const value of values) {
+      if (value.value?.includes(type.name)) {
+        parents[type.name].push(value.name);
+      }
+    }
+    for (const dfn of rootDfns) {
+      if (dfn.value?.includes(type.name)) {
+        parents[type.name].push(dfn.name);
+      }
+    }
+  }
+
+  // Helper functions to reason on the parents index we just created.
+  // Note there may be cycles. For instance, in CSS Images 4, <image> references
+  // <image-set()>, which references <image-set-option>, which references
+  // <image> again:
+  // https://drafts.csswg.org/css-images-4/#typedef-image
+  const isAncestorOf = (ancestor, child) => {
+    let seen = [];
+    const checkChild = c => {
+      let res = ancestor === c;
+      if (!res) {
+        res = parents[c]
+          ?.filter(p => !seen.includes(p))
+          ?.find(p => checkChild(ancestor, p));
+      }
+      seen = seen.concat(parents[c]);
+      return res;
+    }
+    return checkChild(child);
+  };
+  const isDeepestConstruct = (name, list) =>
+    list.every(p => p === name || !isAncestorOf(name, p));
+
+  // We may now associate values with dfns
+  for (const value of values) {
+    value.for.split(',')
+      .map(ref => ref.trim())
+      .filter((ref, _, arr) => isDeepestConstruct(ref, arr))
+      .forEach(ref => {
+        // Look for the referenced definition in root dfns
+        const dfn = rootDfns.find(matchName(ref)) ??
+          rootDfns.find(matchName(ref, { approx: true }));
+        if (dfn) {
+          if (!dfn.values) {
+            dfn.values = [];
+          }
+          dfn.values.push(value);
+        }
+        else {
+          // If the referenced definition is not in root dfns, look in
+          // namespaced dfns as functions/types are sometimes namespaced to a
+          // property, and values may reference these functions/types.
+          let referencedValues = values.filter(matchName(ref));
+          if (referencedValues.length === 0) {
+            referencedValues = values.filter(matchName(ref, { approx: true }));
+          }
+          for (const referencedValue of referencedValues) {
+            if (!referencedValue.values) {
+              referencedValue.values = [];
+            }
+            referencedValue.values.push(value);
+          }
+
+          if (referencedValues.length === 0) {
+            warnings.push(Object.assign(
+              { msg: 'Dangling value' },
+              value,
+              { for: ref }
+            ));
+          }
+        }
+      });
+  }
+
+  // Don't keep the info on whether value comes from a pure syntax section
+  for (const dfn of rootDfns) {
+    delete dfn.pureSyntax;
+  }
+  for (const value of values) {
+    delete value.for;
+    delete value.pureSyntax;
+  }
+
+  // Report warnings
+  if (warnings.length > 0) {
+    res.warnings = warnings;
+  }
+
   return res;
 }
 
@@ -162,32 +424,63 @@ const mergeDfns = (dfn1, dfn2) => {
 
 /**
  * Extract CSS definitions in a spec using the given CSS selector and extractor
+ *
+ * The "duplicates" option controls the behavior of the function when it
+ * encounters a duplicate (or similar) definition for the same thing. Values
+ * may be "reject" to report a warning, "merge" to merge the definitions, and
+ * "push" to keep both definitions separated.
+ *
+ * Merge issues and unexpected duplicates get reported as warnings in the
+ * "warnings" array passed as parameter.
  */
-const extractDfns = (doc, selector, extractor, { unique } = { unique: true }) => {
-  let res = {};
-  [...doc.querySelectorAll(selector)]
+const extractDfns = ({ root = document,
+                       selector,
+                       extractor,
+                       duplicates = 'reject',
+                       mayReturnMultipleDfns = false,
+                       keepDfnType = false,
+                       warnings = [] }) => {
+  const res = [];
+  [...root.querySelectorAll(selector)]
     .filter(el => !el.closest(informativeSelector))
     .filter(el => !el.querySelector('ins, del'))
     .map(extractor)
-    .filter(dfn => !!dfn.name)
-    .map(dfn => dfn.name.split(',').map(name => Object.assign({},
-      dfn, { name: name.trim() })))
+    .filter(dfn => !!dfn?.name)
+    .map(dfn => !mayReturnMultipleDfns ? [dfn] :
+      dfn.name.split(',').map(name => Object.assign({}, dfn, { name: name.trim() })))
     .reduce((acc, val) => acc.concat(val), [])
     .forEach(dfn => {
-      if (res[dfn.name]) {
-        if (unique) {
-          const merged = mergeDfns(res[dfn.name], dfn);
-          if (!merged) {
-            throw new Error(`More than one dfn found for CSS property "${dfn.name}" and dfns cannot be merged`);
+      if (dfn.type && !keepDfnType) {
+        delete dfn.type;
+      }
+      const idx = res.findIndex(e => e.name === dfn.name);
+      if (idx >= 0) {
+        switch (duplicates) {
+        case 'merge':
+          const merged = mergeDfns(res[idx], dfn);
+          if (merged) {
+            res[idx] = merged;
           }
-          res[dfn.name] = merged;
-        }
-        else {
-          res[dfn.name].push(dfn);
+          else {
+            warnings.push(Object.assign(
+              { msg: 'Unmergeable definition' },
+              dfn
+            ));
+          }
+          break;
+
+        case 'push':
+          res[idx].push(dfn);
+
+        default:
+          warnings.push(Object.assign(
+            { msg: 'Duplicate definition' },
+            dfn
+          ));
         }
       }
       else {
-        res[dfn.name] = unique ? dfn : [dfn];
+        res.push(duplicates !== 'push' ? dfn : [dfn]);
       }
     });
   return res;
@@ -195,201 +488,187 @@ const extractDfns = (doc, selector, extractor, { unique } = { unique: true }) =>
 
 
 /**
- * Extract CSS definitions in tables for the given class name
- * (typically one of `propdef` or `descdef`)
+ * Regular expression used to split production rules:
+ * Split on the space that precedes a term immediately before an equal sign
+ * that is not wrapped in quotes (an equal sign wrapped in quotes is part of
+ * actual value syntax)
  */
-const extractTableDfns = (doc, className, options) =>
-  extractDfns(doc, 'table.' + className + ':not(.attrdef)', extractTableDfn, options);
+const reSplitRules = /\s(?=[^\s]+?\s*?=[^'])/;
 
 
 /**
- * Extract CSS definitions in a dl list for the given class name
- * (typically one of `propdef` or `descdef`)
+ * Helper function to parse a production rule. The "pureSyntax" parameter
+ * should be set to indicate that the rule comes from a pure syntactic block
+ * and should have precedence over another value definition that may be
+ * extracted from the prose. For instance, this makes it possible to extract
+ * `<abs()> = abs( <calc-sum> )` from the syntax part in CSS Values instead
+ * of `<abs()> = abs(A)` which is how the function is defined in prose.
  */
-const extractDlDfns = (doc, className, options) =>
-  extractDfns(doc, 'div.' + className + ' dl', extractDlDfn, options);
+const parseProductionRule = (rule, { res = [], pureSyntax = false }) => {
+  const nameAndValue = rule
+    .replace(/\/\*[^]*?\*\//gm, '')  // Drop comments
+    .split(/\s?=\s/)
+    .map(s => s.trim().replace(/\s+/g, ' '));
+
+  const name = nameAndValue[0];
+  const value = nameAndValue[1];
+
+  const normalizedValue = normalize(value);
+  let entry = res.find(e => e.name === name);
+  if (!entry) {
+    entry = { name };
+    res.push(entry);
+  }
+  if (!entry.value || (pureSyntax && !entry.pureSyntax)) {
+    entry.value = normalizedValue;
+    entry.pureSyntax = pureSyntax;
+  }
+  else if (entry.value !== normalizedValue) {
+    // Second definition found. Typically happens for the statement and
+    // block @layer definitions in css-cascade-5. We'll combine the values
+    // as alternative.
+    // Hardcoded exception: re-definitions of rgb() and hsl() are legacy
+    // constructs, stored separately not to pollute `value`.
+    if (name === '<rgb()>' || name === '<hsl()>') {
+      entry.legacyValue = normalizedValue;
+    }
+    else {
+      entry.value += ` | ${normalizedValue}`;
+    }
+  }
+
+  return entry;
+};
 
 
 /**
- * Extract value spaces (non-terminal values) defined in the specification
- *
- * From a definitions data model perspective, non-terminal values are those
- * defined with a `data-dfn-type` attribute equal to `type` or `function`. They
- * form (at least in theory) a single namespace across CSS specs.
- *
- * Definitions with `data-dfn-type` attribute set to `value` are not extracted
- * on purpose as they are typically namespaced to another construct (through a
- * `data-dfn-for` attribute.
- *
- * The function also extracts syntax of at-rules (name starts with '@') defined
- * in "pre.prod" blocks.
+ * Extract the given dfn
  */
-const extractValueSpaces = doc => {
+const extractTypedDfn = dfn => {
   let res = {};
-
-  // Helper function to parse a production rule. The "pureSyntax" parameter
-  // should be set to indicate that the rule comes from a pure syntactic block
-  // and should have precedence over another value definition that may be
-  // extracted from the prose. For instance, this makes it possible to extract
-  // `<abs()> = abs( <calc-sum> )` from the syntax part in CSS Values instead
-  // of `<abs()> = abs(A)` which is how the function is defined in prose.
-  const parseProductionRule = (rule, { pureSyntax = false }) => {
-    const nameAndValue = rule
-      .replace(/\/\*[^]*?\*\//gm, '')  // Drop comments
-      .split(/\s?=\s/)
-      .map(s => s.trim().replace(/\s+/g, ' '));
-
-    function addValuespace(name, value) {
-      const normalizedValue = normalize(value);
-      if (!(name in res)) {
-        res[name] = {};
-      }
-      if (!res[name].value || (pureSyntax && !res[name].pureSyntax)) {
-        res[name].value = normalizedValue;
-        res[name].pureSyntax = pureSyntax;
-      }
-      else if (res[name].value !== normalizedValue) {
-        // Second definition found. Typically happens for the statement and
-        // block @layer definitions in css-cascade-5. We'll combine the values
-        // as alternative.
-        // Hardcoded exception: re-definitions of rgb() and hsl() are legacy
-        // constructs, stored separately not to pollute `value`.
-        if (name === '<rgb()>' || name === '<hsl()>') {
-          res[name].legacyValue = normalizedValue;
-        }
-        else {
-          res[name].value += ` | ${normalizedValue}`;
-        }
-      }
+  const arr = [];
+  const dfnType = dfn.getAttribute('data-dfn-type');
+  const dfnFor = dfn.getAttribute('data-dfn-for');
+  const parent = dfn.parentNode.cloneNode(true);
+
+  // Remove note references as in:
+  // https://drafts.csswg.org/css-syntax-3/#the-anb-type
+  // and remove MDN annotations as well
+  [...parent.querySelectorAll('sup')]
+    .map(sup => sup.parentNode.removeChild(sup));
+  [...parent.querySelectorAll('aside, .mdn-anno')]
+    .map(annotation => annotation.parentNode.removeChild(annotation));
+
+  const text = parent.textContent.trim();
+  if (text.match(/\s?=\s/)) {
+    // Definition appears in a "prod = foo" text, that's all good
+    // ... except in css-easing-2 draft where text also contains another
+    // production rule as a child of the first one:
+    // https://drafts.csswg.org/css-easing-2/#typedef-step-easing-function
+    const prod = text.split(reSplitRules)
+        .find(p => p.trim().startsWith(dfn.textContent.trim()));
+    if (dfn.closest('pre')) {
+      // Don't attempt to parse pre tags at this stage, they are tricky to
+      // split, we'll parse them as text and map them to the right definitions
+      // afterwards.
+      const name = (dfn.getAttribute('data-lt') ?? dfn.textContent).trim();
+      res = { name };
     }
-
-    if (nameAndValue[0].match(/^<.*>$|^.*\(\)$/)) {
-      // Regular valuespace
-      addValuespace(
-        nameAndValue[0].replace(/^(.*\(\))$/, '<$1>'),
-        nameAndValue[1]);
+    else if (prod) {
+      res = parseProductionRule(prod, { pureSyntax: true });
     }
-    else if (nameAndValue[0].match(/^@[a-z\-]+$/)) {
-      // At-rule syntax
-      addValuespace(nameAndValue[0], nameAndValue[1]);
+    else {
+      // "=" may appear in another formula in the body of the text, as in:
+      // https://drafts.csswg.org/css-speech-1/#typedef-voice-volume-decibel
+      // It may be worth checking but not an error per se.
+      console.warn('[reffy]', `Found "=" next to definition of ${dfn.textContent.trim()} but no production rule. Did I miss something?`);
+      const name = (dfn.getAttribute('data-lt') ?? dfn.textContent).trim();
+      res = { name, prose: text.replace(/\s+/g, ' ') };
     }
-  };
-
-  // Regular expression to use to split production rules:
-  // Split on the space that precedes a term immediately before an equal sign
-  // that is not wrapped in quotes (an equal sign wrapped in quotes is part of
-  // actual value syntax)
-  const reSplitRules = /\s(?=[^\s]+?\s*?=[^'])/;
-
-  // Extract all dfns with data-dfn-type="type" or data-dfn-type="function"
-  // but ignore definitions in <pre> as they do not always use dfns, as in
-  // https://drafts.csswg.org/css-values-4/#calc-syntax
-  [...doc.querySelectorAll(
-      'dfn[data-dfn-type=type],dfn[data-dfn-type=function]')]
-    .filter(el => !el.closest(informativeSelector))
-    .filter(el => !el.closest('pre'))
-    .forEach(dfn => {
-      const parent = dfn.parentNode.cloneNode(true);
-
-      // Remove note references as in:
-      // https://drafts.csswg.org/css-syntax-3/#the-anb-type
-      // and remove MDN annotations as well
-      [...parent.querySelectorAll('sup')]
-        .map(sup => sup.parentNode.removeChild(sup));
-      [...parent.querySelectorAll('aside, .mdn-anno')]
-        .map(annotation => annotation.parentNode.removeChild(annotation));
-
-      const text = parent.textContent.trim();
-      if (text.match(/\s?=\s/)) {
-        // Definition appears in a "prod = foo" text, that's all good
-        // ... except in css-easing-2 draft where text also contains another
-        // production rule as a child of the first one:
-        // https://drafts.csswg.org/css-easing-2/#typedef-step-easing-function
-        const prod = text.split(reSplitRules)
-            .find(p => p.trim().startsWith(dfn.textContent.trim()));
-        if (prod) {
-          parseProductionRule(prod, { pureSyntax: true });
-        }
-        else {
-          // "=" may appear in another formula in the body of the text, as in:
-          // https://drafts.csswg.org/css-speech-1/#typedef-voice-volume-decibel
-          // It may be worth checking but not an error per se.
-          console.warn('[reffy]', `Found "=" next to definition of ${dfn.textContent.trim()} but no production rule. Did I miss something?`);
-          const name = (dfn.getAttribute('data-lt') ?? dfn.textContent)
-            .trim().replace(/^<?(.*?)>?$/, '<$1>');
-          if (!(name in res)) {
-            res[name] = {
-              prose: parent.textContent.trim().replace(/\s+/g, ' ')
-            };
-          }
-        }
+  }
+  else if (dfn.textContent.trim().match(/^[a-zA-Z_][a-zA-Z0-9_\-]+\([^\)]+\)$/)) {
+    // Definition is "prod(foo bar)", create a "prod() = prod(foo bar)" entry
+    const fn = dfn.textContent.trim().match(/^([a-zA-Z_][a-zA-Z0-9_\-]+)\([^\)]+\)$/)[1];
+    res = parseProductionRule(`${fn}() = ${dfn.textContent.trim()}`, { pureSyntax: false });
+  }
+  else if (parent.nodeName === 'DT') {
+    // Definition is in a <dt>, look for value in following <dd>
+    let dd = dfn.parentNode;
+    while (dd && (dd.nodeName !== 'DD')) {
+      dd = dd.nextSibling;
+    }
+    if (!dd) {
+      return;
+    }
+    let code = dd.querySelector('p > code, pre.prod');
+    if (code) {
+      if (code.textContent.startsWith(`${text} = `) ||
+          code.textContent.startsWith(`<${text}> = `)) {
+        res = parseProductionRule(code.textContent, { pureSyntax: true });
       }
-      else if (dfn.textContent.trim().match(/^[a-zA-Z_][a-zA-Z0-9_\-]+\([^\)]+\)$/)) {
-        // Definition is "prod(foo bar)", create a "prod() = prod(foo bar)" entry
-        const fn = dfn.textContent.trim().match(/^([a-zA-Z_][a-zA-Z0-9_\-]+)\([^\)]+\)$/)[1];
-        parseProductionRule(`${fn}() = ${dfn.textContent.trim()}`, { pureSyntax: false });
+      else {
+        res = parseProductionRule(`${text} = ${code.textContent}`, { pureSyntax: false });
       }
-      else if (parent.nodeName === 'DT') {
-        // Definition is in a <dt>, look for value in following <dd>
-        let dd = dfn.parentNode;
-        while (dd && (dd.nodeName !== 'DD')) {
-          dd = dd.nextSibling;
-        }
-        if (!dd) {
-          return;
-        }
-        let code = dd.querySelector('p > code, pre.prod');
-        if (code) {
-          if (code.textContent.startsWith(`${text} = `) ||
-              code.textContent.startsWith(`<${text}> = `)) {
-            parseProductionRule(code.textContent, { pureSyntax: true });
-          }
-          else {
-            parseProductionRule(`${text} = ${code.textContent}`, { pureSyntax: false });
-          }
+    }
+    else {
+      // Remove notes, details sections that link to tests, and subsections
+      // that go too much into details
+      dd = dd.cloneNode(true);
+      [...dd.children].forEach(c => {
+        if (c.tagName === 'DETAILS' ||
+            c.tagName === 'DL' ||
+            c.classList.contains('note')) {
+          c.remove();
         }
-        else {
-          // Remove notes, details sections that link to tests, and subsections
-          // that go too much into details
-          dd = dd.cloneNode(true);
-          [...dd.children].forEach(c => {
-            if (c.tagName === 'DETAILS' ||
-                c.tagName === 'DL' ||
-                c.classList.contains('note')) {
-              c.remove();
-            }
-          });
+      });
 
-          const name = (dfn.getAttribute('data-lt') ?? dfn.textContent)
-            .trim().replace(/^<?(.*?)>?$/, '<$1>');
-          if (!(name in res)) {
-            res[name] = {};
-          }
-          if (!res[name].prose) {
-            res[name].prose = dd.textContent.trim().replace(/\s+/g, ' ');
-          }
-        }
-      }
-      else if (parent.nodeName === 'P') {
-        // Definition is in regular prose, extract value from prose.
-        const name = (dfn.getAttribute('data-lt') ?? dfn.textContent)
-          .trim().replace(/^<?(.*?)>?$/, '<$1>');
-        if (!(name in res)) {
-          res[name] = {
-            prose: parent.textContent.trim().replace(/\s+/g, ' ')
-          };
-        }
-      }
-    });
+      const name = (dfn.getAttribute('data-lt') ?? dfn.textContent).trim();
+      res = { name, prose: dd.textContent.trim().replace(/\s+/g, ' ') };
+    }
+  }
+  else if (parent.nodeName === 'P') {
+    // Definition is in regular prose, extract value from prose.
+    const name = (dfn.getAttribute('data-lt') ?? dfn.textContent).trim();
+    res = { name, prose: parent.textContent.trim().replace(/\s+/g, ' ') };
+  }
+  else {
+    // Definition is in a heading or a more complex structure, just list the
+    // name for now.
+    const name = (dfn.getAttribute('data-lt') ?? dfn.textContent).trim();
+    res = { name };
+  }
+
+  res.type = dfnType;
+  if (dfnType === 'value') {
+    res.value = normalize(res.name);
+  }
+  if (dfnFor) {
+    res.for = dfnFor;
+  }
 
-  // Complete with production rules defined in <pre class=prod> tags (some of
-  // which use dfns, while others don't, but all of them are actual production
-  // rules). For <pre> tags that don't have a "prod" class (e.g. in HTML and
-  // css-namespaces), make sure they contain a <dfn> to avoid parsing things
-  // that are not production rules
-  [...doc.querySelectorAll('pre.prod')]
-    .concat([...doc.querySelectorAll('pre:not(.idl)')]
-      .filter(el => el.querySelector('dfn')))
+  return res;
+};
+
+/**
+ * Extract production rules defined in the specification in "<pre>" tags and
+ * complete the result structure received as parameter accordingly.
+ */
+const extractProductionRules = root => {
+  // For <pre> tags that don't have a "prod" class (e.g. in HTML and
+  // css-namespaces), make sure they contain a <dfn> with a valid CSS
+  // data-dfn-type attribute to avoid parsing things that are not production
+  // rules. In all cases, make sure we're not in a changelog with details as in:
+  // https://drafts.csswg.org/css-backgrounds-3/#changes-2017-10
+  const rules = [];
+  [...root.querySelectorAll('pre.prod:not(:has(del)):not(:has(ins))')]
+    .concat([...root.querySelectorAll('pre:not(.idl):not(:has(.idl)):not(:has(del)):not(:has(ins))')]
+      .filter(el => el.querySelector([
+        'dfn[data-dfn-type=at-rule]',
+        'dfn[data-dfn-type=selector]',
+        'dfn[data-dfn-type=value]',
+        'dfn[data-dfn-type=function]',
+        'dfn[data-dfn-type=type]'
+      ].join(','))))
     .filter(el => !el.closest(informativeSelector))
     .map(el => el.cloneNode(true))
     .map(el => {
@@ -402,18 +681,15 @@ const extractValueSpaces = doc => {
     .map(val => val.split(reSplitRules))             // Separate definitions
     .flat()
     .map(text => text.trim())
-    .map(text => {
+    .forEach(text => {
       if (text.match(/\s?=\s/)) {
-        return parseProductionRule(text, { pureSyntax: true });
+        parseProductionRule(text, { res: rules, pureSyntax: true });
       }
       else if (text.startsWith('@')) {
         const name = text.split(' ')[0];
-        return parseProductionRule(`${name} = ${text}`, { pureSyntax: true });
+        parseProductionRule(`${name} = ${text}`, { res: rules, pureSyntax: true });
       }
     });
 
-  // Don't keep the info on whether value comes from a pure syntax section
-  Object.values(res).map(value => delete value.pureSyntax);
-
-  return res;
+  return rules;
 }

package/src/browserlib/extract-dfns.mjs

@@ -24,6 +24,13 @@ import {parse} from "../../node_modules/webidl2/index.js";
  *     "prose" (last one indicates that definition appears in the main body of
  *     the spec)
  *
+ * The extraction ignores definitions with an unknown type. A warning is issued
+ * to the console when that happens.
+ *
+ * The extraction uses the first definition it finds when it bumps into a term
+ * that is defined more than once (same "linkingText", same "type", same "for").
+ * A warning is issued to the console when that happens.
+ *
  * @function
  * @public
  * @return {Array(Object)} An Array of definitions
@@ -99,8 +106,22 @@ function hasValidType(el) {
   return isValid;
 }
 
+// Return true when definition is not already defined in the list,
+// Return false and issue a warning when it is already defined.
+function isNotAlreadyDefined(dfn, idx, list) {
+  const first = list.find(d => d === dfn ||
+      (d.type === dfn.type &&
+      d.linkingText.length === dfn.linkingText.length &&
+      d.linkingText.every(lt => dfn.linkingText.find(t => t == lt)) &&
+      d.for.length === dfn.for.length &&
+      d.for.every(lt => dfn.for.find(t => t === lt))));
+  if (first !== dfn) {
+    console.warn('[reffy]', `Duplicate dfn found for "${dfn.linkingText[0]}", type="${dfn.type}", for="${dfn.for[0]}"`);
+  }
+  return first === dfn;
+}
 
-function definitionMapper(el, idToHeading) {
+function definitionMapper(el, idToHeading, usesDfnDataModel) {
   let definedIn = 'prose';
   const enclosingEl = el.closest('dt,pre,table,h1,h2,h3,h4,h5,h6,.note,.example') || el;
   switch (enclosingEl.nodeName) {
@@ -165,8 +186,10 @@ function definitionMapper(el, idToHeading) {
       [],
 
     // Definition is public if explicitly marked as exportable or if export has
-    // not been explicitly disallowed and its type is not "dfn"
-    access: (el.hasAttribute('data-export') ||
+    // not been explicitly disallowed and its type is not "dfn", or if the spec
+    // is an old spec that does not use the "data-dfn-type" convention.
+    access: (!usesDfnDataModel ||
+             el.hasAttribute('data-export') ||
              (!el.hasAttribute('data-noexport') &&
               el.hasAttribute('data-dfn-type') &&
               el.getAttribute('data-dfn-type') !== 'dfn')) ?
@@ -200,7 +223,6 @@ export default function (spec, idToHeading = {}) {
     'h6[id][data-dfn-type]:not([data-lt=""])'
   ].join(',');
 
-  let extraDefinitions = [];
   const shortname = (typeof spec === 'string') ? spec : spec.shortname;
   switch (shortname) {
   case "html":
@@ -214,7 +236,14 @@ export default function (spec, idToHeading = {}) {
     break;
   }
 
-  return [...document.querySelectorAll(definitionsSelector)]
+  const definitions = [...document.querySelectorAll(definitionsSelector)];
+  const usesDfnDataModel = definitions.some(dfn =>
+    dfn.hasAttribute('data-dfn-type') ||
+    dfn.hasAttribute('data-dfn-for') ||
+    dfn.hasAttribute('data-export') ||
+    dfn.hasAttribute('data-noexport'));
+
+  return definitions
     .map(node => {
       // 2021-06-21: Temporary preprocessing of invalid "idl" dfn type (used for
       // internal slots) while fix for https://github.com/w3c/respec/issues/3644
@@ -237,7 +266,8 @@ export default function (spec, idToHeading = {}) {
       const link = node.querySelector('a[href^="http"]');
       return !link || (node.textContent.trim() !== link.textContent.trim());
     })
-    .map(node => definitionMapper(node, idToHeading));
+    .map(node => definitionMapper(node, idToHeading, usesDfnDataModel))
+    .filter(isNotAlreadyDefined);
 }
 
 function preProcessEcmascript() {

package/src/cli/check-missing-dfns.js

@@ -16,6 +16,9 @@
  * - `format` is the optional output format. Either `json` or `markdown` with
  * `markdown` being the default.
  *
+ * Note: CSS extraction already relies on dfns and reports missing dfns in a
+ * "warnings" property. This checker simply looks at that list.
+ *
  * @module checker
  */
 
@@ -59,61 +62,15 @@ function arraysEqual(a, b) {
  * @return {Array} An array of expected definitions
  */
 function getExpectedDfnsFromCSS(css) {
-  let expected = [];
-
-  // Add the list of expected properties, filtering out properties that define
-  // new values to an existing property (defined elsewhere)
-  expected = expected.concat(
-    Object.values(css.properties || {})
-      .filter(desc => !desc.newValues)
-      .map(desc => {
-        return {
-          linkingText: [desc.name],
-          type: 'property',
-          'for': []
-        };
-      })
-  );
-
-  // Add the list of expected at-rules
-  expected = expected.concat(
-    Object.entries(css.atrules || {}).map(([name, rule]) => {
-      if (rule.value) {
-        return {
-          linkingText: [name],
-          type: 'at-rule',
-          'for': []
-        };
-      }
-    }).filter(dfn => !!dfn)
-  );
-
-  // Add the list of expected descriptors
-  expected = expected.concat(
-    Object.entries(css.atrules || {}).map(([name, rule]) =>
-      (rule.descriptors || []).map(desc => {
-        return {
-          linkingText: [desc.name],
-          type: 'descriptor',
-          'for': [name]
-        };
-      })
-    ).flat()
-  );
-  
-  // Add the list of expected "values".
-  // Note: we don't qualify the "type" of values in valuespaces and don't store
-  // the scope of values either (the "for" property). Definition types can be
-  // "type", "function", "value", etc. in practice. The comparison cannot be
-  // perfect as a result.
-  expected = expected.concat(
-    Object.entries(css.valuespaces || {}).map(([name, desc]) => {
+  const expected = (css.warnings ?? [])
+    .filter(warning => warning.msg === 'Missing definition')
+    .map(warning => {
       return {
-        linkingText: [name],
-        value: desc.value
+        linkingText: [warning.name],
+        type: warning.type,
+        'for': warning.for
       };
-    })
-  );
+    });
 
   return expected;
 }

package/src/lib/nock-server.js

@@ -32,7 +32,7 @@ const mockSpecs = {
     html: `
       <title>WOFF2</title>
       <body>
-        <dfn id='foo'>Foo</dfn>
+        <dfn id='foo' data-dfn-type="dfn">Foo</dfn>
         <a href="https://www.w3.org/TR/bar/#baz">bar</a>
         <ul class='toc'><li><a href='page.html'>page</a></ul>`,
     pages: {

package/src/lib/specs-crawler.js

@@ -241,10 +241,11 @@ async function saveSpecResults(spec, settings) {
 
     // Save CSS dumps
     function defineCSSContent(spec) {
-        return spec.css && (
-            (Object.keys(spec.css.properties || {}).length > 0) ||
-            (Object.keys(spec.css.atrules || {}).length > 0) ||
-            (Object.keys(spec.css.valuespaces || {}).length > 0));
+        return (spec.css?.properties?.length > 0) ||
+               (spec.css?.atrules?.length > 0) ||
+               (spec.css?.selectors?.length > 0) ||
+               (spec.css?.values?.length > 0) ||
+               (spec.css?.warnings?.length > 0);
     }
     if (defineCSSContent(spec)) {
         spec.css = await saveCss(spec);

package/src/postprocessing/csscomplete.js

@@ -20,10 +20,10 @@ module.exports = {
         .filter(dfn => dfn.type == "property" && !dfn.informative)
         .forEach(propDfn => {
           propDfn.linkingText.forEach(lt => {
-            if (!spec.css.properties.hasOwnProperty(lt)) {
-              spec.css.properties[lt] = {
+            if (!spec.css.properties.find(p => p.name === lt)) {
+              spec.css.properties.push({
                 name: lt
-              };
+              });
             }
           });
         });
@@ -31,18 +31,15 @@ module.exports = {
 
     if (spec.css) {
       // Add generated IDL attribute names
-      Object.entries(spec.css.properties || {}).forEach(([prop, dfn]) => {
-        dfn.styleDeclaration = getGeneratedIDLNamesByCSSProperty(prop);
+      spec.css.properties.forEach(dfn => {
+        dfn.styleDeclaration = getGeneratedIDLNamesByCSSProperty(dfn.name);
       });
 
       // Drop the sample definition (property-name) in CSS2 and the custom
       // property definition (--*) in CSS Variables that specs incorrectly flag
       // as real CSS properties.
-      ['property-name', '--*'].forEach(prop => {
-          if ((spec.css.properties || {})[prop]) {
-              delete spec.css.properties[prop];
-          }
-      });
+      spec.css.properties = spec.css.properties.filter(p =>
+        !['property-name', '--*'].includes(p.name));
     }
 
     return spec;