wikipeg 4.0.2 → 6.0.0

package/lib/compiler/charsets.js

@@ -0,0 +1,306 @@
+"use strict";
+const unicode = require("../utils/unicode");
+
+/* Character set utilities. */
+
+// Inverting ranges requires picking MIN/MAX characters
+const CHAR_MIN = 0, CHAR_MAX = 0x10FFFF;
+
+// "CASELESS_RESTRICT" support "suppresses case-insensitive matches
+// between ASCII and non-ASCII characters"; in practice this just means
+// excluding unicode matches for 's' and 'k'.
+let caselessRestrict = false;
+
+// Operations on {start,end} range objects, using unicode code point values.
+const range = {
+
+  // Convert a {start,end} range object to a character class "part"
+  toPart(r) {
+    return (r.start === r.end) ?
+      String.fromCodePoint(r.start) :
+      [String.fromCodePoint(r.start), String.fromCodePoint(r.end)];
+  },
+
+  // Invert a list of {start,end} range objects
+  invert(ranges) {
+    let start = CHAR_MIN;
+    const result = [];
+    for(const range of ranges) {
+      const newRange = { start, end: range.start - 1 };
+      if (newRange.start <= newRange.end) {
+        result.push(newRange);
+      }
+      start = range.end + 1;
+    }
+    const newRange = { start, end: CHAR_MAX };
+    if (newRange.start <= newRange.end) {
+      result.push(newRange);
+    }
+    return result;
+  },
+
+  // Merge the given left and right ranges
+  merge(left, right) {
+    let leftIdx = 0, rightIdx = 0;
+    let newRanges = [];
+    while ((leftIdx < left.length) || (rightIdx < right.length)) {
+      let range;
+      if (leftIdx >= left.length ||
+          (rightIdx < right.length &&
+           right[rightIdx].start < left[leftIdx].start)
+      ) {
+        // Next lowest range comes from the right
+        range = right[rightIdx++];
+      } else {
+        // Next lowest range comes from the left
+        range = left[leftIdx++];
+      }
+      for (;;) {
+        if (leftIdx < left.length && left[leftIdx].start <= (range.end + 1)) {
+          // Merge a range from the left into the current range
+          range.end = Math.max(range.end, left[leftIdx++].end);
+        } else if (rightIdx < right.length && right[rightIdx].start <= (range.end + 1)) {
+          // Merge a range from the right into the current range
+          range.end = Math.max(range.end, right[rightIdx++].end);
+        } else {
+          // No more mergeable ranges
+          break;
+        }
+      }
+      newRanges.push(range);
+    }
+    return newRanges;
+  },
+};
+
+// Operations on character class "parts", which could be a single character
+// or a 2-element array of characters indicating a range.
+const part = {
+
+  // Convert a character class "part", which could be a single character
+  // or a range, to a {start,end} range object.
+  toRange(el) {
+    return Array.isArray(el) ?
+      { start: el[0].codePointAt(0), end: el[1].codePointAt(0) } :
+      { start: el.codePointAt(0), end: el.codePointAt(0) };
+  },
+
+  // The main union/intersection method: merge two lists of sorted parts,
+  // optionally inverting either/both of the inputs or the output.
+  merge(left, leftInvert, right, rightInvert, invertResult) {
+    left = left.map(part.toRange);
+    right = right.map(part.toRange);
+    if (leftInvert) {
+      left = range.invert(left);
+    }
+    if (rightInvert) {
+      right = range.invert(right);
+    }
+    let newRanges = range.merge(left, right);
+    if (invertResult) {
+      newRanges = range.invert(newRanges);
+    }
+    return newRanges.map(range.toPart);
+  },
+};
+
+// Public api -------
+
+// Operations on "class" nodes
+const classNode = {
+  // Option setting
+  setCaselessRestrict(value) {
+    caselessRestrict = value;
+  },
+
+  // Return the `.` class, ie any single character.
+  any() {
+    return {
+      type: "class",
+      parts: [],
+      inverted: true,
+      ignoreCase: false,
+      sorted: true,
+    };
+  },
+
+  // Is this the `.` class, ie any single character?
+  isAny(node) {
+    return node.parts.length === 0 && node.inverted;
+  },
+
+  // Return the empty class, ie will match no character.
+  empty() {
+    return {
+      type: "class",
+      parts: [],
+      inverted: false,
+      ignoreCase: false,
+      sorted: true,
+    };
+  },
+
+  // Is this the empty class, ie will match no character?
+  isEmpty(node) {
+    return node.parts.length === 0 && !node.inverted;
+  },
+
+  // Return a case-sensitive node from the input, which may be
+  // case-insensitive.
+  caseSensitive(node) {
+    if (!node.ignoreCase) {
+      return node;
+    }
+    const newParts = [];
+    for (let range of node.parts.map(part.toRange)) {
+      for (let cp = range.start; cp <= range.end; cp++) {
+        // As per https://tc39.es/ecma262/multipage/text-processing.html
+        // use the unicode "Simple Case Folding" procedure to find all
+        // code points which case fold to the same 'mapped' character that
+        // cp does.
+        const mapped = unicode.simpleCaseFolding[cp] || cp;
+        const expand = unicode.reverseSimpleCaseFolding[mapped] || [mapped];
+        for (let expandedPoint of expand) {
+          if (caselessRestrict && cp <= 0x7F && expandedPoint > 0x7F) {
+            // if caselessRestrict is true, suppress non-ASCII matches when
+            // the original code point was ASCII.
+            continue;
+          }
+          newParts.push(String.fromCodePoint(expandedPoint));
+        }
+      }
+    }
+    // sort and merge newParts
+    return classNode.sort({
+      type: "class",
+      parts: newParts,
+      inverted: node.inverted,
+      ignoreCase: false,
+    });
+  },
+
+  // Compute the union of two character classes
+  union(left, right) {
+    if (classNode.isAny(left) || classNode.isEmpty(right)) {
+      return left;
+    }
+    if (classNode.isAny(right) || classNode.isEmpty(left)) {
+      return right;
+    }
+    left = classNode.caseSensitive(classNode.sort(left));
+    right = classNode.caseSensitive(classNode.sort(right));
+    if (!left.inverted && right.inverted) {
+      return classNode.union(right, left);
+    }
+    // Either both left and right are inverted, or right is not inverted
+    let resultInverted = left.inverted;
+    let newParts = part.merge(
+      left.parts, left.inverted,
+      right.parts, right.inverted,
+      resultInverted
+    );
+    return {
+      type: "class",
+      parts: newParts,
+      inverted: resultInverted,
+      ignoreCase: false,
+      sorted: true,
+    };
+  },
+
+  // Compute the intersection of two character classes
+  intersection(left, right) {
+    if (classNode.isAny(right) || classNode.isEmpty(left)) {
+      return left;
+    }
+    if (classNode.isAny(left) || classNode.isEmpty(right)) {
+      return right;
+    }
+    left = classNode.caseSensitive(classNode.sort(left));
+    right = classNode.caseSensitive(classNode.sort(right));
+    if (!left.inverted && right.inverted) {
+      return classNode.intersection(right, left);
+    }
+    let resultInverted = right.inverted;
+    let newParts = part.merge(
+      left.parts, !left.inverted,
+      right.parts, !right.inverted,
+      !resultInverted
+    );
+    return {
+      type: "class",
+      parts: newParts,
+      inverted: resultInverted,
+      ignoreCase: false,
+      sorted: true,
+    };
+  },
+
+  subtract(left, right) {
+    if (classNode.isAny(right) || classNode.isEmpty(left)) {
+      return classNode.empty();
+    }
+    left = classNode.caseSensitive(classNode.sort(left));
+    right = classNode.caseSensitive(classNode.sort(right));
+    let resultInverted = left.inverted && !right.inverted;
+    let newParts = part.merge(
+      left.parts, !left.inverted,
+      right.parts, right.inverted,
+      !resultInverted
+    );
+    return {
+      type: "class",
+      parts: newParts,
+      inverted: resultInverted,
+      ignoreCase: false,
+      sorted: true,
+    };
+  },
+
+  // Expand ranges in a character class, up to the specified limit of
+  // character entries.
+  expand(input, limit) {
+    input = classNode.sort(input);
+    const newParts = [];
+    for (let range of input.parts.map(part.toRange)) {
+      for (let i = range.start; i <= range.end; i++) {
+        newParts.push(String.fromCodePoint(i));
+        if (newParts.length > limit) {
+          // Exceeded the limit: return the original class (sorted, but
+          // ranges otherwise unmodified).
+          return input;
+        }
+      }
+    }
+    input.parts = newParts;
+    return input;
+  },
+
+  // Sort a character class to make it suitable for input to union, etc.
+  sort(first) {
+    if (!first.sorted) {
+      // firstSet is kept in sorted order and with overlapping ranges merged;
+      // class may not be sorted or merged so preprocess it.
+      first = Object.assign({}, first); // shallow clone
+      first.parts = first.parts.slice(); // deep clone
+      first.parts.sort(function(a,b) {
+        if (Array.isArray(a)) { a = a[0]; }
+        if (Array.isArray(b)) { b = b[0]; }
+        if (a < b) {
+          return -1;
+        } else if (a > b) {
+          return 1;
+        } else {
+          return 0;
+        }
+      });
+      first.parts = part.merge(first.parts, false, [], false, false);
+      first.sorted = true;
+    }
+    return first;
+  },
+};
+
+module.exports = {
+  classNode,
+};

package/lib/compiler/language/javascript.js

@@ -10,6 +10,7 @@ let javascript = {
   maxFailPos: 'peg$maxFailPos',
   inputLength: 'input.length',
   assertionSuccess: 'void 0',
+  advanceInputChar: 'peg$currPos++',
   consumeInputChar: 'input.charAt(peg$currPos++)',
   result: 'peg$result',
   actionArgPrefix: '',
@@ -163,54 +164,123 @@ let javascript = {
       .replace(/[\u1000-\uFFFF]/g,      function(ch) { return '\\u'  + hex(ch); });
   },
 
-  matchClass(node, reg, result) {
-    let regexp;
+  classToRegexp(node) {
+    return '['
+      + (node.inverted ? '^' : '')
+      + node.parts.map(function(part) {
+        return part instanceof Array
+          ? javascript.regexpClassEscape(part[0])
+          + '-'
+          + javascript.regexpClassEscape(part[1])
+          : javascript.regexpClassEscape(part);
+      }).join('')
+      + ']';
+  },
+
+  matchClass(node, reg, result, discard, discardPos) {
+    let regexp, expr;
     if (node.parts.length === 0) {
       if (node.inverted) {
         // Same as .
         result.condition = 'peg$currPos < input.length';
-        result.onSuccess([`${reg} = input.charAt(peg$currPos++);`]);
+        result.onSuccess([discard ? `${reg} = true;` : `${reg} = input.charAt(peg$currPos++);`]);
       } else {
         // Always fail
         result.condition = 'false';
       }
       return;
     }
-    regexp = '/^['
-        + (node.inverted ? '^' : '')
-        + node.parts.map(function(part) {
-          return part instanceof Array
-            ? javascript.regexpClassEscape(part[0])
-            + '-'
-            + javascript.regexpClassEscape(part[1])
-            : javascript.regexpClassEscape(part);
-        }).join('')
-        + ']/' + (node.ignoreCase ? 'i' : '');
-    result.block = [`${reg} = input.charAt(peg$currPos);`];
-    result.condition = `${regexp}.test(${reg})`;
-    result.onSuccess(['peg$currPos++;']);
-  },
-
-  matchLiteral(node, reg, result) {
+    regexp = '/^'
+      + javascript.classToRegexp(node)
+      + '/' + (node.ignoreCase ? 'i' : '');
+    expr = `input.charAt(peg$currPos)`;
+    if (discard) {
+      result.condition = `${regexp}.test(${expr})`;
+      result.onSuccess([`${reg} = true;`]);
+    } else {
+      result.block = [`${reg} = ${expr};`];
+      result.condition = `${regexp}.test(${reg})`;
+    }
+    if (!discardPos) {
+      result.onSuccess(['peg$currPos++;']);
+    }
+  },
+
+  matchRepeatedClass(node, reg, result, atLeastOne, discard, discardPos) {
+    let regexp;
+    if (node.parts.length === 0) {
+      if (node.inverted) {
+        // Same as .* / .+
+        result.condition = atLeastOne ? 'peg$currPos < input.length' : 'true';
+        if (!discard) {
+          result.onSuccess([`${reg} = Array.from(input.substring(peg$currPos));`]);
+        }
+        if (!discardPos) {
+          result.onSuccess([`peg$currPos = input.length;`]);
+        }
+      } else if (atLeastOne) {
+        // Always fail
+        result.condition = 'false';
+      } else {
+        // Zero length match
+        result.condition = 'true';
+        result.onSuccess([`${reg} = [];`]);
+      }
+      return;
+    }
+    regexp = '/'
+        + javascript.classToRegexp(node)
+        + (atLeastOne ? '+' : '*') + '/y' + (node.ignoreCase ? 'i' : '');
+    result.block.push(`${reg} = ${regexp};`);
+    result.block.push(`${reg}.lastIndex = peg$currPos;`);
+    if (discard) {
+      result.condition = `${reg}.exec(input) !== null`;
+      if (!discardPos) {
+        result.onSuccess([`peg$currPos = ${reg}.lastIndex;`]);
+      }
+    } else {
+      result.block.push(`${reg} = ${reg}.exec(input);`);
+      result.condition = `${reg} !== null`;
+      if (!discardPos) {
+        result.onSuccess([`peg$currPos += ${reg}[0].length;`]);
+      }
+      result.onSuccess([`${reg} = Array.from(${reg}[0]);`]);
+    }
+  },
+
+  matchLiteral(node, reg, result, discard, discardPos) {
+    let expr;
     if (node.value.length === 1 && !node.ignoreCase) {
       result.condition = 'input.charCodeAt(peg$currPos) === ' + node.value.charCodeAt(0);
-      result.onSuccess([[reg, ' = ', javascript.stringify(node.value), ';'].join('')]);
+      if (discard) {
+        result.onSuccess([`${reg} = true;`]);
+      } else {
+        result.onSuccess([[reg, ' = ', javascript.stringify(node.value), ';'].join('')]);
+      }
     } else {
       if (node.value.length === 1) {
-        result.block.push([reg, ' = input.charAt(peg$currPos);'].join(''));
+        expr = `input.charAt(peg$currPos)`;
       } else {
-        result.block.push([reg, ' = ',
-          'input.substr(peg$currPos,', node.value.length, ');'].join(''));
+        expr = `input.substr(peg$currPos,${node.value.length})`;
+      }
+      if (!discard) {
+        result.block.push([`${reg} = ${expr};`]);
+        expr = reg;
       }
       if (node.ignoreCase) {
-        result.condition = [reg, '.toLowerCase() === ',
+        result.condition = [expr, '.toLowerCase() === ',
           javascript.stringify(node.value.toLowerCase())].join('');
       } else {
-        result.condition = [reg, ' === ',
+        result.condition = [expr, ' === ',
           javascript.stringify(node.value)].join('');
       }
+      if (discard) {
+        result.onSuccess([`${reg} = true;`]);
+      }
+    }
+    if (!discardPos) {
+      result.onSuccess([`peg$currPos += ${node.value.length};`]);
     }
-    result.onSuccess([['peg$currPos += ', node.value.length, ';'].join('')]);
   },
 
   initCache(/*opts*/) {
@@ -225,7 +295,7 @@ let javascript = {
     if (opts.params.length) {
       keyParts = keyParts.concat(opts.params);
     }
-    let storeRefs = opts.storeRefs.filter(function(part) {
+    const storeRefs = opts.storeRefs.filter(function(part) {
       return part !== '';
     }).map(function(part) {
       return '  ' + part;
@@ -256,17 +326,21 @@ let javascript = {
     return `    if (cached.hasOwnProperty(${encName})) param_${name}.value = cached.$${name};`;
   },
 
-  cacheStoreRef(name, store) {
-    if (!store) return '';
-    return `if (saved_${name} !== param_${name}.value) cached.$${name} = param_${name}.value;`;
+  cacheStoreRef(reg, name) {
+    if (!reg) { return ''; }
+    return `if (${reg} !== param_${name}.value) cached.$${name} = param_${name}.value;`;
+  },
+
+  cacheRestoreRef(reg, name) {
+    return `param_${name}.value = ${reg};`;
   },
 
   /**
    * Get a block which saves ref values to a temporary variable for later
-   * comparison in getCacheStoreRefs().
+   * comparison in cacheStoreRefs() / cacheRestoreRef().
    */
-  cacheSaveRef(name) {
-    return `var saved_${name}=param_${name}.value;`;
+  cacheSaveRef(reg, name) {
+    return `${reg} = param_${name}.value;`;
   }
 };