wikipeg 4.0.2 → 6.0.0

package/lib/compiler/passes/inline-simple-rules.js

@@ -0,0 +1,96 @@
+"use strict";
+
+var visitor = require("../visitor"),
+    asts = require("../asts");
+
+/*
+ * Inline simple rules, like character classes and string constants.
+ */
+function inlineSimpleRules(ast, options) {
+
+  const startRules = options.allowedStartRules.concat(options.allowedStreamRules);
+
+  function isSimpleRule(rule) {
+    function isClassOrLiteral(node) {
+      return node.type === 'class' || node.type === 'literal';
+    }
+
+    function isRepeatedClassOrLiteral(node) {
+      return (node.type === 'zero_or_more' || node.type === 'one_or_more') &&
+        isClassOrLiteral(node.expression);
+    }
+
+    function isTextRepeatedClassOrLiteral(node) {
+      return node.type === 'text' && (
+        isClassOrLiteral(node.expression) ||
+          isRepeatedClassOrLiteral(node.expression)
+      );
+    }
+
+    // By default don't inline rules which have attributes
+    // (other than 'inline')
+    if ((rule.attributes || []).some((attr) => attr.name !== 'inline')) {
+      return false;
+    }
+    return isClassOrLiteral(rule.expression) ||
+      isRepeatedClassOrLiteral(rule.expression) ||
+      isTextRepeatedClassOrLiteral(rule.expression);
+  }
+
+  function isInlineRule(rule) {
+    let inline = asts.getRuleAttributeValue(rule, "inline", undefined);
+    if (inline !== undefined) {
+      return inline;
+    }
+    if (options.noInlining) {
+      return false;
+    }
+    return isSimpleRule(rule);
+  }
+
+  function replaceRuleRefs(ast, from, toNode) {
+    var removedAll = true;
+    var replace = visitor.build({
+      rule_ref: function(node) {
+        if (node.name === from) {
+          // skip inlining if this reference has parameters
+          if (node.assignments.length > 0) {
+            removedAll = false;
+            return;
+          }
+          const wasPicked = node.picked;
+          // Remove all properties of node
+          Object.keys(node).forEach((name) => delete node[name]);
+          // Deep-copy all properties from toNode
+          Object.keys(toNode).forEach((name) => {
+            // A poor man's clone
+            node[name] = JSON.parse(JSON.stringify(toNode[name]));
+          });
+          if (wasPicked) {
+            node.picked = wasPicked;
+          }
+        }
+      }
+    });
+
+    replace(ast);
+    return removedAll;
+  }
+
+  var indices = [];
+
+  ast.rules.forEach( function(rule, i) {
+    if (isInlineRule(rule)) {
+      var removedAll = replaceRuleRefs(ast, rule.name, rule.expression);
+      if (removedAll && !startRules.includes(rule.name)) {
+        indices.push(i);
+      }
+    }
+  });
+
+  indices.reverse();
+
+  indices.forEach((i) => ast.rules.splice(i, 1));
+}
+
+module.exports = inlineSimpleRules;

package/lib/compiler/passes/optimize-character-class.js

@@ -0,0 +1,147 @@
+"use strict";
+/*eslint no-unused-vars: ["error", { "argsIgnorePattern": "^_" }] */
+
+var visitor        = require("../visitor"),
+    classNode      = require("../charsets").classNode,
+    objects        = require("../../utils/objects");
+
+// Optimize character classes:
+//   [...] / [...] => union
+//  ![...] [...] => subtraction
+
+function optimizeCharacterClass(ast, options) {
+  options = options || {};
+
+  if (options.noOptimizeCharacterClass) {
+    return;
+  }
+
+  function isSingleChar(node) {
+    return node.type === 'class' ||
+      node.type === 'any' ||
+      (node.type === 'literal' && node.value.length === 1);
+  }
+
+  function toDesc(node) {
+    if (node.type === 'class') {
+      return node.rawText;
+    } else if (node.type === 'any') {
+      return ".";
+    } else if (node.type === 'literal') {
+      return "\"" + node.value + "\"";
+    } else {
+      throw new Error("unreachable");
+    }
+  }
+
+  function toList(arr) {
+    return (arr.length > 1) ?
+      (arr.slice(0, -1).join(", ") + " or " + arr[arr.length - 1]) :
+      arr[0];
+  }
+
+  function toCharset(node) {
+    if (node.type === 'class') {
+      return classNode.sort(objects.clone(node));
+    } else if (node.type === 'any') {
+      return classNode.any();
+    } else if (node.type === 'literal') {
+      return classNode.sort({
+        type: "class",
+        parts: [node.value],
+        inverted: false,
+        ignoreCase: node.ignoreCase,
+      });
+    }
+    throw new Error('unreachable');
+  }
+
+  const checkCharacterClass = visitor.build({
+    choice: function(node) {
+      // First optimize children.
+      node.alternatives.forEach((alt) => checkCharacterClass(alt));
+      // Check for [...] / [... ]
+      if (node.alternatives.every((alt) => isSingleChar(alt))) {
+        let newClass = node.alternatives.reduce(
+          (acc, alt) => classNode.union(acc, toCharset(alt)),
+          classNode.empty()
+        );
+        let newDesc = toList(node.alternatives.map((alt) => toDesc(alt)));
+        // Remove all properties of node
+        Object.keys(node).forEach((name) => {
+          if (name !== 'location' && name !== 'picked') {
+            delete node[name];
+          }
+        });
+        // Create a new class
+        Object.assign(node, newClass);
+        node.rawText = newDesc;
+        return;
+      }
+    },
+    sequence: function(node) {
+      // First optimize children.
+      node.elements.forEach((el) => checkCharacterClass(el));
+      // Check for ![...] [...]
+      let removed = 0;
+      for (let i = node.elements.length - 2; i >= 0; i--) {
+        let next = i + 1;
+        while (node.elements[next].optimizeCharClass) {
+          // Skip over synthetic nodes previously added.
+          next++;
+        }
+        if (node.elements[i].type === 'simple_not' &&
+            !node.elements[i].picked &&
+            isSingleChar(node.elements[i].expression) &&
+            isSingleChar(node.elements[next])) {
+          let newClass = classNode.subtract(
+            toCharset(node.elements[next]),
+            toCharset(node.elements[i].expression)
+          );
+          let newDesc =
+              toDesc(node.elements[next]) + ' but not ' + toDesc(node.elements[i].expression);
+          let newPicked = node.elements[next].picked;
+          // create a trivial node just to ensure the same # of elements
+          // in the sequence.
+          let newTest = {
+            type: 'simple_and',
+            location: node.elements[i].location,
+            expression: {
+              type: 'literal',
+              value: '',
+              location: node.elements[i].location,
+            },
+            optimizeCharClass: true, // mark synthetic node
+          };
+          let newChar = Object.assign({
+            location: node.elements[next].location,
+            rawText: newDesc,
+            picked: newPicked,
+          }, newClass);
+          node.elements[i] = newTest;
+          node.elements[next] = newChar;
+          removed++;
+        }
+      }
+      // optimize sequence if picked to allow repeated char class
+      let last = node.elements[node.elements.length - 1];
+      if (node.elements.length === (removed + 1) && last.picked) {
+        // Remove all properties of node.
+        Object.keys(node).forEach((name) => {
+          if (name !== 'location' && name !== 'picked') {
+            delete node[name];
+          }
+        });
+        // Create a new class
+        delete last.picked;
+        delete last.location;
+        Object.assign(node, last);
+        return;
+      }
+    },
+  });
+
+  checkCharacterClass(ast);
+}
+
+module.exports = optimizeCharacterClass;

package/lib/compiler/passes/optimize-failure-reporting.js

@@ -0,0 +1,65 @@
+"use strict";
+
+var visitor        = require("../visitor"),
+    asts           = require("../asts");
+
+// Find rules that never report failures and silence them.
+// 1. A start rule always reports failure.
+// 2. A non-start rule does not report failure if every caller is either:
+//    A) a named rule
+//       (this is because named rules manually report failure, attributing
+//        the failure to the named rule instead of its children), or
+//    B) inside an assertion (and (&) or not (!) expressions), or
+//    C) (transitively) a rule which does not report failure
+
+function optimizeFailureReporting(ast, options) {
+
+  const startRules = options.allowedStartRules.concat(options.allowedStreamRules);
+
+  // Disable failure reporting for rules by default, and find rules for which
+  // conditions 2A, 2B, or 2C are false.
+  ast.rules.forEach( (node) => {
+    node.reportsFailure = false;
+  });
+
+  // Enable failure reporting for start rules (condition 1)
+  const failReportingRules = startRules.map((name) =>
+    asts.findRule( ast, name )
+  );
+
+  // Selectively enable failure reporting for rules in start rules' call graph
+  const skipChildren = () => {};
+  const check = visitor.build ({
+    rule: function(node) {
+      node.reportsFailure = true;
+      let named = asts.getRuleAttributeValue(node, "name");
+      if (named !== undefined) {
+        // Break AST traversing because failure reporting is disabled in a
+        // named rule (2A)
+        return;
+      }
+      // We are visiting this rule because we've found that it may report
+      // failures.  All rules referenced may also report failures.
+      check(node.expression);
+    },
+
+    // Never reports failure, so break AST traversing (2B)
+    simple_and: skipChildren,
+    simple_not: skipChildren,
+
+    rule_ref: function(node) {
+      const rule = asts.findRule(ast, node.name);
+      // This function is only reached when the parent rule reports failures.
+      // Recheck all rules called by the referenced rule (2C)
+      if (!rule.reportsFailure) {
+        failReportingRules.push( rule );
+      }
+    },
+  });
+
+  while (failReportingRules.length) {
+    check(failReportingRules.shift());
+  }
+}
+
+module.exports = optimizeFailureReporting;

package/lib/compiler/passes/remove-proxy-rules.js

@@ -1,12 +1,14 @@
 "use strict";
 
-var arrays  = require("../../utils/arrays"),
-    visitor = require("../visitor");
+var visitor = require("../visitor");
 
 /*
  * Removes proxy rules -- that is, rules that only delegate to other rule.
  */
 function removeProxyRules(ast, options) {
+
+  const startRules = options.allowedStartRules.concat(options.allowedStreamRules);
+
   function isProxyRule(node) {
     return node.type === "rule"
       && node.expression.type === "rule_ref"
@@ -27,10 +29,10 @@ function removeProxyRules(ast, options) {
 
   var indices = [];
 
-  arrays.each(ast.rules, function(rule, i) {
+  ast.rules.forEach( function(rule, i) {
     if (isProxyRule(rule)) {
       replaceRuleRefs(ast, rule.name, rule.expression.name);
-      if (!arrays.contains(options.allowedStartRules, rule.name)) {
+      if (!startRules.includes(rule.name)) {
         indices.push(i);
       }
     }
@@ -38,7 +40,7 @@ function removeProxyRules(ast, options) {
 
   indices.reverse();
 
-  arrays.each(indices, function(i) { ast.rules.splice(i, 1); });
+  indices.forEach( (i) => { ast.rules.splice(i, 1); });
 }
 
 module.exports = removeProxyRules;

package/lib/compiler/passes/report-infinite-loops.js

@@ -8,7 +8,10 @@ var GrammarError = require("../../grammar-error"),
  * Reports expressions that don't consume any input inside |*| or |+| in the
  * grammar, which prevents infinite loops in the generated parser.
  */
-function reportInfiniteLoops(ast) {
+function reportInfiniteLoops(ast, options) {
+  if (options && options.allowLoops) {
+    return;
+  }
   var check = visitor.build({
     zero_or_more: function(node) {
       if (asts.matchesEmpty(ast, node.expression)) {

package/lib/compiler/passes/report-left-recursion.js

@@ -1,7 +1,6 @@
 "use strict";
 
-var arrays       = require("../../utils/arrays"),
-    GrammarError = require("../../grammar-error"),
+var GrammarError = require("../../grammar-error"),
     asts         = require("../asts"),
     visitor      = require("../visitor");
 
@@ -24,7 +23,7 @@ function reportLeftRecursion(ast) {
     },
 
     sequence: function(node, visitedRules) {
-      arrays.every(node.elements, function(element) {
+      node.elements.every( (element) => {
         if (element.type === "rule_ref") {
           check(element, visitedRules);
         }
@@ -34,7 +33,7 @@ function reportLeftRecursion(ast) {
     },
 
     rule_ref: function(node, visitedRules) {
-      if (arrays.contains(visitedRules, node.name)) {
+      if (visitedRules.includes(node.name)) {
         throw new GrammarError(
           "Left recursion detected for rule \"" + node.name + "\".",
           node.location

package/lib/compiler/passes/report-unknown-attributes.js

@@ -0,0 +1,39 @@
+"use strict";
+
+var GrammarError = require("../../grammar-error"),
+    visitor      = require("../visitor");
+
+/* Checks that all rule attributes are known. (Catches typos.) */
+function reportUnknownAttributes(ast) {
+  const KNOWN_ATTRIBUTES = {
+    name: 'string',
+    inline: 'boolean',
+    cache: 'boolean',
+    empty: 'boolean',
+    unreachable: 'boolean',
+  };
+
+  var check = visitor.build({
+    rule: function(rule) {
+      for (const attr of rule.attributes || []) {
+        if (!KNOWN_ATTRIBUTES[attr.name]) {
+          throw new GrammarError(
+            "Rule \"" + rule.name + "\" contains unknown attribute \"" + attr.name + "\".",
+            attr.location
+          );
+        }
+        if (typeof(attr.value) !== KNOWN_ATTRIBUTES[attr.name]) {
+          throw new GrammarError(
+            "Rule \"" + rule.name + "\" attribute \"" + attr.name + "\" has " +
+            typeof(attr.value) + " value but expected " + KNOWN_ATTRIBUTES[attr.name] + ".",
+            attr.location
+          );
+        }
+      }
+    }
+  });
+
+  check(ast);
+}
+
+module.exports = reportUnknownAttributes;

package/lib/compiler/passes/transform-common-lang.js

@@ -8,7 +8,7 @@ var visitor        = require("../visitor");
  * Used for testing.
  */
 function transformCommonLang(ast, options) {
-  if (!options.commonLang) {
+  if (!(options && options.commonLang)) {
     return;
   }
 

package/lib/compiler/traverser.js

@@ -61,7 +61,6 @@ Traverser.prototype = {
   defaultHandlers: {
     initializer:   traverseNop,
     rule:          traverseExpression,
-    named:         traverseExpression,
     choice:        traverseChildren("alternatives"),
     action:        traverseExpression,
     sequence:      traverseChildren("elements"),
@@ -79,7 +78,7 @@ Traverser.prototype = {
     labeled_param: traverseNop,
     rule_ref:      traverseRuleRef,
     literal:       traverseNop,
-    "class":       traverseNop,
+    class:         traverseNop,
     any:           traverseNop
   }
 };

package/lib/compiler/visitor.js

@@ -1,7 +1,6 @@
 "use strict";
 
-var objects = require("../utils/objects"),
-    arrays  = require("../utils/arrays");
+var objects = require("../utils/objects");
 
 /* Simple AST node visitor builder. */
 var visitor = {
@@ -22,7 +21,7 @@ var visitor = {
       return function(node) {
         var extraArgs = Array.prototype.slice.call(arguments, 1);
 
-        arrays.each(node[property], function(child) {
+        node[property].forEach( (child) => {
           visit.apply(null, [child].concat(extraArgs));
         });
       };
@@ -34,7 +33,7 @@ var visitor = {
 
         if (node.initializer) {
           if (Array.isArray(node.initializer)) {
-            arrays.each(node.initializer, function(initializer) {
+            node.initializer.forEach( (initializer) => {
               visit.apply(null, [initializer].concat(extraArgs));
             });
           } else {
@@ -42,14 +41,13 @@ var visitor = {
           }
         }
 
-        arrays.each(node.rules, function(rule) {
+        node.rules.forEach( (rule) => {
           visit.apply(null, [rule].concat(extraArgs));
         });
       },
 
       initializer:   visitNop,
       rule:          visitExpression,
-      named:         visitExpression,
       choice:        visitChildren("alternatives"),
       action:        visitExpression,
       sequence:      visitChildren("elements"),
@@ -67,7 +65,7 @@ var visitor = {
       labeled_param: visitNop,
       rule_ref:      visitNop,
       literal:       visitNop,
-      "class":       visitNop,
+      class:         visitNop,
       any:           visitNop
     };
 

package/lib/compiler.js

@@ -1,7 +1,6 @@
 "use strict";
 
-var arrays  = require("./utils/arrays"),
-    objects = require("./utils/objects");
+const objects = require("./utils/objects");
 
 var compiler = {
   /*
@@ -15,11 +14,20 @@ var compiler = {
     check: {
       reportMissingRules:  require("./compiler/passes/report-missing-rules"),
       reportLeftRecursion: require("./compiler/passes/report-left-recursion"),
-      reportInfiniteLoops: require("./compiler/passes/report-infinite-loops")
+      reportInfiniteLoops: require("./compiler/passes/report-infinite-loops"),
+      reportUnknownAttributes:
+                           require("./compiler/passes/report-unknown-attributes")
     },
     transform: {
       removeProxyRules:    require("./compiler/passes/remove-proxy-rules"),
+      inlineSimpleRules:   require("./compiler/passes/inline-simple-rules"),
+      optimizeCharacterClass:
+                           require("./compiler/passes/optimize-character-class"),
       analyzeParams:       require("./compiler/passes/analyze-params"),
+      analyzeAlwaysMatch:  require("./compiler/passes/analyze-always-match"),
+      analyzeFirst:        require("./compiler/passes/analyze-first"),
+      optimizeFailureReporting:
+                           require("./compiler/passes/optimize-failure-reporting"),
       transformCommonLang: require("./compiler/passes/transform-common-lang"),
     },
     generate: {
@@ -34,8 +42,7 @@ var compiler = {
    * cause its malfunction.
    */
   compile: function(ast, passes) {
-    var options = arguments.length > 2 ? objects.clone(arguments[2]) : {},
-        stage;
+    let options = arguments.length > 2 ? objects.clone(arguments[2]) : {};
 
     objects.defaults(options, {
       allowedStartRules:  [ast.rules[0].name],
@@ -43,14 +50,21 @@ var compiler = {
       cache:              false,
       trace:              false,
       optimize:           "speed",
-      output:             "parser"
+      output:             "parser",
     });
 
-    for (stage in passes) {
-      if (passes.hasOwnProperty(stage)) {
-        arrays.each(passes[stage], function(p) { p(ast, options); });
+    // Ensure the 'caseless restrict' setting matches the compiler options
+    require("./compiler/charsets").classNode.setCaselessRestrict(
+      options.caselessRestrict || false
+    );
+
+    Object.getOwnPropertyNames(passes).forEach((stage) => {
+      let stagePasses = passes[stage];
+      if (typeof(stagePasses) === 'object') {
+        stagePasses = Object.values(stagePasses);
       }
-    }
+      stagePasses.forEach( (p) => { p(ast, options); });
+    });
 
     switch (options.output) {
       case "parser": return eval(ast.code);