wikipeg 4.0.2 → 6.0.0

package/lib/compiler/passes/ast-to-code.js

@@ -4,10 +4,16 @@ var js      = require("../language/javascript"),
     php     = require("../language/php"),
     visitor = require("../visitor"),
     objects = require('../../utils/objects'),
+    classNode = require("../charsets").classNode,
     asts    = require("../asts"),
     fs      = require("fs");
 
 function generateJavascript(ast, options) {
+  /**
+   * How much to expand ranges in character classes before giving up.
+   */
+  const CHARSET_EXPAND = 16;
+
   var rulesToGenerate = [];
   var generatedRuleNames = {};
 
@@ -88,6 +94,22 @@ function generateJavascript(ast, options) {
     namespace = `namespace ${matches[1]};`;
   }
 
+  // Look for 'cache' attributes on rules.  If some rule has [cache] on it,
+  // but options.cache was false, then set options.cache but default all
+  // rules to [cache=false], so only the explicitly [cache=true] rules
+  // will be cached.
+  if (ast.rules.some(
+    (rule) => asts.getRuleAttributeValue(rule, "cache") === true
+  ) && !options.cache) {
+    options.cache = true;
+    ast.rules.forEach((rule) => {
+      if (asts.findRuleAttribute(rule, "cache") === undefined) {
+        rule.attributes = rule.attributes || [];
+        rule.attributes.push({ name: "cache", type: "boolean", value: false });
+      }
+    });
+  }
+
   var refsSet = {};
   var getRefs = visitor.build({
     rule_ref: function(node) {
@@ -136,6 +158,9 @@ function generateJavascript(ast, options) {
     this.resultReg_ = false;
     this.silence_ = language.silence;
     this.discard_ = false;
+    this.discardPos_ = false;
+    this.savedPosRefState_ = null;
+    this.shouldFreePosRefState_ = false;
   }
   Context.prototype = {
     clone: function() {
@@ -221,10 +246,11 @@ function generateJavascript(ast, options) {
     },
 
     /**
-     * Clone the object, and set the discard flag in the cloned object. This
-     * indicates that the caller is only interested in success or failure, and
-     * some subexpressions will use this information to return true instead of
-     * the match result.
+     * Clone the object, and set the "discard result" flag in the
+     * cloned object. When true this indicates that the caller is only
+     * interested in success or failure, and some subexpressions will
+     * use this information to return true instead of the match
+     * result.
      */
     discard: function(value) {
       var obj = this.clone();
@@ -236,11 +262,117 @@ function generateJavascript(ast, options) {
     },
 
     /**
-     * Get the discard flag.
+     * Clone the object, and set the "discard position" flag in the
+     * cloned object. When true this indicates that the caller will take care
+     * of setting/restoring the current parse position and the callee
+     * needn't bother to ensure it is set to the end of its match.
+     * (Typically used during lookahead matches.)
+     */
+    discardPos: function(value) {
+      var obj = this.clone();
+      if (typeof value === 'undefined') {
+        value = true;
+      }
+      obj.discardPos_ = value;
+      return obj;
+    },
+
+    /**
+     * Get the "discard result" flag.
      */
     getDiscard: function() {
       return this.discard_;
-    }
+    },
+
+    /**
+     * Get the "discard position" flag.
+     */
+    getDiscardPos: function() {
+      return this.discardPos_;
+    },
+
+    /**
+     * Return a saved pos/ref state, or null if none can be reused.
+     */
+    getPosRefState() {
+      return this.savedPosRefState_;
+    },
+
+    /**
+     * Indicate that the pos/ref state of this context can't be reused by
+     * children.
+     */
+    resetPosRefState() {
+      if (this.savedPosRefState_ === null) {
+        return this;
+      }
+      var obj = this.clone();
+      obj.savedPosRefState_ = null;
+      return obj;
+    },
+
+    /**
+     * Ensure we have a saved pos/ref state, reusing the saved parent state
+     * where that can be reused.
+     */
+    savePosRefState(node, result) {
+      let obj;
+      if (this.savedPosRefState_ !== null) {
+        if (this.shouldFreePosRefState_) {
+          // We can reuse the parent state, but don't free it in the child!
+          obj = this.clone();
+          obj.shouldFreePosRefState_ = false;
+          return obj;
+        }
+        return this;
+      }
+      // Save current position
+      // (should be later freed with a matching call to freePosRefState)
+      let saved = {
+        dst: allocPosReg(),
+        src: language.currPos,
+        refs: [],
+      };
+      let rule = currentRule;
+      // Add reference variables
+      for (let name in rule.passedParams) {
+        if (rule.passedParams[name].type === 'reference') {
+          saved.refs.push({reg: allocReg([]), name: name});
+        }
+      }
+      result.block.push(`${saved.dst} = ${saved.src};`);
+      for(let ref of saved.refs) {
+        result.block.push(language.cacheSaveRef(ref.reg, ref.name));
+      }
+      obj = this.clone();
+      obj.savedPosRefState_ = saved;
+      obj.shouldFreePosRefState_ = true;
+      return obj;
+    },
+
+    /**
+     * Restore the saved pos/ref state.
+     */
+    restorePosRefState() {
+      let saved = this.savedPosRefState_;
+      let block = [];
+      block.push(`${saved.src} = ${saved.dst};`);
+      for(let ref of saved.refs) {
+        block.push(language.cacheRestoreRef(ref.reg, ref.name));
+      }
+      return block;
+    },
+
+    /**
+     * Free this context's pos/ref state, if necessary.
+     */
+    freePosRefState(result) {
+      if (this.savedPosRefState_ !== null && this.shouldFreePosRefState_) {
+        let saved = this.savedPosRefState_;
+        let regList = [ saved.dst ].concat(saved.refs.map((ref)=>ref.reg));
+        freeReg(regList, result);
+      }
+    },
 
   };
 
@@ -461,7 +593,7 @@ function generateJavascript(ast, options) {
    */
   function makeActionFunc(code, context) {
     var argNames = [];
-    objects.keys(context.env).forEach(function(argName) {
+    Object.keys(context.env).forEach(function(argName) {
       if (context.envTypes[argName] === 'reference') {
         argNames.push(language.refArgActionDeclarator(argName));
       } else {
@@ -484,7 +616,7 @@ function generateJavascript(ast, options) {
    * funcId is the index of the function returned by makeActionFunc().
    */
   function makeActionCall(funcId, context) {
-    return language.actionCall(funcId, objects.values(context.env));
+    return language.actionCall(funcId, Object.values(context.env));
   }
 
   /**
@@ -493,22 +625,14 @@ function generateJavascript(ast, options) {
    * compile time, this returns an empty string.
    */
   function makeFailCall(value, context) {
-    var silence;
-    if (context) {
-      silence = context.getSilence();
-    } else {
-      silence = 'false';
-    }
+    var silence = context.getSilence();
     if (silence === 'true') {
+      // This should not be reached in practice.
       return '';
     }
     var expectation = addExpectation(value);
     var call = language.libraryCall('fail', [expectation]);
-    if (silence === 'false') {
-      return call + ';';
-    } else {
-      return ['if (!', silence, ') {', call, ';}'].join('');
-    }
+    return `if (!${silence}) { ${call}; }`;
   }
 
   /**
@@ -580,10 +704,9 @@ function generateJavascript(ast, options) {
   function buildSimplePredicate(node, context) {
     var result = new Result();
     var negate = node.type === 'simple_not';
-    var posReg = allocPosReg();
     var reg = context.getResultReg(result);
-    result.block = [`${posReg} = ${language.currPos};`];
-    var newContext = context.silence().cloneEnv().discard();
+    var newContext = context.silence().cloneEnv().discard().discardPos()
+      .savePosRefState(node, result);
     result.append(recurse(node.expression, newContext));
     if (negate) {
       result.resolveBlock();
@@ -591,12 +714,18 @@ function generateJavascript(ast, options) {
       result.onFailure([`${reg} = ${language.failed};`]);
     }
     result.onSuccess([`${reg} = ${language.assertionSuccess};`]);
+    if (node.isFirstSetTest && context.getSilence() !== 'true') {
+      result.onFailure([makeFailCall({
+        type: 'other',
+        description: node.isFirstSetTest,
+      }, context)]);
+    }
     if (negate) {
-      result.onFailure([`${language.currPos} = ${posReg};`]);
+      result.onFailure(newContext.restorePosRefState());
     } else {
-      result.onSuccess([`${language.currPos} = ${posReg};`]);
+      result.onSuccess(newContext.restorePosRefState());
     }
-    freeReg(posReg, result);
+    newContext.freePosRefState(result);
     return result;
   }
 
@@ -644,6 +773,25 @@ function generateJavascript(ast, options) {
     return result;
   }
 
+  /**
+   * Handler for one_or_more of a character class
+   */
+  function buildRepeatedCharacterClass(node, context, atLeastOne) {
+    var result = new Result();
+    var reg = context.getResultReg(result);
+    var classExpr = classNode.expand(classNode.caseSensitive(node.expression), CHARSET_EXPAND);
+    language.matchRepeatedClass(classExpr, reg, result, atLeastOne, context.getDiscard(), context.getDiscardPos());
+    result.onFailure([`${reg} = ${language.failed};`]);
+    if (context.getSilence() !== 'true') {
+      result.onFailure([makeFailCall({
+        type: "class",
+        value: node.expression.rawText,
+        description: node.expression.rawText
+      }, context)]);
+    }
+    return result;
+  }
+
   function makeGenerator(node, context) {
     if (node.type !== 'zero_or_more') {
       throw new Error('Iterable rules must be a single starred subexpression');
@@ -655,7 +803,7 @@ function generateJavascript(ast, options) {
     subresult.onSuccess([`yield ${subresult.expression};`]);
     subresult.onFailure([
       `if (${language.currPos} < ${language.inputLength}) {`,
-      indent2( makeFailCall({ type: "end", description: 'end of input' }) ),
+      indent2( makeFailCall({ type: "end", description: 'end of input' }, newContext) ),
       indent2(`throw ${language.libraryCall('buildParseException')};`),
       '}',
       'break;'
@@ -673,8 +821,14 @@ function generateJavascript(ast, options) {
   /**
    * Get the names of the arguments to the given rule function
    */
-  function getRuleArgNames(rule) {
-    var args = [language.silence];
+  function getRuleArgNames(rule, discard) {
+    var args;
+    if (discard || !rule.reportsFailure) {
+      // these rules never report failure, so silence argument isn't needed
+      args = [];
+    } else {
+      args = [language.silence];
+    }
 
     if (rule.hasBoolParams) {
       args.push(language.boolParams);
@@ -699,7 +853,7 @@ function generateJavascript(ast, options) {
    */
   function getStartArgs(ruleName) {
     var rule = asts.findRule(ast, ruleName);
-    var argNames = getRuleArgNames(rule);
+    var argNames = getRuleArgNames(rule, false);
     var args = [];
 
     for (let i = 0; i < argNames.length; i++) {
@@ -777,32 +931,16 @@ function generateJavascript(ast, options) {
    * Get the list of expressions or statements returned by cacheStoreRef() for
    * refs which may have changed.
    */
-  function getCacheStoreRefs(rule) {
-    var store = {};
-    for (let name in rule.passedParams) {
-      if (rule.passedParams[name].type === 'reference') {
-        store[name] = true;
-      }
+  function getCacheStoreRefs(rule, saved) {
+    var regMap = {};
+    for (let ref of saved.refs) {
+      regMap[ref.name] = ref.reg;
     }
     return references.map(function(name) {
-      return language.cacheStoreRef(name, store[name]);
+      return language.cacheStoreRef(regMap[name], name);
     });
   }
 
-  /**
-   * Get a block which saves ref values to a temporary variable for later
-   * comparison in getCacheStoreRefs().
-   */
-  function getCacheSaveRefs(rule) {
-    var parts = [];
-    for (let name in rule.passedParams) {
-      if (rule.passedParams[name].type === 'reference') {
-        parts.push(language.cacheSaveRef(name));
-      }
-    }
-    return parts.join('\n');
-  }
-
   function expandTemplate(template, vars) {
     for (let name in vars) {
       let value = vars[name];
@@ -836,14 +974,35 @@ function generateJavascript(ast, options) {
       choiceIndex = 0;
       currentRule = node;
 
-      // Generate the Result
+      // Set up the Context and save the position and reference state
       var context = (new Context()).discard(discard);
+      if (discard || !node.reportsFailure) {
+        context = context.silence();
+      }
+      var outerContext = context;
+      let nodeName = asts.getRuleAttributeValue(node, "name");
+      if (nodeName !== undefined) {
+        // Named rules suppress failure reporting in their subexpression.
+        context = context.silence();
+      }
+
+      var topSaveState = new Result();
+      let cacheThisRule = asts.getRuleAttributeValue(node, "cache", options.cache);
+      if (!iterable && cacheThisRule) {
+        context = context.savePosRefState(node, topSaveState);
+      }
+
+      // Generate the Result
       var result;
       if (iterable) {
         result = makeGenerator(node.expression, context);
       } else {
         result = recurse(node.expression, context);
       }
+      if (nodeName !== undefined && outerContext.getSilence() !== 'true') {
+        // Failure in a named rule is reported at this node.
+        result.onFailure([makeFailCall({type: 'other', description: nodeName}, outerContext)]);
+      }
       result.resolveBlock();
 
       // Make the function body
@@ -856,7 +1015,7 @@ function generateJavascript(ast, options) {
       } else {
         var ruleIndexCode = asts.indexOfRule(ast, node.name);
         var cacheBits;
-        if (options.cache) {
+        if (cacheThisRule) {
           var cacheFunc = options.cacheRuleHook || generateCacheRule;
           cacheBits = cacheFunc({
             startPos: language.currPos,
@@ -869,22 +1028,22 @@ function generateJavascript(ast, options) {
             result: result.expression,
             params: getParamsForCacheKey(node),
             loadRefs: getCacheLoadRefs(node),
-            saveRefs: getCacheSaveRefs(node),
-            storeRefs: getCacheStoreRefs(node),
+            storeRefs: getCacheStoreRefs(node, context.getPosRefState()),
+            saveRefs: topSaveState.block.join('\n'),
             className: className,
           });
           body.push(cacheBits.start);
         }
         body.push(result.block.join('\n'));
 
-        if (options.cache) {
+        if (cacheThisRule) {
           body.push(cacheBits.store);
         }
         body.push(`return ${result.expression};`);
       }
       body = indent2(body.join('\n'));
 
-      let argNames = getRuleArgNames(node);
+      let argNames = getRuleArgNames(node, discard);
       let args = argNames.join(', ');
 
       // Wrap the function body in a trace decorator if requested.
@@ -907,7 +1066,6 @@ function generateJavascript(ast, options) {
 
     rule_ref: function(node, context) {
       var result = new Result();
-      var reg = context.getResultReg(result);
 
       var newParamValues = {};
       var boolSetMask = 0;
@@ -948,7 +1106,7 @@ function generateJavascript(ast, options) {
       }
 
       let rule = asts.findRule(ast, node.name);
-      let argNameList = getRuleArgNames(rule);
+      let argNameList = getRuleArgNames(rule, context.getDiscard());
       let args = [];
       for (let i = 0; i < argNameList.length; i++) {
         let argName = argNameList[i];
@@ -986,15 +1144,12 @@ function generateJavascript(ast, options) {
       }
 
       let funcName = addRule(node.name, context.getDiscard());
-      result.block = [`${reg} = ${language.ruleFuncCall(funcName, args)};`];
-      return result;
-    },
-
-    named: function(node, context) {
-      var result = new Result();
-      result.append(recurse(node.expression, context.silence()));
-      if (context.getSilence() !== 'true') {
-        result.onFailure([makeFailCall({type: 'other', description: node.name}, context)]);
+      if (context.getDiscard() && node.alwaysMatch) {
+        result.block = [`${language.ruleFuncCall(funcName, args)};`];
+        result.expression = 'true';
+      } else {
+        const reg = context.getResultReg(result);
+        result.block = [`${reg} = ${language.ruleFuncCall(funcName, args)};`];
       }
       return result;
     },
@@ -1024,16 +1179,27 @@ function generateJavascript(ast, options) {
     action: function(node, context) {
       var result = new Result();
       var reg = context.getResultReg(result);
-      var newContext = context.cloneEnv().discard();
-      var savedPos = allocPosReg();
+      var newContext = context.cloneEnv().discard().discardPos(false);
+      var saved = context.getPosRefState();
+      var savedPos, freePos = [];
+      if (saved !== null) {
+        savedPos = saved.dst;
+      } else {
+        savedPos = allocPosReg();
+        freePos = [ savedPos ];
+        result.block = [`${savedPos} = ${language.currPos};`];
+      }
       var subresult = recurse(node.expression, newContext);
       var funcId = makeActionFunc(node.code, newContext);
-      result.block = [`${savedPos} = ${language.currPos};`];
       result.append(subresult);
+      if (node.alwaysMatch) {
+        result.condition = 'true';
+      }
       result.onSuccess([
         `${language.savedPos} = ${savedPos};`,
         `${reg} = ${makeActionCall(funcId, newContext)};`
       ]);
+      freeReg(freePos, result);
       return result;
     },
 
@@ -1041,43 +1207,72 @@ function generateJavascript(ast, options) {
       if (node.elements.length === 1) {
         return recurse(node.elements[0], context);
       } else {
-        var posReg = allocPosReg();
         var result = new Result();
         var resultReg = context.getResultReg(result);
         var label = `seq_${++seqIndex}`;
         result.block = [
           language.blockStart(label),
-          `${posReg} = ${language.currPos};`];
-        var parts = [], i;
+        ];
+        var parts = [], partFree = [], i;
+        var subcontext;
 
+        if (!node.alwaysMatch) {
+          context = context.savePosRefState(node, result);
+        }
+        subcontext = context;
         for (i = 0; i < node.elements.length; i++) {
-          var subresult = recurse(node.elements[i], context.noPassThru());
-          subresult.free = [];
+          var isPicked = node.elements[i].picked || node.numPicked === undefined;
+          var ctxt = isPicked ?
+          // if there's only one node picked, use this result register for
+          // the subcontext, otherwise allocate a new result register.
+            (node.numPicked === 1 ? subcontext : subcontext.noPassThru()):
+          // set 'discard' if this subexpression is not picked.
+            subcontext.noPassThru().discard();
+          // Unless this is the last element, we need the position
+          // (last element inherits the parent's need)
+          if (i < node.elements.length - 1) {
+            ctxt = ctxt.discardPos(false);
+          }
+          var subresult = recurse(node.elements[i], ctxt);
+          if (isPicked) {
+            partFree.push(...subresult.free);
+            subresult.free = [];
+            parts.push(subresult.expression);
+          }
           result.append(subresult);
-          parts.push(subresult.expression);
 
-          // On failure, backtrack to the start of the sequence. If this is
-          // the first element of the sequence, it's not necessary to backtrack
-          // since failing subexpressions do not increment the position.
-          if (i > 0) {
-            result.onFailure([`${language.currPos} = ${posReg};`]);
+          if (i === 0) {
+            // After the first element, it's not safe to reuse parent pos/ref
+            // state any more.
+            subcontext = subcontext.resetPosRefState();
+          }
+          if (!(node.alwaysMatch || node.elements[i].alwaysMatch)) {
+            if (i > 0) {
+              // On failure, backtrack to the start of the sequence. If this is
+              // the first element of the sequence, it's not necessary to backtrack
+              // since failing subexpressions do not increment the position
+              // nor affect reference parameters
+              result.onFailure(context.restorePosRefState());
+            }
+            // On failure, set the result register and exit the sequence
+            result.onFailure([
+              `${resultReg} = ${language.failed};`,
+              language.gotoBlockEnd(label)
+            ]);
           }
-          // On failure, set the result register and exit the sequence
-          result.onFailure([
-            `${resultReg} = ${language.failed};`,
-            language.gotoBlockEnd(label)
-          ]);
           result.resolveBlock();
         }
-        if (context.getDiscard()) {
-          result.block.push(`${resultReg} = true;`);
-        } else {
-          result.block.push(`${resultReg} = [${parts.join(',')}];`);
+        if (node.numPicked !== 1) {
+          if (context.getDiscard()) {
+            result.block.push(`${resultReg} = true;`);
+          } else {
+            result.block.push(`${resultReg} = [${parts.join(',')}];`);
+          }
         }
         result.block.push(language.blockEnd(label));
         result.expression = resultReg;
-        freeReg(result.free.concat([posReg]), result);
-        result.free = [];
+        freeReg(partFree, result);
+        context.freePosRefState(result);
         return result;
       }
     },
@@ -1106,11 +1301,16 @@ function generateJavascript(ast, options) {
     },
 
     text: function(node, context) {
+      if (context.getDiscard()) {
+        // text does nothing if we're already discarding
+        return recurse(node.expression, context.cloneEnv());
+      }
       var startPos = allocPosReg();
       var result = new Result();
       var reg = context.getResultReg(result);
       result.block = [`${startPos} = ${language.currPos};`];
-      result.append(recurse(node.expression, context.cloneEnv().discard()));
+      // Note that here we are 'discarding' but not 'silenced'
+      result.append(recurse(node.expression, context.cloneEnv().discard().discardPos(false)));
       result.onSuccess([
         `${reg} = ${language.inputSubstring(startPos, language.currPos)};`
       ]);
@@ -1136,6 +1336,10 @@ function generateJavascript(ast, options) {
     },
 
     zero_or_more: function(node, context) {
+      // Special case zero_or_more of a character class
+      if (node.expression.type === 'class') {
+        return buildRepeatedCharacterClass(node, context, false);
+      }
       // Pseudocode for the non-discard case:
       //
       // let r1 = [];
@@ -1161,7 +1365,7 @@ function generateJavascript(ast, options) {
       var result = new Result();
       var resultReg = context.getResultReg(result);
       var partReg = allocReg([]);
-      var newContext = context.resultReg(partReg).cloneEnv();
+      var newContext = context.resultReg(partReg).cloneEnv().discardPos(false).resetPosRefState();
       var subresult = recurse(node.expression, newContext);
       if (!context.getDiscard()) {
         result.block.push(`${resultReg} = [];`);
@@ -1185,6 +1389,10 @@ function generateJavascript(ast, options) {
     },
 
     one_or_more: function(node, context) {
+      // Special case one_or_more of a character class
+      if (node.expression.type === 'class') {
+        return buildRepeatedCharacterClass(node, context, true);
+      }
       // Pseudocode for the non-discard case:
       //
       // let r1 = [];
@@ -1217,7 +1425,7 @@ function generateJavascript(ast, options) {
       var initialFree = result.free;
       result.free = [];
       var partReg = allocReg([]);
-      var newContext = context.resultReg(partReg).cloneEnv();
+      var newContext = context.resultReg(partReg).cloneEnv().discardPos(false).resetPosRefState();
       var subresult = recurse(node.expression, newContext);
       if (!context.getDiscard()) {
         result.block.push(`${resultReg} = [];`);
@@ -1258,13 +1466,13 @@ function generateJavascript(ast, options) {
       var result = new Result();
       // Special case: empty string always matches
       if (node.value.length === 0) {
-        result.expression = "''";
+        result.expression = context.getDiscard() ? "true" : "''";
         result.condition = 'true';
         return result;
       }
 
       var reg = context.getResultReg(result);
-      language.matchLiteral(node, reg, result);
+      language.matchLiteral(node, reg, result, context.getDiscard(), context.getDiscardPos());
       if (context.getSilence() !== 'true') {
         result.onFailure([
           makeFailCall({
@@ -1277,10 +1485,11 @@ function generateJavascript(ast, options) {
       return result;
     },
 
-    "class": function(node, context) {
+    class: function(node, context) {
       var result = new Result();
       var reg = context.getResultReg(result);
-      language.matchClass(node, reg, result);
+      var classExpr = classNode.expand(classNode.caseSensitive(node), CHARSET_EXPAND);
+      language.matchClass(classExpr, reg, result, context.getDiscard(), context.getDiscardPos());
       result.onFailure([`${reg} = ${language.failed};`]);
       if (context.getSilence() !== 'true') {
         result.onFailure([makeFailCall({
@@ -1296,7 +1505,14 @@ function generateJavascript(ast, options) {
       var result = new Result();
       var reg = context.getResultReg(result);
       result.condition = `${language.currPos} < ${language.inputLength}`;
-      result.onSuccess([`${reg} = ${language.consumeInputChar};`]);
+      if (context.getDiscard()) {
+        if (!context.getDiscardPos()) {
+          result.onSuccess([`${language.advanceInputChar};`]);
+        }
+        result.onSuccess([`${reg} = true;`]);
+      } else {
+        result.onSuccess([`${reg} = ${language.consumeInputChar};`]);
+      }
       result.onFailure([`${reg} = ${language.failed};`]);
       if (context.getSilence() !== 'true') {
         result.onFailure([makeFailCall({