wikipeg 2.0.5 → 4.0.0

package/README.md

@@ -67,6 +67,8 @@ You can tweak the generated parser with several options:
   * `--extra-options-file` — file with additional options (in JSON format) to
     pass to `PEG.buildParser`
   * `--trace` — makes the parser trace its progress
+  * `--header-comment-file` — file containing a well-formatted comment, used
+    to customize the comment at the top of the generated file
 
 ### JavaScript API
 

package/VERSION

@@ -1 +1 @@
-2.0.5
+4.0.0

package/bin/wikipeg

@@ -47,6 +47,11 @@ Options:
                                      to PEG.buildParser
       --extra-options-file <file>    file with additional options (in JSON
                                      format) to pass to PEG.buildParser
+      --header-comment-file <file>   file containing the comment to add at the top
+                                     of the generated file to document where it
+                                     comes from. Must contain all formatting
+                                     necessary for said comment in the target
+                                     language.
   -v, --version                      print version information and exit
   -h, --help                         print help and exit
 `);
@@ -210,6 +215,19 @@ while (args.length > 0 && isOption(args[0])) {
       addExtraOptions(options, json);
       break;
 
+    case "--header-comment-file":
+      nextArg();
+      if (args.length === 0) {
+        abort("Missing parameter of the --header-comment-file option.");
+      }
+      try {
+        var comment = fs.readFileSync(args[0]);
+      } catch(e) {
+        abort("Can't read from file \"" + args[0] + "\".");
+      }
+      options.headerComment = comment;
+      break;
+
     case "-v":
     case "--version":
       printVersion();

package/lib/compiler/language/javascript.js

@@ -225,6 +225,11 @@ let javascript = {
     if (opts.params.length) {
       keyParts = keyParts.concat(opts.params);
     }
+    let storeRefs = opts.storeRefs.filter(function(part) {
+      return part !== '';
+    }).map(function(part) {
+      return '  ' + part;
+    }).join('\n');
     return {
       start: [
         `var key = [${keyParts.join(',')}].map(String).join(":");`,
@@ -241,7 +246,7 @@ let javascript = {
         '  nextPos: peg$currPos, ',
         `  result: ${opts.result}, `,
         '};',
-        opts.storeRefs
+        storeRefs
       ].join('\n')
     };
   },
@@ -251,7 +256,8 @@ let javascript = {
     return `    if (cached.hasOwnProperty(${encName})) param_${name}.value = cached.$${name};`;
   },
 
-  cacheStoreRef(name) {
+  cacheStoreRef(name, store) {
+    if (!store) return '';
     return `if (saved_${name} !== param_${name}.value) cached.$${name} = param_${name}.value;`;
   },
 

package/lib/compiler/language/php.js

@@ -424,33 +424,38 @@ let php = {
     if (opts.params.length) {
       keyParts = keyParts.concat(opts.params);
     }
+    let storeRefs = opts.storeRefs.map(function(part) {
+      return '  ' + part;
+    }).join(',\n');
     return {
       start: [
         `$key = json_encode([${keyParts.join(',')}]);`,
         `$cached = $this->cache[$key] ?? null;`,
         `if ($cached) {`,
-        `  $this->currPos = $cached['nextPos'];`,
+        `  $this->currPos = $cached->nextPos;`,
         opts.loadRefs,
-        `  return $cached['result'];`,
+        `  return $cached->result;`,
         '}',
         opts.saveRefs,
       ].join('\n'),
       store: [
-        `$cached = ['nextPos' => $this->currPos, 'result' => ${opts.result}];`,
-        opts.storeRefs,
-        `$this->cache[$key] = $cached;`
+        `$this->cache[$key] = new ${opts.className}CacheEntry(`,
+        '  $this->currPos,',
+        `  ${opts.result + (opts.storeRefs.length > 0 ? ',' : '')}`,
+        storeRefs,
+        `);`
       ].join('\n')
     };
   },
 
   cacheLoadRef(name) {
-    let encName = php.stringify('$' + name);
-    return `if (array_key_exists(${encName}, $cached)) $param_${name} = $cached[${encName}];`;
+    return `if ($cached->${name} !== self::$UNDEFINED) { $param_${name} = $cached->${name}; }`;
   },
 
-  cacheStoreRef(name) {
-    let encName = php.stringify('$' + name);
-    return `if ($saved_${name} !== $param_${name}) $cached[${encName}] = $param_${name};`;
+  cacheStoreRef(name, store) {
+    return store ?
+      `$saved_${name} !== $param_${name} ? $param_${name} : self::$UNDEFINED` :
+      'self::$UNDEFINED';
   },
 
   /**

package/lib/compiler/passes/ast-to-code.js

@@ -80,6 +80,28 @@ function generateJavascript(ast, options) {
     language = js;
   }
 
+  var className = options.className || 'PEGParser';
+  var namespace = '';
+  var matches = className.match(/^(.*)\\([^\\]*)$/);
+  if (matches) {
+    className = matches[2];
+    namespace = `namespace ${matches[1]};`;
+  }
+
+  var refsSet = {};
+  var getRefs = visitor.build({
+    rule_ref: function(node) {
+      for (let i = 0; i < node.assignments.length; i++) {
+        let assignment = node.assignments[i];
+        if (assignment.isref) {
+          refsSet[assignment.name] = true;
+        }
+      }
+    }
+  });
+  getRefs(ast);
+  var references = Object.keys(refsSet);
+
   /**
    * The default init cache hook
    */
@@ -91,7 +113,6 @@ function generateJavascript(ast, options) {
   var generateCacheRule = language.generateCacheRule;
 
   function indent2(code)  { return code.replace(/^(.+)$/gm, '  $1');       }
-  function indent4(code)  { return code.replace(/^(.+)$/gm, '    $1');       }
 
   /**
    * The Context class.
@@ -749,21 +770,23 @@ function generateJavascript(ast, options) {
         block.push(language.cacheLoadRef(name));
       }
     }
-    return indent4(block.join('\n'));
+    return indent2(block.join('\n'));
   }
 
   /**
-   * Get a block which determines which refs have changed, if any. Any changed
-   * refs are stored in the `cached` object.
+   * Get the list of expressions or statements returned by cacheStoreRef() for
+   * refs which may have changed.
    */
   function getCacheStoreRefs(rule) {
-    var block = [];
+    var store = {};
     for (let name in rule.passedParams) {
       if (rule.passedParams[name].type === 'reference') {
-        block.push(language.cacheStoreRef(name));
+        store[name] = true;
       }
     }
-    return indent2(block.join('\n'));
+    return references.map(function(name) {
+      return language.cacheStoreRef(name, store[name]);
+    });
   }
 
   /**
@@ -774,10 +797,10 @@ function generateJavascript(ast, options) {
     var parts = [];
     for (let name in rule.passedParams) {
       if (rule.passedParams[name].type === 'reference') {
-        parts.push(indent2(language.cacheSaveRef(name)));
+        parts.push(language.cacheSaveRef(name));
       }
     }
-    return indent2(parts.join('\n'));
+    return parts.join('\n');
   }
 
   function expandTemplate(template, vars) {
@@ -848,6 +871,7 @@ function generateJavascript(ast, options) {
             loadRefs: getCacheLoadRefs(node),
             saveRefs: getCacheSaveRefs(node),
             storeRefs: getCacheStoreRefs(node),
+            className: className,
           });
           body.push(cacheBits.start);
         }
@@ -1356,15 +1380,8 @@ function generateJavascript(ast, options) {
   }
   templateVars['/*GENERATED*/'] = generated.join('\n');
 
-  var className = options.className || 'PEGParser';
-  var matches = className.match(/^(.*)\\([^\\]*)$/);
-  if (matches) {
-    templateVars['CLASS_NAME'] = matches[2];
-    templateVars['/*NAMESPACE*/'] = `namespace ${matches[1]};`;
-  } else {
-    templateVars['CLASS_NAME'] = className;
-    templateVars['/*NAMESPACE*/'] = '';
-  }
+  templateVars['CLASS_NAME'] = className;
+  templateVars['/*NAMESPACE*/'] = namespace;
 
   var cacheInitCode = '';
   var cacheInitHook;
@@ -1377,9 +1394,35 @@ function generateJavascript(ast, options) {
   }
   templateVars['/*CACHE_INIT*/'] = cacheInitCode;
 
+  var headerComment = '';
+  if (options.headerComment) {
+    headerComment = options.headerComment + '\n';
+  } else {
+    headerComment = '/*\n' +
+                    ' * Generated by WikiPEG\n' +
+                    ' */\n\n';
+  }
+  templateVars['/*HEADER_COMMENT*/'] = headerComment;
+
   var template;
   if (options.language === 'php') {
     template = fs.readFileSync(__dirname + '/../../runtime/template.php', 'utf8');
+    if (options.cache) {
+      templateVars['/*CACHE_ENTRY_BEGIN*/'] = '';
+      templateVars['/*CACHE_ENTRY_END*/'] = '';
+      templateVars['/*CACHE_ENTRY_DECLARE*/'] = '';
+      templateVars['/*CACHE_ENTRY_ARGS*/'] = '';
+      templateVars['/*CACHE_ENTRY_INIT*/'] = '';
+      references.forEach(function(name) {
+        templateVars['/*CACHE_ENTRY_DECLARE*/'] += `\tpublic $${name};\n`;
+        templateVars['/*CACHE_ENTRY_ARGS*/'] += `, $${name}`;
+        templateVars['/*CACHE_ENTRY_INIT*/'] += `\t\t$this->${name} = $${name};\n`;
+      });
+    } else {
+      template = template.replace(
+        /\/\*CACHE_ENTRY_BEGIN\*\/[^]+?\/\*CACHE_ENTRY_END\*\//, ''
+      );
+    }
   } else {
     template = fs.readFileSync(__dirname + '/../../runtime/template.js', 'utf8');
   }

package/lib/parser.js

@@ -5,6 +5,8 @@ module.exports = ( function () {
  * Generated by WikiPEG
  */
 
+
+
 function peg$subclass(child, parent) {
   function ctor() { this.constructor = child; }
   ctor.prototype = parent.prototype;

package/lib/peg.js

@@ -5,7 +5,7 @@ var arrays  = require("./utils/arrays"),
 
 var PEG = {
   /* WikiPEG version (uses semantic versioning). */
-  VERSION: "2.0.5",
+  VERSION: "4.0.0",
 
   GrammarError: require("./grammar-error"),
   parser:       require("./parser"),

package/lib/runtime/template.js

@@ -1,9 +1,7 @@
 ( function () {
 
 "use strict";
-/*
- * Generated by WikiPEG
- */
+/*HEADER_COMMENT*/
 
 function peg$subclass(child, parent) {
   function ctor() { this.constructor = child; }

package/lib/runtime/template.php

@@ -1,53 +1,67 @@
 <?php
-
+/*HEADER_COMMENT*/
 /*NAMESPACE*/
 
 /*INITIALIZER0*/
 
-class CLASS_NAME extends \WikiPEG\PEGParserBase {
-  // initializer
-  /*INITIALIZER*/
-
-  // cache init
-  /*CACHE_INIT*/
-
-  // expectations
-  protected $expectations = [
-    /*EXPECTATIONS*/
-  ];
-
-  // actions
-  /*ACTIONS*/
-
-  // generated
-  /*GENERATED*/
-
-  public function parse($input, $options = []) {
-    $this->initInternal($input, $options);
-    $startRule = $options['startRule'] ?? '(DEFAULT)';
-    $result = null;
-
-    if (!empty($options['stream'])) {
-      switch ($startRule) {
-        /*STREAM_CASES*/
-        default:
-          throw new \WikiPEG\InternalError("Can't stream rule $startRule.");
-      }
-    } else {
-      switch ($startRule) {
-        /*START_CASES*/
-        default:
-          throw new \WikiPEG\InternalError("Can't start parsing from rule $startRule.");
-      }
-    }
-
-    if ($result !== self::$FAILED && $this->currPos === $this->inputLength) {
-      return $result;
-    } else {
-      if ($result !== self::$FAILED && $this->currPos < $this->inputLength) {
-        $this->fail(/*END_EXPECTATION*/);
-      }
-      throw $this->buildParseException();
-    }
-  }
+/*CACHE_ENTRY_BEGIN*/
+class CLASS_NAMECacheEntry {
+	public $nextPos;
+	public $result;
+/*CACHE_ENTRY_DECLARE*/
+
+	public function __construct( $nextPos, $result/*CACHE_ENTRY_ARGS*/ ) {
+		$this->nextPos = $nextPos;
+		$this->result = $result;
+/*CACHE_ENTRY_INIT*/
+	}
+}
+/*CACHE_ENTRY_END*/
+
+class CLASS_NAME extends \Wikimedia\WikiPEG\PEGParserBase {
+	// initializer
+	/*INITIALIZER*/
+
+	// cache init
+	/*CACHE_INIT*/
+
+	// expectations
+	protected $expectations = [
+		/*EXPECTATIONS*/
+	];
+
+	// actions
+	/*ACTIONS*/
+
+	// generated
+	/*GENERATED*/
+
+	public function parse( $input, $options = [] ) {
+		$this->initInternal( $input, $options );
+		$startRule = $options['startRule'] ?? '(DEFAULT)';
+		$result = null;
+
+		if ( !empty( $options['stream'] ) ) {
+			switch ( $startRule ) {
+				/*STREAM_CASES*/
+			default:
+				throw new \Wikimedia\WikiPEG\InternalError( "Can't stream rule $startRule." );
+			}
+		} else {
+			switch ( $startRule ) {
+				/*START_CASES*/
+			default:
+				throw new \Wikimedia\WikiPEG\InternalError( "Can't start parsing from rule $startRule." );
+			}
+		}
+
+		if ( $result !== self::$FAILED && $this->currPos === $this->inputLength ) {
+			return $result;
+		} else {
+			if ( $result !== self::$FAILED && $this->currPos < $this->inputLength ) {
+				$this->fail( /*END_EXPECTATION*/ );
+			}
+			throw $this->buildParseException();
+		}
+	}
 }

package/package.json

@@ -1,49 +1,49 @@
 {
-  "name": "wikipeg",
-  "version": "2.0.5",
-  "description": "Parser generator for JavaScript and PHP",
-  "license": "MIT",
-  "homepage": "https://gerrit.wikimedia.org/r/plugins/gitiles/wikipeg/",
-  "contributors": [
-    {
-      "name": "David Majda",
-      "email": "david@majda.cz",
-      "url": "http://majda.cz/"
-    },
-    {
-      "name": "Tim Starling",
-      "email": "tstarling@wikimedia.org"
-    }
-  ],
-  "files": [
-    "CHANGELOG.md",
-    "LICENSE",
-    "README.md",
-    "VERSION",
-    "bin/wikipeg",
-    "examples/*.pegjs",
-    "examples/*.pegphp",
-    "lib/**/*.js",
-    "lib/**/*.php",
-    "src/**/*.php",
-    "tools/build-browser.js",
-    "package.json"
-  ],
-  "main": "lib/peg",
-  "bin": "bin/wikipeg",
-  "scripts": {
-    "eslint": "make eslint",
-    "test": "make eslint && make test"
-  },
-  "repository": {
-    "type": "git",
-    "url": "https://gerrit.wikimedia.org/r/wikipeg"
-  },
-  "devDependencies": {
-    "jasmine-node": "3.0.0",
-    "eslint": "5.14.1"
-  },
-  "engines": {
-    "node": ">= 6.0.0"
-  }
+	"name": "wikipeg",
+	"version": "4.0.0",
+	"description": "Parser generator for JavaScript and PHP",
+	"license": "MIT",
+	"homepage": "https://gerrit.wikimedia.org/r/plugins/gitiles/wikipeg/",
+	"contributors": [
+		{
+			"name": "David Majda",
+			"email": "david@majda.cz",
+			"url": "http://majda.cz/"
+		},
+		{
+			"name": "Tim Starling",
+			"email": "tstarling@wikimedia.org"
+		}
+	],
+	"files": [
+		"CHANGELOG.md",
+		"LICENSE",
+		"README.md",
+		"VERSION",
+		"bin/wikipeg",
+		"examples/*.pegjs",
+		"examples/*.pegphp",
+		"lib/**/*.js",
+		"lib/**/*.php",
+		"src/**/*.php",
+		"tools/build-browser.js",
+		"package.json"
+	],
+	"main": "lib/peg",
+	"bin": "bin/wikipeg",
+	"scripts": {
+		"eslint": "make eslint",
+		"test": "make eslint && make test"
+	},
+	"repository": {
+		"type": "git",
+		"url": "https://gerrit.wikimedia.org/r/wikipeg"
+	},
+	"devDependencies": {
+		"eslint": "8.31.0",
+		"jasmine-node": "3.0.0"
+	},
+	"engines": {
+		"node": ">= 6.0.0"
+	}
 }

package/src/DefaultTracer.php

@@ -1,72 +1,73 @@
 <?php
 
-namespace WikiPEG;
+namespace Wikimedia\WikiPEG;
+
+use InvalidArgumentException;
 
 class DefaultTracer implements Tracer {
-  private $indentLevel = 0;
+	private $indentLevel = 0;
 
-  public function trace($event) {
-    switch ($event['type']) {
-      case 'rule.enter':
-        $this->log($event);
-        $this->indentLevel++;
-        break;
+	public function trace( $event ) {
+		switch ( $event['type'] ) {
+		case 'rule.enter':
+			$this->log( $event );
+			$this->indentLevel++;
+			break;
 
-      case 'rule.match':
-        $this->indentLevel--;
-        $this->log($event);
-        break;
+		case 'rule.match':
+			$this->indentLevel--;
+			$this->log( $event );
+			break;
 
-      case 'rule.fail':
-        $this->indentLevel--;
-        $this->log($event);
-        break;
+		case 'rule.fail':
+			$this->indentLevel--;
+			$this->log( $event );
+			break;
 
-      default:
-        throw new \Exception("Invalid event type {$event['type']}");
-    }
-  }
+		default:
+			throw new InvalidArgumentException( "Invalid event type {$event['type']}" );
+		}
+	}
 
-  private function log($event) {
-    print
-      str_pad(
-        ''.$event['location'],
-        20
-      )
-      . str_pad($event['type'], 10) . ' '
-      . str_repeat(' ', $this->indentLevel) . $event['rule']
-      . $this->formatArgs($event['args'] ?? null)
-      . "\n";
-  }
+	private function log( $event ) {
+		print str_pad(
+			'' . $event['location'],
+			20
+		)
+			. str_pad( $event['type'], 10 ) . ' '
+			. str_repeat( ' ', $this->indentLevel ) . $event['rule']
+			. $this->formatArgs( $event['args'] ?? null )
+			. "\n";
+	}
 
-  private function formatArgs($argMap) {
-    if (!$argMap) {
-      return '';
-    }
+	private function formatArgs( $argMap ) {
+		if ( !$argMap ) {
+			return '';
+		}
 
-    $argParts = [];
-    foreach ($argMap as $argName => $argValue) {
-      if ($argName === '$silence') {
-        continue;
-      }
-      if ($argName === '$boolParams') {
-        $argParts[] = '0x' . base_convert($argValue, 10, 16);
-      } else {
-        $displayName = str_replace( '$param_', '', $argName);
-        if ( $displayName[0] === '&' ) {
-          $displayName = substr( $displayName, 1 );
-          $ref = '&';
-        } else {
-          $ref = '';
-        }
-        $argParts[] = "$displayName=$ref" .
-          json_encode( $argValue, JSON_UNESCAPED_SLASHES |  JSON_UNESCAPED_UNICODE );
-      }
-    }
-    if ($argParts) {
-      return '<' . implode(', ', $argParts) . '>';
-    } else {
-      return '';
-    }
-  }
+		$argParts = [];
+		foreach ( $argMap as $argName => $argValue ) {
+			if ( $argName === '$silence' ) {
+				continue;
+			}
+			if ( $argName === '$boolParams' ) {
+				$argParts[] = '0x' . base_convert( $argValue, 10, 16 );
+			} else {
+				$displayName = str_replace( '$param_', '', $argName );
+				if ( $displayName[0] === '&' ) {
+					$displayName = substr( $displayName, 1 );
+					$ref = '&';
+				} else {
+					$ref = '';
+				}
+				$argParts[] = "$displayName=$ref" .
+						   json_encode( $argValue, JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE );
+			}
+		}
+		if ( $argParts ) {
+			return '<' . implode( ', ', $argParts ) . '>';
+		} else {
+			return '';
+		}
+	}
 }