tarsec 0.3.2 → 0.4.1

package/README.md

@@ -43,7 +43,7 @@ parser("hello there"); // failure
 - Derived types: tarsec will generate TypeScript types for your parser
 - [Debug mode](/tutorials/debugging.md) that prints what's happening step-by-step
 - Tools to debug your parser's [performance](/tutorials/performance.md)
-- Partial [backtracking](/tutorials/backtracking.md) support
+- `peek` / `not` lookahead for [disambiguating grammars](/tutorials/backtracking.md) without backtracking
 - A way to make your parser more [secure](/tutorials/security.md).
 - [Pretty error messages](/tutorials/pretty-errors.md)
 

package/dist/combinators.d.ts

@@ -58,20 +58,6 @@ export declare function many1WithJoin(parser: Parser<string>): Parser<string>;
  * const parser = or(capture(digit, "num"), capture(word, "name"));
  * ```
  *
- * `or` supports backtracking by returning a `nextParser`:
- *
- * ```ts
- * const parser = or(str("hello"), str("hello!"));
- *
- * // this will match the first parser
- * const result = parser("hello");
- *
- * // but or returns the untried parsers as a new parser
- * result.nextParser("hello!"); // works
- *
- * // result.nextParser is the same as or(str("hello!"))
- * ```
- *
  * @param parsers - parsers to try
  * @returns - a parser that tries each parser in order. Returns the result of the first parser that succeeds.
  */
@@ -96,6 +82,28 @@ export declare function optional<T>(parser: Parser<T>): Parser<T | null>;
  * If it succeeds, returns a failure.
  */
 export declare function not(parser: Parser<any>): Parser<null>;
+/**
+ * Positive lookahead. Runs the given parser without consuming any input.
+ * On success, returns the parser's result with `rest` set to the original input.
+ * On failure, returns the underlying failure (also with `rest` reset to the original input).
+ *
+ * Useful for disambiguating alternatives without backtracking:
+ *
+ * ```ts
+ * const parser = or(
+ *   seqR(peek(str("hello!")), str("hello!")),
+ *   str("hello"),
+ * );
+ * ```
+ *
+ * The `peek` decides which branch to commit to; the real parser then consumes.
+ * Captures are preserved when the inner parser is a `CaptureParser`.
+ *
+ * @param parser - parser to look ahead with
+ * @returns - a parser that runs the given parser without consuming input
+ */
+export declare function peek<T>(parser: Parser<T>): Parser<T>;
+export declare function peek<T, C extends PlainObject>(parser: CaptureParser<T, C>): CaptureParser<T, C>;
 /**
  * Takes three parsers, `open`, `close`, and `parser`.
  * `between` matches multiple instances of `parser`,
@@ -357,7 +365,6 @@ export declare function search<T>(parser: Parser<T>): Parser<T[]>;
  * Finally, you don't need to use seq at all. You can just hand write the logic.
  * But you'll need to do the error handling
  * and pass the remaining input to the next parser yourself.
- * seq also does some backtracking for you that you will need to do yourself.
  *
  * Also see `seqR` and `seqC` for convenience functions that return the results or captures respectively.
  *
@@ -490,3 +497,21 @@ export type OperatorInfo<T> = {
  * @returns - a parser that handles the full expression grammar
  */
 export declare function buildExpressionParser<T>(atom: Parser<T>, operatorTable: OperatorInfo<T>[][], parenParser?: Parser<T>): Parser<T>;
+/**
+ * Wraps a parser with a per-input cache. Useful for parsers that may be invoked
+ * many times at the same position (e.g. recursive grammars where the same
+ * sub-parser is consulted from multiple alternatives).
+ *
+ * Both successes and failures are cached. The cache is keyed by the input
+ * string the parser is called with — because the rest passed between parsers
+ * is value-equal across paths, identical sub-parses share a cache entry.
+ *
+ * `memo` assumes its wrapped parser is a pure function of its input. Don't
+ * memoize parsers that consult mutable external state.
+ *
+ * @param parser - parser to memoize
+ * @param name - optional debug name (shown in `parserDebug` counts/times as `memo(name)`)
+ * @returns - memoized parser
+ */
+export declare function memo<T>(parser: Parser<T>, name?: string): Parser<T>;
+export declare function memo<T, C extends PlainObject>(parser: CaptureParser<T, C>, name?: string): CaptureParser<T, C>;

package/dist/combinators.js

@@ -1,8 +1,8 @@
 import { within } from "./parsers/within.js";
 import { TarsecError } from "./tarsecError.js";
 import { getDiagnostics, trace } from "./trace.js";
-import { captureSuccess, createTree, failure, isCaptureResult, isSuccess, success, } from "./types.js";
-import { escape, findAncestorWithNextParser, popMany } from "./utils.js";
+import { captureSuccess, failure, isCaptureResult, isSuccess, success, } from "./types.js";
+import { escape } from "./utils.js";
 /**
  * Takes a parser and runs it zero or more times, returning the results as an array.
  * If the parser is a capture parser, it returns the captures as an array in this form:
@@ -148,20 +148,6 @@ export function many1WithJoin(parser) {
  * const parser = or(capture(digit, "num"), capture(word, "name"));
  * ```
  *
- * `or` supports backtracking by returning a `nextParser`:
- *
- * ```ts
- * const parser = or(str("hello"), str("hello!"));
- *
- * // this will match the first parser
- * const result = parser("hello");
- *
- * // but or returns the untried parsers as a new parser
- * result.nextParser("hello!"); // works
- *
- * // result.nextParser is the same as or(str("hello!"))
- * ```
- *
  * @param parsers - parsers to try
  * @returns - a parser that tries each parser in order. Returns the result of the first parser that succeeds.
  */
@@ -170,11 +156,7 @@ export function or(...parsers) {
         for (let i = 0; i < parsers.length; i++) {
             let result = parsers[i](input);
             if (result.success) {
-                if (i === parsers.length - 1)
-                    return result;
-                const nextParser = or(...parsers.slice(i + 1));
-                /* console.log({ nextParser }, parsers.slice(i + 1)); */
-                return Object.assign(Object.assign({}, result), { nextParser });
+                return result;
             }
         }
         return failure(`all parsers failed`, input);
@@ -220,6 +202,15 @@ export function not(parser) {
         return success(null, input);
     });
 }
+export function peek(parser) {
+    return trace("peek", (input) => {
+        const result = parser(input);
+        if (!result.success) {
+            return Object.assign(Object.assign({}, result), { rest: input });
+        }
+        return Object.assign(Object.assign({}, result), { rest: input });
+    });
+}
 /**
  * Takes three parsers, `open`, `close`, and `parser`.
  * `between` matches multiple instances of `parser`,
@@ -624,64 +615,6 @@ export function search(parser) {
         return success([], input);
     });
 }
-/*
-To add backtracking support requires a fairly big change. Here's an example that needs backtracking.
-
-```ts
-  const parser = seq([
-        str("hello "),
-        or(str("world"), str("world!")),
-        optional("?")
-    ], getResults);
-```
-
-If we try to parse `"hello world!"`, the first parser in the OR will succeed, but then we'll get stuck at the `optional`. Instead, we need to go back up the tree and try the second parser in the OR. A few things need to happen.
-
-1. instead of just processing these parsers sequentially in a for loop, we need to model them as a tree
-2. the OR parser needs to let us know that there are other branches to try.
-
-For #2, there's an optional `nextParser` key on a parser success. The or parser can use this to say "a parser succeeded and here's the result, but there are other parsers that could be tried". `nextParser` is a parser that runs the remaining branches. So in this example, the OR would return a success with `nextParser = or(str("world"))`.
-
-Next, we need to model this as a tree. Each node in the tree has a parent and child and the parser for that node.
-
-```ts
-  parent: Node;
-  parser: GeneralParser<any, any> | null;
-  child: Node;
-```
-
-Hopefully that is self-explanatory. We start at the root of the tree, try the parser there, then use `.child` to go to the next node and so on. We don't model multiple paths as multiple children. To keep the code simple, we do something else.
-
-Each node also has a `closed` key. Once we've run the parser for a node, we mark it `closed`. Closed means there are no more branches here. UNLESS, the parser returns a `nextParser`. In that case, we *don't* mark it closed because there are still other options to try. In that case, we also *replace* the parser on that node with nextParser.
-
-So, going back to the hello world example, let's say we're stuck at the `optional`:
-
-```ts
-  const parser = seq([
-        str("hello "),
-        or(str("world"), str("world!")),
-        optional("?")
-    ], getResults);
-```
-
-We use `.parent` to go back up the tree. We're looking for a node that isn't closed. If we find one, we start again from there. In this case, we'd find an open node at the or with parser `or(str("world"))`. We can restart from there, but there's a bunch of state to reset.
-
-1. From the new `or` parser, we need to go to the optional parser. We're doing it all again in the same order. This is one reason why it's easier to model this without multiple children. Otherwise, all the children would have to point to the next level, the next level would have to point to all the children in the previous level, and you'd have multiple parents, which is awful to deal with.
-
-2. We have consumed input and added to the results. We need to undo that. At this point, the input is `!`, because we've consumed `hello world`. And the results array is `["hello ", "world"]`. We need to rewind both of those.
-
-To do that, I count how many levels up we've gone to find another branch, and just pop that many elements off the results array. So results is now `["hello "]`. The input is trickier. How would I keep track of what the input was when we were at the OR the last time?
-
-This is where the final key on a tree node comes in. Nodes also have an optional `input` key.
-
-IF a parser succeeds, and
-IF there's a nextParser,
-We know we may come back to this node. So we save the current input as `.input` on the node.
-
-This approach has some issues. Notably, it doesn't work if you need to backtrack at multiple points in the tree. The test `backtracking-deep.test.ts` shows this.
-
-The code is also complex and it would be easy to have bugs in this logic. I wish there was a cleaner solution for rewinding state.
-*/
 /**
  * seq takes an array of parsers and runs them sequentially.
  * If any of the parsers fail, seq fails without consuming any input.
@@ -699,7 +632,6 @@ The code is also complex and it would be easy to have bugs in this logic. I wish
  * Finally, you don't need to use seq at all. You can just hand write the logic.
  * But you'll need to do the error handling
  * and pass the remaining input to the next parser yourself.
- * seq also does some backtracking for you that you will need to do yourself.
  *
  * Also see `seqR` and `seqC` for convenience functions that return the results or captures respectively.
  *
@@ -713,44 +645,18 @@ export function seq(parsers, transform, debugName = "") {
         const results = [];
         let rest = input;
         const captures = {};
-        const rootNode = createTree(parsers);
-        let current = rootNode;
-        while (current) {
-            const parser = current.parser;
-            if (!parser) {
-                console.log({ current, parser, results, captures });
-                throw new Error("parser is null");
-            }
-            const parsed = parser(rest);
-            current.closed = true;
-            /*       console.log({ parsed }); */
+        for (let i = 0; i < parsers.length; i++) {
+            const parsed = parsers[i](rest);
             if (!parsed.success) {
-                const [ancestor, count] = findAncestorWithNextParser(current);
-                if (ancestor) {
-                    current = ancestor;
-                    rest = ancestor.input;
-                    popMany(results, count);
-                    continue;
-                }
-                else {
-                    // don't consume input if we're failing
-                    return Object.assign(Object.assign({}, parsed), { rest: input });
-                }
+                return Object.assign(Object.assign({}, parsed), { rest: input });
             }
             results.push(parsed.result);
-            if (parsed.nextParser) {
-                /* console.log("setting next parser", parsed.nextParser); */
-                current.parser = parsed.nextParser;
-                current.input = rest;
-                current.closed = false;
-            }
             rest = parsed.rest;
             if (isCaptureResult(parsed)) {
                 for (const key in parsed.captures) {
                     captures[key] = parsed.captures[key];
                 }
             }
-            current = current.child;
         }
         const result = transform(results, captures);
         return success(result, rest);
@@ -1026,3 +932,20 @@ function tryOps(ops, input) {
     }
     return null;
 }
+const DEFAULT_MEMO_LIMIT = 10000;
+export function memo(parser, name) {
+    const cache = new Map();
+    return trace(name ? `memo(${name})` : "memo", (input) => {
+        const hit = cache.get(input);
+        if (hit !== undefined)
+            return hit;
+        const result = parser(input);
+        if (cache.size >= DEFAULT_MEMO_LIMIT) {
+            const oldest = cache.keys().next().value;
+            if (oldest !== undefined)
+                cache.delete(oldest);
+        }
+        cache.set(input, result);
+        return result;
+    });
+}

package/dist/types.d.ts

@@ -5,15 +5,13 @@ export type ParserSuccess<T> = {
     success: true;
     result: T;
     rest: string;
-    nextParser?: Parser<any>;
 };
-/** Represents a parse success with captures. Notice nextParser is also a CaptureParser. */
+/** Represents a parse success with captures. */
 export type CaptureParserSuccess<T, C extends PlainObject> = {
     success: true;
     result: T;
     rest: string;
     captures: C;
-    nextParser?: CaptureParser<any, any>;
 };
 /** Represents a parse failure. */
 export type ParserFailure = {
@@ -98,23 +96,6 @@ export type InferManyReturnType<T extends GeneralParser<any, any>> = T extends C
     captures: C[];
 }> : T extends Parser<infer R> ? Parser<R[]> : never;
 export type MergedResults<T extends readonly GeneralParser<any, any>[]> = ExtractResults<T[number]>;
-/** Used to create a parser tree for backtracking. */
-export type Node = ParserNode | EmptyNode;
-export type ParserNode = {
-    parent: Node;
-    parser: GeneralParser<any, any> | null;
-    input?: string;
-    child: Node;
-    closed: boolean;
-};
-export type EmptyNode = null;
-/** Convenience function to create a ParserNode. */
-export declare function createNode(parent: Node | null, parser: GeneralParser<any, any>): ParserNode;
-/** Convenience function where, given an array of parsers, it creates a tree we can use for backtracking.
- * This tree is what `seq` use. It's used to keep track of the parsers we've tried so far,
- * so we can backtrack if we need to.
- */
-export declare function createTree(parsers: readonly GeneralParser<any, any>[]): Node;
 /** Used by `within`. */
 export type Matched<T> = {
     type: "matched";

package/dist/types.js

@@ -30,28 +30,3 @@ export function captureSuccess(result, rest, captures) {
 export function failure(message, rest) {
     return { success: false, message, rest };
 }
-/** Convenience function to create a ParserNode. */
-export function createNode(parent, parser) {
-    return {
-        parent,
-        parser,
-        child: null,
-        closed: false,
-    };
-}
-/** Convenience function where, given an array of parsers, it creates a tree we can use for backtracking.
- * This tree is what `seq` use. It's used to keep track of the parsers we've tried so far,
- * so we can backtrack if we need to.
- */
-export function createTree(parsers) {
-    if (parsers.length === 0) {
-        return null;
-    }
-    const rootNode = createNode(null, parsers[0]);
-    let currentNode = rootNode;
-    for (let i = 1; i < parsers.length; i++) {
-        currentNode.child = createNode(currentNode, parsers[i]);
-        currentNode = currentNode.child;
-    }
-    return rootNode;
-}

package/dist/utils.d.ts

@@ -1,8 +1,5 @@
-import { Node } from "./types.js";
 export declare function escape(str: any): string;
 export declare function merge(a: any | any[], b: any | any[]): any[];
 export declare function mergeCaptures(a: Record<string, any>, b: Record<string, any>): Record<string, any>;
-export declare function findAncestorWithNextParser(node: Node, count?: number): [Node, number];
-export declare function popMany(arr: any[], count: number): void;
 export declare function round(num: number, places?: number): number;
 export declare function shorten(str: string, length?: number): string;

package/dist/utils.js

@@ -30,22 +30,6 @@ export function mergeCaptures(a, b) {
     });
     return result;
 }
-export function findAncestorWithNextParser(node, count = 0) {
-    if (node === null)
-        return [null, count];
-    if (!node.closed) {
-        return [node, count];
-    }
-    if (node.parent) {
-        return findAncestorWithNextParser(node.parent, count + 1);
-    }
-    return [null, count];
-}
-export function popMany(arr, count) {
-    for (let i = 0; i < count; i++) {
-        arr.pop();
-    }
-}
 export function round(num, places = 2) {
     return Math.round(num * 10 ** places) / 10 ** places;
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tarsec",
-  "version": "0.3.2",
+  "version": "0.4.1",
   "description": "A parser combinator library for TypeScript, inspired by Parsec.",
   "homepage": "https://github.com/egonSchiele/tarsec",
   "scripts": {

package/dist/combinators/seq.d.ts

@@ -1 +0,0 @@
-export {};

package/dist/combinators/seq.js

@@ -1 +0,0 @@
-export {};