yaml 2.0.0-4 → 2.0.0-5

package/dist/parse/lexer.js

@@ -1,6 +1,6 @@
 'use strict';
 
-var tokens = require('./tokens.js');
+var cst = require('./cst.js');
 
 /*
 START -> stream
@@ -100,13 +100,7 @@ const isNotIdentifierChar = (ch) => !ch || invalidIdentifierChars.includes(ch);
  * - `\u{FEFF}` (Byte order mark): Emitted separately outside documents
  */
 class Lexer {
-    /**
-     * Define/initialise a YAML lexer. `push` will be called separately with each
-     * token when `lex()` is passed an input string.
-     *
-     * @public
-     */
-    constructor(push) {
+    constructor() {
         /**
          * Flag indicating whether the end of the current buffer marks the end of
          * all input
@@ -144,22 +138,20 @@ class Lexer {
         this.next = null;
         /** A pointer to `buffer`; the current position of the lexer. */
         this.pos = 0;
-        this.push = push;
     }
     /**
-     * Read YAML tokens from the `source` string, calling the callback
-     * defined in the constructor for each one. If `incomplete`, a part
-     * of the last line may be left as a buffer for the next call.
+     * Generate YAML tokens from the `source` string. If `incomplete`,
+     * a part of the last line may be left as a buffer for the next call.
      *
-     * @public
+     * @returns A generator of lexical tokens
      */
-    lex(source, incomplete) {
+    *lex(source, incomplete = false) {
         if (source)
             this.buffer = this.buffer ? this.buffer + source : source;
         this.atEnd = !incomplete;
         let next = this.next || 'stream';
         while (next && (incomplete || this.hasChars(1)))
-            next = this.parseNext(next);
+            next = yield* this.parseNext(next);
     }
     atLineEnd() {
         let i = this.pos;
@@ -217,32 +209,32 @@ class Lexer {
     peek(n) {
         return this.buffer.substr(this.pos, n);
     }
-    parseNext(next) {
+    *parseNext(next) {
         switch (next) {
             case 'stream':
-                return this.parseStream();
+                return yield* this.parseStream();
             case 'line-start':
-                return this.parseLineStart();
+                return yield* this.parseLineStart();
             case 'block-start':
-                return this.parseBlockStart();
+                return yield* this.parseBlockStart();
             case 'doc':
-                return this.parseDocument();
+                return yield* this.parseDocument();
             case 'flow':
-                return this.parseFlowCollection();
+                return yield* this.parseFlowCollection();
             case 'quoted-scalar':
-                return this.parseQuotedScalar();
+                return yield* this.parseQuotedScalar();
             case 'block-scalar':
-                return this.parseBlockScalar();
+                return yield* this.parseBlockScalar();
             case 'plain-scalar':
-                return this.parsePlainScalar();
+                return yield* this.parsePlainScalar();
         }
     }
-    parseStream() {
+    *parseStream() {
         let line = this.getLine();
         if (line === null)
             return this.setNext('stream');
-        if (line[0] === tokens.BOM) {
-            this.pushCount(1);
+        if (line[0] === cst.BOM) {
+            yield* this.pushCount(1);
             line = line.substring(1);
         }
         if (line[0] === '%') {
@@ -260,21 +252,21 @@ class Lexer {
                 else
                     break;
             }
-            const n = this.pushCount(dirEnd) + this.pushSpaces(true);
-            this.pushCount(line.length - n); // possible comment
+            const n = (yield* this.pushCount(dirEnd)) + (yield* this.pushSpaces(true));
+            yield* this.pushCount(line.length - n); // possible comment
             this.pushNewline();
             return 'stream';
         }
         if (this.atLineEnd()) {
-            const sp = this.pushSpaces(true);
-            this.pushCount(line.length - sp);
-            this.pushNewline();
+            const sp = yield* this.pushSpaces(true);
+            yield* this.pushCount(line.length - sp);
+            yield* this.pushNewline();
             return 'stream';
         }
-        this.push(tokens.DOCUMENT);
-        return this.parseLineStart();
+        yield cst.DOCUMENT;
+        return yield* this.parseLineStart();
     }
-    parseLineStart() {
+    *parseLineStart() {
         const ch = this.charAt(0);
         if (!ch && !this.atEnd)
             return this.setNext('line-start');
@@ -283,80 +275,80 @@ class Lexer {
                 return this.setNext('line-start');
             const s = this.peek(3);
             if (s === '---' && isEmpty(this.charAt(3))) {
-                this.pushCount(3);
+                yield* this.pushCount(3);
                 this.indentValue = 0;
                 this.indentNext = 0;
                 return 'doc';
             }
             else if (s === '...' && isEmpty(this.charAt(3))) {
-                this.pushCount(3);
+                yield* this.pushCount(3);
                 return 'stream';
             }
         }
-        this.indentValue = this.pushSpaces(false);
+        this.indentValue = yield* this.pushSpaces(false);
         if (this.indentNext > this.indentValue && !isEmpty(this.charAt(1)))
             this.indentNext = this.indentValue;
-        return this.parseBlockStart();
+        return yield* this.parseBlockStart();
     }
-    parseBlockStart() {
+    *parseBlockStart() {
         const [ch0, ch1] = this.peek(2);
         if (!ch1 && !this.atEnd)
             return this.setNext('block-start');
         if ((ch0 === '-' || ch0 === '?' || ch0 === ':') && isEmpty(ch1)) {
-            const n = this.pushCount(1) + this.pushSpaces(true);
+            const n = (yield* this.pushCount(1)) + (yield* this.pushSpaces(true));
             this.indentNext = this.indentValue + 1;
             this.indentValue += n;
-            return this.parseBlockStart();
+            return yield* this.parseBlockStart();
         }
         return 'doc';
     }
-    parseDocument() {
-        this.pushSpaces(true);
+    *parseDocument() {
+        yield* this.pushSpaces(true);
         const line = this.getLine();
         if (line === null)
             return this.setNext('doc');
-        let n = this.pushIndicators();
+        let n = yield* this.pushIndicators();
         switch (line[n]) {
             case '#':
-                this.pushCount(line.length - n);
+                yield* this.pushCount(line.length - n);
             // fallthrough
             case undefined:
-                this.pushNewline();
-                return this.parseLineStart();
+                yield* this.pushNewline();
+                return yield* this.parseLineStart();
             case '{':
             case '[':
-                this.pushCount(1);
+                yield* this.pushCount(1);
                 this.flowKey = false;
                 this.flowLevel = 1;
                 return 'flow';
             case '}':
             case ']':
                 // this is an error
-                this.pushCount(1);
+                yield* this.pushCount(1);
                 return 'doc';
             case '*':
-                this.pushUntil(isNotIdentifierChar);
+                yield* this.pushUntil(isNotIdentifierChar);
                 return 'doc';
             case '"':
             case "'":
-                return this.parseQuotedScalar();
+                return yield* this.parseQuotedScalar();
             case '|':
             case '>':
-                n += this.parseBlockScalarHeader();
-                n += this.pushSpaces(true);
-                this.pushCount(line.length - n);
-                this.pushNewline();
-                return this.parseBlockScalar();
+                n += yield* this.parseBlockScalarHeader();
+                n += yield* this.pushSpaces(true);
+                yield* this.pushCount(line.length - n);
+                yield* this.pushNewline();
+                return yield* this.parseBlockScalar();
             default:
-                return this.parsePlainScalar();
+                return yield* this.parsePlainScalar();
         }
     }
-    parseFlowCollection() {
+    *parseFlowCollection() {
         let nl, sp;
         let indent = -1;
         do {
-            nl = this.pushNewline();
-            sp = this.pushSpaces(true);
+            nl = yield* this.pushNewline();
+            sp = yield* this.pushSpaces(true);
             if (nl > 0)
                 this.indentValue = indent = sp;
         } while (nl + sp > 0);
@@ -376,54 +368,54 @@ class Lexer {
             if (!atFlowEndMarker) {
                 // this is an error
                 this.flowLevel = 0;
-                this.push(tokens.FLOW_END);
-                return this.parseLineStart();
+                yield cst.FLOW_END;
+                return yield* this.parseLineStart();
             }
         }
         let n = 0;
         while (line[n] === ',')
-            n += this.pushCount(1) + this.pushSpaces(true);
-        n += this.pushIndicators();
+            n += (yield* this.pushCount(1)) + (yield* this.pushSpaces(true));
+        n += yield* this.pushIndicators();
         switch (line[n]) {
             case undefined:
                 return 'flow';
             case '#':
-                this.pushCount(line.length - n);
+                yield* this.pushCount(line.length - n);
                 return 'flow';
             case '{':
             case '[':
-                this.pushCount(1);
+                yield* this.pushCount(1);
                 this.flowKey = false;
                 this.flowLevel += 1;
                 return 'flow';
             case '}':
             case ']':
-                this.pushCount(1);
+                yield* this.pushCount(1);
                 this.flowKey = true;
                 this.flowLevel -= 1;
                 return this.flowLevel ? 'flow' : 'doc';
             case '*':
-                this.pushUntil(isNotIdentifierChar);
+                yield* this.pushUntil(isNotIdentifierChar);
                 return 'flow';
             case '"':
             case "'":
                 this.flowKey = true;
-                return this.parseQuotedScalar();
+                return yield* this.parseQuotedScalar();
             case ':': {
                 const next = this.charAt(1);
                 if (this.flowKey || isEmpty(next) || next === ',') {
-                    this.pushCount(1);
-                    this.pushSpaces(true);
+                    yield* this.pushCount(1);
+                    yield* this.pushSpaces(true);
                     return 'flow';
                 }
             }
             // fallthrough
             default:
                 this.flowKey = false;
-                return this.parsePlainScalar();
+                return yield* this.parsePlainScalar();
         }
     }
-    parseQuotedScalar() {
+    *parseQuotedScalar() {
         const quote = this.charAt(0);
         let end = this.buffer.indexOf(quote, this.pos + 1);
         if (quote === "'") {
@@ -459,10 +451,10 @@ class Lexer {
                 return this.setNext('quoted-scalar');
             end = this.buffer.length;
         }
-        this.pushToIndex(end + 1, false);
+        yield* this.pushToIndex(end + 1, false);
         return this.flowLevel ? 'flow' : 'doc';
     }
-    parseBlockScalarHeader() {
+    *parseBlockScalarHeader() {
         this.blockScalarIndent = -1;
         this.blockScalarKeep = false;
         let i = this.pos;
@@ -475,9 +467,9 @@ class Lexer {
             else if (ch !== '-')
                 break;
         }
-        return this.pushUntil(ch => isEmpty(ch) || ch === '#');
+        return yield* this.pushUntil(ch => isEmpty(ch) || ch === '#');
     }
-    parseBlockScalar() {
+    *parseBlockScalar() {
         let nl = this.pos - 1; // may be -1 if this.pos === 0
         let indent = 0;
         let ch;
@@ -534,11 +526,11 @@ class Lexer {
                     break;
             } while (true);
         }
-        this.push(tokens.SCALAR);
-        this.pushToIndex(nl + 1, true);
-        return this.parseLineStart();
+        yield cst.SCALAR;
+        yield* this.pushToIndex(nl + 1, true);
+        return yield* this.parseLineStart();
     }
-    parsePlainScalar() {
+    *parsePlainScalar() {
         const inFlow = this.flowLevel > 0;
         let end = this.pos - 1;
         let i = this.pos - 1;
@@ -577,69 +569,71 @@ class Lexer {
         }
         if (!ch && !this.atEnd)
             return this.setNext('plain-scalar');
-        this.push(tokens.SCALAR);
-        this.pushToIndex(end + 1, true);
+        yield cst.SCALAR;
+        yield* this.pushToIndex(end + 1, true);
         return inFlow ? 'flow' : 'doc';
     }
-    pushCount(n) {
+    *pushCount(n) {
         if (n > 0) {
-            this.push(this.buffer.substr(this.pos, n));
+            yield this.buffer.substr(this.pos, n);
             this.pos += n;
             return n;
         }
         return 0;
     }
-    pushToIndex(i, allowEmpty) {
+    *pushToIndex(i, allowEmpty) {
         const s = this.buffer.slice(this.pos, i);
         if (s) {
-            this.push(s);
+            yield s;
             this.pos += s.length;
             return s.length;
         }
         else if (allowEmpty)
-            this.push('');
+            yield '';
         return 0;
     }
-    pushIndicators() {
+    *pushIndicators() {
         switch (this.charAt(0)) {
             case '!':
                 if (this.charAt(1) === '<')
-                    return (this.pushVerbatimTag() +
-                        this.pushSpaces(true) +
-                        this.pushIndicators());
+                    return ((yield* this.pushVerbatimTag()) +
+                        (yield* this.pushSpaces(true)) +
+                        (yield* this.pushIndicators()));
             // fallthrough
             case '&':
-                return (this.pushUntil(isNotIdentifierChar) +
-                    this.pushSpaces(true) +
-                    this.pushIndicators());
+                return ((yield* this.pushUntil(isNotIdentifierChar)) +
+                    (yield* this.pushSpaces(true)) +
+                    (yield* this.pushIndicators()));
             case ':':
             case '?': // this is an error outside flow collections
             case '-': // this is an error
                 if (isEmpty(this.charAt(1))) {
                     if (this.flowLevel === 0)
                         this.indentNext = this.indentValue + 1;
-                    return (this.pushCount(1) + this.pushSpaces(true) + this.pushIndicators());
+                    return ((yield* this.pushCount(1)) +
+                        (yield* this.pushSpaces(true)) +
+                        (yield* this.pushIndicators()));
                 }
         }
         return 0;
     }
-    pushVerbatimTag() {
+    *pushVerbatimTag() {
         let i = this.pos + 2;
         let ch = this.buffer[i];
         while (!isEmpty(ch) && ch !== '>')
             ch = this.buffer[++i];
-        return this.pushToIndex(ch === '>' ? i + 1 : i, false);
+        return yield* this.pushToIndex(ch === '>' ? i + 1 : i, false);
     }
-    pushNewline() {
+    *pushNewline() {
         const ch = this.buffer[this.pos];
         if (ch === '\n')
-            return this.pushCount(1);
+            return yield* this.pushCount(1);
         else if (ch === '\r' && this.charAt(1) === '\n')
-            return this.pushCount(2);
+            return yield* this.pushCount(2);
         else
             return 0;
     }
-    pushSpaces(allowTabs) {
+    *pushSpaces(allowTabs) {
         let i = this.pos - 1;
         let ch;
         do {
@@ -647,17 +641,17 @@ class Lexer {
         } while (ch === ' ' || (allowTabs && ch === '\t'));
         const n = i - this.pos;
         if (n > 0) {
-            this.push(this.buffer.substr(this.pos, n));
+            yield this.buffer.substr(this.pos, n);
             this.pos = i;
         }
         return n;
     }
-    pushUntil(test) {
+    *pushUntil(test) {
         let i = this.pos;
         let ch = this.buffer[i];
         while (!test(ch))
             ch = this.buffer[++i];
-        return this.pushToIndex(i, false);
+        return yield* this.pushToIndex(i, false);
     }
 }
 

package/dist/parse/parser.d.ts

@@ -1,23 +1,32 @@
-import { Token } from './tokens.js';
+import { Token } from './cst.js';
 /**
  * A YAML concrete syntax tree (CST) parser
  *
- * While the `parse()` method provides an API for parsing a source string
- * directly, the parser may also be used with a user-provided lexer:
- *
  * ```ts
- * const cst: Token[] = []
- * const parser = new Parser(tok => cst.push(tok))
  * const src: string = ...
+ * for (const token of new Parser().parse(src)) {
+ *   // token: Token
+ * }
+ * ```
  *
- * // The following would be equivalent to `parser.parse(src, false)`
- * const lexer = new Lexer(parser.next)
- * lexer.lex(src, false)
- * parser.end()
+ * To use the parser with a user-provided lexer:
+ *
+ * ```ts
+ * function* parse(source: string, lexer: Lexer) {
+ *   const parser = new Parser()
+ *   for (const lexeme of lexer.lex(source))
+ *     yield* parser.next(lexeme)
+ *   yield* parser.end()
+ * }
+ *
+ * const src: string = ...
+ * const lexer = new Lexer()
+ * for (const token of parse(src, lexer)) {
+ *   // token: Token
+ * }
  * ```
  */
 export declare class Parser {
-    private push;
     private onNewLine?;
     /** If true, space and sequence indicators count as indentation */
     private atNewLine;
@@ -36,29 +45,26 @@ export declare class Parser {
     /** The type of the current token, set in parse() */
     private type;
     /**
-     * @param push - Called separately with each parsed token
      * @param onNewLine - If defined, called separately with the start position of
      *   each new line (in `parse()`, including the start of input).
-     * @public
      */
-    constructor(push: (token: Token) => void, onNewLine?: (offset: number) => void);
+    constructor(onNewLine?: (offset: number) => void);
     /**
-     * Parse `source` as a YAML stream, calling `push` with each directive,
-     * document and other structure as it is completely parsed. If `incomplete`,
-     * a part of the last line may be left as a buffer for the next call.
+     * Parse `source` as a YAML stream.
+     * If `incomplete`, a part of the last line may be left as a buffer for the next call.
+     *
+     * Errors are not thrown, but yielded as `{ type: 'error', message }` tokens.
      *
-     * Errors are not thrown, but pushed out as `{ type: 'error', message }` tokens.
-     * @public
+     * @returns A generator of tokens representing each directive, document, and other structure.
      */
-    parse(source: string, incomplete?: boolean): void;
+    parse(source: string, incomplete?: boolean): Generator<Token, void, unknown>;
     /**
-     * Advance the parser by the `source` of one lexical token. Bound to the
-     * Parser instance, so may be used directly as a callback function.
+     * Advance the parser by the `source` of one lexical token.
      */
-    next: (source: string) => void;
+    next(source: string): Generator<Token, void, unknown>;
     private lexer;
     /** Call at end of input to push out any remaining constructions */
-    end(): void;
+    end(): Generator<Token, void, unknown>;
     private get sourceToken();
     private step;
     private peek;