yaml 2.0.0-4 → 2.0.0-8

package/dist/parse/parser.js

@@ -1,7 +1,7 @@
 'use strict';
 
+var cst = require('./cst.js');
 var lexer = require('./lexer.js');
-var tokens = require('./tokens.js');
 
 function includesToken(list, type) {
     for (let i = 0; i < list.length; ++i)
@@ -22,38 +22,6 @@ function includesNonEmpty(list) {
     }
     return false;
 }
-function atFirstEmptyLineAfterComments(start) {
-    let hasComment = false;
-    for (let i = 0; i < start.length; ++i) {
-        switch (start[i].type) {
-            case 'space':
-                break;
-            case 'comment':
-                hasComment = true;
-                break;
-            case 'newline':
-                if (!hasComment)
-                    return false;
-                break;
-            default:
-                return false;
-        }
-    }
-    if (hasComment) {
-        for (let i = start.length - 1; i >= 0; --i) {
-            switch (start[i].type) {
-                /* istanbul ignore next */
-                case 'space':
-                    break;
-                case 'newline':
-                    return true;
-                default:
-                    return false;
-            }
-        }
-    }
-    return false;
-}
 function isFlowToken(token) {
     switch (token === null || token === void 0 ? void 0 : token.type) {
         case 'alias':
@@ -89,6 +57,7 @@ function getFirstKeyStartProps(prev) {
     let i = prev.length;
     loop: while (--i >= 0) {
         switch (prev[i].type) {
+            case 'doc-start':
             case 'explicit-key-ind':
             case 'map-value-ind':
             case 'seq-item-ind':
@@ -101,31 +70,62 @@ function getFirstKeyStartProps(prev) {
     }
     return prev.splice(i, prev.length);
 }
+function fixFlowSeqItems(fc) {
+    if (fc.start.type === 'flow-seq-start') {
+        for (const it of fc.items) {
+            if (it.sep &&
+                !it.value &&
+                !includesToken(it.start, 'explicit-key-ind') &&
+                !includesToken(it.sep, 'map-value-ind')) {
+                if (it.key)
+                    it.value = it.key;
+                delete it.key;
+                if (isFlowToken(it.value)) {
+                    if (it.value.end)
+                        Array.prototype.push.apply(it.value.end, it.sep);
+                    else
+                        it.value.end = it.sep;
+                }
+                else
+                    Array.prototype.push.apply(it.start, it.sep);
+                delete it.sep;
+            }
+        }
+    }
+}
 /**
  * A YAML concrete syntax tree (CST) parser
  *
- * While the `parse()` method provides an API for parsing a source string
- * directly, the parser may also be used with a user-provided lexer:
- *
  * ```ts
- * const cst: Token[] = []
- * const parser = new Parser(tok => cst.push(tok))
  * const src: string = ...
+ * for (const token of new Parser().parse(src)) {
+ *   // token: Token
+ * }
+ * ```
+ *
+ * To use the parser with a user-provided lexer:
+ *
+ * ```ts
+ * function* parse(source: string, lexer: Lexer) {
+ *   const parser = new Parser()
+ *   for (const lexeme of lexer.lex(source))
+ *     yield* parser.next(lexeme)
+ *   yield* parser.end()
+ * }
  *
- * // The following would be equivalent to `parser.parse(src, false)`
- * const lexer = new Lexer(parser.next)
- * lexer.lex(src, false)
- * parser.end()
+ * const src: string = ...
+ * const lexer = new Lexer()
+ * for (const token of parse(src, lexer)) {
+ *   // token: Token
+ * }
  * ```
  */
 class Parser {
     /**
-     * @param push - Called separately with each parsed token
      * @param onNewLine - If defined, called separately with the start position of
      *   each new line (in `parse()`, including the start of input).
-     * @public
      */
-    constructor(push, onNewLine) {
+    constructor(onNewLine) {
         /** If true, space and sequence indicators count as indentation */
         this.atNewLine = true;
         /** If true, next token is a scalar value */
@@ -142,96 +142,97 @@ class Parser {
         this.source = '';
         /** The type of the current token, set in parse() */
         this.type = '';
-        /**
-         * Advance the parser by the `source` of one lexical token. Bound to the
-         * Parser instance, so may be used directly as a callback function.
-         */
-        this.next = (source) => {
-            this.source = source;
-            if (process.env.LOG_TOKENS)
-                console.log('|', tokens.prettyToken(source));
-            if (this.atScalar) {
-                this.atScalar = false;
-                this.step();
-                this.offset += source.length;
-                return;
-            }
-            const type = tokens.tokenType(source);
-            if (!type) {
-                const message = `Not a YAML token: ${source}`;
-                this.pop({ type: 'error', offset: this.offset, message, source });
-                this.offset += source.length;
-            }
-            else if (type === 'scalar') {
-                this.atNewLine = false;
-                this.atScalar = true;
-                this.type = 'scalar';
-            }
-            else {
-                this.type = type;
-                this.step();
-                switch (type) {
-                    case 'newline':
-                        this.atNewLine = true;
-                        this.indent = 0;
-                        if (this.onNewLine)
-                            this.onNewLine(this.offset + source.length);
-                        break;
-                    case 'space':
-                        if (this.atNewLine && source[0] === ' ')
-                            this.indent += source.length;
-                        break;
-                    case 'explicit-key-ind':
-                    case 'map-value-ind':
-                    case 'seq-item-ind':
-                        if (this.atNewLine)
-                            this.indent += source.length;
-                        break;
-                    case 'doc-mode':
-                        return;
-                    default:
-                        this.atNewLine = false;
-                }
-                this.offset += source.length;
-            }
-        };
         // Must be defined after `next()`
-        this.lexer = new lexer.Lexer(this.next);
-        this.push = push;
+        this.lexer = new lexer.Lexer();
         this.onNewLine = onNewLine;
     }
     /**
-     * Parse `source` as a YAML stream, calling `push` with each directive,
-     * document and other structure as it is completely parsed. If `incomplete`,
-     * a part of the last line may be left as a buffer for the next call.
+     * Parse `source` as a YAML stream.
+     * If `incomplete`, a part of the last line may be left as a buffer for the next call.
      *
-     * Errors are not thrown, but pushed out as `{ type: 'error', message }` tokens.
-     * @public
+     * Errors are not thrown, but yielded as `{ type: 'error', message }` tokens.
+     *
+     * @returns A generator of tokens representing each directive, document, and other structure.
      */
-    parse(source, incomplete = false) {
+    *parse(source, incomplete = false) {
         if (this.onNewLine && this.offset === 0)
             this.onNewLine(0);
-        this.lexer.lex(source, incomplete);
+        for (const lexeme of this.lexer.lex(source, incomplete))
+            yield* this.next(lexeme);
         if (!incomplete)
-            this.end();
+            yield* this.end();
+    }
+    /**
+     * Advance the parser by the `source` of one lexical token.
+     */
+    *next(source) {
+        this.source = source;
+        if (process.env.LOG_TOKENS)
+            console.log('|', cst.prettyToken(source));
+        if (this.atScalar) {
+            this.atScalar = false;
+            yield* this.step();
+            this.offset += source.length;
+            return;
+        }
+        const type = cst.tokenType(source);
+        if (!type) {
+            const message = `Not a YAML token: ${source}`;
+            yield* this.pop({ type: 'error', offset: this.offset, message, source });
+            this.offset += source.length;
+        }
+        else if (type === 'scalar') {
+            this.atNewLine = false;
+            this.atScalar = true;
+            this.type = 'scalar';
+        }
+        else {
+            this.type = type;
+            yield* this.step();
+            switch (type) {
+                case 'newline':
+                    this.atNewLine = true;
+                    this.indent = 0;
+                    if (this.onNewLine)
+                        this.onNewLine(this.offset + source.length);
+                    break;
+                case 'space':
+                    if (this.atNewLine && source[0] === ' ')
+                        this.indent += source.length;
+                    break;
+                case 'explicit-key-ind':
+                case 'map-value-ind':
+                case 'seq-item-ind':
+                    if (this.atNewLine)
+                        this.indent += source.length;
+                    break;
+                case 'doc-mode':
+                    return;
+                default:
+                    this.atNewLine = false;
+            }
+            this.offset += source.length;
+        }
     }
     /** Call at end of input to push out any remaining constructions */
-    end() {
+    *end() {
         while (this.stack.length > 0)
-            this.pop();
+            yield* this.pop();
     }
     get sourceToken() {
-        return {
+        const st = {
             type: this.type,
+            offset: this.offset,
             indent: this.indent,
             source: this.source
         };
+        return st;
     }
-    step() {
+    *step() {
         const top = this.peek(1);
         if (this.type === 'doc-end' && (!top || top.type !== 'doc-end')) {
             while (this.stack.length > 0)
-                this.pop();
+                yield* this.pop();
             this.stack.push({
                 type: 'doc-end',
                 offset: this.offset,
@@ -240,47 +241,49 @@ class Parser {
             return;
         }
         if (!top)
-            return this.stream();
+            return yield* this.stream();
         switch (top.type) {
             case 'document':
-                return this.document(top);
+                return yield* this.document(top);
             case 'alias':
             case 'scalar':
             case 'single-quoted-scalar':
             case 'double-quoted-scalar':
-                return this.scalar(top);
+                return yield* this.scalar(top);
             case 'block-scalar':
-                return this.blockScalar(top);
+                return yield* this.blockScalar(top);
             case 'block-map':
-                return this.blockMap(top);
+                return yield* this.blockMap(top);
             case 'block-seq':
-                return this.blockSequence(top);
+                return yield* this.blockSequence(top);
             case 'flow-collection':
-                return this.flowCollection(top);
+                return yield* this.flowCollection(top);
             case 'doc-end':
-                return this.documentEnd(top);
+                return yield* this.documentEnd(top);
         }
         /* istanbul ignore next should not happen */
-        this.pop();
+        yield* this.pop();
     }
     peek(n) {
         return this.stack[this.stack.length - n];
     }
-    pop(error) {
+    *pop(error) {
         const token = error || this.stack.pop();
         /* istanbul ignore if should not happen */
         if (!token) {
             const message = 'Tried to pop an empty stack';
-            this.push({ type: 'error', source: '', message });
+            yield { type: 'error', offset: this.offset, source: '', message };
         }
         else if (this.stack.length === 0) {
-            this.push(token);
+            yield token;
         }
         else {
             const top = this.peek(1);
             // For these, parent indent is needed instead of own
             if (token.type === 'block-scalar' || token.type === 'flow-collection')
                 token.indent = 'indent' in top ? top.indent : -1;
+            if (token.type === 'flow-collection')
+                fixFlowSeqItems(token);
             switch (top.type) {
                 case 'document':
                     top.value = token;
@@ -313,13 +316,20 @@ class Parser {
                         it.value = token;
                     break;
                 }
-                case 'flow-collection':
-                    top.items.push(token);
-                    break;
+                case 'flow-collection': {
+                    const it = top.items[top.items.length - 1];
+                    if (!it || it.value)
+                        top.items.push({ start: [], key: token, sep: [] });
+                    else if (it.sep)
+                        it.value = token;
+                    else
+                        Object.assign(it, { key: token, sep: [] });
+                    return;
+                }
                 /* istanbul ignore next should not happen */
                 default:
-                    this.pop();
-                    this.pop(token);
+                    yield* this.pop();
+                    yield* this.pop(token);
             }
             if ((top.type === 'document' ||
                 top.type === 'block-map' ||
@@ -342,16 +352,16 @@ class Parser {
             }
         }
     }
-    stream() {
+    *stream() {
         switch (this.type) {
             case 'directive-line':
-                this.push({ type: 'directive', source: this.source });
+                yield { type: 'directive', offset: this.offset, source: this.source };
                 return;
             case 'byte-order-mark':
             case 'space':
             case 'comment':
             case 'newline':
-                this.push(this.sourceToken);
+                yield this.sourceToken;
                 return;
             case 'doc-mode':
             case 'doc-start': {
@@ -366,21 +376,21 @@ class Parser {
                 return;
             }
         }
-        this.push({
+        yield {
             type: 'error',
             offset: this.offset,
             message: `Unexpected ${this.type} token in YAML stream`,
             source: this.source
-        });
+        };
     }
-    document(doc) {
+    *document(doc) {
         if (doc.value)
-            return this.lineEnd(doc);
+            return yield* this.lineEnd(doc);
         switch (this.type) {
             case 'doc-start': {
                 if (includesNonEmpty(doc.start)) {
-                    this.pop();
-                    this.step();
+                    yield* this.pop();
+                    yield* this.step();
                 }
                 else
                     doc.start.push(this.sourceToken);
@@ -398,15 +408,15 @@ class Parser {
         if (bv)
             this.stack.push(bv);
         else {
-            this.push({
+            yield {
                 type: 'error',
                 offset: this.offset,
                 message: `Unexpected ${this.type} token in YAML document`,
                 source: this.source
-            });
+            };
         }
     }
-    scalar(scalar) {
+    *scalar(scalar) {
         if (this.type === 'map-value-ind') {
             const prev = getPrevProps(this.peek(2));
             const start = getFirstKeyStartProps(prev);
@@ -428,9 +438,9 @@ class Parser {
             this.stack[this.stack.length - 1] = map;
         }
         else
-            this.lineEnd(scalar);
+            yield* this.lineEnd(scalar);
     }
-    blockScalar(scalar) {
+    *blockScalar(scalar) {
         switch (this.type) {
             case 'space':
             case 'comment':
@@ -449,39 +459,53 @@ class Parser {
                         nl = this.source.indexOf('\n', nl) + 1;
                     }
                 }
-                this.pop();
+                yield* this.pop();
                 break;
             /* istanbul ignore next should not happen */
             default:
-                this.pop();
-                this.step();
+                yield* this.pop();
+                yield* this.step();
         }
     }
-    blockMap(map) {
+    *blockMap(map) {
         var _a;
         const it = map.items[map.items.length - 1];
         // it.sep is true-ish if pair already has key or : separator
         switch (this.type) {
             case 'newline':
                 this.onKeyLine = false;
-                if (!it.sep && atFirstEmptyLineAfterComments(it.start)) {
-                    const prev = map.items[map.items.length - 2];
-                    const end = (_a = prev === null || prev === void 0 ? void 0 : prev.value) === null || _a === void 0 ? void 0 : _a.end;
-                    if (Array.isArray(end)) {
-                        Array.prototype.push.apply(end, it.start);
-                        it.start = [this.sourceToken];
-                        return;
-                    }
+                if (it.value) {
+                    const end = 'end' in it.value ? it.value.end : undefined;
+                    const last = Array.isArray(end) ? end[end.length - 1] : undefined;
+                    if ((last === null || last === void 0 ? void 0 : last.type) === 'comment')
+                        end === null || end === void 0 ? void 0 : end.push(this.sourceToken);
+                    else
+                        map.items.push({ start: [this.sourceToken] });
                 }
-            // fallthrough
+                else if (it.sep)
+                    it.sep.push(this.sourceToken);
+                else
+                    it.start.push(this.sourceToken);
+                return;
             case 'space':
             case 'comment':
                 if (it.value)
                     map.items.push({ start: [this.sourceToken] });
                 else if (it.sep)
                     it.sep.push(this.sourceToken);
-                else
+                else {
+                    if (this.atIndentedComment(it.start, map.indent)) {
+                        const prev = map.items[map.items.length - 2];
+                        const end = (_a = prev === null || prev === void 0 ? void 0 : prev.value) === null || _a === void 0 ? void 0 : _a.end;
+                        if (Array.isArray(end)) {
+                            Array.prototype.push.apply(end, it.start);
+                            end.push(this.sourceToken);
+                            map.items.pop();
+                            return;
+                        }
+                    }
                     it.start.push(this.sourceToken);
+                }
                 return;
         }
         if (this.indent >= map.indent) {
@@ -578,30 +602,42 @@ class Parser {
                 }
             }
         }
-        this.pop();
-        this.step();
+        yield* this.pop();
+        yield* this.step();
     }
-    blockSequence(seq) {
+    *blockSequence(seq) {
         var _a;
         const it = seq.items[seq.items.length - 1];
         switch (this.type) {
             case 'newline':
-                if (!it.value && atFirstEmptyLineAfterComments(it.start)) {
-                    const prev = seq.items[seq.items.length - 2];
-                    const end = (_a = prev === null || prev === void 0 ? void 0 : prev.value) === null || _a === void 0 ? void 0 : _a.end;
-                    if (Array.isArray(end)) {
-                        Array.prototype.push.apply(end, it.start);
-                        it.start = [this.sourceToken];
-                        return;
-                    }
+                if (it.value) {
+                    const end = 'end' in it.value ? it.value.end : undefined;
+                    const last = Array.isArray(end) ? end[end.length - 1] : undefined;
+                    if ((last === null || last === void 0 ? void 0 : last.type) === 'comment')
+                        end === null || end === void 0 ? void 0 : end.push(this.sourceToken);
+                    else
+                        seq.items.push({ start: [this.sourceToken] });
                 }
-            // fallthrough
+                else
+                    it.start.push(this.sourceToken);
+                return;
             case 'space':
             case 'comment':
                 if (it.value)
                     seq.items.push({ start: [this.sourceToken] });
-                else
+                else {
+                    if (this.atIndentedComment(it.start, seq.indent)) {
+                        const prev = seq.items[seq.items.length - 2];
+                        const end = (_a = prev === null || prev === void 0 ? void 0 : prev.value) === null || _a === void 0 ? void 0 : _a.end;
+                        if (Array.isArray(end)) {
+                            Array.prototype.push.apply(end, it.start);
+                            end.push(this.sourceToken);
+                            seq.items.pop();
+                            return;
+                        }
+                    }
                     it.start.push(this.sourceToken);
+                }
                 return;
             case 'anchor':
             case 'tag':
@@ -620,38 +656,65 @@ class Parser {
         }
         if (this.indent > seq.indent) {
             const bv = this.startBlockValue(seq);
-            if (bv)
-                return this.stack.push(bv);
+            if (bv) {
+                this.stack.push(bv);
+                return;
+            }
         }
-        this.pop();
-        this.step();
+        yield* this.pop();
+        yield* this.step();
     }
-    flowCollection(fc) {
+    *flowCollection(fc) {
+        const it = fc.items[fc.items.length - 1];
         if (this.type === 'flow-error-end') {
             let top;
             do {
-                this.pop();
+                yield* this.pop();
                 top = this.peek(1);
             } while (top && top.type === 'flow-collection');
         }
         else if (fc.end.length === 0) {
             switch (this.type) {
-                case 'space':
-                case 'comment':
-                case 'newline':
                 case 'comma':
                 case 'explicit-key-ind':
+                    if (!it || it.sep)
+                        fc.items.push({ start: [this.sourceToken] });
+                    else
+                        it.start.push(this.sourceToken);
+                    return;
                 case 'map-value-ind':
+                    if (!it || it.value)
+                        fc.items.push({ start: [], key: null, sep: [this.sourceToken] });
+                    else if (it.sep)
+                        it.sep.push(this.sourceToken);
+                    else
+                        Object.assign(it, { key: null, sep: [this.sourceToken] });
+                    return;
+                case 'space':
+                case 'comment':
+                case 'newline':
                 case 'anchor':
                 case 'tag':
-                    fc.items.push(this.sourceToken);
+                    if (!it || it.value)
+                        fc.items.push({ start: [this.sourceToken] });
+                    else if (it.sep)
+                        it.sep.push(this.sourceToken);
+                    else
+                        it.start.push(this.sourceToken);
                     return;
                 case 'alias':
                 case 'scalar':
                 case 'single-quoted-scalar':
-                case 'double-quoted-scalar':
-                    fc.items.push(this.flowScalar(this.type));
+                case 'double-quoted-scalar': {
+                    const fs = this.flowScalar(this.type);
+                    if (!it || it.value)
+                        fc.items.push({ start: [], key: fs, sep: [] });
+                    else if (it.sep)
+                        this.stack.push(fs);
+                    else
+                        Object.assign(it, { key: fs, sep: [] });
                     return;
+                }
                 case 'flow-map-end':
                 case 'flow-seq-end':
                     fc.end.push(this.sourceToken);
@@ -660,10 +723,10 @@ class Parser {
             const bv = this.startBlockValue(fc);
             /* istanbul ignore else should not happen */
             if (bv)
-                return this.stack.push(bv);
+                this.stack.push(bv);
             else {
-                this.pop();
-                this.step();
+                yield* this.pop();
+                yield* this.step();
             }
         }
         else {
@@ -672,13 +735,14 @@ class Parser {
                 (this.type === 'map-value-ind' ||
                     (this.type === 'newline' &&
                         !parent.items[parent.items.length - 1].sep))) {
-                this.pop();
-                this.step();
+                yield* this.pop();
+                yield* this.step();
             }
             else if (this.type === 'map-value-ind' &&
                 parent.type !== 'flow-collection') {
                 const prev = getPrevProps(parent);
                 const start = getFirstKeyStartProps(prev);
+                fixFlowSeqItems(fc);
                 const sep = fc.end.splice(1, fc.end.length);
                 sep.push(this.sourceToken);
                 const map = {
@@ -691,7 +755,7 @@ class Parser {
                 this.stack[this.stack.length - 1] = map;
             }
             else {
-                this.lineEnd(fc);
+                yield* this.lineEnd(fc);
             }
         }
     }
@@ -722,7 +786,8 @@ class Parser {
                     type: 'block-scalar',
                     offset: this.offset,
                     indent: this.indent,
-                    props: [this.sourceToken]
+                    props: [this.sourceToken],
+                    source: ''
                 };
             case 'flow-map-start':
             case 'flow-seq-start':
@@ -741,14 +806,18 @@ class Parser {
                     indent: this.indent,
                     items: [{ start: [this.sourceToken] }]
                 };
-            case 'explicit-key-ind':
+            case 'explicit-key-ind': {
                 this.onKeyLine = true;
+                const prev = getPrevProps(parent);
+                const start = getFirstKeyStartProps(prev);
+                start.push(this.sourceToken);
                 return {
                     type: 'block-map',
                     offset: this.offset,
                     indent: this.indent,
-                    items: [{ start: [this.sourceToken] }]
+                    items: [{ start }]
                 };
+            }
             case 'map-value-ind': {
                 this.onKeyLine = true;
                 const prev = getPrevProps(parent);
@@ -763,17 +832,24 @@ class Parser {
         }
         return null;
     }
-    documentEnd(docEnd) {
+    atIndentedComment(start, indent) {
+        if (this.type !== 'comment')
+            return false;
+        if (this.indent <= indent)
+            return false;
+        return start.every(st => st.type === 'newline' || st.type === 'space');
+    }
+    *documentEnd(docEnd) {
         if (this.type !== 'doc-mode') {
             if (docEnd.end)
                 docEnd.end.push(this.sourceToken);
             else
                 docEnd.end = [this.sourceToken];
             if (this.type === 'newline')
-                this.pop();
+                yield* this.pop();
         }
     }
-    lineEnd(token) {
+    *lineEnd(token) {
         switch (this.type) {
             case 'comma':
             case 'doc-start':
@@ -781,8 +857,8 @@ class Parser {
             case 'flow-seq-end':
             case 'flow-map-end':
             case 'map-value-ind':
-                this.pop();
-                this.step();
+                yield* this.pop();
+                yield* this.step();
                 break;
             case 'newline':
                 this.onKeyLine = false;
@@ -796,7 +872,7 @@ class Parser {
                 else
                     token.end = [this.sourceToken];
                 if (this.type === 'newline')
-                    this.pop();
+                    yield* this.pop();
         }
     }
 }