yaml 2.0.0-4 → 2.0.0-5

package/dist/parse/parser.js

@@ -1,7 +1,7 @@
 'use strict';
 
+var cst = require('./cst.js');
 var lexer = require('./lexer.js');
-var tokens = require('./tokens.js');
 
 function includesToken(list, type) {
     for (let i = 0; i < list.length; ++i)
@@ -89,6 +89,7 @@ function getFirstKeyStartProps(prev) {
     let i = prev.length;
     loop: while (--i >= 0) {
         switch (prev[i].type) {
+            case 'doc-start':
             case 'explicit-key-ind':
             case 'map-value-ind':
             case 'seq-item-ind':
@@ -101,31 +102,62 @@ function getFirstKeyStartProps(prev) {
     }
     return prev.splice(i, prev.length);
 }
+function fixFlowSeqItems(fc) {
+    if (fc.start.type === 'flow-seq-start') {
+        for (const it of fc.items) {
+            if (it.sep &&
+                !it.value &&
+                !includesToken(it.start, 'explicit-key-ind') &&
+                !includesToken(it.sep, 'map-value-ind')) {
+                if (it.key)
+                    it.value = it.key;
+                delete it.key;
+                if (isFlowToken(it.value)) {
+                    if (it.value.end)
+                        Array.prototype.push.apply(it.value.end, it.sep);
+                    else
+                        it.value.end = it.sep;
+                }
+                else
+                    Array.prototype.push.apply(it.start, it.sep);
+                delete it.sep;
+            }
+        }
+    }
+}
 /**
  * A YAML concrete syntax tree (CST) parser
  *
- * While the `parse()` method provides an API for parsing a source string
- * directly, the parser may also be used with a user-provided lexer:
- *
  * ```ts
- * const cst: Token[] = []
- * const parser = new Parser(tok => cst.push(tok))
  * const src: string = ...
+ * for (const token of new Parser().parse(src)) {
+ *   // token: Token
+ * }
+ * ```
+ *
+ * To use the parser with a user-provided lexer:
  *
- * // The following would be equivalent to `parser.parse(src, false)`
- * const lexer = new Lexer(parser.next)
- * lexer.lex(src, false)
- * parser.end()
+ * ```ts
+ * function* parse(source: string, lexer: Lexer) {
+ *   const parser = new Parser()
+ *   for (const lexeme of lexer.lex(source))
+ *     yield* parser.next(lexeme)
+ *   yield* parser.end()
+ * }
+ *
+ * const src: string = ...
+ * const lexer = new Lexer()
+ * for (const token of parse(src, lexer)) {
+ *   // token: Token
+ * }
  * ```
  */
 class Parser {
     /**
-     * @param push - Called separately with each parsed token
      * @param onNewLine - If defined, called separately with the start position of
      *   each new line (in `parse()`, including the start of input).
-     * @public
      */
-    constructor(push, onNewLine) {
+    constructor(onNewLine) {
         /** If true, space and sequence indicators count as indentation */
         this.atNewLine = true;
         /** If true, next token is a scalar value */
@@ -142,96 +174,97 @@ class Parser {
         this.source = '';
         /** The type of the current token, set in parse() */
         this.type = '';
-        /**
-         * Advance the parser by the `source` of one lexical token. Bound to the
-         * Parser instance, so may be used directly as a callback function.
-         */
-        this.next = (source) => {
-            this.source = source;
-            if (process.env.LOG_TOKENS)
-                console.log('|', tokens.prettyToken(source));
-            if (this.atScalar) {
-                this.atScalar = false;
-                this.step();
-                this.offset += source.length;
-                return;
-            }
-            const type = tokens.tokenType(source);
-            if (!type) {
-                const message = `Not a YAML token: ${source}`;
-                this.pop({ type: 'error', offset: this.offset, message, source });
-                this.offset += source.length;
-            }
-            else if (type === 'scalar') {
-                this.atNewLine = false;
-                this.atScalar = true;
-                this.type = 'scalar';
-            }
-            else {
-                this.type = type;
-                this.step();
-                switch (type) {
-                    case 'newline':
-                        this.atNewLine = true;
-                        this.indent = 0;
-                        if (this.onNewLine)
-                            this.onNewLine(this.offset + source.length);
-                        break;
-                    case 'space':
-                        if (this.atNewLine && source[0] === ' ')
-                            this.indent += source.length;
-                        break;
-                    case 'explicit-key-ind':
-                    case 'map-value-ind':
-                    case 'seq-item-ind':
-                        if (this.atNewLine)
-                            this.indent += source.length;
-                        break;
-                    case 'doc-mode':
-                        return;
-                    default:
-                        this.atNewLine = false;
-                }
-                this.offset += source.length;
-            }
-        };
         // Must be defined after `next()`
-        this.lexer = new lexer.Lexer(this.next);
-        this.push = push;
+        this.lexer = new lexer.Lexer();
         this.onNewLine = onNewLine;
     }
     /**
-     * Parse `source` as a YAML stream, calling `push` with each directive,
-     * document and other structure as it is completely parsed. If `incomplete`,
-     * a part of the last line may be left as a buffer for the next call.
+     * Parse `source` as a YAML stream.
+     * If `incomplete`, a part of the last line may be left as a buffer for the next call.
      *
-     * Errors are not thrown, but pushed out as `{ type: 'error', message }` tokens.
-     * @public
+     * Errors are not thrown, but yielded as `{ type: 'error', message }` tokens.
+     *
+     * @returns A generator of tokens representing each directive, document, and other structure.
      */
-    parse(source, incomplete = false) {
+    *parse(source, incomplete = false) {
         if (this.onNewLine && this.offset === 0)
             this.onNewLine(0);
-        this.lexer.lex(source, incomplete);
+        for (const lexeme of this.lexer.lex(source, incomplete))
+            yield* this.next(lexeme);
         if (!incomplete)
-            this.end();
+            yield* this.end();
+    }
+    /**
+     * Advance the parser by the `source` of one lexical token.
+     */
+    *next(source) {
+        this.source = source;
+        if (process.env.LOG_TOKENS)
+            console.log('|', cst.prettyToken(source));
+        if (this.atScalar) {
+            this.atScalar = false;
+            yield* this.step();
+            this.offset += source.length;
+            return;
+        }
+        const type = cst.tokenType(source);
+        if (!type) {
+            const message = `Not a YAML token: ${source}`;
+            yield* this.pop({ type: 'error', offset: this.offset, message, source });
+            this.offset += source.length;
+        }
+        else if (type === 'scalar') {
+            this.atNewLine = false;
+            this.atScalar = true;
+            this.type = 'scalar';
+        }
+        else {
+            this.type = type;
+            yield* this.step();
+            switch (type) {
+                case 'newline':
+                    this.atNewLine = true;
+                    this.indent = 0;
+                    if (this.onNewLine)
+                        this.onNewLine(this.offset + source.length);
+                    break;
+                case 'space':
+                    if (this.atNewLine && source[0] === ' ')
+                        this.indent += source.length;
+                    break;
+                case 'explicit-key-ind':
+                case 'map-value-ind':
+                case 'seq-item-ind':
+                    if (this.atNewLine)
+                        this.indent += source.length;
+                    break;
+                case 'doc-mode':
+                    return;
+                default:
+                    this.atNewLine = false;
+            }
+            this.offset += source.length;
+        }
     }
     /** Call at end of input to push out any remaining constructions */
-    end() {
+    *end() {
         while (this.stack.length > 0)
-            this.pop();
+            yield* this.pop();
     }
     get sourceToken() {
-        return {
+        const st = {
             type: this.type,
+            offset: this.offset,
             indent: this.indent,
             source: this.source
         };
+        return st;
     }
-    step() {
+    *step() {
         const top = this.peek(1);
         if (this.type === 'doc-end' && (!top || top.type !== 'doc-end')) {
             while (this.stack.length > 0)
-                this.pop();
+                yield* this.pop();
             this.stack.push({
                 type: 'doc-end',
                 offset: this.offset,
@@ -240,47 +273,49 @@ class Parser {
             return;
         }
         if (!top)
-            return this.stream();
+            return yield* this.stream();
         switch (top.type) {
             case 'document':
-                return this.document(top);
+                return yield* this.document(top);
             case 'alias':
             case 'scalar':
             case 'single-quoted-scalar':
             case 'double-quoted-scalar':
-                return this.scalar(top);
+                return yield* this.scalar(top);
             case 'block-scalar':
-                return this.blockScalar(top);
+                return yield* this.blockScalar(top);
             case 'block-map':
-                return this.blockMap(top);
+                return yield* this.blockMap(top);
             case 'block-seq':
-                return this.blockSequence(top);
+                return yield* this.blockSequence(top);
             case 'flow-collection':
-                return this.flowCollection(top);
+                return yield* this.flowCollection(top);
             case 'doc-end':
-                return this.documentEnd(top);
+                return yield* this.documentEnd(top);
         }
         /* istanbul ignore next should not happen */
-        this.pop();
+        yield* this.pop();
     }
     peek(n) {
         return this.stack[this.stack.length - n];
     }
-    pop(error) {
+    *pop(error) {
         const token = error || this.stack.pop();
         /* istanbul ignore if should not happen */
         if (!token) {
             const message = 'Tried to pop an empty stack';
-            this.push({ type: 'error', source: '', message });
+            yield { type: 'error', offset: this.offset, source: '', message };
         }
         else if (this.stack.length === 0) {
-            this.push(token);
+            yield token;
         }
         else {
             const top = this.peek(1);
             // For these, parent indent is needed instead of own
             if (token.type === 'block-scalar' || token.type === 'flow-collection')
                 token.indent = 'indent' in top ? top.indent : -1;
+            if (token.type === 'flow-collection')
+                fixFlowSeqItems(token);
             switch (top.type) {
                 case 'document':
                     top.value = token;
@@ -313,13 +348,20 @@ class Parser {
                         it.value = token;
                     break;
                 }
-                case 'flow-collection':
-                    top.items.push(token);
-                    break;
+                case 'flow-collection': {
+                    const it = top.items[top.items.length - 1];
+                    if (!it || it.value)
+                        top.items.push({ start: [], key: token, sep: [] });
+                    else if (it.sep)
+                        it.value = token;
+                    else
+                        Object.assign(it, { key: token, sep: [] });
+                    return;
+                }
                 /* istanbul ignore next should not happen */
                 default:
-                    this.pop();
-                    this.pop(token);
+                    yield* this.pop();
+                    yield* this.pop(token);
             }
             if ((top.type === 'document' ||
                 top.type === 'block-map' ||
@@ -342,16 +384,16 @@ class Parser {
             }
         }
     }
-    stream() {
+    *stream() {
         switch (this.type) {
             case 'directive-line':
-                this.push({ type: 'directive', source: this.source });
+                yield { type: 'directive', offset: this.offset, source: this.source };
                 return;
             case 'byte-order-mark':
             case 'space':
             case 'comment':
             case 'newline':
-                this.push(this.sourceToken);
+                yield this.sourceToken;
                 return;
             case 'doc-mode':
             case 'doc-start': {
@@ -366,21 +408,21 @@ class Parser {
                 return;
             }
         }
-        this.push({
+        yield {
             type: 'error',
             offset: this.offset,
             message: `Unexpected ${this.type} token in YAML stream`,
             source: this.source
-        });
+        };
     }
-    document(doc) {
+    *document(doc) {
         if (doc.value)
-            return this.lineEnd(doc);
+            return yield* this.lineEnd(doc);
         switch (this.type) {
             case 'doc-start': {
                 if (includesNonEmpty(doc.start)) {
-                    this.pop();
-                    this.step();
+                    yield* this.pop();
+                    yield* this.step();
                 }
                 else
                     doc.start.push(this.sourceToken);
@@ -398,15 +440,15 @@ class Parser {
         if (bv)
             this.stack.push(bv);
         else {
-            this.push({
+            yield {
                 type: 'error',
                 offset: this.offset,
                 message: `Unexpected ${this.type} token in YAML document`,
                 source: this.source
-            });
+            };
         }
     }
-    scalar(scalar) {
+    *scalar(scalar) {
         if (this.type === 'map-value-ind') {
             const prev = getPrevProps(this.peek(2));
             const start = getFirstKeyStartProps(prev);
@@ -428,9 +470,9 @@ class Parser {
             this.stack[this.stack.length - 1] = map;
         }
         else
-            this.lineEnd(scalar);
+            yield* this.lineEnd(scalar);
     }
-    blockScalar(scalar) {
+    *blockScalar(scalar) {
         switch (this.type) {
             case 'space':
             case 'comment':
@@ -449,15 +491,15 @@ class Parser {
                         nl = this.source.indexOf('\n', nl) + 1;
                     }
                 }
-                this.pop();
+                yield* this.pop();
                 break;
             /* istanbul ignore next should not happen */
             default:
-                this.pop();
-                this.step();
+                yield* this.pop();
+                yield* this.step();
         }
     }
-    blockMap(map) {
+    *blockMap(map) {
         var _a;
         const it = map.items[map.items.length - 1];
         // it.sep is true-ish if pair already has key or : separator
@@ -578,10 +620,10 @@ class Parser {
                 }
             }
         }
-        this.pop();
-        this.step();
+        yield* this.pop();
+        yield* this.step();
     }
-    blockSequence(seq) {
+    *blockSequence(seq) {
         var _a;
         const it = seq.items[seq.items.length - 1];
         switch (this.type) {
@@ -620,38 +662,65 @@ class Parser {
         }
         if (this.indent > seq.indent) {
             const bv = this.startBlockValue(seq);
-            if (bv)
-                return this.stack.push(bv);
+            if (bv) {
+                this.stack.push(bv);
+                return;
+            }
         }
-        this.pop();
-        this.step();
+        yield* this.pop();
+        yield* this.step();
     }
-    flowCollection(fc) {
+    *flowCollection(fc) {
+        const it = fc.items[fc.items.length - 1];
         if (this.type === 'flow-error-end') {
             let top;
             do {
-                this.pop();
+                yield* this.pop();
                 top = this.peek(1);
             } while (top && top.type === 'flow-collection');
         }
         else if (fc.end.length === 0) {
             switch (this.type) {
-                case 'space':
-                case 'comment':
-                case 'newline':
                 case 'comma':
                 case 'explicit-key-ind':
+                    if (!it || it.sep)
+                        fc.items.push({ start: [this.sourceToken] });
+                    else
+                        it.start.push(this.sourceToken);
+                    return;
                 case 'map-value-ind':
+                    if (!it || it.value)
+                        fc.items.push({ start: [], key: null, sep: [this.sourceToken] });
+                    else if (it.sep)
+                        it.sep.push(this.sourceToken);
+                    else
+                        Object.assign(it, { key: null, sep: [this.sourceToken] });
+                    return;
+                case 'space':
+                case 'comment':
+                case 'newline':
                 case 'anchor':
                 case 'tag':
-                    fc.items.push(this.sourceToken);
+                    if (!it || it.value)
+                        fc.items.push({ start: [this.sourceToken] });
+                    else if (it.sep)
+                        it.sep.push(this.sourceToken);
+                    else
+                        it.start.push(this.sourceToken);
                     return;
                 case 'alias':
                 case 'scalar':
                 case 'single-quoted-scalar':
-                case 'double-quoted-scalar':
-                    fc.items.push(this.flowScalar(this.type));
+                case 'double-quoted-scalar': {
+                    const fs = this.flowScalar(this.type);
+                    if (!it || it.value)
+                        fc.items.push({ start: [], key: fs, sep: [] });
+                    else if (it.sep)
+                        this.stack.push(fs);
+                    else
+                        Object.assign(it, { key: fs, sep: [] });
                     return;
+                }
                 case 'flow-map-end':
                 case 'flow-seq-end':
                     fc.end.push(this.sourceToken);
@@ -660,10 +729,10 @@ class Parser {
             const bv = this.startBlockValue(fc);
             /* istanbul ignore else should not happen */
             if (bv)
-                return this.stack.push(bv);
+                this.stack.push(bv);
             else {
-                this.pop();
-                this.step();
+                yield* this.pop();
+                yield* this.step();
             }
         }
         else {
@@ -672,13 +741,14 @@ class Parser {
                 (this.type === 'map-value-ind' ||
                     (this.type === 'newline' &&
                         !parent.items[parent.items.length - 1].sep))) {
-                this.pop();
-                this.step();
+                yield* this.pop();
+                yield* this.step();
             }
             else if (this.type === 'map-value-ind' &&
                 parent.type !== 'flow-collection') {
                 const prev = getPrevProps(parent);
                 const start = getFirstKeyStartProps(prev);
+                fixFlowSeqItems(fc);
                 const sep = fc.end.splice(1, fc.end.length);
                 sep.push(this.sourceToken);
                 const map = {
@@ -691,7 +761,7 @@ class Parser {
                 this.stack[this.stack.length - 1] = map;
             }
             else {
-                this.lineEnd(fc);
+                yield* this.lineEnd(fc);
             }
         }
     }
@@ -722,7 +792,8 @@ class Parser {
                     type: 'block-scalar',
                     offset: this.offset,
                     indent: this.indent,
-                    props: [this.sourceToken]
+                    props: [this.sourceToken],
+                    source: ''
                 };
             case 'flow-map-start':
             case 'flow-seq-start':
@@ -741,14 +812,18 @@ class Parser {
                     indent: this.indent,
                     items: [{ start: [this.sourceToken] }]
                 };
-            case 'explicit-key-ind':
+            case 'explicit-key-ind': {
                 this.onKeyLine = true;
+                const prev = getPrevProps(parent);
+                const start = getFirstKeyStartProps(prev);
+                start.push(this.sourceToken);
                 return {
                     type: 'block-map',
                     offset: this.offset,
                     indent: this.indent,
-                    items: [{ start: [this.sourceToken] }]
+                    items: [{ start }]
                 };
+            }
             case 'map-value-ind': {
                 this.onKeyLine = true;
                 const prev = getPrevProps(parent);
@@ -763,17 +838,17 @@ class Parser {
         }
         return null;
     }
-    documentEnd(docEnd) {
+    *documentEnd(docEnd) {
         if (this.type !== 'doc-mode') {
             if (docEnd.end)
                 docEnd.end.push(this.sourceToken);
             else
                 docEnd.end = [this.sourceToken];
             if (this.type === 'newline')
-                this.pop();
+                yield* this.pop();
         }
     }
-    lineEnd(token) {
+    *lineEnd(token) {
         switch (this.type) {
             case 'comma':
             case 'doc-start':
@@ -781,8 +856,8 @@ class Parser {
             case 'flow-seq-end':
             case 'flow-map-end':
             case 'map-value-ind':
-                this.pop();
-                this.step();
+                yield* this.pop();
+                yield* this.step();
                 break;
             case 'newline':
                 this.onKeyLine = false;
@@ -796,7 +871,7 @@ class Parser {
                 else
                     token.end = [this.sourceToken];
                 if (this.type === 'newline')
-                    this.pop();
+                    yield* this.pop();
         }
     }
 }

package/dist/public-api.d.ts

@@ -2,7 +2,7 @@ import { Composer } from './compose/composer.js';
 import type { Reviver } from './doc/applyReviver.js';
 import { Document, Replacer } from './doc/Document.js';
 import type { ParsedNode } from './nodes/Node.js';
-import type { Options, ToJSOptions, ToStringOptions } from './options.js';
+import type { CreateNodeOptions, DocumentOptions, ParseOptions, SchemaOptions, ToJSOptions, ToStringOptions } from './options.js';
 export interface EmptyStream extends Array<Document.Parsed>, ReturnType<Composer['streamInfo']> {
     empty: true;
 }
@@ -15,9 +15,9 @@ export interface EmptyStream extends Array<Document.Parsed>, ReturnType<Composer
  *   EmptyStream and contain additional stream information. In
  *   TypeScript, you should use `'empty' in docs` as a type guard for it.
  */
-export declare function parseAllDocuments<T extends ParsedNode = ParsedNode>(source: string, options?: Options): Document.Parsed<T>[] | EmptyStream;
+export declare function parseAllDocuments<T extends ParsedNode = ParsedNode>(source: string, options?: ParseOptions & DocumentOptions & SchemaOptions): Document.Parsed<T>[] | EmptyStream;
 /** Parse an input string into a single YAML.Document */
-export declare function parseDocument<T extends ParsedNode = ParsedNode>(source: string, options?: Options): Document.Parsed<T>;
+export declare function parseDocument<T extends ParsedNode = ParsedNode>(source: string, options?: ParseOptions & DocumentOptions & SchemaOptions): Document.Parsed<T>;
 /**
  * Parse an input string into JavaScript.
  *
@@ -31,13 +31,13 @@ export declare function parseDocument<T extends ParsedNode = ParsedNode>(source:
  *   document, so Maps become objects, Sequences arrays, and scalars result in
  *   nulls, booleans, numbers and strings.
  */
-export declare function parse(src: string, options?: Options & ToJSOptions): any;
-export declare function parse(src: string, reviver: Reviver, options?: Options & ToJSOptions): any;
+export declare function parse(src: string, options?: ParseOptions & DocumentOptions & SchemaOptions & ToJSOptions): any;
+export declare function parse(src: string, reviver: Reviver, options?: ParseOptions & DocumentOptions & SchemaOptions & ToJSOptions): any;
 /**
  * Stringify a value as a YAML document.
  *
  * @param replacer - A replacer array or function, as in `JSON.stringify()`
  * @returns Will always include `\n` as the last character, as is expected of YAML documents.
  */
-export declare function stringify(value: any, options?: Options & ToStringOptions): string;
-export declare function stringify(value: any, replacer?: Replacer | null, options?: string | number | (Options & ToStringOptions)): string;
+export declare function stringify(value: any, options?: DocumentOptions & SchemaOptions & ParseOptions & CreateNodeOptions & ToStringOptions): string;
+export declare function stringify(value: any, replacer?: Replacer | null, options?: string | number | (DocumentOptions & SchemaOptions & ParseOptions & CreateNodeOptions & ToStringOptions)): string;