yaml 2.0.0-4 → 2.0.0-8

package/browser/dist/parse/lexer.js

@@ -1,4 +1,4 @@
-import { BOM, DOCUMENT, FLOW_END, SCALAR } from './tokens.js';
+import { BOM, DOCUMENT, FLOW_END, SCALAR } from './cst.js';
 
 /*
 START -> stream
@@ -79,9 +79,11 @@ function isEmpty(ch) {
             return false;
     }
 }
-const invalidFlowScalarChars = [',', '[', ']', '{', '}'];
-const invalidIdentifierChars = [' ', ',', '[', ']', '{', '}', '\n', '\r', '\t'];
-const isNotIdentifierChar = (ch) => !ch || invalidIdentifierChars.includes(ch);
+const hexDigits = '0123456789ABCDEFabcdef'.split('');
+const tagChars = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz-#;/?:@&=+$_.!~*'()".split('');
+const invalidFlowScalarChars = ',[]{}'.split('');
+const invalidAnchorChars = ' ,[]{}\n\r\t'.split('');
+const isNotAnchorChar = (ch) => !ch || invalidAnchorChars.includes(ch);
 /**
  * Splits an input string into lexical tokens, i.e. smaller strings that are
  * easily identifiable by `tokens.tokenType()`.
@@ -98,13 +100,7 @@ const isNotIdentifierChar = (ch) => !ch || invalidIdentifierChars.includes(ch);
  * - `\u{FEFF}` (Byte order mark): Emitted separately outside documents
  */
 class Lexer {
-    /**
-     * Define/initialise a YAML lexer. `push` will be called separately with each
-     * token when `lex()` is passed an input string.
-     *
-     * @public
-     */
-    constructor(push) {
+    constructor() {
         /**
          * Flag indicating whether the end of the current buffer marks the end of
          * all input
@@ -138,26 +134,28 @@ class Lexer {
         this.indentNext = 0;
         /** Indentation level of the current line. */
         this.indentValue = 0;
+        /** Position of the next \n character. */
+        this.lineEndPos = null;
         /** Stores the state of the lexer if reaching the end of incpomplete input */
         this.next = null;
         /** A pointer to `buffer`; the current position of the lexer. */
         this.pos = 0;
-        this.push = push;
     }
     /**
-     * Read YAML tokens from the `source` string, calling the callback
-     * defined in the constructor for each one. If `incomplete`, a part
-     * of the last line may be left as a buffer for the next call.
+     * Generate YAML tokens from the `source` string. If `incomplete`,
+     * a part of the last line may be left as a buffer for the next call.
      *
-     * @public
+     * @returns A generator of lexical tokens
      */
-    lex(source, incomplete) {
-        if (source)
+    *lex(source, incomplete = false) {
+        if (source) {
             this.buffer = this.buffer ? this.buffer + source : source;
+            this.lineEndPos = null;
+        }
         this.atEnd = !incomplete;
         let next = this.next || 'stream';
         while (next && (incomplete || this.hasChars(1)))
-            next = this.parseNext(next);
+            next = yield* this.parseNext(next);
     }
     atLineEnd() {
         let i = this.pos;
@@ -196,7 +194,11 @@ class Lexer {
         return offset;
     }
     getLine() {
-        let end = this.buffer.indexOf('\n', this.pos);
+        let end = this.lineEndPos;
+        if (typeof end !== 'number' || (end !== -1 && end < this.pos)) {
+            end = this.buffer.indexOf('\n', this.pos);
+            this.lineEndPos = end;
+        }
         if (end === -1)
             return this.atEnd ? this.buffer.substring(this.pos) : null;
         if (this.buffer[end - 1] === '\r')
@@ -209,38 +211,39 @@ class Lexer {
     setNext(state) {
         this.buffer = this.buffer.substring(this.pos);
         this.pos = 0;
+        this.lineEndPos = null;
         this.next = state;
         return null;
     }
     peek(n) {
         return this.buffer.substr(this.pos, n);
     }
-    parseNext(next) {
+    *parseNext(next) {
         switch (next) {
             case 'stream':
-                return this.parseStream();
+                return yield* this.parseStream();
             case 'line-start':
-                return this.parseLineStart();
+                return yield* this.parseLineStart();
             case 'block-start':
-                return this.parseBlockStart();
+                return yield* this.parseBlockStart();
             case 'doc':
-                return this.parseDocument();
+                return yield* this.parseDocument();
             case 'flow':
-                return this.parseFlowCollection();
+                return yield* this.parseFlowCollection();
             case 'quoted-scalar':
-                return this.parseQuotedScalar();
+                return yield* this.parseQuotedScalar();
             case 'block-scalar':
-                return this.parseBlockScalar();
+                return yield* this.parseBlockScalar();
             case 'plain-scalar':
-                return this.parsePlainScalar();
+                return yield* this.parsePlainScalar();
         }
     }
-    parseStream() {
+    *parseStream() {
         let line = this.getLine();
         if (line === null)
             return this.setNext('stream');
         if (line[0] === BOM) {
-            this.pushCount(1);
+            yield* this.pushCount(1);
             line = line.substring(1);
         }
         if (line[0] === '%') {
@@ -258,21 +261,21 @@ class Lexer {
                 else
                     break;
             }
-            const n = this.pushCount(dirEnd) + this.pushSpaces(true);
-            this.pushCount(line.length - n); // possible comment
+            const n = (yield* this.pushCount(dirEnd)) + (yield* this.pushSpaces(true));
+            yield* this.pushCount(line.length - n); // possible comment
             this.pushNewline();
             return 'stream';
         }
         if (this.atLineEnd()) {
-            const sp = this.pushSpaces(true);
-            this.pushCount(line.length - sp);
-            this.pushNewline();
+            const sp = yield* this.pushSpaces(true);
+            yield* this.pushCount(line.length - sp);
+            yield* this.pushNewline();
             return 'stream';
         }
-        this.push(DOCUMENT);
-        return this.parseLineStart();
+        yield DOCUMENT;
+        return yield* this.parseLineStart();
     }
-    parseLineStart() {
+    *parseLineStart() {
         const ch = this.charAt(0);
         if (!ch && !this.atEnd)
             return this.setNext('line-start');
@@ -281,80 +284,80 @@ class Lexer {
                 return this.setNext('line-start');
             const s = this.peek(3);
             if (s === '---' && isEmpty(this.charAt(3))) {
-                this.pushCount(3);
+                yield* this.pushCount(3);
                 this.indentValue = 0;
                 this.indentNext = 0;
                 return 'doc';
             }
             else if (s === '...' && isEmpty(this.charAt(3))) {
-                this.pushCount(3);
+                yield* this.pushCount(3);
                 return 'stream';
             }
         }
-        this.indentValue = this.pushSpaces(false);
+        this.indentValue = yield* this.pushSpaces(false);
         if (this.indentNext > this.indentValue && !isEmpty(this.charAt(1)))
             this.indentNext = this.indentValue;
-        return this.parseBlockStart();
+        return yield* this.parseBlockStart();
     }
-    parseBlockStart() {
+    *parseBlockStart() {
         const [ch0, ch1] = this.peek(2);
         if (!ch1 && !this.atEnd)
             return this.setNext('block-start');
         if ((ch0 === '-' || ch0 === '?' || ch0 === ':') && isEmpty(ch1)) {
-            const n = this.pushCount(1) + this.pushSpaces(true);
+            const n = (yield* this.pushCount(1)) + (yield* this.pushSpaces(true));
             this.indentNext = this.indentValue + 1;
             this.indentValue += n;
-            return this.parseBlockStart();
+            return yield* this.parseBlockStart();
         }
         return 'doc';
     }
-    parseDocument() {
-        this.pushSpaces(true);
+    *parseDocument() {
+        yield* this.pushSpaces(true);
         const line = this.getLine();
         if (line === null)
             return this.setNext('doc');
-        let n = this.pushIndicators();
+        let n = yield* this.pushIndicators();
         switch (line[n]) {
             case '#':
-                this.pushCount(line.length - n);
+                yield* this.pushCount(line.length - n);
             // fallthrough
             case undefined:
-                this.pushNewline();
-                return this.parseLineStart();
+                yield* this.pushNewline();
+                return yield* this.parseLineStart();
             case '{':
             case '[':
-                this.pushCount(1);
+                yield* this.pushCount(1);
                 this.flowKey = false;
                 this.flowLevel = 1;
                 return 'flow';
             case '}':
             case ']':
                 // this is an error
-                this.pushCount(1);
+                yield* this.pushCount(1);
                 return 'doc';
             case '*':
-                this.pushUntil(isNotIdentifierChar);
+                yield* this.pushUntil(isNotAnchorChar);
                 return 'doc';
             case '"':
             case "'":
-                return this.parseQuotedScalar();
+                return yield* this.parseQuotedScalar();
             case '|':
             case '>':
-                n += this.parseBlockScalarHeader();
-                n += this.pushSpaces(true);
-                this.pushCount(line.length - n);
-                this.pushNewline();
-                return this.parseBlockScalar();
+                n += yield* this.parseBlockScalarHeader();
+                n += yield* this.pushSpaces(true);
+                yield* this.pushCount(line.length - n);
+                yield* this.pushNewline();
+                return yield* this.parseBlockScalar();
             default:
-                return this.parsePlainScalar();
+                return yield* this.parsePlainScalar();
         }
     }
-    parseFlowCollection() {
+    *parseFlowCollection() {
         let nl, sp;
         let indent = -1;
         do {
-            nl = this.pushNewline();
-            sp = this.pushSpaces(true);
+            nl = yield* this.pushNewline();
+            sp = yield* this.pushSpaces(true);
             if (nl > 0)
                 this.indentValue = indent = sp;
         } while (nl + sp > 0);
@@ -374,54 +377,54 @@ class Lexer {
             if (!atFlowEndMarker) {
                 // this is an error
                 this.flowLevel = 0;
-                this.push(FLOW_END);
-                return this.parseLineStart();
+                yield FLOW_END;
+                return yield* this.parseLineStart();
             }
         }
         let n = 0;
         while (line[n] === ',')
-            n += this.pushCount(1) + this.pushSpaces(true);
-        n += this.pushIndicators();
+            n += (yield* this.pushCount(1)) + (yield* this.pushSpaces(true));
+        n += yield* this.pushIndicators();
         switch (line[n]) {
             case undefined:
                 return 'flow';
             case '#':
-                this.pushCount(line.length - n);
+                yield* this.pushCount(line.length - n);
                 return 'flow';
             case '{':
             case '[':
-                this.pushCount(1);
+                yield* this.pushCount(1);
                 this.flowKey = false;
                 this.flowLevel += 1;
                 return 'flow';
             case '}':
             case ']':
-                this.pushCount(1);
+                yield* this.pushCount(1);
                 this.flowKey = true;
                 this.flowLevel -= 1;
                 return this.flowLevel ? 'flow' : 'doc';
             case '*':
-                this.pushUntil(isNotIdentifierChar);
+                yield* this.pushUntil(isNotAnchorChar);
                 return 'flow';
             case '"':
             case "'":
                 this.flowKey = true;
-                return this.parseQuotedScalar();
+                return yield* this.parseQuotedScalar();
             case ':': {
                 const next = this.charAt(1);
                 if (this.flowKey || isEmpty(next) || next === ',') {
-                    this.pushCount(1);
-                    this.pushSpaces(true);
+                    yield* this.pushCount(1);
+                    yield* this.pushSpaces(true);
                     return 'flow';
                 }
             }
             // fallthrough
             default:
                 this.flowKey = false;
-                return this.parsePlainScalar();
+                return yield* this.parsePlainScalar();
         }
     }
-    parseQuotedScalar() {
+    *parseQuotedScalar() {
         const quote = this.charAt(0);
         let end = this.buffer.indexOf(quote, this.pos + 1);
         if (quote === "'") {
@@ -439,17 +442,19 @@ class Lexer {
                 end = this.buffer.indexOf('"', end + 1);
             }
         }
-        let nl = this.buffer.indexOf('\n', this.pos);
-        if (nl !== -1 && nl < end) {
-            while (nl !== -1 && nl < end) {
+        // Only looking for newlines within the quotes
+        const qb = this.buffer.substring(0, end);
+        let nl = qb.indexOf('\n', this.pos);
+        if (nl !== -1) {
+            while (nl !== -1) {
                 const cs = this.continueScalar(nl + 1);
                 if (cs === -1)
                     break;
-                nl = this.buffer.indexOf('\n', cs);
+                nl = qb.indexOf('\n', cs);
             }
-            if (nl !== -1 && nl < end) {
+            if (nl !== -1) {
                 // this is an error caused by an unexpected unindent
-                end = nl - 1;
+                end = nl - (qb[nl - 1] === '\r' ? 2 : 1);
             }
         }
         if (end === -1) {
@@ -457,10 +462,10 @@ class Lexer {
                 return this.setNext('quoted-scalar');
             end = this.buffer.length;
         }
-        this.pushToIndex(end + 1, false);
+        yield* this.pushToIndex(end + 1, false);
         return this.flowLevel ? 'flow' : 'doc';
     }
-    parseBlockScalarHeader() {
+    *parseBlockScalarHeader() {
         this.blockScalarIndent = -1;
         this.blockScalarKeep = false;
         let i = this.pos;
@@ -473,9 +478,9 @@ class Lexer {
             else if (ch !== '-')
                 break;
         }
-        return this.pushUntil(ch => isEmpty(ch) || ch === '#');
+        return yield* this.pushUntil(ch => isEmpty(ch) || ch === '#');
     }
-    parseBlockScalar() {
+    *parseBlockScalar() {
         let nl = this.pos - 1; // may be -1 if this.pos === 0
         let indent = 0;
         let ch;
@@ -532,11 +537,11 @@ class Lexer {
                     break;
             } while (true);
         }
-        this.push(SCALAR);
-        this.pushToIndex(nl + 1, true);
-        return this.parseLineStart();
+        yield SCALAR;
+        yield* this.pushToIndex(nl + 1, true);
+        return yield* this.parseLineStart();
     }
-    parsePlainScalar() {
+    *parsePlainScalar() {
         const inFlow = this.flowLevel > 0;
         let end = this.pos - 1;
         let i = this.pos - 1;
@@ -549,17 +554,18 @@ class Lexer {
                 end = i;
             }
             else if (isEmpty(ch)) {
-                const next = this.buffer[i + 1];
-                if (next === '#' || (inFlow && invalidFlowScalarChars.includes(next)))
-                    break;
+                let next = this.buffer[i + 1];
                 if (ch === '\r') {
                     if (next === '\n') {
                         i += 1;
                         ch = '\n';
+                        next = this.buffer[i + 1];
                     }
                     else
                         end = i;
                 }
+                if (next === '#' || (inFlow && invalidFlowScalarChars.includes(next)))
+                    break;
                 if (ch === '\n') {
                     const cs = this.continueScalar(i + 1);
                     if (cs === -1)
@@ -575,69 +581,87 @@ class Lexer {
         }
         if (!ch && !this.atEnd)
             return this.setNext('plain-scalar');
-        this.push(SCALAR);
-        this.pushToIndex(end + 1, true);
+        yield SCALAR;
+        yield* this.pushToIndex(end + 1, true);
         return inFlow ? 'flow' : 'doc';
     }
-    pushCount(n) {
+    *pushCount(n) {
         if (n > 0) {
-            this.push(this.buffer.substr(this.pos, n));
+            yield this.buffer.substr(this.pos, n);
             this.pos += n;
             return n;
         }
         return 0;
     }
-    pushToIndex(i, allowEmpty) {
+    *pushToIndex(i, allowEmpty) {
         const s = this.buffer.slice(this.pos, i);
         if (s) {
-            this.push(s);
+            yield s;
             this.pos += s.length;
             return s.length;
         }
         else if (allowEmpty)
-            this.push('');
+            yield '';
         return 0;
     }
-    pushIndicators() {
+    *pushIndicators() {
         switch (this.charAt(0)) {
             case '!':
-                if (this.charAt(1) === '<')
-                    return (this.pushVerbatimTag() +
-                        this.pushSpaces(true) +
-                        this.pushIndicators());
-            // fallthrough
+                return ((yield* this.pushTag()) +
+                    (yield* this.pushSpaces(true)) +
+                    (yield* this.pushIndicators()));
             case '&':
-                return (this.pushUntil(isNotIdentifierChar) +
-                    this.pushSpaces(true) +
-                    this.pushIndicators());
+                return ((yield* this.pushUntil(isNotAnchorChar)) +
+                    (yield* this.pushSpaces(true)) +
+                    (yield* this.pushIndicators()));
             case ':':
             case '?': // this is an error outside flow collections
             case '-': // this is an error
                 if (isEmpty(this.charAt(1))) {
                     if (this.flowLevel === 0)
                         this.indentNext = this.indentValue + 1;
-                    return (this.pushCount(1) + this.pushSpaces(true) + this.pushIndicators());
+                    return ((yield* this.pushCount(1)) +
+                        (yield* this.pushSpaces(true)) +
+                        (yield* this.pushIndicators()));
                 }
         }
         return 0;
     }
-    pushVerbatimTag() {
-        let i = this.pos + 2;
-        let ch = this.buffer[i];
-        while (!isEmpty(ch) && ch !== '>')
-            ch = this.buffer[++i];
-        return this.pushToIndex(ch === '>' ? i + 1 : i, false);
+    *pushTag() {
+        if (this.charAt(1) === '<') {
+            let i = this.pos + 2;
+            let ch = this.buffer[i];
+            while (!isEmpty(ch) && ch !== '>')
+                ch = this.buffer[++i];
+            return yield* this.pushToIndex(ch === '>' ? i + 1 : i, false);
+        }
+        else {
+            let i = this.pos + 1;
+            let ch = this.buffer[i];
+            while (ch) {
+                if (tagChars.includes(ch))
+                    ch = this.buffer[++i];
+                else if (ch === '%' &&
+                    hexDigits.includes(this.buffer[i + 1]) &&
+                    hexDigits.includes(this.buffer[i + 2])) {
+                    ch = this.buffer[(i += 3)];
+                }
+                else
+                    break;
+            }
+            return yield* this.pushToIndex(i, false);
+        }
     }
-    pushNewline() {
+    *pushNewline() {
         const ch = this.buffer[this.pos];
         if (ch === '\n')
-            return this.pushCount(1);
+            return yield* this.pushCount(1);
         else if (ch === '\r' && this.charAt(1) === '\n')
-            return this.pushCount(2);
+            return yield* this.pushCount(2);
         else
             return 0;
     }
-    pushSpaces(allowTabs) {
+    *pushSpaces(allowTabs) {
         let i = this.pos - 1;
         let ch;
         do {
@@ -645,17 +669,17 @@ class Lexer {
         } while (ch === ' ' || (allowTabs && ch === '\t'));
         const n = i - this.pos;
         if (n > 0) {
-            this.push(this.buffer.substr(this.pos, n));
+            yield this.buffer.substr(this.pos, n);
             this.pos = i;
         }
         return n;
     }
-    pushUntil(test) {
+    *pushUntil(test) {
         let i = this.pos;
         let ch = this.buffer[i];
         while (!test(ch))
             ch = this.buffer[++i];
-        return this.pushToIndex(i, false);
+        return yield* this.pushToIndex(i, false);
     }
 }