web-csv-toolbox 0.0.0-next-20240601094527 → 0.0.0-next-20240609055757

package/dist/es/Lexer.js

@@ -9,6 +9,13 @@ class Lexer {
   #buffer = "";
   #flush = false;
   #matcher;
+  #fieldDelimiterLength;
+  #cursor = {
+    line: 1,
+    column: 1,
+    offset: 0
+  };
+  #rowNumber = 1;
   /**
    * Constructs a new Lexer instance.
    * @param options - The common options for the lexer.
@@ -20,6 +27,7 @@ class Lexer {
     assertCommonOptions({ delimiter, quotation });
     this.#delimiter = delimiter;
     this.#quotation = quotation;
+    this.#fieldDelimiterLength = delimiter.length;
     const d = escapeRegExp(delimiter);
     const q = escapeRegExp(quotation);
     this.#matcher = new RegExp(
@@ -56,39 +64,22 @@ class Lexer {
   *#tokens() {
     if (this.#flush) {
       if (this.#buffer.endsWith(CRLF)) {
-        this.#buffer = this.#buffer.slice(0, -CRLF.length);
+        this.#buffer = this.#buffer.slice(
+          0,
+          -2
+          /* -CRLF.length */
+        );
       } else if (this.#buffer.endsWith(LF)) {
-        this.#buffer = this.#buffer.slice(0, -LF.length);
+        this.#buffer = this.#buffer.slice(
+          0,
+          -1
+          /* -LF.length */
+        );
       }
     }
-    let currentField = null;
-    for (let token; token = this.#nextToken(); ) {
-      switch (token) {
-        case FieldDelimiter:
-          if (currentField) {
-            yield currentField;
-            currentField = null;
-          }
-          yield token;
-          break;
-        case RecordDelimiter:
-          if (currentField) {
-            yield currentField;
-            currentField = null;
-          }
-          yield token;
-          break;
-        default:
-          if (currentField) {
-            currentField.value += token.value;
-          } else {
-            currentField = token;
-          }
-          break;
-      }
-    }
-    if (currentField) {
-      yield currentField;
+    let token;
+    while (token = this.#nextToken()) {
+      yield token;
     }
   }
   /**
@@ -104,62 +95,121 @@ class Lexer {
     }
     if (this.#buffer.startsWith(CRLF)) {
       this.#buffer = this.#buffer.slice(2);
-      return RecordDelimiter;
+      const start = { ...this.#cursor };
+      this.#cursor.line++;
+      this.#cursor.column = 1;
+      this.#cursor.offset += 2;
+      const token = {
+        type: RecordDelimiter,
+        value: CRLF,
+        location: {
+          start,
+          end: { ...this.#cursor },
+          rowNumber: this.#rowNumber++
+        }
+      };
+      return token;
     }
     if (this.#buffer.startsWith(LF)) {
       this.#buffer = this.#buffer.slice(1);
-      return RecordDelimiter;
+      const start = { ...this.#cursor };
+      this.#cursor.line++;
+      this.#cursor.column = 1;
+      this.#cursor.offset += 1;
+      const token = {
+        type: RecordDelimiter,
+        value: LF,
+        location: {
+          start,
+          end: { ...this.#cursor },
+          rowNumber: this.#rowNumber++
+        }
+      };
+      return token;
     }
     if (this.#buffer.startsWith(this.#delimiter)) {
       this.#buffer = this.#buffer.slice(1);
-      return FieldDelimiter;
+      const start = { ...this.#cursor };
+      this.#cursor.column += this.#fieldDelimiterLength;
+      this.#cursor.offset += this.#fieldDelimiterLength;
+      return {
+        type: FieldDelimiter,
+        value: this.#delimiter,
+        location: {
+          start,
+          end: { ...this.#cursor },
+          rowNumber: this.#rowNumber
+        }
+      };
     }
     if (this.#buffer.startsWith(this.#quotation)) {
-      if (this.#flush === false && this.#buffer.endsWith(this.#quotation)) {
-        return null;
-      }
-      return this.#extractQuotedString();
+      let value = "";
+      let offset = 1;
+      let column = 2;
+      let line = 0;
+      let cur = this.#buffer[offset];
+      let next = this.#buffer[offset + 1];
+      do {
+        if (cur === this.#quotation) {
+          if (next === this.#quotation) {
+            value += this.#quotation;
+            offset += 2;
+            cur = this.#buffer[offset];
+            next = this.#buffer[offset + 1];
+            column += 2;
+            continue;
+          }
+          if (next === void 0 && this.#flush === false) {
+            return null;
+          }
+          offset++;
+          this.#buffer = this.#buffer.slice(offset);
+          const start = { ...this.#cursor };
+          this.#cursor.column += column;
+          this.#cursor.offset += offset;
+          this.#cursor.line += line;
+          return {
+            type: Field,
+            value,
+            location: {
+              start,
+              end: { ...this.#cursor },
+              rowNumber: this.#rowNumber
+            }
+          };
+        }
+        value += cur;
+        if (cur === LF) {
+          line++;
+          column = 1;
+        } else {
+          column++;
+        }
+        offset++;
+        cur = next;
+        next = this.#buffer[offset + 1];
+      } while (cur !== void 0);
+      return null;
     }
     const match = this.#matcher.exec(this.#buffer);
     if (match) {
       if (this.#flush === false && match[0].length === this.#buffer.length) {
         return null;
       }
-      this.#buffer = this.#buffer.slice(match[0].length);
-      return { type: Field, value: match[0] };
-    }
-    return null;
-  }
-  /**
-   * Extracts a quoted string token from the buffered CSV data.
-   * @returns The quoted string token or null if the string is not properly quoted.
-   */
-  #extractQuotedString() {
-    let end = 1;
-    let value = "";
-    while (end < this.#buffer.length) {
-      if (this.#buffer.slice(end, end + 1) === this.#quotation && this.#buffer.slice(end + 1, end + 1 * 2) === this.#quotation) {
-        value += this.#quotation;
-        end += 1 * 2;
-        continue;
-      }
-      if (this.#buffer.slice(end, end + 1) === this.#quotation) {
-        if (this.#flush === false && end + 1 < this.#buffer.length && this.#buffer.slice(end + 1, 1) !== this.#delimiter && this.#buffer.slice(
-          end + 1,
-          end + 1 + 2
-          /** CRLF.length */
-        ) !== CRLF && this.#buffer.slice(
-          end + 1,
-          end + 1 + 1
-          /** LF.length */
-        ) !== LF) {
-          return null;
+      const value = match[1];
+      this.#buffer = this.#buffer.slice(value.length);
+      const start = { ...this.#cursor };
+      this.#cursor.column += value.length;
+      this.#cursor.offset += value.length;
+      return {
+        type: Field,
+        value,
+        location: {
+          start,
+          end: { ...this.#cursor },
+          rowNumber: this.#rowNumber
         }
-        this.#buffer = this.#buffer.slice(end + 1);
-        return { type: Field, value };
-      }
-      value += this.#buffer[end];
-      end++;
+      };
     }
     return null;
   }

package/dist/es/Lexer.js.map

@@ -1 +1 @@
-{"version":3,"file":"Lexer.js","sources":["../../src/Lexer.ts"],"sourcesContent":["import { assertCommonOptions } from \"./assertCommonOptions.ts\";\nimport { Field, FieldDelimiter, RecordDelimiter } from \"./common/constants.ts\";\nimport type { CommonOptions, Token } from \"./common/types.ts\";\nimport { COMMA, CRLF, DOUBLE_QUOTE, LF } from \"./constants.ts\";\nimport { escapeRegExp } from \"./utils/escapeRegExp.ts\";\n\n/**\n * CSV Lexer.\n *\n * Lexter tokenizes CSV data into fields and records.\n */\nexport class Lexer {\n  #delimiter: string;\n  #quotation: string;\n  #buffer = \"\";\n  #flush = false;\n  #matcher: RegExp;\n\n  /**\n   * Constructs a new Lexer instance.\n   * @param options - The common options for the lexer.\n   */\n  constructor({\n    delimiter = COMMA,\n    quotation = DOUBLE_QUOTE,\n  }: CommonOptions = {}) {\n    assertCommonOptions({ delimiter, quotation });\n    this.#delimiter = delimiter;\n    this.#quotation = quotation;\n    const d = escapeRegExp(delimiter);\n    const q = escapeRegExp(quotation);\n    this.#matcher = new RegExp(\n      `^(?:(?!${q})(?!${d})(?![\\\\r\\\\n]))([\\\\S\\\\s\\\\uFEFF\\\\xA0]+?)(?=${q}|${d}|\\\\r|\\\\n|$)`,\n    );\n  }\n\n  /**\n   * Lexes the given chunk of CSV data.\n   * @param chunk - The chunk of CSV data to be lexed.\n   * @param buffering - Indicates whether the lexer is buffering or not.\n   * @returns An iterable iterator of tokens.\n   */\n  public lex(chunk: string | null, buffering = false): IterableIterator<Token> {\n    if (!buffering) {\n      this.#flush = true;\n    }\n    if (typeof chunk === \"string\" && chunk.length !== 0) {\n      this.#buffer += chunk;\n    }\n\n    return this.#tokens();\n  }\n\n  /**\n   * Flushes the lexer and returns any remaining tokens.\n   * @returns An array of tokens.\n   */\n  public flush(): Token[] {\n    this.#flush = true;\n    return [...this.#tokens()];\n  }\n\n  /**\n   * Generates tokens from the buffered CSV data.\n   * @yields Tokens from the buffered CSV data.\n   */\n  *#tokens(): Generator<Token> {\n    if (this.#flush) {\n      // Trim the last CRLF or LF\n      if (this.#buffer.endsWith(CRLF)) {\n        this.#buffer = this.#buffer.slice(0, -CRLF.length);\n      } else if (this.#buffer.endsWith(LF)) {\n        this.#buffer = this.#buffer.slice(0, -LF.length);\n      }\n    }\n    let currentField: Token | null = null;\n    for (let token: Token | null; (token = this.#nextToken()); ) {\n      switch (token) {\n        case FieldDelimiter:\n          if (currentField) {\n            yield currentField;\n            currentField = null;\n          }\n          yield token;\n          break;\n        case RecordDelimiter:\n          if (currentField) {\n            yield currentField;\n            currentField = null;\n          }\n          yield token;\n          break;\n        default:\n          // If currentField is not null, append the new token's value to it\n          if (currentField) {\n            currentField.value += token.value;\n          } else {\n            currentField = token;\n          }\n          break;\n      }\n    }\n    if (currentField) {\n      yield currentField;\n    }\n  }\n\n  /**\n   * Retrieves the next token from the buffered CSV data.\n   * @returns The next token or null if there are no more tokens.\n   */\n  #nextToken(): Token | null {\n    if (this.#buffer.length === 0) {\n      return null;\n    }\n    // Buffer is Record Delimiter, defer to the next iteration.\n    if (\n      this.#flush === false &&\n      (this.#buffer === CRLF || this.#buffer === LF)\n    ) {\n      return null;\n    }\n\n    // Check for CRLF\n    if (this.#buffer.startsWith(CRLF)) {\n      this.#buffer = this.#buffer.slice(2);\n      return RecordDelimiter;\n    }\n\n    // Check for LF\n    if (this.#buffer.startsWith(LF)) {\n      this.#buffer = this.#buffer.slice(1);\n      return RecordDelimiter;\n    }\n\n    // Check for Delimiter\n    if (this.#buffer.startsWith(this.#delimiter)) {\n      this.#buffer = this.#buffer.slice(1);\n      return FieldDelimiter;\n    }\n\n    // Check for Quoted String\n    if (this.#buffer.startsWith(this.#quotation)) {\n      // If not flushing and the buffer doesn't end with a quote, then return null.\n      if (this.#flush === false && this.#buffer.endsWith(this.#quotation)) {\n        return null;\n      }\n      return this.#extractQuotedString();\n    }\n\n    // Check for Unquoted String\n    const match = this.#matcher.exec(this.#buffer);\n    if (match) {\n      // If we're flushing and the match doesn't consume the entire buffer,\n      // then return null\n      if (this.#flush === false && match[0].length === this.#buffer.length) {\n        return null;\n      }\n      this.#buffer = this.#buffer.slice(match[0].length);\n      return { type: Field, value: match[0] };\n    }\n\n    // Otherwise, return null\n    return null;\n  }\n\n  /**\n   * Extracts a quoted string token from the buffered CSV data.\n   * @returns The quoted string token or null if the string is not properly quoted.\n   */\n  #extractQuotedString(): Token | null {\n    let end = 1; // Skip the opening quote\n    let value = \"\";\n\n    while (end < this.#buffer.length) {\n      // Escaped quote\n      if (\n        this.#buffer.slice(end, end + 1) === this.#quotation &&\n        this.#buffer.slice(end + 1, end + 1 * 2) === this.#quotation\n      ) {\n        value += this.#quotation;\n        end += 1 * 2;\n        continue;\n      }\n\n      // Closing quote\n      if (this.#buffer.slice(end, end + 1) === this.#quotation) {\n        // If flushing and the buffer doesn't end with a quote, then return null\n        if (\n          this.#flush === false &&\n          end + 1 < this.#buffer.length &&\n          this.#buffer.slice(end + 1, 1) !== this.#delimiter &&\n          this.#buffer.slice(end + 1, end + 1 + 2 /** CRLF.length */) !==\n            CRLF &&\n          this.#buffer.slice(end + 1, end + 1 + 1 /** LF.length */) !== LF\n        ) {\n          return null;\n        }\n\n        // Otherwise, return the quoted string\n        this.#buffer = this.#buffer.slice(end + 1);\n        return { type: Field, value };\n      }\n\n      value += this.#buffer[end];\n      end++;\n    }\n\n    // If we get here, we've reached the end of the buffer\n    return null;\n  }\n}\n"],"names":[],"mappings":";;;;;AAWO,MAAM,KAAM,CAAA;AAAA,EACjB,UAAA,CAAA;AAAA,EACA,UAAA,CAAA;AAAA,EACA,OAAU,GAAA,EAAA,CAAA;AAAA,EACV,MAAS,GAAA,KAAA,CAAA;AAAA,EACT,QAAA,CAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,WAAY,CAAA;AAAA,IACV,SAAY,GAAA,KAAA;AAAA,IACZ,SAAY,GAAA,YAAA;AAAA,GACd,GAAmB,EAAI,EAAA;AACrB,IAAoB,mBAAA,CAAA,EAAE,SAAW,EAAA,SAAA,EAAW,CAAA,CAAA;AAC5C,IAAA,IAAA,CAAK,UAAa,GAAA,SAAA,CAAA;AAClB,IAAA,IAAA,CAAK,UAAa,GAAA,SAAA,CAAA;AAClB,IAAM,MAAA,CAAA,GAAI,aAAa,SAAS,CAAA,CAAA;AAChC,IAAM,MAAA,CAAA,GAAI,aAAa,SAAS,CAAA,CAAA;AAChC,IAAA,IAAA,CAAK,WAAW,IAAI,MAAA;AAAA,MAClB,UAAU,CAAC,CAAA,IAAA,EAAO,CAAC,CAA4C,yCAAA,EAAA,CAAC,IAAI,CAAC,CAAA,WAAA,CAAA;AAAA,KACvE,CAAA;AAAA,GACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAQO,GAAA,CAAI,KAAsB,EAAA,SAAA,GAAY,KAAgC,EAAA;AAC3E,IAAA,IAAI,CAAC,SAAW,EAAA;AACd,MAAA,IAAA,CAAK,MAAS,GAAA,IAAA,CAAA;AAAA,KAChB;AACA,IAAA,IAAI,OAAO,KAAA,KAAU,QAAY,IAAA,KAAA,CAAM,WAAW,CAAG,EAAA;AACnD,MAAA,IAAA,CAAK,OAAW,IAAA,KAAA,CAAA;AAAA,KAClB;AAEA,IAAA,OAAO,KAAK,OAAQ,EAAA,CAAA;AAAA,GACtB;AAAA;AAAA;AAAA;AAAA;AAAA,EAMO,KAAiB,GAAA;AACtB,IAAA,IAAA,CAAK,MAAS,GAAA,IAAA,CAAA;AACd,IAAA,OAAO,CAAC,GAAG,IAAK,CAAA,OAAA,EAAS,CAAA,CAAA;AAAA,GAC3B;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,CAAC,OAA4B,GAAA;AAC3B,IAAA,IAAI,KAAK,MAAQ,EAAA;AAEf,MAAA,IAAI,IAAK,CAAA,OAAA,CAAQ,QAAS,CAAA,IAAI,CAAG,EAAA;AAC/B,QAAA,IAAA,CAAK,UAAU,IAAK,CAAA,OAAA,CAAQ,MAAM,CAAG,EAAA,CAAC,KAAK,MAAM,CAAA,CAAA;AAAA,OACxC,MAAA,IAAA,IAAA,CAAK,OAAQ,CAAA,QAAA,CAAS,EAAE,CAAG,EAAA;AACpC,QAAA,IAAA,CAAK,UAAU,IAAK,CAAA,OAAA,CAAQ,MAAM,CAAG,EAAA,CAAC,GAAG,MAAM,CAAA,CAAA;AAAA,OACjD;AAAA,KACF;AACA,IAAA,IAAI,YAA6B,GAAA,IAAA,CAAA;AACjC,IAAA,KAAA,IAAS,KAAsB,EAAA,KAAA,GAAQ,IAAK,CAAA,UAAA,EAAiB,IAAA;AAC3D,MAAA,QAAQ,KAAO;AAAA,QACb,KAAK,cAAA;AACH,UAAA,IAAI,YAAc,EAAA;AAChB,YAAM,MAAA,YAAA,CAAA;AACN,YAAe,YAAA,GAAA,IAAA,CAAA;AAAA,WACjB;AACA,UAAM,MAAA,KAAA,CAAA;AACN,UAAA,MAAA;AAAA,QACF,KAAK,eAAA;AACH,UAAA,IAAI,YAAc,EAAA;AAChB,YAAM,MAAA,YAAA,CAAA;AACN,YAAe,YAAA,GAAA,IAAA,CAAA;AAAA,WACjB;AACA,UAAM,MAAA,KAAA,CAAA;AACN,UAAA,MAAA;AAAA,QACF;AAEE,UAAA,IAAI,YAAc,EAAA;AAChB,YAAA,YAAA,CAAa,SAAS,KAAM,CAAA,KAAA,CAAA;AAAA,WACvB,MAAA;AACL,YAAe,YAAA,GAAA,KAAA,CAAA;AAAA,WACjB;AACA,UAAA,MAAA;AAAA,OACJ;AAAA,KACF;AACA,IAAA,IAAI,YAAc,EAAA;AAChB,MAAM,MAAA,YAAA,CAAA;AAAA,KACR;AAAA,GACF;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,UAA2B,GAAA;AACzB,IAAI,IAAA,IAAA,CAAK,OAAQ,CAAA,MAAA,KAAW,CAAG,EAAA;AAC7B,MAAO,OAAA,IAAA,CAAA;AAAA,KACT;AAEA,IACE,IAAA,IAAA,CAAK,WAAW,KACf,KAAA,IAAA,CAAK,YAAY,IAAQ,IAAA,IAAA,CAAK,YAAY,EAC3C,CAAA,EAAA;AACA,MAAO,OAAA,IAAA,CAAA;AAAA,KACT;AAGA,IAAA,IAAI,IAAK,CAAA,OAAA,CAAQ,UAAW,CAAA,IAAI,CAAG,EAAA;AACjC,MAAA,IAAA,CAAK,OAAU,GAAA,IAAA,CAAK,OAAQ,CAAA,KAAA,CAAM,CAAC,CAAA,CAAA;AACnC,MAAO,OAAA,eAAA,CAAA;AAAA,KACT;AAGA,IAAA,IAAI,IAAK,CAAA,OAAA,CAAQ,UAAW,CAAA,EAAE,CAAG,EAAA;AAC/B,MAAA,IAAA,CAAK,OAAU,GAAA,IAAA,CAAK,OAAQ,CAAA,KAAA,CAAM,CAAC,CAAA,CAAA;AACnC,MAAO,OAAA,eAAA,CAAA;AAAA,KACT;AAGA,IAAA,IAAI,IAAK,CAAA,OAAA,CAAQ,UAAW,CAAA,IAAA,CAAK,UAAU,CAAG,EAAA;AAC5C,MAAA,IAAA,CAAK,OAAU,GAAA,IAAA,CAAK,OAAQ,CAAA,KAAA,CAAM,CAAC,CAAA,CAAA;AACnC,MAAO,OAAA,cAAA,CAAA;AAAA,KACT;AAGA,IAAA,IAAI,IAAK,CAAA,OAAA,CAAQ,UAAW,CAAA,IAAA,CAAK,UAAU,CAAG,EAAA;AAE5C,MAAI,IAAA,IAAA,CAAK,WAAW,KAAS,IAAA,IAAA,CAAK,QAAQ,QAAS,CAAA,IAAA,CAAK,UAAU,CAAG,EAAA;AACnE,QAAO,OAAA,IAAA,CAAA;AAAA,OACT;AACA,MAAA,OAAO,KAAK,oBAAqB,EAAA,CAAA;AAAA,KACnC;AAGA,IAAA,MAAM,KAAQ,GAAA,IAAA,CAAK,QAAS,CAAA,IAAA,CAAK,KAAK,OAAO,CAAA,CAAA;AAC7C,IAAA,IAAI,KAAO,EAAA;AAGT,MAAI,IAAA,IAAA,CAAK,WAAW,KAAS,IAAA,KAAA,CAAM,CAAC,CAAE,CAAA,MAAA,KAAW,IAAK,CAAA,OAAA,CAAQ,MAAQ,EAAA;AACpE,QAAO,OAAA,IAAA,CAAA;AAAA,OACT;AACA,MAAA,IAAA,CAAK,UAAU,IAAK,CAAA,OAAA,CAAQ,MAAM,KAAM,CAAA,CAAC,EAAE,MAAM,CAAA,CAAA;AACjD,MAAA,OAAO,EAAE,IAAM,EAAA,KAAA,EAAO,KAAO,EAAA,KAAA,CAAM,CAAC,CAAE,EAAA,CAAA;AAAA,KACxC;AAGA,IAAO,OAAA,IAAA,CAAA;AAAA,GACT;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,oBAAqC,GAAA;AACnC,IAAA,IAAI,GAAM,GAAA,CAAA,CAAA;AACV,IAAA,IAAI,KAAQ,GAAA,EAAA,CAAA;AAEZ,IAAO,OAAA,GAAA,GAAM,IAAK,CAAA,OAAA,CAAQ,MAAQ,EAAA;AAEhC,MAAA,IACE,KAAK,OAAQ,CAAA,KAAA,CAAM,KAAK,GAAM,GAAA,CAAC,MAAM,IAAK,CAAA,UAAA,IAC1C,KAAK,OAAQ,CAAA,KAAA,CAAM,MAAM,CAAG,EAAA,GAAA,GAAM,IAAI,CAAC,CAAA,KAAM,KAAK,UAClD,EAAA;AACA,QAAA,KAAA,IAAS,IAAK,CAAA,UAAA,CAAA;AACd,QAAA,GAAA,IAAO,CAAI,GAAA,CAAA,CAAA;AACX,QAAA,SAAA;AAAA,OACF;AAGA,MAAI,IAAA,IAAA,CAAK,QAAQ,KAAM,CAAA,GAAA,EAAK,MAAM,CAAC,CAAA,KAAM,KAAK,UAAY,EAAA;AAExD,QAAA,IACE,KAAK,MAAW,KAAA,KAAA,IAChB,MAAM,CAAI,GAAA,IAAA,CAAK,QAAQ,MACvB,IAAA,IAAA,CAAK,OAAQ,CAAA,KAAA,CAAM,MAAM,CAAG,EAAA,CAAC,MAAM,IAAK,CAAA,UAAA,IACxC,KAAK,OAAQ,CAAA,KAAA;AAAA,UAAM,GAAM,GAAA,CAAA;AAAA,UAAG,MAAM,CAAI,GAAA,CAAA;AAAA;AAAA,SAAoB,KACxD,IACF,IAAA,IAAA,CAAK,OAAQ,CAAA,KAAA;AAAA,UAAM,GAAM,GAAA,CAAA;AAAA,UAAG,MAAM,CAAI,GAAA,CAAA;AAAA;AAAA,cAAwB,EAC9D,EAAA;AACA,UAAO,OAAA,IAAA,CAAA;AAAA,SACT;AAGA,QAAA,IAAA,CAAK,OAAU,GAAA,IAAA,CAAK,OAAQ,CAAA,KAAA,CAAM,MAAM,CAAC,CAAA,CAAA;AACzC,QAAO,OAAA,EAAE,IAAM,EAAA,KAAA,EAAO,KAAM,EAAA,CAAA;AAAA,OAC9B;AAEA,MAAS,KAAA,IAAA,IAAA,CAAK,QAAQ,GAAG,CAAA,CAAA;AACzB,MAAA,GAAA,EAAA,CAAA;AAAA,KACF;AAGA,IAAO,OAAA,IAAA,CAAA;AAAA,GACT;AACF;;;;"}
+{"version":3,"file":"Lexer.js","sources":["../../src/Lexer.ts"],"sourcesContent":["import { assertCommonOptions } from \"./assertCommonOptions.ts\";\nimport { Field, FieldDelimiter, RecordDelimiter } from \"./common/constants.ts\";\nimport type {\n  CommonOptions,\n  Position,\n  RecordDelimiterToken,\n  Token,\n} from \"./common/types.ts\";\nimport { COMMA, CRLF, DOUBLE_QUOTE, LF } from \"./constants.ts\";\nimport { escapeRegExp } from \"./utils/escapeRegExp.ts\";\n\n/**\n * CSV Lexer.\n *\n * Lexter tokenizes CSV data into fields and records.\n */\nexport class Lexer {\n  #delimiter: string;\n  #quotation: string;\n  #buffer = \"\";\n  #flush = false;\n  #matcher: RegExp;\n  #fieldDelimiterLength: number;\n\n  #cursor: Position = {\n    line: 1,\n    column: 1,\n    offset: 0,\n  };\n  #rowNumber = 1;\n\n  /**\n   * Constructs a new Lexer instance.\n   * @param options - The common options for the lexer.\n   */\n  constructor({\n    delimiter = COMMA,\n    quotation = DOUBLE_QUOTE,\n  }: CommonOptions = {}) {\n    assertCommonOptions({ delimiter, quotation });\n    this.#delimiter = delimiter;\n    this.#quotation = quotation;\n    this.#fieldDelimiterLength = delimiter.length;\n    const d = escapeRegExp(delimiter);\n    const q = escapeRegExp(quotation);\n    this.#matcher = new RegExp(\n      `^(?:(?!${q})(?!${d})(?![\\\\r\\\\n]))([\\\\S\\\\s\\\\uFEFF\\\\xA0]+?)(?=${q}|${d}|\\\\r|\\\\n|$)`,\n    );\n  }\n\n  /**\n   * Lexes the given chunk of CSV data.\n   * @param chunk - The chunk of CSV data to be lexed.\n   * @param buffering - Indicates whether the lexer is buffering or not.\n   * @returns An iterable iterator of tokens.\n   */\n  public lex(chunk: string | null, buffering = false): IterableIterator<Token> {\n    if (!buffering) {\n      this.#flush = true;\n    }\n    if (typeof chunk === \"string\" && chunk.length !== 0) {\n      this.#buffer += chunk;\n    }\n\n    return this.#tokens();\n  }\n\n  /**\n   * Flushes the lexer and returns any remaining tokens.\n   * @returns An array of tokens.\n   */\n  public flush(): Token[] {\n    this.#flush = true;\n    return [...this.#tokens()];\n  }\n\n  /**\n   * Generates tokens from the buffered CSV data.\n   * @yields Tokens from the buffered CSV data.\n   */\n  *#tokens(): Generator<Token> {\n    if (this.#flush) {\n      // Trim the last CRLF or LF\n      if (this.#buffer.endsWith(CRLF)) {\n        this.#buffer = this.#buffer.slice(0, -2 /* -CRLF.length */);\n      } else if (this.#buffer.endsWith(LF)) {\n        this.#buffer = this.#buffer.slice(0, -1 /* -LF.length */);\n      }\n    }\n    let token: Token | null;\n    while ((token = this.#nextToken())) {\n      yield token;\n    }\n  }\n\n  /**\n   * Retrieves the next token from the buffered CSV data.\n   * @returns The next token or null if there are no more tokens.\n   */\n  #nextToken(): Token | null {\n    if (this.#buffer.length === 0) {\n      return null;\n    }\n    // Buffer is Record Delimiter, defer to the next iteration.\n    if (\n      this.#flush === false &&\n      (this.#buffer === CRLF || this.#buffer === LF)\n    ) {\n      return null;\n    }\n\n    // Check for CRLF\n    if (this.#buffer.startsWith(CRLF)) {\n      this.#buffer = this.#buffer.slice(2);\n      const start: Position = { ...this.#cursor };\n      this.#cursor.line++;\n      this.#cursor.column = 1;\n      this.#cursor.offset += 2; // CRLF.length\n      const token: RecordDelimiterToken = {\n        type: RecordDelimiter,\n        value: CRLF,\n        location: {\n          start,\n          end: { ...this.#cursor },\n          rowNumber: this.#rowNumber++,\n        },\n      };\n      return token;\n    }\n\n    // Check for LF\n    if (this.#buffer.startsWith(LF)) {\n      this.#buffer = this.#buffer.slice(1);\n      const start: Position = { ...this.#cursor };\n      this.#cursor.line++;\n      this.#cursor.column = 1;\n      this.#cursor.offset += 1; // LF.length\n      const token: RecordDelimiterToken = {\n        type: RecordDelimiter,\n        value: LF,\n        location: {\n          start,\n          end: { ...this.#cursor },\n          rowNumber: this.#rowNumber++,\n        },\n      };\n      return token;\n    }\n\n    // Check for Delimiter\n    if (this.#buffer.startsWith(this.#delimiter)) {\n      this.#buffer = this.#buffer.slice(1);\n      const start: Position = { ...this.#cursor };\n      this.#cursor.column += this.#fieldDelimiterLength;\n      this.#cursor.offset += this.#fieldDelimiterLength;\n      return {\n        type: FieldDelimiter,\n        value: this.#delimiter,\n        location: {\n          start,\n          end: { ...this.#cursor },\n          rowNumber: this.#rowNumber,\n        },\n      };\n    }\n\n    // Check for Quoted String\n    if (this.#buffer.startsWith(this.#quotation)) {\n      /**\n       * Extract Quoted field.\n       *\n       * The following code is equivalent to the following:\n       *\n       * If the next character is a quote:\n       * - If the character after that is a quote, then append a quote to the value and skip two characters.\n       * - Otherwise, return the quoted string.\n       * Otherwise, append the character to the value and skip one character.\n       *\n       * ```plaintext\n       * | `i`        | `i + 1`    | `i + 2`  |\n       * |------------|------------|----------|\n       * | cur        | next       |          | => Variable names\n       * | #quotation | #quotation |          | => Escaped quote\n       * | #quotation | (EOF)      |          | => Closing quote\n       * | #quotation | undefined  |          | => End of buffer\n       * | undefined  |            |          | => End of buffer\n       * ```\n       */\n      let value = \"\";\n      let offset = 1; // Skip the opening quote\n      let column = 2; // Skip the opening quote\n      let line = 0;\n\n      // Define variables\n      let cur: string = this.#buffer[offset];\n      let next: string | undefined = this.#buffer[offset + 1];\n      do {\n        // If the current character is a quote, check the next characters for closing quotes.\n        if (cur === this.#quotation) {\n          // If the cur character is a quote and the next character is a quote,\n          // then append a quote to the value and skip two characters.\n          if (next === this.#quotation) {\n            // Append a quote to the value and skip two characters.\n            value += this.#quotation;\n            offset += 2;\n            cur = this.#buffer[offset];\n            next = this.#buffer[offset + 1];\n\n            // Update the diff\n            column += 2;\n            continue;\n          }\n\n          // If the cur character is a quote and the next character is undefined,\n          // then return null.\n          if (next === undefined && this.#flush === false) {\n            return null;\n          }\n\n          // Otherwise, return the quoted string.\n          // Update the buffer and return the token\n          offset++;\n          this.#buffer = this.#buffer.slice(offset);\n          const start: Position = { ...this.#cursor };\n          this.#cursor.column += column;\n          this.#cursor.offset += offset;\n          this.#cursor.line += line;\n          return {\n            type: Field,\n            value,\n            location: {\n              start,\n              end: { ...this.#cursor },\n              rowNumber: this.#rowNumber,\n            },\n          };\n          // return this.#field(value, { column, offset, line });\n        }\n\n        // Append the character to the value.\n        value += cur;\n\n        // Prepare for the next iteration\n        if (cur === LF) {\n          // If the current character is a LF,\n          // then increment the line number and reset the column number.\n          line++;\n          column = 1;\n        } else {\n          // Otherwise, increment the column number and offset.\n          column++;\n        }\n\n        offset++;\n        cur = next;\n        next = this.#buffer[offset + 1];\n      } while (cur !== undefined);\n\n      // If we get here, we've reached the end of the buffer\n      return null;\n      // TODO: If flash is true, the buffer is exiting unquoted and an exception should be raised.\n    }\n\n    // Check for Unquoted String\n    const match = this.#matcher.exec(this.#buffer);\n    if (match) {\n      // If we're flushing and the match doesn't consume the entire buffer,\n      // then return null\n      if (this.#flush === false && match[0].length === this.#buffer.length) {\n        return null;\n      }\n      const value = match[1];\n      this.#buffer = this.#buffer.slice(value.length);\n      const start: Position = { ...this.#cursor };\n      this.#cursor.column += value.length;\n      this.#cursor.offset += value.length;\n      return {\n        type: Field,\n        value,\n        location: {\n          start,\n          end: { ...this.#cursor },\n          rowNumber: this.#rowNumber,\n        },\n      };\n    }\n\n    // Otherwise, return null\n    return null;\n  }\n}\n"],"names":[],"mappings":";;;;;AAgBO,MAAM,KAAM,CAAA;AAAA,EACjB,UAAA,CAAA;AAAA,EACA,UAAA,CAAA;AAAA,EACA,OAAU,GAAA,EAAA,CAAA;AAAA,EACV,MAAS,GAAA,KAAA,CAAA;AAAA,EACT,QAAA,CAAA;AAAA,EACA,qBAAA,CAAA;AAAA,EAEA,OAAoB,GAAA;AAAA,IAClB,IAAM,EAAA,CAAA;AAAA,IACN,MAAQ,EAAA,CAAA;AAAA,IACR,MAAQ,EAAA,CAAA;AAAA,GACV,CAAA;AAAA,EACA,UAAa,GAAA,CAAA,CAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAMb,WAAY,CAAA;AAAA,IACV,SAAY,GAAA,KAAA;AAAA,IACZ,SAAY,GAAA,YAAA;AAAA,GACd,GAAmB,EAAI,EAAA;AACrB,IAAoB,mBAAA,CAAA,EAAE,SAAW,EAAA,SAAA,EAAW,CAAA,CAAA;AAC5C,IAAA,IAAA,CAAK,UAAa,GAAA,SAAA,CAAA;AAClB,IAAA,IAAA,CAAK,UAAa,GAAA,SAAA,CAAA;AAClB,IAAA,IAAA,CAAK,wBAAwB,SAAU,CAAA,MAAA,CAAA;AACvC,IAAM,MAAA,CAAA,GAAI,aAAa,SAAS,CAAA,CAAA;AAChC,IAAM,MAAA,CAAA,GAAI,aAAa,SAAS,CAAA,CAAA;AAChC,IAAA,IAAA,CAAK,WAAW,IAAI,MAAA;AAAA,MAClB,UAAU,CAAC,CAAA,IAAA,EAAO,CAAC,CAA4C,yCAAA,EAAA,CAAC,IAAI,CAAC,CAAA,WAAA,CAAA;AAAA,KACvE,CAAA;AAAA,GACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAQO,GAAA,CAAI,KAAsB,EAAA,SAAA,GAAY,KAAgC,EAAA;AAC3E,IAAA,IAAI,CAAC,SAAW,EAAA;AACd,MAAA,IAAA,CAAK,MAAS,GAAA,IAAA,CAAA;AAAA,KAChB;AACA,IAAA,IAAI,OAAO,KAAA,KAAU,QAAY,IAAA,KAAA,CAAM,WAAW,CAAG,EAAA;AACnD,MAAA,IAAA,CAAK,OAAW,IAAA,KAAA,CAAA;AAAA,KAClB;AAEA,IAAA,OAAO,KAAK,OAAQ,EAAA,CAAA;AAAA,GACtB;AAAA;AAAA;AAAA;AAAA;AAAA,EAMO,KAAiB,GAAA;AACtB,IAAA,IAAA,CAAK,MAAS,GAAA,IAAA,CAAA;AACd,IAAA,OAAO,CAAC,GAAG,IAAK,CAAA,OAAA,EAAS,CAAA,CAAA;AAAA,GAC3B;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,CAAC,OAA4B,GAAA;AAC3B,IAAA,IAAI,KAAK,MAAQ,EAAA;AAEf,MAAA,IAAI,IAAK,CAAA,OAAA,CAAQ,QAAS,CAAA,IAAI,CAAG,EAAA;AAC/B,QAAK,IAAA,CAAA,OAAA,GAAU,KAAK,OAAQ,CAAA,KAAA;AAAA,UAAM,CAAA;AAAA,UAAG,CAAA,CAAA;AAAA;AAAA,SAAqB,CAAA;AAAA,OACjD,MAAA,IAAA,IAAA,CAAK,OAAQ,CAAA,QAAA,CAAS,EAAE,CAAG,EAAA;AACpC,QAAK,IAAA,CAAA,OAAA,GAAU,KAAK,OAAQ,CAAA,KAAA;AAAA,UAAM,CAAA;AAAA,UAAG,CAAA,CAAA;AAAA;AAAA,SAAmB,CAAA;AAAA,OAC1D;AAAA,KACF;AACA,IAAI,IAAA,KAAA,CAAA;AACJ,IAAQ,OAAA,KAAA,GAAQ,IAAK,CAAA,UAAA,EAAe,EAAA;AAClC,MAAM,MAAA,KAAA,CAAA;AAAA,KACR;AAAA,GACF;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,UAA2B,GAAA;AACzB,IAAI,IAAA,IAAA,CAAK,OAAQ,CAAA,MAAA,KAAW,CAAG,EAAA;AAC7B,MAAO,OAAA,IAAA,CAAA;AAAA,KACT;AAEA,IACE,IAAA,IAAA,CAAK,WAAW,KACf,KAAA,IAAA,CAAK,YAAY,IAAQ,IAAA,IAAA,CAAK,YAAY,EAC3C,CAAA,EAAA;AACA,MAAO,OAAA,IAAA,CAAA;AAAA,KACT;AAGA,IAAA,IAAI,IAAK,CAAA,OAAA,CAAQ,UAAW,CAAA,IAAI,CAAG,EAAA;AACjC,MAAA,IAAA,CAAK,OAAU,GAAA,IAAA,CAAK,OAAQ,CAAA,KAAA,CAAM,CAAC,CAAA,CAAA;AACnC,MAAA,MAAM,KAAkB,GAAA,EAAE,GAAG,IAAA,CAAK,OAAQ,EAAA,CAAA;AAC1C,MAAA,IAAA,CAAK,OAAQ,CAAA,IAAA,EAAA,CAAA;AACb,MAAA,IAAA,CAAK,QAAQ,MAAS,GAAA,CAAA,CAAA;AACtB,MAAA,IAAA,CAAK,QAAQ,MAAU,IAAA,CAAA,CAAA;AACvB,MAAA,MAAM,KAA8B,GAAA;AAAA,QAClC,IAAM,EAAA,eAAA;AAAA,QACN,KAAO,EAAA,IAAA;AAAA,QACP,QAAU,EAAA;AAAA,UACR,KAAA;AAAA,UACA,GAAK,EAAA,EAAE,GAAG,IAAA,CAAK,OAAQ,EAAA;AAAA,UACvB,WAAW,IAAK,CAAA,UAAA,EAAA;AAAA,SAClB;AAAA,OACF,CAAA;AACA,MAAO,OAAA,KAAA,CAAA;AAAA,KACT;AAGA,IAAA,IAAI,IAAK,CAAA,OAAA,CAAQ,UAAW,CAAA,EAAE,CAAG,EAAA;AAC/B,MAAA,IAAA,CAAK,OAAU,GAAA,IAAA,CAAK,OAAQ,CAAA,KAAA,CAAM,CAAC,CAAA,CAAA;AACnC,MAAA,MAAM,KAAkB,GAAA,EAAE,GAAG,IAAA,CAAK,OAAQ,EAAA,CAAA;AAC1C,MAAA,IAAA,CAAK,OAAQ,CAAA,IAAA,EAAA,CAAA;AACb,MAAA,IAAA,CAAK,QAAQ,MAAS,GAAA,CAAA,CAAA;AACtB,MAAA,IAAA,CAAK,QAAQ,MAAU,IAAA,CAAA,CAAA;AACvB,MAAA,MAAM,KAA8B,GAAA;AAAA,QAClC,IAAM,EAAA,eAAA;AAAA,QACN,KAAO,EAAA,EAAA;AAAA,QACP,QAAU,EAAA;AAAA,UACR,KAAA;AAAA,UACA,GAAK,EAAA,EAAE,GAAG,IAAA,CAAK,OAAQ,EAAA;AAAA,UACvB,WAAW,IAAK,CAAA,UAAA,EAAA;AAAA,SAClB;AAAA,OACF,CAAA;AACA,MAAO,OAAA,KAAA,CAAA;AAAA,KACT;AAGA,IAAA,IAAI,IAAK,CAAA,OAAA,CAAQ,UAAW,CAAA,IAAA,CAAK,UAAU,CAAG,EAAA;AAC5C,MAAA,IAAA,CAAK,OAAU,GAAA,IAAA,CAAK,OAAQ,CAAA,KAAA,CAAM,CAAC,CAAA,CAAA;AACnC,MAAA,MAAM,KAAkB,GAAA,EAAE,GAAG,IAAA,CAAK,OAAQ,EAAA,CAAA;AAC1C,MAAK,IAAA,CAAA,OAAA,CAAQ,UAAU,IAAK,CAAA,qBAAA,CAAA;AAC5B,MAAK,IAAA,CAAA,OAAA,CAAQ,UAAU,IAAK,CAAA,qBAAA,CAAA;AAC5B,MAAO,OAAA;AAAA,QACL,IAAM,EAAA,cAAA;AAAA,QACN,OAAO,IAAK,CAAA,UAAA;AAAA,QACZ,QAAU,EAAA;AAAA,UACR,KAAA;AAAA,UACA,GAAK,EAAA,EAAE,GAAG,IAAA,CAAK,OAAQ,EAAA;AAAA,UACvB,WAAW,IAAK,CAAA,UAAA;AAAA,SAClB;AAAA,OACF,CAAA;AAAA,KACF;AAGA,IAAA,IAAI,IAAK,CAAA,OAAA,CAAQ,UAAW,CAAA,IAAA,CAAK,UAAU,CAAG,EAAA;AAqB5C,MAAA,IAAI,KAAQ,GAAA,EAAA,CAAA;AACZ,MAAA,IAAI,MAAS,GAAA,CAAA,CAAA;AACb,MAAA,IAAI,MAAS,GAAA,CAAA,CAAA;AACb,MAAA,IAAI,IAAO,GAAA,CAAA,CAAA;AAGX,MAAI,IAAA,GAAA,GAAc,IAAK,CAAA,OAAA,CAAQ,MAAM,CAAA,CAAA;AACrC,MAAA,IAAI,IAA2B,GAAA,IAAA,CAAK,OAAQ,CAAA,MAAA,GAAS,CAAC,CAAA,CAAA;AACtD,MAAG,GAAA;AAED,QAAI,IAAA,GAAA,KAAQ,KAAK,UAAY,EAAA;AAG3B,UAAI,IAAA,IAAA,KAAS,KAAK,UAAY,EAAA;AAE5B,YAAA,KAAA,IAAS,IAAK,CAAA,UAAA,CAAA;AACd,YAAU,MAAA,IAAA,CAAA,CAAA;AACV,YAAM,GAAA,GAAA,IAAA,CAAK,QAAQ,MAAM,CAAA,CAAA;AACzB,YAAO,IAAA,GAAA,IAAA,CAAK,OAAQ,CAAA,MAAA,GAAS,CAAC,CAAA,CAAA;AAG9B,YAAU,MAAA,IAAA,CAAA,CAAA;AACV,YAAA,SAAA;AAAA,WACF;AAIA,UAAA,IAAI,IAAS,KAAA,KAAA,CAAA,IAAa,IAAK,CAAA,MAAA,KAAW,KAAO,EAAA;AAC/C,YAAO,OAAA,IAAA,CAAA;AAAA,WACT;AAIA,UAAA,MAAA,EAAA,CAAA;AACA,UAAA,IAAA,CAAK,OAAU,GAAA,IAAA,CAAK,OAAQ,CAAA,KAAA,CAAM,MAAM,CAAA,CAAA;AACxC,UAAA,MAAM,KAAkB,GAAA,EAAE,GAAG,IAAA,CAAK,OAAQ,EAAA,CAAA;AAC1C,UAAA,IAAA,CAAK,QAAQ,MAAU,IAAA,MAAA,CAAA;AACvB,UAAA,IAAA,CAAK,QAAQ,MAAU,IAAA,MAAA,CAAA;AACvB,UAAA,IAAA,CAAK,QAAQ,IAAQ,IAAA,IAAA,CAAA;AACrB,UAAO,OAAA;AAAA,YACL,IAAM,EAAA,KAAA;AAAA,YACN,KAAA;AAAA,YACA,QAAU,EAAA;AAAA,cACR,KAAA;AAAA,cACA,GAAK,EAAA,EAAE,GAAG,IAAA,CAAK,OAAQ,EAAA;AAAA,cACvB,WAAW,IAAK,CAAA,UAAA;AAAA,aAClB;AAAA,WACF,CAAA;AAAA,SAEF;AAGA,QAAS,KAAA,IAAA,GAAA,CAAA;AAGT,QAAA,IAAI,QAAQ,EAAI,EAAA;AAGd,UAAA,IAAA,EAAA,CAAA;AACA,UAAS,MAAA,GAAA,CAAA,CAAA;AAAA,SACJ,MAAA;AAEL,UAAA,MAAA,EAAA,CAAA;AAAA,SACF;AAEA,QAAA,MAAA,EAAA,CAAA;AACA,QAAM,GAAA,GAAA,IAAA,CAAA;AACN,QAAO,IAAA,GAAA,IAAA,CAAK,OAAQ,CAAA,MAAA,GAAS,CAAC,CAAA,CAAA;AAAA,eACvB,GAAQ,KAAA,KAAA,CAAA,EAAA;AAGjB,MAAO,OAAA,IAAA,CAAA;AAAA,KAET;AAGA,IAAA,MAAM,KAAQ,GAAA,IAAA,CAAK,QAAS,CAAA,IAAA,CAAK,KAAK,OAAO,CAAA,CAAA;AAC7C,IAAA,IAAI,KAAO,EAAA;AAGT,MAAI,IAAA,IAAA,CAAK,WAAW,KAAS,IAAA,KAAA,CAAM,CAAC,CAAE,CAAA,MAAA,KAAW,IAAK,CAAA,OAAA,CAAQ,MAAQ,EAAA;AACpE,QAAO,OAAA,IAAA,CAAA;AAAA,OACT;AACA,MAAM,MAAA,KAAA,GAAQ,MAAM,CAAC,CAAA,CAAA;AACrB,MAAA,IAAA,CAAK,OAAU,GAAA,IAAA,CAAK,OAAQ,CAAA,KAAA,CAAM,MAAM,MAAM,CAAA,CAAA;AAC9C,MAAA,MAAM,KAAkB,GAAA,EAAE,GAAG,IAAA,CAAK,OAAQ,EAAA,CAAA;AAC1C,MAAK,IAAA,CAAA,OAAA,CAAQ,UAAU,KAAM,CAAA,MAAA,CAAA;AAC7B,MAAK,IAAA,CAAA,OAAA,CAAQ,UAAU,KAAM,CAAA,MAAA,CAAA;AAC7B,MAAO,OAAA;AAAA,QACL,IAAM,EAAA,KAAA;AAAA,QACN,KAAA;AAAA,QACA,QAAU,EAAA;AAAA,UACR,KAAA;AAAA,UACA,GAAK,EAAA,EAAE,GAAG,IAAA,CAAK,OAAQ,EAAA;AAAA,UACvB,WAAW,IAAK,CAAA,UAAA;AAAA,SAClB;AAAA,OACF,CAAA;AAAA,KACF;AAGA,IAAO,OAAA,IAAA,CAAA;AAAA,GACT;AACF;;;;"}

package/dist/es/RecordAssembler.js

@@ -12,7 +12,7 @@ class RecordAssembler {
   }
   *assemble(tokens, flush = true) {
     for (const token of tokens) {
-      switch (token) {
+      switch (token.type) {
         case FieldDelimiter:
           this.#fieldIndex++;
           this.#dirty = true;

package/dist/es/RecordAssembler.js.map

@@ -1 +1 @@
-{"version":3,"file":"RecordAssembler.js","sources":["../../src/RecordAssembler.ts"],"sourcesContent":["import { FieldDelimiter, RecordDelimiter } from \"./common/constants.ts\";\nimport type {\n  CSVRecord,\n  RecordAssemblerOptions,\n  Token,\n} from \"./common/types.ts\";\n\nexport class RecordAssembler<Header extends ReadonlyArray<string>> {\n  #fieldIndex = 0;\n  #row: string[] = [];\n  #header: Header | undefined;\n  #dirty = false;\n\n  constructor(options: RecordAssemblerOptions<Header> = {}) {\n    if (options.header !== undefined && Array.isArray(options.header)) {\n      this.#setHeader(options.header);\n    }\n  }\n\n  public *assemble(\n    tokens: Iterable<Token>,\n    flush = true,\n  ): IterableIterator<CSVRecord<Header>> {\n    for (const token of tokens) {\n      switch (token) {\n        case FieldDelimiter:\n          this.#fieldIndex++;\n          this.#dirty = true;\n          break;\n        case RecordDelimiter:\n          if (this.#header === undefined) {\n            this.#setHeader(this.#row as unknown as Header);\n          } else {\n            if (this.#dirty) {\n              yield Object.fromEntries(\n                this.#header.map((header, index) => [\n                  header,\n                  this.#row.at(index),\n                ]),\n              ) as unknown as CSVRecord<Header>;\n            } else {\n              yield Object.fromEntries(\n                this.#header.map((header) => [header, \"\"]),\n              ) as CSVRecord<Header>;\n            }\n          }\n          // Reset the row fields buffer.\n          this.#fieldIndex = 0;\n          this.#row = new Array(this.#header?.length).fill(\"\");\n          this.#dirty = false;\n          break;\n        default:\n          this.#dirty = true;\n          this.#row[this.#fieldIndex] = token.value;\n          break;\n      }\n    }\n\n    if (flush) {\n      yield* this.flush();\n    }\n  }\n\n  public *flush(): Generator<CSVRecord<Header>> {\n    if (this.#header !== undefined) {\n      if (this.#dirty) {\n        yield Object.fromEntries(\n          this.#header\n            .filter((v) => v)\n            .map((header, index) => [header, this.#row.at(index)]),\n        ) as unknown as CSVRecord<Header>;\n      }\n    }\n  }\n\n  #setHeader(header: Header) {\n    this.#header = header;\n    if (this.#header.length === 0) {\n      throw new Error(\"The header must not be empty.\");\n    }\n    if (new Set(this.#header).size !== this.#header.length) {\n      throw new Error(\"The header must not contain duplicate fields.\");\n    }\n  }\n}\n"],"names":[],"mappings":";;AAOO,MAAM,eAAsD,CAAA;AAAA,EACjE,WAAc,GAAA,CAAA,CAAA;AAAA,EACd,OAAiB,EAAC,CAAA;AAAA,EAClB,OAAA,CAAA;AAAA,EACA,MAAS,GAAA,KAAA,CAAA;AAAA,EAET,WAAA,CAAY,OAA0C,GAAA,EAAI,EAAA;AACxD,IAAA,IAAI,QAAQ,MAAW,KAAA,KAAA,CAAA,IAAa,MAAM,OAAQ,CAAA,OAAA,CAAQ,MAAM,CAAG,EAAA;AACjE,MAAK,IAAA,CAAA,UAAA,CAAW,QAAQ,MAAM,CAAA,CAAA;AAAA,KAChC;AAAA,GACF;AAAA,EAEA,CAAQ,QAAA,CACN,MACA,EAAA,KAAA,GAAQ,IAC6B,EAAA;AACrC,IAAA,KAAA,MAAW,SAAS,MAAQ,EAAA;AAC1B,MAAA,QAAQ,KAAO;AAAA,QACb,KAAK,cAAA;AACH,UAAK,IAAA,CAAA,WAAA,EAAA,CAAA;AACL,UAAA,IAAA,CAAK,MAAS,GAAA,IAAA,CAAA;AACd,UAAA,MAAA;AAAA,QACF,KAAK,eAAA;AACH,UAAI,IAAA,IAAA,CAAK,YAAY,KAAW,CAAA,EAAA;AAC9B,YAAK,IAAA,CAAA,UAAA,CAAW,KAAK,IAAyB,CAAA,CAAA;AAAA,WACzC,MAAA;AACL,YAAA,IAAI,KAAK,MAAQ,EAAA;AACf,cAAA,MAAM,MAAO,CAAA,WAAA;AAAA,gBACX,IAAK,CAAA,OAAA,CAAQ,GAAI,CAAA,CAAC,QAAQ,KAAU,KAAA;AAAA,kBAClC,MAAA;AAAA,kBACA,IAAA,CAAK,IAAK,CAAA,EAAA,CAAG,KAAK,CAAA;AAAA,iBACnB,CAAA;AAAA,eACH,CAAA;AAAA,aACK,MAAA;AACL,cAAA,MAAM,MAAO,CAAA,WAAA;AAAA,gBACX,IAAA,CAAK,QAAQ,GAAI,CAAA,CAAC,WAAW,CAAC,MAAA,EAAQ,EAAE,CAAC,CAAA;AAAA,eAC3C,CAAA;AAAA,aACF;AAAA,WACF;AAEA,UAAA,IAAA,CAAK,WAAc,GAAA,CAAA,CAAA;AACnB,UAAK,IAAA,CAAA,IAAA,GAAO,IAAI,KAAM,CAAA,IAAA,CAAK,SAAS,MAAM,CAAA,CAAE,KAAK,EAAE,CAAA,CAAA;AACnD,UAAA,IAAA,CAAK,MAAS,GAAA,KAAA,CAAA;AACd,UAAA,MAAA;AAAA,QACF;AACE,UAAA,IAAA,CAAK,MAAS,GAAA,IAAA,CAAA;AACd,UAAA,IAAA,CAAK,IAAK,CAAA,IAAA,CAAK,WAAW,CAAA,GAAI,KAAM,CAAA,KAAA,CAAA;AACpC,UAAA,MAAA;AAAA,OACJ;AAAA,KACF;AAEA,IAAA,IAAI,KAAO,EAAA;AACT,MAAA,OAAO,KAAK,KAAM,EAAA,CAAA;AAAA,KACpB;AAAA,GACF;AAAA,EAEA,CAAQ,KAAsC,GAAA;AAC5C,IAAI,IAAA,IAAA,CAAK,YAAY,KAAW,CAAA,EAAA;AAC9B,MAAA,IAAI,KAAK,MAAQ,EAAA;AACf,QAAA,MAAM,MAAO,CAAA,WAAA;AAAA,UACX,KAAK,OACF,CAAA,MAAA,CAAO,CAAC,CAAM,KAAA,CAAC,EACf,GAAI,CAAA,CAAC,MAAQ,EAAA,KAAA,KAAU,CAAC,MAAQ,EAAA,IAAA,CAAK,KAAK,EAAG,CAAA,KAAK,CAAC,CAAC,CAAA;AAAA,SACzD,CAAA;AAAA,OACF;AAAA,KACF;AAAA,GACF;AAAA,EAEA,WAAW,MAAgB,EAAA;AACzB,IAAA,IAAA,CAAK,OAAU,GAAA,MAAA,CAAA;AACf,IAAI,IAAA,IAAA,CAAK,OAAQ,CAAA,MAAA,KAAW,CAAG,EAAA;AAC7B,MAAM,MAAA,IAAI,MAAM,+BAA+B,CAAA,CAAA;AAAA,KACjD;AACA,IAAI,IAAA,IAAI,IAAI,IAAK,CAAA,OAAO,EAAE,IAAS,KAAA,IAAA,CAAK,QAAQ,MAAQ,EAAA;AACtD,MAAM,MAAA,IAAI,MAAM,+CAA+C,CAAA,CAAA;AAAA,KACjE;AAAA,GACF;AACF;;;;"}
+{"version":3,"file":"RecordAssembler.js","sources":["../../src/RecordAssembler.ts"],"sourcesContent":["import { FieldDelimiter, RecordDelimiter } from \"./common/constants.ts\";\nimport type {\n  CSVRecord,\n  RecordAssemblerOptions,\n  Token,\n} from \"./common/types.ts\";\n\nexport class RecordAssembler<Header extends ReadonlyArray<string>> {\n  #fieldIndex = 0;\n  #row: string[] = [];\n  #header: Header | undefined;\n  #dirty = false;\n\n  constructor(options: RecordAssemblerOptions<Header> = {}) {\n    if (options.header !== undefined && Array.isArray(options.header)) {\n      this.#setHeader(options.header);\n    }\n  }\n\n  public *assemble(\n    tokens: Iterable<Token>,\n    flush = true,\n  ): IterableIterator<CSVRecord<Header>> {\n    for (const token of tokens) {\n      switch (token.type) {\n        case FieldDelimiter:\n          this.#fieldIndex++;\n          this.#dirty = true;\n          break;\n        case RecordDelimiter:\n          if (this.#header === undefined) {\n            this.#setHeader(this.#row as unknown as Header);\n          } else {\n            if (this.#dirty) {\n              yield Object.fromEntries(\n                this.#header.map((header, index) => [\n                  header,\n                  this.#row.at(index),\n                ]),\n              ) as unknown as CSVRecord<Header>;\n            } else {\n              yield Object.fromEntries(\n                this.#header.map((header) => [header, \"\"]),\n              ) as CSVRecord<Header>;\n            }\n          }\n          // Reset the row fields buffer.\n          this.#fieldIndex = 0;\n          this.#row = new Array(this.#header?.length).fill(\"\");\n          this.#dirty = false;\n          break;\n        default:\n          this.#dirty = true;\n          this.#row[this.#fieldIndex] = token.value;\n          break;\n      }\n    }\n\n    if (flush) {\n      yield* this.flush();\n    }\n  }\n\n  public *flush(): Generator<CSVRecord<Header>> {\n    if (this.#header !== undefined) {\n      if (this.#dirty) {\n        yield Object.fromEntries(\n          this.#header\n            .filter((v) => v)\n            .map((header, index) => [header, this.#row.at(index)]),\n        ) as unknown as CSVRecord<Header>;\n      }\n    }\n  }\n\n  #setHeader(header: Header) {\n    this.#header = header;\n    if (this.#header.length === 0) {\n      throw new Error(\"The header must not be empty.\");\n    }\n    if (new Set(this.#header).size !== this.#header.length) {\n      throw new Error(\"The header must not contain duplicate fields.\");\n    }\n  }\n}\n"],"names":[],"mappings":";;AAOO,MAAM,eAAsD,CAAA;AAAA,EACjE,WAAc,GAAA,CAAA,CAAA;AAAA,EACd,OAAiB,EAAC,CAAA;AAAA,EAClB,OAAA,CAAA;AAAA,EACA,MAAS,GAAA,KAAA,CAAA;AAAA,EAET,WAAA,CAAY,OAA0C,GAAA,EAAI,EAAA;AACxD,IAAA,IAAI,QAAQ,MAAW,KAAA,KAAA,CAAA,IAAa,MAAM,OAAQ,CAAA,OAAA,CAAQ,MAAM,CAAG,EAAA;AACjE,MAAK,IAAA,CAAA,UAAA,CAAW,QAAQ,MAAM,CAAA,CAAA;AAAA,KAChC;AAAA,GACF;AAAA,EAEA,CAAQ,QAAA,CACN,MACA,EAAA,KAAA,GAAQ,IAC6B,EAAA;AACrC,IAAA,KAAA,MAAW,SAAS,MAAQ,EAAA;AAC1B,MAAA,QAAQ,MAAM,IAAM;AAAA,QAClB,KAAK,cAAA;AACH,UAAK,IAAA,CAAA,WAAA,EAAA,CAAA;AACL,UAAA,IAAA,CAAK,MAAS,GAAA,IAAA,CAAA;AACd,UAAA,MAAA;AAAA,QACF,KAAK,eAAA;AACH,UAAI,IAAA,IAAA,CAAK,YAAY,KAAW,CAAA,EAAA;AAC9B,YAAK,IAAA,CAAA,UAAA,CAAW,KAAK,IAAyB,CAAA,CAAA;AAAA,WACzC,MAAA;AACL,YAAA,IAAI,KAAK,MAAQ,EAAA;AACf,cAAA,MAAM,MAAO,CAAA,WAAA;AAAA,gBACX,IAAK,CAAA,OAAA,CAAQ,GAAI,CAAA,CAAC,QAAQ,KAAU,KAAA;AAAA,kBAClC,MAAA;AAAA,kBACA,IAAA,CAAK,IAAK,CAAA,EAAA,CAAG,KAAK,CAAA;AAAA,iBACnB,CAAA;AAAA,eACH,CAAA;AAAA,aACK,MAAA;AACL,cAAA,MAAM,MAAO,CAAA,WAAA;AAAA,gBACX,IAAA,CAAK,QAAQ,GAAI,CAAA,CAAC,WAAW,CAAC,MAAA,EAAQ,EAAE,CAAC,CAAA;AAAA,eAC3C,CAAA;AAAA,aACF;AAAA,WACF;AAEA,UAAA,IAAA,CAAK,WAAc,GAAA,CAAA,CAAA;AACnB,UAAK,IAAA,CAAA,IAAA,GAAO,IAAI,KAAM,CAAA,IAAA,CAAK,SAAS,MAAM,CAAA,CAAE,KAAK,EAAE,CAAA,CAAA;AACnD,UAAA,IAAA,CAAK,MAAS,GAAA,KAAA,CAAA;AACd,UAAA,MAAA;AAAA,QACF;AACE,UAAA,IAAA,CAAK,MAAS,GAAA,IAAA,CAAA;AACd,UAAA,IAAA,CAAK,IAAK,CAAA,IAAA,CAAK,WAAW,CAAA,GAAI,KAAM,CAAA,KAAA,CAAA;AACpC,UAAA,MAAA;AAAA,OACJ;AAAA,KACF;AAEA,IAAA,IAAI,KAAO,EAAA;AACT,MAAA,OAAO,KAAK,KAAM,EAAA,CAAA;AAAA,KACpB;AAAA,GACF;AAAA,EAEA,CAAQ,KAAsC,GAAA;AAC5C,IAAI,IAAA,IAAA,CAAK,YAAY,KAAW,CAAA,EAAA;AAC9B,MAAA,IAAI,KAAK,MAAQ,EAAA;AACf,QAAA,MAAM,MAAO,CAAA,WAAA;AAAA,UACX,KAAK,OACF,CAAA,MAAA,CAAO,CAAC,CAAM,KAAA,CAAC,EACf,GAAI,CAAA,CAAC,MAAQ,EAAA,KAAA,KAAU,CAAC,MAAQ,EAAA,IAAA,CAAK,KAAK,EAAG,CAAA,KAAK,CAAC,CAAC,CAAA;AAAA,SACzD,CAAA;AAAA,OACF;AAAA,KACF;AAAA,GACF;AAAA,EAEA,WAAW,MAAgB,EAAA;AACzB,IAAA,IAAA,CAAK,OAAU,GAAA,MAAA,CAAA;AACf,IAAI,IAAA,IAAA,CAAK,OAAQ,CAAA,MAAA,KAAW,CAAG,EAAA;AAC7B,MAAM,MAAA,IAAI,MAAM,+BAA+B,CAAA,CAAA;AAAA,KACjD;AACA,IAAI,IAAA,IAAI,IAAI,IAAK,CAAA,OAAO,EAAE,IAAS,KAAA,IAAA,CAAK,QAAQ,MAAQ,EAAA;AACtD,MAAM,MAAA,IAAI,MAAM,+CAA+C,CAAA,CAAA;AAAA,KACjE;AAAA,GACF;AACF;;;;"}