web-csv-toolbox 0.10.3-next-10321180d04c1d67f731d55f6c6190691c0d7ba6 → 0.11.0-next-15ec1363a1f8064b33306871989088ab40f44a94
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +72 -7
- package/dist/cjs/Lexer.cjs +1 -1
- package/dist/cjs/Lexer.cjs.map +1 -1
- package/dist/cjs/RecordAssembler.cjs +1 -1
- package/dist/cjs/RecordAssembler.cjs.map +1 -1
- package/dist/cjs/_virtual/web_csv_toolbox_wasm_bg.wasm.cjs +1 -1
- package/dist/es/Lexer.js +7 -1
- package/dist/es/Lexer.js.map +1 -1
- package/dist/es/RecordAssembler.js +5 -0
- package/dist/es/RecordAssembler.js.map +1 -1
- package/dist/es/_virtual/web_csv_toolbox_wasm_bg.wasm.js +1 -1
- package/dist/types/Lexer.d.ts +2 -2
- package/dist/types/RecordAssembler.test.d.ts +1 -0
- package/dist/types/common/types.d.ts +54 -2
- package/dist/web-csv-toolbox.umd.cjs +1 -1
- package/dist/web-csv-toolbox.umd.cjs.map +1 -1
- package/dist/web_csv_toolbox_wasm_bg.wasm +0 -0
- package/package.json +1 -1
package/dist/es/Lexer.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"Lexer.js","sources":["../../src/Lexer.ts"],"sourcesContent":["import { assertCommonOptions } from \"./assertCommonOptions.ts\";\nimport { Field, FieldDelimiter, RecordDelimiter } from \"./common/constants.ts\";\nimport { ParseError } from \"./common/errors.ts\";\nimport type {\n CommonOptions,\n Position,\n RecordDelimiterToken,\n Token,\n} from \"./common/types.ts\";\nimport { COMMA, CRLF, DOUBLE_QUOTE, LF } from \"./constants.ts\";\nimport { escapeRegExp } from \"./utils/escapeRegExp.ts\";\n\n/**\n * CSV Lexer.\n *\n * Lexter tokenizes CSV data into fields and records.\n */\nexport class Lexer {\n #delimiter: string;\n #quotation: string;\n #buffer = \"\";\n #flush = false;\n #matcher: RegExp;\n #fieldDelimiterLength: number;\n\n #cursor: Position = {\n line: 1,\n column: 1,\n offset: 0,\n };\n #rowNumber = 1;\n\n /**\n * Constructs a new Lexer instance.\n * @param options - The common options for the lexer.\n */\n constructor({\n delimiter = COMMA,\n quotation = DOUBLE_QUOTE,\n }: CommonOptions = {}) {\n assertCommonOptions({ delimiter, quotation });\n this.#delimiter = delimiter;\n this.#quotation = quotation;\n this.#fieldDelimiterLength = delimiter.length;\n const d = escapeRegExp(delimiter);\n const q = escapeRegExp(quotation);\n this.#matcher = new RegExp(\n `^(?:(?!${q})(?!${d})(?![\\\\r\\\\n]))([\\\\S\\\\s\\\\uFEFF\\\\xA0]+?)(?=${q}|${d}|\\\\r|\\\\n|$)`,\n );\n }\n\n /**\n * Lexes the given chunk of CSV data.\n * @param chunk - The chunk of CSV data to be lexed.\n * @param buffering - Indicates whether the lexer is buffering or not.\n * @returns An iterable iterator of tokens.\n */\n public lex(chunk: string | null, buffering = false): IterableIterator<Token> {\n if (!buffering) {\n this.#flush = true;\n }\n if (typeof chunk === \"string\" && chunk.length !== 0) {\n this.#buffer += chunk;\n }\n\n return this.#tokens();\n }\n\n /**\n * Flushes the lexer and returns any remaining tokens.\n * @returns An array of tokens.\n */\n public flush(): Token[] {\n this.#flush = true;\n return [...this.#tokens()];\n }\n\n /**\n * Generates tokens from the buffered CSV data.\n * @yields Tokens from the buffered CSV data.\n */\n *#tokens(): Generator<Token> {\n if (this.#flush) {\n // Trim the last CRLF or LF\n if (this.#buffer.endsWith(CRLF)) {\n this.#buffer = this.#buffer.slice(0, -2 /* -CRLF.length */);\n } else if (this.#buffer.endsWith(LF)) {\n this.#buffer = this.#buffer.slice(0, -1 /* -LF.length */);\n }\n }\n let token: Token | null;\n while ((token = this.#nextToken())) {\n yield token;\n }\n }\n\n /**\n * Retrieves the next token from the buffered CSV data.\n * @returns The next token or null if there are no more tokens.\n */\n #nextToken(): Token | null {\n if (this.#buffer.length === 0) {\n return null;\n }\n // Buffer is Record Delimiter, defer to the next iteration.\n if (\n this.#flush === false &&\n (this.#buffer === CRLF || this.#buffer === LF)\n ) {\n return null;\n }\n\n // Check for CRLF\n if (this.#buffer.startsWith(CRLF)) {\n this.#buffer = this.#buffer.slice(2);\n const start: Position = { ...this.#cursor };\n this.#cursor.line++;\n this.#cursor.column = 1;\n this.#cursor.offset += 2; // CRLF.length\n const token: RecordDelimiterToken = {\n type: RecordDelimiter,\n value: CRLF,\n location: {\n start,\n end: { ...this.#cursor },\n rowNumber: this.#rowNumber++,\n },\n };\n return token;\n }\n\n // Check for LF\n if (this.#buffer.startsWith(LF)) {\n this.#buffer = this.#buffer.slice(1);\n const start: Position = { ...this.#cursor };\n this.#cursor.line++;\n this.#cursor.column = 1;\n this.#cursor.offset += 1; // LF.length\n const token: RecordDelimiterToken = {\n type: RecordDelimiter,\n value: LF,\n location: {\n start,\n end: { ...this.#cursor },\n rowNumber: this.#rowNumber++,\n },\n };\n return token;\n }\n\n // Check for Delimiter\n if (this.#buffer.startsWith(this.#delimiter)) {\n this.#buffer = this.#buffer.slice(1);\n const start: Position = { ...this.#cursor };\n this.#cursor.column += this.#fieldDelimiterLength;\n this.#cursor.offset += this.#fieldDelimiterLength;\n return {\n type: FieldDelimiter,\n value: this.#delimiter,\n location: {\n start,\n end: { ...this.#cursor },\n rowNumber: this.#rowNumber,\n },\n };\n }\n\n // Check for Quoted String\n if (this.#buffer.startsWith(this.#quotation)) {\n /**\n * Extract Quoted field.\n *\n * The following code is equivalent to the following:\n *\n * If the next character is a quote:\n * - If the character after that is a quote, then append a quote to the value and skip two characters.\n * - Otherwise, return the quoted string.\n * Otherwise, append the character to the value and skip one character.\n *\n * ```plaintext\n * | `i` | `i + 1` | `i + 2` |\n * |------------|------------|----------|\n * | cur | next | | => Variable names\n * | #quotation | #quotation | | => Escaped quote\n * | #quotation | (EOF) | | => Closing quote\n * | #quotation | undefined | | => End of buffer\n * | undefined | | | => End of buffer\n * ```\n */\n let value = \"\";\n let offset = 1; // Skip the opening quote\n let column = 2; // Skip the opening quote\n let line = 0;\n\n // Define variables\n let cur: string = this.#buffer[offset];\n let next: string | undefined = this.#buffer[offset + 1];\n do {\n // If the current character is a quote, check the next characters for closing quotes.\n if (cur === this.#quotation) {\n // If the cur character is a quote and the next character is a quote,\n // then append a quote to the value and skip two characters.\n if (next === this.#quotation) {\n // Append a quote to the value and skip two characters.\n value += this.#quotation;\n offset += 2;\n cur = this.#buffer[offset];\n next = this.#buffer[offset + 1];\n\n // Update the diff\n column += 2;\n continue;\n }\n\n // If the cur character is a quote and the next character is undefined,\n // then return null.\n if (next === undefined && this.#flush === false) {\n return null;\n }\n\n // Otherwise, return the quoted string.\n // Update the buffer and return the token\n offset++;\n this.#buffer = this.#buffer.slice(offset);\n const start: Position = { ...this.#cursor };\n this.#cursor.column += column;\n this.#cursor.offset += offset;\n this.#cursor.line += line;\n return {\n type: Field,\n value,\n location: {\n start,\n end: { ...this.#cursor },\n rowNumber: this.#rowNumber,\n },\n };\n }\n\n // Append the character to the value.\n value += cur;\n\n // Prepare for the next iteration\n if (cur === LF) {\n // If the current character is a LF,\n // then increment the line number and reset the column number.\n line++;\n column = 1;\n } else {\n // Otherwise, increment the column number and offset.\n column++;\n }\n\n offset++;\n cur = next;\n next = this.#buffer[offset + 1];\n } while (cur !== undefined);\n\n if (this.#flush) {\n throw new ParseError(\"Unexpected EOF while parsing quoted field.\", {\n position: { ...this.#cursor },\n });\n }\n return null;\n }\n\n // Check for Unquoted String\n const match = this.#matcher.exec(this.#buffer);\n if (match) {\n // If we're flushing and the match doesn't consume the entire buffer,\n // then return null\n if (this.#flush === false && match[0].length === this.#buffer.length) {\n return null;\n }\n const value = match[1];\n this.#buffer = this.#buffer.slice(value.length);\n const start: Position = { ...this.#cursor };\n this.#cursor.column += value.length;\n this.#cursor.offset += value.length;\n return {\n type: Field,\n value,\n location: {\n start,\n end: { ...this.#cursor },\n rowNumber: this.#rowNumber,\n },\n };\n }\n\n // Otherwise, return null\n return null;\n }\n}\n"],"names":[],"mappings":";;;;;;AAiBO,MAAM,KAAM,CAAA;AAAA,EACjB,UAAA,CAAA;AAAA,EACA,UAAA,CAAA;AAAA,EACA,OAAU,GAAA,EAAA,CAAA;AAAA,EACV,MAAS,GAAA,KAAA,CAAA;AAAA,EACT,QAAA,CAAA;AAAA,EACA,qBAAA,CAAA;AAAA,EAEA,OAAoB,GAAA;AAAA,IAClB,IAAM,EAAA,CAAA;AAAA,IACN,MAAQ,EAAA,CAAA;AAAA,IACR,MAAQ,EAAA,CAAA;AAAA,GACV,CAAA;AAAA,EACA,UAAa,GAAA,CAAA,CAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAMb,WAAY,CAAA;AAAA,IACV,SAAY,GAAA,KAAA;AAAA,IACZ,SAAY,GAAA,YAAA;AAAA,GACd,GAAmB,EAAI,EAAA;AACrB,IAAoB,mBAAA,CAAA,EAAE,SAAW,EAAA,SAAA,EAAW,CAAA,CAAA;AAC5C,IAAA,IAAA,CAAK,UAAa,GAAA,SAAA,CAAA;AAClB,IAAA,IAAA,CAAK,UAAa,GAAA,SAAA,CAAA;AAClB,IAAA,IAAA,CAAK,wBAAwB,SAAU,CAAA,MAAA,CAAA;AACvC,IAAM,MAAA,CAAA,GAAI,aAAa,SAAS,CAAA,CAAA;AAChC,IAAM,MAAA,CAAA,GAAI,aAAa,SAAS,CAAA,CAAA;AAChC,IAAA,IAAA,CAAK,WAAW,IAAI,MAAA;AAAA,MAClB,UAAU,CAAC,CAAA,IAAA,EAAO,CAAC,CAA4C,yCAAA,EAAA,CAAC,IAAI,CAAC,CAAA,WAAA,CAAA;AAAA,KACvE,CAAA;AAAA,GACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAQO,GAAA,CAAI,KAAsB,EAAA,SAAA,GAAY,KAAgC,EAAA;AAC3E,IAAA,IAAI,CAAC,SAAW,EAAA;AACd,MAAA,IAAA,CAAK,MAAS,GAAA,IAAA,CAAA;AAAA,KAChB;AACA,IAAA,IAAI,OAAO,KAAA,KAAU,QAAY,IAAA,KAAA,CAAM,WAAW,CAAG,EAAA;AACnD,MAAA,IAAA,CAAK,OAAW,IAAA,KAAA,CAAA;AAAA,KAClB;AAEA,IAAA,OAAO,KAAK,OAAQ,EAAA,CAAA;AAAA,GACtB;AAAA;AAAA;AAAA;AAAA;AAAA,EAMO,KAAiB,GAAA;AACtB,IAAA,IAAA,CAAK,MAAS,GAAA,IAAA,CAAA;AACd,IAAA,OAAO,CAAC,GAAG,IAAK,CAAA,OAAA,EAAS,CAAA,CAAA;AAAA,GAC3B;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,CAAC,OAA4B,GAAA;AAC3B,IAAA,IAAI,KAAK,MAAQ,EAAA;AAEf,MAAA,IAAI,IAAK,CAAA,OAAA,CAAQ,QAAS,CAAA,IAAI,CAAG,EAAA;AAC/B,QAAK,IAAA,CAAA,OAAA,GAAU,KAAK,OAAQ,CAAA,KAAA;AAAA,UAAM,CAAA;AAAA,UAAG,CAAA,CAAA;AAAA;AAAA,SAAqB,CAAA;AAAA,OACjD,MAAA,IAAA,IAAA,CAAK,OAAQ,CAAA,QAAA,CAAS,EAAE,CAAG,EAAA;AACpC,QAAK,IAAA,CAAA,OAAA,GAAU,KAAK,OAAQ,CAAA,KAAA;AAAA,UAAM,CAAA;AAAA,UAAG,CAAA,CAAA;AAAA;AAAA,SAAmB,CAAA;AAAA,OAC1D;AAAA,KACF;AACA,IAAI,IAAA,KAAA,CAAA;AACJ,IAAQ,OAAA,KAAA,GAAQ,IAAK,CAAA,UAAA,EAAe,EAAA;AAClC,MAAM,MAAA,KAAA,CAAA;AAAA,KACR;AAAA,GACF;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,UAA2B,GAAA;AACzB,IAAI,IAAA,IAAA,CAAK,OAAQ,CAAA,MAAA,KAAW,CAAG,EAAA;AAC7B,MAAO,OAAA,IAAA,CAAA;AAAA,KACT;AAEA,IACE,IAAA,IAAA,CAAK,WAAW,KACf,KAAA,IAAA,CAAK,YAAY,IAAQ,IAAA,IAAA,CAAK,YAAY,EAC3C,CAAA,EAAA;AACA,MAAO,OAAA,IAAA,CAAA;AAAA,KACT;AAGA,IAAA,IAAI,IAAK,CAAA,OAAA,CAAQ,UAAW,CAAA,IAAI,CAAG,EAAA;AACjC,MAAA,IAAA,CAAK,OAAU,GAAA,IAAA,CAAK,OAAQ,CAAA,KAAA,CAAM,CAAC,CAAA,CAAA;AACnC,MAAA,MAAM,KAAkB,GAAA,EAAE,GAAG,IAAA,CAAK,OAAQ,EAAA,CAAA;AAC1C,MAAA,IAAA,CAAK,OAAQ,CAAA,IAAA,EAAA,CAAA;AACb,MAAA,IAAA,CAAK,QAAQ,MAAS,GAAA,CAAA,CAAA;AACtB,MAAA,IAAA,CAAK,QAAQ,MAAU,IAAA,CAAA,CAAA;AACvB,MAAA,MAAM,KAA8B,GAAA;AAAA,QAClC,IAAM,EAAA,eAAA;AAAA,QACN,KAAO,EAAA,IAAA;AAAA,QACP,QAAU,EAAA;AAAA,UACR,KAAA;AAAA,UACA,GAAK,EAAA,EAAE,GAAG,IAAA,CAAK,OAAQ,EAAA;AAAA,UACvB,WAAW,IAAK,CAAA,UAAA,EAAA;AAAA,SAClB;AAAA,OACF,CAAA;AACA,MAAO,OAAA,KAAA,CAAA;AAAA,KACT;AAGA,IAAA,IAAI,IAAK,CAAA,OAAA,CAAQ,UAAW,CAAA,EAAE,CAAG,EAAA;AAC/B,MAAA,IAAA,CAAK,OAAU,GAAA,IAAA,CAAK,OAAQ,CAAA,KAAA,CAAM,CAAC,CAAA,CAAA;AACnC,MAAA,MAAM,KAAkB,GAAA,EAAE,GAAG,IAAA,CAAK,OAAQ,EAAA,CAAA;AAC1C,MAAA,IAAA,CAAK,OAAQ,CAAA,IAAA,EAAA,CAAA;AACb,MAAA,IAAA,CAAK,QAAQ,MAAS,GAAA,CAAA,CAAA;AACtB,MAAA,IAAA,CAAK,QAAQ,MAAU,IAAA,CAAA,CAAA;AACvB,MAAA,MAAM,KAA8B,GAAA;AAAA,QAClC,IAAM,EAAA,eAAA;AAAA,QACN,KAAO,EAAA,EAAA;AAAA,QACP,QAAU,EAAA;AAAA,UACR,KAAA;AAAA,UACA,GAAK,EAAA,EAAE,GAAG,IAAA,CAAK,OAAQ,EAAA;AAAA,UACvB,WAAW,IAAK,CAAA,UAAA,EAAA;AAAA,SAClB;AAAA,OACF,CAAA;AACA,MAAO,OAAA,KAAA,CAAA;AAAA,KACT;AAGA,IAAA,IAAI,IAAK,CAAA,OAAA,CAAQ,UAAW,CAAA,IAAA,CAAK,UAAU,CAAG,EAAA;AAC5C,MAAA,IAAA,CAAK,OAAU,GAAA,IAAA,CAAK,OAAQ,CAAA,KAAA,CAAM,CAAC,CAAA,CAAA;AACnC,MAAA,MAAM,KAAkB,GAAA,EAAE,GAAG,IAAA,CAAK,OAAQ,EAAA,CAAA;AAC1C,MAAK,IAAA,CAAA,OAAA,CAAQ,UAAU,IAAK,CAAA,qBAAA,CAAA;AAC5B,MAAK,IAAA,CAAA,OAAA,CAAQ,UAAU,IAAK,CAAA,qBAAA,CAAA;AAC5B,MAAO,OAAA;AAAA,QACL,IAAM,EAAA,cAAA;AAAA,QACN,OAAO,IAAK,CAAA,UAAA;AAAA,QACZ,QAAU,EAAA;AAAA,UACR,KAAA;AAAA,UACA,GAAK,EAAA,EAAE,GAAG,IAAA,CAAK,OAAQ,EAAA;AAAA,UACvB,WAAW,IAAK,CAAA,UAAA;AAAA,SAClB;AAAA,OACF,CAAA;AAAA,KACF;AAGA,IAAA,IAAI,IAAK,CAAA,OAAA,CAAQ,UAAW,CAAA,IAAA,CAAK,UAAU,CAAG,EAAA;AAqB5C,MAAA,IAAI,KAAQ,GAAA,EAAA,CAAA;AACZ,MAAA,IAAI,MAAS,GAAA,CAAA,CAAA;AACb,MAAA,IAAI,MAAS,GAAA,CAAA,CAAA;AACb,MAAA,IAAI,IAAO,GAAA,CAAA,CAAA;AAGX,MAAI,IAAA,GAAA,GAAc,IAAK,CAAA,OAAA,CAAQ,MAAM,CAAA,CAAA;AACrC,MAAA,IAAI,IAA2B,GAAA,IAAA,CAAK,OAAQ,CAAA,MAAA,GAAS,CAAC,CAAA,CAAA;AACtD,MAAG,GAAA;AAED,QAAI,IAAA,GAAA,KAAQ,KAAK,UAAY,EAAA;AAG3B,UAAI,IAAA,IAAA,KAAS,KAAK,UAAY,EAAA;AAE5B,YAAA,KAAA,IAAS,IAAK,CAAA,UAAA,CAAA;AACd,YAAU,MAAA,IAAA,CAAA,CAAA;AACV,YAAM,GAAA,GAAA,IAAA,CAAK,QAAQ,MAAM,CAAA,CAAA;AACzB,YAAO,IAAA,GAAA,IAAA,CAAK,OAAQ,CAAA,MAAA,GAAS,CAAC,CAAA,CAAA;AAG9B,YAAU,MAAA,IAAA,CAAA,CAAA;AACV,YAAA,SAAA;AAAA,WACF;AAIA,UAAA,IAAI,IAAS,KAAA,KAAA,CAAA,IAAa,IAAK,CAAA,MAAA,KAAW,KAAO,EAAA;AAC/C,YAAO,OAAA,IAAA,CAAA;AAAA,WACT;AAIA,UAAA,MAAA,EAAA,CAAA;AACA,UAAA,IAAA,CAAK,OAAU,GAAA,IAAA,CAAK,OAAQ,CAAA,KAAA,CAAM,MAAM,CAAA,CAAA;AACxC,UAAA,MAAM,KAAkB,GAAA,EAAE,GAAG,IAAA,CAAK,OAAQ,EAAA,CAAA;AAC1C,UAAA,IAAA,CAAK,QAAQ,MAAU,IAAA,MAAA,CAAA;AACvB,UAAA,IAAA,CAAK,QAAQ,MAAU,IAAA,MAAA,CAAA;AACvB,UAAA,IAAA,CAAK,QAAQ,IAAQ,IAAA,IAAA,CAAA;AACrB,UAAO,OAAA;AAAA,YACL,IAAM,EAAA,KAAA;AAAA,YACN,KAAA;AAAA,YACA,QAAU,EAAA;AAAA,cACR,KAAA;AAAA,cACA,GAAK,EAAA,EAAE,GAAG,IAAA,CAAK,OAAQ,EAAA;AAAA,cACvB,WAAW,IAAK,CAAA,UAAA;AAAA,aAClB;AAAA,WACF,CAAA;AAAA,SACF;AAGA,QAAS,KAAA,IAAA,GAAA,CAAA;AAGT,QAAA,IAAI,QAAQ,EAAI,EAAA;AAGd,UAAA,IAAA,EAAA,CAAA;AACA,UAAS,MAAA,GAAA,CAAA,CAAA;AAAA,SACJ,MAAA;AAEL,UAAA,MAAA,EAAA,CAAA;AAAA,SACF;AAEA,QAAA,MAAA,EAAA,CAAA;AACA,QAAM,GAAA,GAAA,IAAA,CAAA;AACN,QAAO,IAAA,GAAA,IAAA,CAAK,OAAQ,CAAA,MAAA,GAAS,CAAC,CAAA,CAAA;AAAA,eACvB,GAAQ,KAAA,KAAA,CAAA,EAAA;AAEjB,MAAA,IAAI,KAAK,MAAQ,EAAA;AACf,QAAM,MAAA,IAAI,WAAW,4CAA8C,EAAA;AAAA,UACjE,QAAU,EAAA,EAAE,GAAG,IAAA,CAAK,OAAQ,EAAA;AAAA,SAC7B,CAAA,CAAA;AAAA,OACH;AACA,MAAO,OAAA,IAAA,CAAA;AAAA,KACT;AAGA,IAAA,MAAM,KAAQ,GAAA,IAAA,CAAK,QAAS,CAAA,IAAA,CAAK,KAAK,OAAO,CAAA,CAAA;AAC7C,IAAA,IAAI,KAAO,EAAA;AAGT,MAAI,IAAA,IAAA,CAAK,WAAW,KAAS,IAAA,KAAA,CAAM,CAAC,CAAE,CAAA,MAAA,KAAW,IAAK,CAAA,OAAA,CAAQ,MAAQ,EAAA;AACpE,QAAO,OAAA,IAAA,CAAA;AAAA,OACT;AACA,MAAM,MAAA,KAAA,GAAQ,MAAM,CAAC,CAAA,CAAA;AACrB,MAAA,IAAA,CAAK,OAAU,GAAA,IAAA,CAAK,OAAQ,CAAA,KAAA,CAAM,MAAM,MAAM,CAAA,CAAA;AAC9C,MAAA,MAAM,KAAkB,GAAA,EAAE,GAAG,IAAA,CAAK,OAAQ,EAAA,CAAA;AAC1C,MAAK,IAAA,CAAA,OAAA,CAAQ,UAAU,KAAM,CAAA,MAAA,CAAA;AAC7B,MAAK,IAAA,CAAA,OAAA,CAAQ,UAAU,KAAM,CAAA,MAAA,CAAA;AAC7B,MAAO,OAAA;AAAA,QACL,IAAM,EAAA,KAAA;AAAA,QACN,KAAA;AAAA,QACA,QAAU,EAAA;AAAA,UACR,KAAA;AAAA,UACA,GAAK,EAAA,EAAE,GAAG,IAAA,CAAK,OAAQ,EAAA;AAAA,UACvB,WAAW,IAAK,CAAA,UAAA;AAAA,SAClB;AAAA,OACF,CAAA;AAAA,KACF;AAGA,IAAO,OAAA,IAAA,CAAA;AAAA,GACT;AACF;;;;"}
|
|
1
|
+
{"version":3,"file":"Lexer.js","sources":["../../src/Lexer.ts"],"sourcesContent":["import { assertCommonOptions } from \"./assertCommonOptions.ts\";\nimport { Field, FieldDelimiter, RecordDelimiter } from \"./common/constants.ts\";\nimport { ParseError } from \"./common/errors.ts\";\nimport type {\n AbortSignalOptions,\n CommonOptions,\n Position,\n RecordDelimiterToken,\n Token,\n} from \"./common/types.ts\";\nimport { COMMA, CRLF, DOUBLE_QUOTE, LF } from \"./constants.ts\";\nimport { escapeRegExp } from \"./utils/escapeRegExp.ts\";\n\n/**\n * CSV Lexer.\n *\n * Lexter tokenizes CSV data into fields and records.\n */\nexport class Lexer {\n #delimiter: string;\n #quotation: string;\n #buffer = \"\";\n #flush = false;\n #matcher: RegExp;\n #fieldDelimiterLength: number;\n\n #cursor: Position = {\n line: 1,\n column: 1,\n offset: 0,\n };\n #rowNumber = 1;\n\n #signal?: AbortSignal;\n\n /**\n * Constructs a new Lexer instance.\n * @param options - The common options for the lexer.\n */\n constructor({\n delimiter = COMMA,\n quotation = DOUBLE_QUOTE,\n signal,\n }: CommonOptions & AbortSignalOptions = {}) {\n assertCommonOptions({ delimiter, quotation });\n this.#delimiter = delimiter;\n this.#quotation = quotation;\n this.#fieldDelimiterLength = delimiter.length;\n const d = escapeRegExp(delimiter);\n const q = escapeRegExp(quotation);\n this.#matcher = new RegExp(\n `^(?:(?!${q})(?!${d})(?![\\\\r\\\\n]))([\\\\S\\\\s\\\\uFEFF\\\\xA0]+?)(?=${q}|${d}|\\\\r|\\\\n|$)`,\n );\n if (signal) {\n this.#signal = signal;\n }\n }\n\n /**\n * Lexes the given chunk of CSV data.\n * @param chunk - The chunk of CSV data to be lexed.\n * @param buffering - Indicates whether the lexer is buffering or not.\n * @returns An iterable iterator of tokens.\n */\n public lex(chunk: string | null, buffering = false): IterableIterator<Token> {\n if (!buffering) {\n this.#flush = true;\n }\n if (typeof chunk === \"string\" && chunk.length !== 0) {\n this.#buffer += chunk;\n }\n\n return this.#tokens();\n }\n\n /**\n * Flushes the lexer and returns any remaining tokens.\n * @returns An array of tokens.\n */\n public flush(): Token[] {\n this.#flush = true;\n return [...this.#tokens()];\n }\n\n /**\n * Generates tokens from the buffered CSV data.\n * @yields Tokens from the buffered CSV data.\n */\n *#tokens(): Generator<Token> {\n if (this.#flush) {\n // Trim the last CRLF or LF\n if (this.#buffer.endsWith(CRLF)) {\n this.#buffer = this.#buffer.slice(0, -2 /* -CRLF.length */);\n } else if (this.#buffer.endsWith(LF)) {\n this.#buffer = this.#buffer.slice(0, -1 /* -LF.length */);\n }\n }\n let token: Token | null;\n while ((token = this.#nextToken())) {\n yield token;\n }\n }\n\n /**\n * Retrieves the next token from the buffered CSV data.\n * @returns The next token or null if there are no more tokens.\n */\n #nextToken(): Token | null {\n this.#signal?.throwIfAborted();\n if (this.#buffer.length === 0) {\n return null;\n }\n // Buffer is Record Delimiter, defer to the next iteration.\n if (\n this.#flush === false &&\n (this.#buffer === CRLF || this.#buffer === LF)\n ) {\n return null;\n }\n\n // Check for CRLF\n if (this.#buffer.startsWith(CRLF)) {\n this.#buffer = this.#buffer.slice(2);\n const start: Position = { ...this.#cursor };\n this.#cursor.line++;\n this.#cursor.column = 1;\n this.#cursor.offset += 2; // CRLF.length\n const token: RecordDelimiterToken = {\n type: RecordDelimiter,\n value: CRLF,\n location: {\n start,\n end: { ...this.#cursor },\n rowNumber: this.#rowNumber++,\n },\n };\n return token;\n }\n\n // Check for LF\n if (this.#buffer.startsWith(LF)) {\n this.#buffer = this.#buffer.slice(1);\n const start: Position = { ...this.#cursor };\n this.#cursor.line++;\n this.#cursor.column = 1;\n this.#cursor.offset += 1; // LF.length\n const token: RecordDelimiterToken = {\n type: RecordDelimiter,\n value: LF,\n location: {\n start,\n end: { ...this.#cursor },\n rowNumber: this.#rowNumber++,\n },\n };\n return token;\n }\n\n // Check for Delimiter\n if (this.#buffer.startsWith(this.#delimiter)) {\n this.#buffer = this.#buffer.slice(1);\n const start: Position = { ...this.#cursor };\n this.#cursor.column += this.#fieldDelimiterLength;\n this.#cursor.offset += this.#fieldDelimiterLength;\n return {\n type: FieldDelimiter,\n value: this.#delimiter,\n location: {\n start,\n end: { ...this.#cursor },\n rowNumber: this.#rowNumber,\n },\n };\n }\n\n // Check for Quoted String\n if (this.#buffer.startsWith(this.#quotation)) {\n /**\n * Extract Quoted field.\n *\n * The following code is equivalent to the following:\n *\n * If the next character is a quote:\n * - If the character after that is a quote, then append a quote to the value and skip two characters.\n * - Otherwise, return the quoted string.\n * Otherwise, append the character to the value and skip one character.\n *\n * ```plaintext\n * | `i` | `i + 1` | `i + 2` |\n * |------------|------------|----------|\n * | cur | next | | => Variable names\n * | #quotation | #quotation | | => Escaped quote\n * | #quotation | (EOF) | | => Closing quote\n * | #quotation | undefined | | => End of buffer\n * | undefined | | | => End of buffer\n * ```\n */\n let value = \"\";\n let offset = 1; // Skip the opening quote\n let column = 2; // Skip the opening quote\n let line = 0;\n\n // Define variables\n let cur: string = this.#buffer[offset];\n let next: string | undefined = this.#buffer[offset + 1];\n do {\n // If the current character is a quote, check the next characters for closing quotes.\n if (cur === this.#quotation) {\n // If the cur character is a quote and the next character is a quote,\n // then append a quote to the value and skip two characters.\n if (next === this.#quotation) {\n // Append a quote to the value and skip two characters.\n value += this.#quotation;\n offset += 2;\n cur = this.#buffer[offset];\n next = this.#buffer[offset + 1];\n\n // Update the diff\n column += 2;\n continue;\n }\n\n // If the cur character is a quote and the next character is undefined,\n // then return null.\n if (next === undefined && this.#flush === false) {\n return null;\n }\n\n // Otherwise, return the quoted string.\n // Update the buffer and return the token\n offset++;\n this.#buffer = this.#buffer.slice(offset);\n const start: Position = { ...this.#cursor };\n this.#cursor.column += column;\n this.#cursor.offset += offset;\n this.#cursor.line += line;\n return {\n type: Field,\n value,\n location: {\n start,\n end: { ...this.#cursor },\n rowNumber: this.#rowNumber,\n },\n };\n }\n\n // Append the character to the value.\n value += cur;\n\n // Prepare for the next iteration\n if (cur === LF) {\n // If the current character is a LF,\n // then increment the line number and reset the column number.\n line++;\n column = 1;\n } else {\n // Otherwise, increment the column number and offset.\n column++;\n }\n\n offset++;\n cur = next;\n next = this.#buffer[offset + 1];\n } while (cur !== undefined);\n\n if (this.#flush) {\n throw new ParseError(\"Unexpected EOF while parsing quoted field.\", {\n position: { ...this.#cursor },\n });\n }\n return null;\n }\n\n // Check for Unquoted String\n const match = this.#matcher.exec(this.#buffer);\n if (match) {\n // If we're flushing and the match doesn't consume the entire buffer,\n // then return null\n if (this.#flush === false && match[0].length === this.#buffer.length) {\n return null;\n }\n const value = match[1];\n this.#buffer = this.#buffer.slice(value.length);\n const start: Position = { ...this.#cursor };\n this.#cursor.column += value.length;\n this.#cursor.offset += value.length;\n return {\n type: Field,\n value,\n location: {\n start,\n end: { ...this.#cursor },\n rowNumber: this.#rowNumber,\n },\n };\n }\n\n // Otherwise, return null\n return null;\n }\n}\n"],"names":[],"mappings":";;;;;;AAkBO,MAAM,KAAM,CAAA;AAAA,EACjB,UAAA,CAAA;AAAA,EACA,UAAA,CAAA;AAAA,EACA,OAAU,GAAA,EAAA,CAAA;AAAA,EACV,MAAS,GAAA,KAAA,CAAA;AAAA,EACT,QAAA,CAAA;AAAA,EACA,qBAAA,CAAA;AAAA,EAEA,OAAoB,GAAA;AAAA,IAClB,IAAM,EAAA,CAAA;AAAA,IACN,MAAQ,EAAA,CAAA;AAAA,IACR,MAAQ,EAAA,CAAA;AAAA,GACV,CAAA;AAAA,EACA,UAAa,GAAA,CAAA,CAAA;AAAA,EAEb,OAAA,CAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,WAAY,CAAA;AAAA,IACV,SAAY,GAAA,KAAA;AAAA,IACZ,SAAY,GAAA,YAAA;AAAA,IACZ,MAAA;AAAA,GACF,GAAwC,EAAI,EAAA;AAC1C,IAAoB,mBAAA,CAAA,EAAE,SAAW,EAAA,SAAA,EAAW,CAAA,CAAA;AAC5C,IAAA,IAAA,CAAK,UAAa,GAAA,SAAA,CAAA;AAClB,IAAA,IAAA,CAAK,UAAa,GAAA,SAAA,CAAA;AAClB,IAAA,IAAA,CAAK,wBAAwB,SAAU,CAAA,MAAA,CAAA;AACvC,IAAM,MAAA,CAAA,GAAI,aAAa,SAAS,CAAA,CAAA;AAChC,IAAM,MAAA,CAAA,GAAI,aAAa,SAAS,CAAA,CAAA;AAChC,IAAA,IAAA,CAAK,WAAW,IAAI,MAAA;AAAA,MAClB,UAAU,CAAC,CAAA,IAAA,EAAO,CAAC,CAA4C,yCAAA,EAAA,CAAC,IAAI,CAAC,CAAA,WAAA,CAAA;AAAA,KACvE,CAAA;AACA,IAAA,IAAI,MAAQ,EAAA;AACV,MAAA,IAAA,CAAK,OAAU,GAAA,MAAA,CAAA;AAAA,KACjB;AAAA,GACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAQO,GAAA,CAAI,KAAsB,EAAA,SAAA,GAAY,KAAgC,EAAA;AAC3E,IAAA,IAAI,CAAC,SAAW,EAAA;AACd,MAAA,IAAA,CAAK,MAAS,GAAA,IAAA,CAAA;AAAA,KAChB;AACA,IAAA,IAAI,OAAO,KAAA,KAAU,QAAY,IAAA,KAAA,CAAM,WAAW,CAAG,EAAA;AACnD,MAAA,IAAA,CAAK,OAAW,IAAA,KAAA,CAAA;AAAA,KAClB;AAEA,IAAA,OAAO,KAAK,OAAQ,EAAA,CAAA;AAAA,GACtB;AAAA;AAAA;AAAA;AAAA;AAAA,EAMO,KAAiB,GAAA;AACtB,IAAA,IAAA,CAAK,MAAS,GAAA,IAAA,CAAA;AACd,IAAA,OAAO,CAAC,GAAG,IAAK,CAAA,OAAA,EAAS,CAAA,CAAA;AAAA,GAC3B;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,CAAC,OAA4B,GAAA;AAC3B,IAAA,IAAI,KAAK,MAAQ,EAAA;AAEf,MAAA,IAAI,IAAK,CAAA,OAAA,CAAQ,QAAS,CAAA,IAAI,CAAG,EAAA;AAC/B,QAAK,IAAA,CAAA,OAAA,GAAU,KAAK,OAAQ,CAAA,KAAA;AAAA,UAAM,CAAA;AAAA,UAAG,CAAA,CAAA;AAAA;AAAA,SAAqB,CAAA;AAAA,OACjD,MAAA,IAAA,IAAA,CAAK,OAAQ,CAAA,QAAA,CAAS,EAAE,CAAG,EAAA;AACpC,QAAK,IAAA,CAAA,OAAA,GAAU,KAAK,OAAQ,CAAA,KAAA;AAAA,UAAM,CAAA;AAAA,UAAG,CAAA,CAAA;AAAA;AAAA,SAAmB,CAAA;AAAA,OAC1D;AAAA,KACF;AACA,IAAI,IAAA,KAAA,CAAA;AACJ,IAAQ,OAAA,KAAA,GAAQ,IAAK,CAAA,UAAA,EAAe,EAAA;AAClC,MAAM,MAAA,KAAA,CAAA;AAAA,KACR;AAAA,GACF;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,UAA2B,GAAA;AACzB,IAAA,IAAA,CAAK,SAAS,cAAe,EAAA,CAAA;AAC7B,IAAI,IAAA,IAAA,CAAK,OAAQ,CAAA,MAAA,KAAW,CAAG,EAAA;AAC7B,MAAO,OAAA,IAAA,CAAA;AAAA,KACT;AAEA,IACE,IAAA,IAAA,CAAK,WAAW,KACf,KAAA,IAAA,CAAK,YAAY,IAAQ,IAAA,IAAA,CAAK,YAAY,EAC3C,CAAA,EAAA;AACA,MAAO,OAAA,IAAA,CAAA;AAAA,KACT;AAGA,IAAA,IAAI,IAAK,CAAA,OAAA,CAAQ,UAAW,CAAA,IAAI,CAAG,EAAA;AACjC,MAAA,IAAA,CAAK,OAAU,GAAA,IAAA,CAAK,OAAQ,CAAA,KAAA,CAAM,CAAC,CAAA,CAAA;AACnC,MAAA,MAAM,KAAkB,GAAA,EAAE,GAAG,IAAA,CAAK,OAAQ,EAAA,CAAA;AAC1C,MAAA,IAAA,CAAK,OAAQ,CAAA,IAAA,EAAA,CAAA;AACb,MAAA,IAAA,CAAK,QAAQ,MAAS,GAAA,CAAA,CAAA;AACtB,MAAA,IAAA,CAAK,QAAQ,MAAU,IAAA,CAAA,CAAA;AACvB,MAAA,MAAM,KAA8B,GAAA;AAAA,QAClC,IAAM,EAAA,eAAA;AAAA,QACN,KAAO,EAAA,IAAA;AAAA,QACP,QAAU,EAAA;AAAA,UACR,KAAA;AAAA,UACA,GAAK,EAAA,EAAE,GAAG,IAAA,CAAK,OAAQ,EAAA;AAAA,UACvB,WAAW,IAAK,CAAA,UAAA,EAAA;AAAA,SAClB;AAAA,OACF,CAAA;AACA,MAAO,OAAA,KAAA,CAAA;AAAA,KACT;AAGA,IAAA,IAAI,IAAK,CAAA,OAAA,CAAQ,UAAW,CAAA,EAAE,CAAG,EAAA;AAC/B,MAAA,IAAA,CAAK,OAAU,GAAA,IAAA,CAAK,OAAQ,CAAA,KAAA,CAAM,CAAC,CAAA,CAAA;AACnC,MAAA,MAAM,KAAkB,GAAA,EAAE,GAAG,IAAA,CAAK,OAAQ,EAAA,CAAA;AAC1C,MAAA,IAAA,CAAK,OAAQ,CAAA,IAAA,EAAA,CAAA;AACb,MAAA,IAAA,CAAK,QAAQ,MAAS,GAAA,CAAA,CAAA;AACtB,MAAA,IAAA,CAAK,QAAQ,MAAU,IAAA,CAAA,CAAA;AACvB,MAAA,MAAM,KAA8B,GAAA;AAAA,QAClC,IAAM,EAAA,eAAA;AAAA,QACN,KAAO,EAAA,EAAA;AAAA,QACP,QAAU,EAAA;AAAA,UACR,KAAA;AAAA,UACA,GAAK,EAAA,EAAE,GAAG,IAAA,CAAK,OAAQ,EAAA;AAAA,UACvB,WAAW,IAAK,CAAA,UAAA,EAAA;AAAA,SAClB;AAAA,OACF,CAAA;AACA,MAAO,OAAA,KAAA,CAAA;AAAA,KACT;AAGA,IAAA,IAAI,IAAK,CAAA,OAAA,CAAQ,UAAW,CAAA,IAAA,CAAK,UAAU,CAAG,EAAA;AAC5C,MAAA,IAAA,CAAK,OAAU,GAAA,IAAA,CAAK,OAAQ,CAAA,KAAA,CAAM,CAAC,CAAA,CAAA;AACnC,MAAA,MAAM,KAAkB,GAAA,EAAE,GAAG,IAAA,CAAK,OAAQ,EAAA,CAAA;AAC1C,MAAK,IAAA,CAAA,OAAA,CAAQ,UAAU,IAAK,CAAA,qBAAA,CAAA;AAC5B,MAAK,IAAA,CAAA,OAAA,CAAQ,UAAU,IAAK,CAAA,qBAAA,CAAA;AAC5B,MAAO,OAAA;AAAA,QACL,IAAM,EAAA,cAAA;AAAA,QACN,OAAO,IAAK,CAAA,UAAA;AAAA,QACZ,QAAU,EAAA;AAAA,UACR,KAAA;AAAA,UACA,GAAK,EAAA,EAAE,GAAG,IAAA,CAAK,OAAQ,EAAA;AAAA,UACvB,WAAW,IAAK,CAAA,UAAA;AAAA,SAClB;AAAA,OACF,CAAA;AAAA,KACF;AAGA,IAAA,IAAI,IAAK,CAAA,OAAA,CAAQ,UAAW,CAAA,IAAA,CAAK,UAAU,CAAG,EAAA;AAqB5C,MAAA,IAAI,KAAQ,GAAA,EAAA,CAAA;AACZ,MAAA,IAAI,MAAS,GAAA,CAAA,CAAA;AACb,MAAA,IAAI,MAAS,GAAA,CAAA,CAAA;AACb,MAAA,IAAI,IAAO,GAAA,CAAA,CAAA;AAGX,MAAI,IAAA,GAAA,GAAc,IAAK,CAAA,OAAA,CAAQ,MAAM,CAAA,CAAA;AACrC,MAAA,IAAI,IAA2B,GAAA,IAAA,CAAK,OAAQ,CAAA,MAAA,GAAS,CAAC,CAAA,CAAA;AACtD,MAAG,GAAA;AAED,QAAI,IAAA,GAAA,KAAQ,KAAK,UAAY,EAAA;AAG3B,UAAI,IAAA,IAAA,KAAS,KAAK,UAAY,EAAA;AAE5B,YAAA,KAAA,IAAS,IAAK,CAAA,UAAA,CAAA;AACd,YAAU,MAAA,IAAA,CAAA,CAAA;AACV,YAAM,GAAA,GAAA,IAAA,CAAK,QAAQ,MAAM,CAAA,CAAA;AACzB,YAAO,IAAA,GAAA,IAAA,CAAK,OAAQ,CAAA,MAAA,GAAS,CAAC,CAAA,CAAA;AAG9B,YAAU,MAAA,IAAA,CAAA,CAAA;AACV,YAAA,SAAA;AAAA,WACF;AAIA,UAAA,IAAI,IAAS,KAAA,KAAA,CAAA,IAAa,IAAK,CAAA,MAAA,KAAW,KAAO,EAAA;AAC/C,YAAO,OAAA,IAAA,CAAA;AAAA,WACT;AAIA,UAAA,MAAA,EAAA,CAAA;AACA,UAAA,IAAA,CAAK,OAAU,GAAA,IAAA,CAAK,OAAQ,CAAA,KAAA,CAAM,MAAM,CAAA,CAAA;AACxC,UAAA,MAAM,KAAkB,GAAA,EAAE,GAAG,IAAA,CAAK,OAAQ,EAAA,CAAA;AAC1C,UAAA,IAAA,CAAK,QAAQ,MAAU,IAAA,MAAA,CAAA;AACvB,UAAA,IAAA,CAAK,QAAQ,MAAU,IAAA,MAAA,CAAA;AACvB,UAAA,IAAA,CAAK,QAAQ,IAAQ,IAAA,IAAA,CAAA;AACrB,UAAO,OAAA;AAAA,YACL,IAAM,EAAA,KAAA;AAAA,YACN,KAAA;AAAA,YACA,QAAU,EAAA;AAAA,cACR,KAAA;AAAA,cACA,GAAK,EAAA,EAAE,GAAG,IAAA,CAAK,OAAQ,EAAA;AAAA,cACvB,WAAW,IAAK,CAAA,UAAA;AAAA,aAClB;AAAA,WACF,CAAA;AAAA,SACF;AAGA,QAAS,KAAA,IAAA,GAAA,CAAA;AAGT,QAAA,IAAI,QAAQ,EAAI,EAAA;AAGd,UAAA,IAAA,EAAA,CAAA;AACA,UAAS,MAAA,GAAA,CAAA,CAAA;AAAA,SACJ,MAAA;AAEL,UAAA,MAAA,EAAA,CAAA;AAAA,SACF;AAEA,QAAA,MAAA,EAAA,CAAA;AACA,QAAM,GAAA,GAAA,IAAA,CAAA;AACN,QAAO,IAAA,GAAA,IAAA,CAAK,OAAQ,CAAA,MAAA,GAAS,CAAC,CAAA,CAAA;AAAA,eACvB,GAAQ,KAAA,KAAA,CAAA,EAAA;AAEjB,MAAA,IAAI,KAAK,MAAQ,EAAA;AACf,QAAM,MAAA,IAAI,WAAW,4CAA8C,EAAA;AAAA,UACjE,QAAU,EAAA,EAAE,GAAG,IAAA,CAAK,OAAQ,EAAA;AAAA,SAC7B,CAAA,CAAA;AAAA,OACH;AACA,MAAO,OAAA,IAAA,CAAA;AAAA,KACT;AAGA,IAAA,MAAM,KAAQ,GAAA,IAAA,CAAK,QAAS,CAAA,IAAA,CAAK,KAAK,OAAO,CAAA,CAAA;AAC7C,IAAA,IAAI,KAAO,EAAA;AAGT,MAAI,IAAA,IAAA,CAAK,WAAW,KAAS,IAAA,KAAA,CAAM,CAAC,CAAE,CAAA,MAAA,KAAW,IAAK,CAAA,OAAA,CAAQ,MAAQ,EAAA;AACpE,QAAO,OAAA,IAAA,CAAA;AAAA,OACT;AACA,MAAM,MAAA,KAAA,GAAQ,MAAM,CAAC,CAAA,CAAA;AACrB,MAAA,IAAA,CAAK,OAAU,GAAA,IAAA,CAAK,OAAQ,CAAA,KAAA,CAAM,MAAM,MAAM,CAAA,CAAA;AAC9C,MAAA,MAAM,KAAkB,GAAA,EAAE,GAAG,IAAA,CAAK,OAAQ,EAAA,CAAA;AAC1C,MAAK,IAAA,CAAA,OAAA,CAAQ,UAAU,KAAM,CAAA,MAAA,CAAA;AAC7B,MAAK,IAAA,CAAA,OAAA,CAAQ,UAAU,KAAM,CAAA,MAAA,CAAA;AAC7B,MAAO,OAAA;AAAA,QACL,IAAM,EAAA,KAAA;AAAA,QACN,KAAA;AAAA,QACA,QAAU,EAAA;AAAA,UACR,KAAA;AAAA,UACA,GAAK,EAAA,EAAE,GAAG,IAAA,CAAK,OAAQ,EAAA;AAAA,UACvB,WAAW,IAAK,CAAA,UAAA;AAAA,SAClB;AAAA,OACF,CAAA;AAAA,KACF;AAGA,IAAO,OAAA,IAAA,CAAA;AAAA,GACT;AACF;;;;"}
|
|
@@ -6,13 +6,18 @@ class RecordAssembler {
|
|
|
6
6
|
#row = [];
|
|
7
7
|
#header;
|
|
8
8
|
#dirty = false;
|
|
9
|
+
#signal;
|
|
9
10
|
constructor(options = {}) {
|
|
10
11
|
if (options.header !== void 0 && Array.isArray(options.header)) {
|
|
11
12
|
this.#setHeader(options.header);
|
|
12
13
|
}
|
|
14
|
+
if (options.signal) {
|
|
15
|
+
this.#signal = options.signal;
|
|
16
|
+
}
|
|
13
17
|
}
|
|
14
18
|
*assemble(tokens, flush = true) {
|
|
15
19
|
for (const token of tokens) {
|
|
20
|
+
this.#signal?.throwIfAborted();
|
|
16
21
|
switch (token.type) {
|
|
17
22
|
case FieldDelimiter:
|
|
18
23
|
this.#fieldIndex++;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"RecordAssembler.js","sources":["../../src/RecordAssembler.ts"],"sourcesContent":["import { FieldDelimiter, RecordDelimiter } from \"./common/constants.ts\";\nimport { ParseError } from \"./common/errors.ts\";\nimport type {\n CSVRecord,\n RecordAssemblerOptions,\n Token,\n} from \"./common/types.ts\";\n\nexport class RecordAssembler<Header extends ReadonlyArray<string>> {\n #fieldIndex = 0;\n #row: string[] = [];\n #header: Header | undefined;\n #dirty = false;\n\n constructor(options: RecordAssemblerOptions<Header> = {}) {\n if (options.header !== undefined && Array.isArray(options.header)) {\n this.#setHeader(options.header);\n }\n }\n\n public *assemble(\n tokens: Iterable<Token>,\n flush = true,\n ): IterableIterator<CSVRecord<Header>> {\n for (const token of tokens) {\n switch (token.type) {\n case FieldDelimiter:\n this.#fieldIndex++;\n this.#dirty = true;\n break;\n case RecordDelimiter:\n if (this.#header === undefined) {\n this.#setHeader(this.#row as unknown as Header);\n } else {\n if (this.#dirty) {\n yield Object.fromEntries(\n this.#header.map((header, index) => [\n header,\n this.#row.at(index),\n ]),\n ) as unknown as CSVRecord<Header>;\n } else {\n yield Object.fromEntries(\n this.#header.map((header) => [header, \"\"]),\n ) as CSVRecord<Header>;\n }\n }\n // Reset the row fields buffer.\n this.#fieldIndex = 0;\n this.#row = new Array(this.#header?.length).fill(\"\");\n this.#dirty = false;\n break;\n default:\n this.#dirty = true;\n this.#row[this.#fieldIndex] = token.value;\n break;\n }\n }\n\n if (flush) {\n yield* this.flush();\n }\n }\n\n public *flush(): Generator<CSVRecord<Header>> {\n if (this.#header !== undefined) {\n if (this.#dirty) {\n yield Object.fromEntries(\n this.#header\n .filter((v) => v)\n .map((header, index) => [header, this.#row.at(index)]),\n ) as unknown as CSVRecord<Header>;\n }\n }\n }\n\n #setHeader(header: Header) {\n this.#header = header;\n if (this.#header.length === 0) {\n throw new ParseError(\"The header must not be empty.\");\n }\n if (new Set(this.#header).size !== this.#header.length) {\n throw new ParseError(\"The header must not contain duplicate fields.\");\n }\n }\n}\n"],"names":[],"mappings":";;;AAQO,MAAM,eAAsD,CAAA;AAAA,EACjE,WAAc,GAAA,CAAA,CAAA;AAAA,EACd,OAAiB,EAAC,CAAA;AAAA,EAClB,OAAA,CAAA;AAAA,EACA,MAAS,GAAA,KAAA,CAAA;AAAA,
|
|
1
|
+
{"version":3,"file":"RecordAssembler.js","sources":["../../src/RecordAssembler.ts"],"sourcesContent":["import { FieldDelimiter, RecordDelimiter } from \"./common/constants.ts\";\nimport { ParseError } from \"./common/errors.ts\";\nimport type {\n CSVRecord,\n RecordAssemblerOptions,\n Token,\n} from \"./common/types.ts\";\n\nexport class RecordAssembler<Header extends ReadonlyArray<string>> {\n #fieldIndex = 0;\n #row: string[] = [];\n #header: Header | undefined;\n #dirty = false;\n #signal?: AbortSignal;\n\n constructor(options: RecordAssemblerOptions<Header> = {}) {\n if (options.header !== undefined && Array.isArray(options.header)) {\n this.#setHeader(options.header);\n }\n if (options.signal) {\n this.#signal = options.signal;\n }\n }\n\n public *assemble(\n tokens: Iterable<Token>,\n flush = true,\n ): IterableIterator<CSVRecord<Header>> {\n for (const token of tokens) {\n this.#signal?.throwIfAborted();\n switch (token.type) {\n case FieldDelimiter:\n this.#fieldIndex++;\n this.#dirty = true;\n break;\n case RecordDelimiter:\n if (this.#header === undefined) {\n this.#setHeader(this.#row as unknown as Header);\n } else {\n if (this.#dirty) {\n yield Object.fromEntries(\n this.#header.map((header, index) => [\n header,\n this.#row.at(index),\n ]),\n ) as unknown as CSVRecord<Header>;\n } else {\n yield Object.fromEntries(\n this.#header.map((header) => [header, \"\"]),\n ) as CSVRecord<Header>;\n }\n }\n // Reset the row fields buffer.\n this.#fieldIndex = 0;\n this.#row = new Array(this.#header?.length).fill(\"\");\n this.#dirty = false;\n break;\n default:\n this.#dirty = true;\n this.#row[this.#fieldIndex] = token.value;\n break;\n }\n }\n\n if (flush) {\n yield* this.flush();\n }\n }\n\n public *flush(): Generator<CSVRecord<Header>> {\n if (this.#header !== undefined) {\n if (this.#dirty) {\n yield Object.fromEntries(\n this.#header\n .filter((v) => v)\n .map((header, index) => [header, this.#row.at(index)]),\n ) as unknown as CSVRecord<Header>;\n }\n }\n }\n\n #setHeader(header: Header) {\n this.#header = header;\n if (this.#header.length === 0) {\n throw new ParseError(\"The header must not be empty.\");\n }\n if (new Set(this.#header).size !== this.#header.length) {\n throw new ParseError(\"The header must not contain duplicate fields.\");\n }\n }\n}\n"],"names":[],"mappings":";;;AAQO,MAAM,eAAsD,CAAA;AAAA,EACjE,WAAc,GAAA,CAAA,CAAA;AAAA,EACd,OAAiB,EAAC,CAAA;AAAA,EAClB,OAAA,CAAA;AAAA,EACA,MAAS,GAAA,KAAA,CAAA;AAAA,EACT,OAAA,CAAA;AAAA,EAEA,WAAA,CAAY,OAA0C,GAAA,EAAI,EAAA;AACxD,IAAA,IAAI,QAAQ,MAAW,KAAA,KAAA,CAAA,IAAa,MAAM,OAAQ,CAAA,OAAA,CAAQ,MAAM,CAAG,EAAA;AACjE,MAAK,IAAA,CAAA,UAAA,CAAW,QAAQ,MAAM,CAAA,CAAA;AAAA,KAChC;AACA,IAAA,IAAI,QAAQ,MAAQ,EAAA;AAClB,MAAA,IAAA,CAAK,UAAU,OAAQ,CAAA,MAAA,CAAA;AAAA,KACzB;AAAA,GACF;AAAA,EAEA,CAAQ,QAAA,CACN,MACA,EAAA,KAAA,GAAQ,IAC6B,EAAA;AACrC,IAAA,KAAA,MAAW,SAAS,MAAQ,EAAA;AAC1B,MAAA,IAAA,CAAK,SAAS,cAAe,EAAA,CAAA;AAC7B,MAAA,QAAQ,MAAM,IAAM;AAAA,QAClB,KAAK,cAAA;AACH,UAAK,IAAA,CAAA,WAAA,EAAA,CAAA;AACL,UAAA,IAAA,CAAK,MAAS,GAAA,IAAA,CAAA;AACd,UAAA,MAAA;AAAA,QACF,KAAK,eAAA;AACH,UAAI,IAAA,IAAA,CAAK,YAAY,KAAW,CAAA,EAAA;AAC9B,YAAK,IAAA,CAAA,UAAA,CAAW,KAAK,IAAyB,CAAA,CAAA;AAAA,WACzC,MAAA;AACL,YAAA,IAAI,KAAK,MAAQ,EAAA;AACf,cAAA,MAAM,MAAO,CAAA,WAAA;AAAA,gBACX,IAAK,CAAA,OAAA,CAAQ,GAAI,CAAA,CAAC,QAAQ,KAAU,KAAA;AAAA,kBAClC,MAAA;AAAA,kBACA,IAAA,CAAK,IAAK,CAAA,EAAA,CAAG,KAAK,CAAA;AAAA,iBACnB,CAAA;AAAA,eACH,CAAA;AAAA,aACK,MAAA;AACL,cAAA,MAAM,MAAO,CAAA,WAAA;AAAA,gBACX,IAAA,CAAK,QAAQ,GAAI,CAAA,CAAC,WAAW,CAAC,MAAA,EAAQ,EAAE,CAAC,CAAA;AAAA,eAC3C,CAAA;AAAA,aACF;AAAA,WACF;AAEA,UAAA,IAAA,CAAK,WAAc,GAAA,CAAA,CAAA;AACnB,UAAK,IAAA,CAAA,IAAA,GAAO,IAAI,KAAM,CAAA,IAAA,CAAK,SAAS,MAAM,CAAA,CAAE,KAAK,EAAE,CAAA,CAAA;AACnD,UAAA,IAAA,CAAK,MAAS,GAAA,KAAA,CAAA;AACd,UAAA,MAAA;AAAA,QACF;AACE,UAAA,IAAA,CAAK,MAAS,GAAA,IAAA,CAAA;AACd,UAAA,IAAA,CAAK,IAAK,CAAA,IAAA,CAAK,WAAW,CAAA,GAAI,KAAM,CAAA,KAAA,CAAA;AACpC,UAAA,MAAA;AAAA,OACJ;AAAA,KACF;AAEA,IAAA,IAAI,KAAO,EAAA;AACT,MAAA,OAAO,KAAK,KAAM,EAAA,CAAA;AAAA,KACpB;AAAA,GACF;AAAA,EAEA,CAAQ,KAAsC,GAAA;AAC5C,IAAI,IAAA,IAAA,CAAK,YAAY,KAAW,CAAA,EAAA;AAC9B,MAAA,IAAI,KAAK,MAAQ,EAAA;AACf,QAAA,MAAM,MAAO,CAAA,WAAA;AAAA,UACX,KAAK,OACF,CAAA,MAAA,CAAO,CAAC,CAAM,KAAA,CAAC,EACf,GAAI,CAAA,CAAC,MAAQ,EAAA,KAAA,KAAU,CAAC,MAAQ,EAAA,IAAA,CAAK,KAAK,EAAG,CAAA,KAAK,CAAC,CAAC,CAAA;AAAA,SACzD,CAAA;AAAA,OACF;AAAA,KACF;AAAA,GACF;AAAA,EAEA,WAAW,MAAgB,EAAA;AACzB,IAAA,IAAA,CAAK,OAAU,GAAA,MAAA,CAAA;AACf,IAAI,IAAA,IAAA,CAAK,OAAQ,CAAA,MAAA,KAAW,CAAG,EAAA;AAC7B,MAAM,MAAA,IAAI,WAAW,+BAA+B,CAAA,CAAA;AAAA,KACtD;AACA,IAAI,IAAA,IAAI,IAAI,IAAK,CAAA,OAAO,EAAE,IAAS,KAAA,IAAA,CAAK,QAAQ,MAAQ,EAAA;AACtD,MAAM,MAAA,IAAI,WAAW,+CAA+C,CAAA,CAAA;AAAA,KACtE;AAAA,GACF;AACF;;;;"}
|