web-csv-toolbox 0.4.0 → 0.5.0

package/lib/index.js

@@ -5,83 +5,76 @@ const CR = "\r";
 const CRLF = "\r\n";
 const LF = "\n";
 const COMMA = ",";
-const DOUBLE_QUATE = '"';
+const DOUBLE_QUOTE = '"';
 function assertCommonOptions(options) {
   if (typeof options.quotation === "string" && options.quotation.length === 0) {
     throw new Error("quotation must not be empty");
   }
-  if (typeof options.demiliter === "string" && options.demiliter.length === 0) {
-    throw new Error("demiliter must not be empty");
+  if (typeof options.delimiter === "string" && options.delimiter.length === 0) {
+    throw new Error("delimiter must not be empty");
   }
   if (options.quotation.includes(LF) || options.quotation.includes(CR)) {
     throw new Error("quotation must not include CR or LF");
   }
-  if (options.demiliter.includes(LF) || options.demiliter.includes(CR)) {
-    throw new Error("demiliter must not include CR or LF");
+  if (options.delimiter.includes(LF) || options.delimiter.includes(CR)) {
+    throw new Error("delimiter must not include CR or LF");
   }
   if (
-    options.demiliter.includes(options.quotation) ||
-    options.quotation.includes(options.demiliter)
+    options.delimiter.includes(options.quotation) ||
+    options.quotation.includes(options.delimiter)
   ) {
     throw new Error(
-      "demiliter and quotation must not include each other as a substring",
+      "delimiter and quotation must not include each other as a substring",
     );
   }
 }
 function escapeRegExp(v) {
   return v.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
 }
-class LexerTransformer extends TransformStream {
-  #demiliter;
-  #demiliterLength;
+class Lexer {
+  #delimiter;
+  #delimiterLength;
   #quotation;
   #quotationLength;
   #matcher;
   #buffer = "";
-  get demiliter() {
-    return this.#demiliter;
-  }
-  get quotation() {
-    return this.#quotation;
-  }
-  constructor({ demiliter = COMMA, quotation = DOUBLE_QUATE } = {}) {
-    assertCommonOptions({ demiliter, quotation });
-    super({
-      transform: (chunk, controller) => {
-        if (chunk.length !== 0) {
-          this.#buffer += chunk;
-          for (const token of this.#tokens({ flush: false })) {
-            controller.enqueue(token);
-          }
-        }
-      },
-      flush: (controller) => {
-        for (const token of this.#tokens({ flush: true })) {
-          controller.enqueue(token);
-        }
-      },
-    });
-    this.#demiliter = demiliter;
-    this.#demiliterLength = demiliter.length;
+  #flush = false;
+  constructor({ delimiter = COMMA, quotation = DOUBLE_QUOTE } = {}) {
+    assertCommonOptions({ delimiter, quotation });
+    this.#delimiter = delimiter;
+    this.#delimiterLength = delimiter.length;
     this.#quotation = quotation;
     this.#quotationLength = quotation.length;
-    const d = escapeRegExp(demiliter);
+    const d = escapeRegExp(delimiter);
     const q = escapeRegExp(quotation);
     this.#matcher = new RegExp(
       `^(?:(?!${q})(?!${d})(?![\\r\\n]))([\\S\\s\\uFEFF\\xA0]+?)(?=${q}|${d}|\\r|\\n|$)`,
     );
   }
-  *#tokens({ flush }) {
+  lex(chunk, buffering = false) {
+    if (!buffering) {
+      this.#flush = true;
+    }
+    if (typeof chunk === "string" && chunk.length !== 0) {
+      this.#buffer += chunk;
+    }
+    return this.#tokens();
+  }
+  flush() {
+    this.#flush = true;
+    return [...this.#tokens()];
+  }
+  *#tokens() {
+    if (this.#flush) {
+      if (this.#buffer.endsWith(CRLF)) {
+        this.#buffer = this.#buffer.slice(0, -CRLF.length);
+      } else if (this.#buffer.endsWith(LF)) {
+        this.#buffer = this.#buffer.slice(0, -LF.length);
+      }
+    }
     let currentField = null;
-    for (let token; (token = this.#nextToken({ flush })); ) {
-      switch (token.type) {
-        case Field:
-          if (currentField) {
-            currentField.value += token.value;
-          } else {
-            currentField = token;
-          }
-          break;
+    for (let token; (token = this.#nextToken()); ) {
+      switch (token) {
         case FieldDelimiter:
           if (currentField) {
             yield currentField;
@@ -96,37 +89,50 @@ class LexerTransformer extends TransformStream {
           }
           yield token;
           break;
+        default:
+          if (currentField) {
+            currentField.value += token.value;
+          } else {
+            currentField = token;
+          }
+          break;
       }
     }
     if (currentField) {
       yield currentField;
     }
   }
-  #nextToken({ flush = false } = {}) {
+  #nextToken() {
     if (this.#buffer.length === 0) {
       return null;
     }
+    if (
+      this.#flush === false &&
+      (this.#buffer === CRLF || this.#buffer === LF)
+    ) {
+      return null;
+    }
     if (this.#buffer.startsWith(CRLF)) {
       this.#buffer = this.#buffer.slice(2);
-      return { type: RecordDelimiter, value: CRLF };
+      return RecordDelimiter;
     }
     if (this.#buffer.startsWith(LF)) {
       this.#buffer = this.#buffer.slice(1);
-      return { type: RecordDelimiter, value: LF };
+      return RecordDelimiter;
     }
-    if (this.#buffer.startsWith(this.#demiliter)) {
-      this.#buffer = this.#buffer.slice(this.#demiliterLength);
-      return { type: FieldDelimiter, value: this.#demiliter };
+    if (this.#buffer.startsWith(this.#delimiter)) {
+      this.#buffer = this.#buffer.slice(this.#delimiterLength);
+      return FieldDelimiter;
     }
     if (this.#buffer.startsWith(this.#quotation)) {
-      if (flush === false && this.#buffer.endsWith(this.#quotation)) {
+      if (this.#flush === false && this.#buffer.endsWith(this.#quotation)) {
         return null;
       }
-      return this.extractQuotedString(flush);
+      return this.#extractQuotedString();
     }
     const match = this.#matcher.exec(this.#buffer);
     if (match) {
-      if (flush === false && match[0].length === this.#buffer.length) {
+      if (this.#flush === false && match[0].length === this.#buffer.length) {
         return null;
       }
       this.#buffer = this.#buffer.slice(match[0].length);
@@ -134,32 +140,32 @@ class LexerTransformer extends TransformStream {
     }
     return null;
   }
-  extractQuotedString(flush) {
+  #extractQuotedString() {
     let end = this.#quotationLength;
     let value = "";
     while (end < this.#buffer.length) {
       if (
         this.#buffer.slice(end, end + this.#quotationLength) ===
-          this.quotation &&
+          this.#quotation &&
         this.#buffer.slice(
           end + this.#quotationLength,
           end + this.#quotationLength * 2,
-        ) === this.quotation
+        ) === this.#quotation
       ) {
-        value += this.quotation;
+        value += this.#quotation;
         end += this.#quotationLength * 2;
         continue;
       }
       if (
-        this.#buffer.slice(end, end + this.#quotationLength) === this.quotation
+        this.#buffer.slice(end, end + this.#quotationLength) === this.#quotation
       ) {
         if (
-          flush === false &&
+          this.#flush === false &&
           end + this.#quotationLength < this.#buffer.length &&
           this.#buffer.slice(
             end + this.#quotationLength,
-            this.#demiliterLength,
-          ) !== this.demiliter &&
+            this.#delimiterLength,
+          ) !== this.#delimiter &&
           this.#buffer.slice(
             end + this.#quotationLength,
             end + this.#quotationLength + 2,
@@ -180,58 +186,80 @@ class LexerTransformer extends TransformStream {
     return null;
   }
 }
-class RecordAssemblerTransformar extends TransformStream {
-  #fieldIndex = 0;
-  #row = [];
-  #header;
-  #darty = false;
+class LexerTransformer extends TransformStream {
   constructor(options = {}) {
+    const lexer = new Lexer(options);
     super({
-      transform: (token, controller) => {
-        switch (token.type) {
-          case Field:
-            this.#darty = true;
-            this.#row[this.#fieldIndex] = token.value;
-            break;
-          case FieldDelimiter:
-            this.#fieldIndex++;
-            break;
-          case RecordDelimiter:
-            if (this.#header === undefined) {
-              this.#setHeader(this.#row);
-            } else {
-              if (this.#darty) {
-                const record = Object.fromEntries(
-                  this.#header
-                    .filter((v) => v)
-                    .map((header, index) => [header, this.#row.at(index)]),
-                );
-                controller.enqueue(record);
-              }
-            }
-            this.#fieldIndex = 0;
-            this.#row = new Array(this.#header?.length);
-            this.#darty = false;
-            break;
+      transform: (chunk, controller) => {
+        if (chunk.length !== 0) {
+          controller.enqueue([...lexer.lex(chunk, true)]);
         }
       },
       flush: (controller) => {
-        if (this.#fieldIndex !== 0 && this.#header !== undefined) {
-          if (this.#darty) {
-            const record = Object.fromEntries(
-              this.#header
-                .filter((v) => v)
-                .map((header, index) => [header, this.#row.at(index)]),
-            );
-            controller.enqueue(record);
-          }
-        }
+        controller.enqueue(lexer.flush());
       },
     });
+  }
+}
+class RecordAssembler {
+  #fieldIndex = 0;
+  #row = [];
+  #header;
+  #dirty = false;
+  constructor(options = {}) {
     if (options.header !== undefined && Array.isArray(options.header)) {
       this.#setHeader(options.header);
     }
   }
+  *assemble(tokens, flush = true) {
+    for (const token of tokens) {
+      switch (token) {
+        case FieldDelimiter:
+          this.#fieldIndex++;
+          this.#dirty = true;
+          break;
+        case RecordDelimiter:
+          if (this.#header === undefined) {
+            this.#setHeader(this.#row);
+          } else {
+            if (this.#dirty) {
+              yield Object.fromEntries(
+                this.#header.map((header, index) => [
+                  header,
+                  this.#row.at(index),
+                ]),
+              );
+            } else {
+              yield Object.fromEntries(
+                this.#header.map((header) => [header, ""]),
+              );
+            }
+          }
+          this.#fieldIndex = 0;
+          this.#row = new Array(this.#header?.length).fill("");
+          this.#dirty = false;
+          break;
+        default:
+          this.#dirty = true;
+          this.#row[this.#fieldIndex] = token.value;
+          break;
+      }
+    }
+    if (flush) {
+      yield* this.flush();
+    }
+  }
+  *flush() {
+    if (this.#header !== undefined) {
+      if (this.#dirty) {
+        yield Object.fromEntries(
+          this.#header
+            .filter((v) => v)
+            .map((header, index) => [header, this.#row.at(index)]),
+        );
+      }
+    }
+  }
   #setHeader(header) {
     this.#header = header;
     if (this.#header.length === 0) {
@@ -242,16 +270,48 @@ class RecordAssemblerTransformar extends TransformStream {
     }
   }
 }
-class SingleValueReadableStream extends ReadableStream {
-  constructor(value) {
+class RecordAssemblerTransformer extends TransformStream {
+  constructor(options = {}) {
+    const assembler = new RecordAssembler(options);
     super({
-      start(controller) {
-        controller.enqueue(value);
-        controller.close();
+      transform: (tokens, controller) => {
+        for (const token of assembler.assemble(tokens, false)) {
+          controller.enqueue(token);
+        }
+      },
+      flush: (controller) => {
+        for (const token of assembler.flush()) {
+          controller.enqueue(token);
+        }
       },
     });
   }
 }
+function parseStringToArraySync(csv, options) {
+  const lexer = new Lexer(options);
+  const assembler = new RecordAssembler(options);
+  const tokens = lexer.lex(csv);
+  return [...assembler.assemble(tokens)];
+}
+function parseStringToIterableIterator(csv, options) {
+  const lexer = new Lexer(options);
+  const assembler = new RecordAssembler(options);
+  const tokens = lexer.lex(csv);
+  return assembler.assemble(tokens);
+}
+function parseStringToStream(csv, options) {
+  const lexer = new Lexer(options);
+  const assembler = new RecordAssembler(options);
+  return new ReadableStream({
+    start(controller) {
+      const tokens = lexer.lex(csv);
+      for (const record of assembler.assemble(tokens)) {
+        controller.enqueue(record);
+      }
+      controller.close();
+    },
+  });
+}
 async function toArray(...args) {
   const rows = [];
   for await (const row of this(...args)) {
@@ -259,57 +319,132 @@ async function toArray(...args) {
   }
   return rows;
 }
-async function* parseStringStream(stream, options) {
-  let controller;
-  const readable = new ReadableStream({
-    start: (controller_) => (controller = controller_),
-  });
-  await stream
-    .pipeThrough(new LexerTransformer(options))
-    .pipeThrough(new RecordAssemblerTransformar(options))
-    .pipeTo(
-      new WritableStream({
-        write: (row) => controller.enqueue(row),
-        close: () => controller.close(),
-      }),
-    );
-  const reader = readable.getReader();
-  try {
-    while (true) {
-      const { value, done } = await reader.read();
-      if (done) break;
-      yield value;
-    }
-  } finally {
-    reader.releaseLock();
-  }
+async function* parseString(csv, options) {
+  yield* parseStringToIterableIterator(csv, options);
 }
-((parseStringStream) => {
-  Object.defineProperty(parseStringStream, "toArray", {
+((parseString) => {
+  Object.defineProperty(parseString, "toArray", {
     enumerable: true,
     writable: false,
     value: toArray,
   });
-})(parseStringStream || (parseStringStream = {}));
-async function* parseString(csv, options) {
-  yield* parseStringStream(new SingleValueReadableStream(csv), options);
+  Object.defineProperty(parseString, "toArraySync", {
+    enumerable: true,
+    writable: false,
+    value: parseStringToArraySync,
+  });
+  Object.defineProperty(parseString, "toIterableIterator", {
+    enumerable: true,
+    writable: false,
+    value: parseStringToIterableIterator,
+  });
+  Object.defineProperty(parseString, "toStream", {
+    enumerable: true,
+    writable: false,
+    value: parseStringToStream,
+  });
+})(parseString || (parseString = {}));
+function convertBinaryToString(binary, options) {
+  return new TextDecoder(options?.charset, {
+    ignoreBOM: options?.ignoreBOM,
+    fatal: options?.fatal,
+  }).decode(binary instanceof ArrayBuffer ? new Uint8Array(binary) : binary);
 }
-((parseString) => {
-  Object.defineProperty(parseString, "toArray", {
+function parseBinaryToArraySync(binary, options = {}) {
+  const csv = convertBinaryToString(binary, options);
+  return parseStringToArraySync(csv, options);
+}
+function parseBinaryToIterableIterator(binary, options = {}) {
+  const csv = convertBinaryToString(binary, options);
+  return parseStringToIterableIterator(csv, options);
+}
+function parseBinaryToStream(binary, options = {}) {
+  const csv = convertBinaryToString(binary, options);
+  return parseStringToStream(csv, options);
+}
+function iterableIteratorToAsync(iterator) {
+  return {
+    async next() {
+      const result = iterator.next();
+      return Promise.resolve(result);
+    },
+    [Symbol.asyncIterator]() {
+      return this;
+    },
+  };
+}
+function parseBinary(bytes, options) {
+  const iterator = parseBinaryToIterableIterator(bytes, options);
+  return iterableIteratorToAsync(iterator);
+}
+((parseBinary) => {
+  Object.defineProperty(parseBinary, "toArray", {
     enumerable: true,
     writable: false,
     value: toArray,
   });
-})(parseString || (parseString = {}));
-async function* parseUint8ArrayStream(stream, options) {
+  Object.defineProperty(parseBinary, "toArraySync", {
+    enumerable: true,
+    writable: false,
+    value: parseBinaryToArraySync,
+  });
+  Object.defineProperty(parseBinary, "toIterableIterator", {
+    enumerable: true,
+    writable: false,
+    value: parseBinaryToIterableIterator,
+  });
+  Object.defineProperty(parseBinary, "toStream", {
+    enumerable: true,
+    writable: false,
+    value: parseBinaryToStream,
+  });
+})(parseBinary || (parseBinary = {}));
+function pipeline(stream, ...transformers) {
+  return new ReadableStream({
+    start: (controller) => {
+      (() =>
+        transformers
+          .reduce(
+            (stream, transformer) => stream.pipeThrough(transformer),
+            stream,
+          )
+          .pipeTo(
+            new WritableStream({
+              write: (v) => controller.enqueue(v),
+              close: () => controller.close(),
+            }),
+          ))();
+    },
+  });
+}
+function parseUint8ArrayStreamToStream(stream, options) {
   const { charset, fatal, ignoreBOM, decomposition } = options ?? {};
-  yield* parseStringStream(
-    [
-      ...(decomposition ? [new DecompressionStream(decomposition)] : []),
-      new TextDecoderStream(charset, { fatal, ignoreBOM }),
-    ].reduce((stream, transformer) => stream.pipeThrough(transformer), stream),
-    options,
-  );
+  return decomposition
+    ? pipeline(
+        stream,
+        new DecompressionStream(decomposition),
+        new TextDecoderStream(charset, { fatal, ignoreBOM }),
+        new LexerTransformer(options),
+        new RecordAssemblerTransformer(options),
+      )
+    : pipeline(
+        stream,
+        new TextDecoderStream(charset, { fatal, ignoreBOM }),
+        new LexerTransformer(options),
+        new RecordAssemblerTransformer(options),
+      );
+}
+async function* streamToAsyncIterableIterator(stream) {
+  const reader = stream.getReader();
+  while (true) {
+    const { done, value } = await reader.read();
+    if (done) break;
+    yield value;
+  }
+}
+function parseUint8ArrayStream(stream, options) {
+  const recordStream = parseUint8ArrayStreamToStream(stream, options);
+  return streamToAsyncIterableIterator(recordStream);
 }
 ((parseUint8ArrayStream) => {
   Object.defineProperty(parseUint8ArrayStream, "toArray", {
@@ -317,27 +452,35 @@ async function* parseUint8ArrayStream(stream, options) {
     writable: false,
     value: toArray,
   });
+  Object.defineProperty(parseUint8ArrayStream, "toStream", {
+    enumerable: true,
+    writable: false,
+    value: parseUint8ArrayStreamToStream,
+  });
 })(parseUint8ArrayStream || (parseUint8ArrayStream = {}));
-function parseUint8Array(bytes, options) {
-  return parseUint8ArrayStream(new SingleValueReadableStream(bytes), options);
+function parseStringStreamToStream(stream, options) {
+  return pipeline(
+    stream,
+    new LexerTransformer(options),
+    new RecordAssemblerTransformer(options),
+  );
+}
+function parseStringStream(stream, options) {
+  const recordStream = parseStringStreamToStream(stream, options);
+  return streamToAsyncIterableIterator(recordStream);
 }
-((parseUint8Array) => {
-  Object.defineProperty(parseUint8Array, "toArray", {
+((parseStringStream) => {
+  Object.defineProperty(parseStringStream, "toArray", {
     enumerable: true,
     writable: false,
     value: toArray,
   });
-})(parseUint8Array || (parseUint8Array = {}));
-function parseArrayBuffer(buffer, options) {
-  return parseUint8Array(new Uint8Array(buffer), options);
-}
-((parseArrayBuffer) => {
-  Object.defineProperty(parseArrayBuffer, "toArray", {
+  Object.defineProperty(parseStringStream, "toStream", {
     enumerable: true,
     writable: false,
-    value: toArray,
+    value: parseStringStreamToStream,
   });
-})(parseArrayBuffer || (parseArrayBuffer = {}));
+})(parseStringStream || (parseStringStream = {}));
 function parseMime(contentType) {
   const [type, ...parameters] = contentType.split(";");
   const result = {
@@ -350,7 +493,7 @@ function parseMime(contentType) {
   }
   return result;
 }
-function parseResponse(response, options) {
+function getOptionsFromResponse(response, options = {}) {
   const { headers } = response;
   const contentType = headers.get("content-type") ?? "text/csv";
   const mime = parseMime(contentType);
@@ -359,14 +502,25 @@ function parseResponse(response, options) {
   }
   const decomposition = headers.get("content-encoding") ?? undefined;
   const charset = mime.parameters.charset ?? "utf-8";
-  if (response.body === null) {
-    throw new Error("Response body is null");
-  }
-  return parseUint8ArrayStream(response.body, {
+  return {
     decomposition,
     charset,
     ...options,
-  });
+  };
+}
+function parseResponseToStream(response, options) {
+  const options_ = getOptionsFromResponse(response, options);
+  if (response.body === null) {
+    throw new Error("Response body is null");
+  }
+  return parseUint8ArrayStreamToStream(response.body, options_);
+}
+function parseResponse(response, options) {
+  const options_ = getOptionsFromResponse(response, options);
+  if (response.body === null) {
+    throw new Error("Response body is null");
+  }
+  return parseUint8ArrayStream(response.body, options_);
 }
 ((parseResponse) => {
   Object.defineProperty(parseResponse, "toArray", {
@@ -374,34 +528,27 @@ function parseResponse(response, options) {
     writable: false,
     value: toArray,
   });
-})(parseResponse || (parseResponse = {}));
-async function* parseStream(stream, options) {
-  const [branch1, branch2] = stream.tee();
-  const reader1 = branch1.getReader();
-  const { value: firstChunk } = await reader1.read();
-  reader1.releaseLock();
-  if (typeof firstChunk === "string") {
-    yield* parseStringStream(branch2, options);
-  } else if (firstChunk instanceof Uint8Array) {
-    yield* parseUint8ArrayStream(branch2, options);
-  }
-}
-((parseStream) => {
-  Object.defineProperty(parseStream, "toArray", {
+  Object.defineProperty(parseResponse, "toStream", {
     enumerable: true,
     writable: false,
-    value: toArray,
+    value: parseResponseToStream,
   });
-})(parseStream || (parseStream = {}));
+})(parseResponse || (parseResponse = {}));
 async function* parse(csv, options) {
   if (typeof csv === "string") {
     yield* parseString(csv, options);
-  } else if (csv instanceof Uint8Array) {
-    yield* parseUint8Array(csv, options);
-  } else if (csv instanceof ArrayBuffer) {
-    yield* parseArrayBuffer(csv, options);
+  } else if (csv instanceof Uint8Array || csv instanceof ArrayBuffer) {
+    yield* parseBinary(csv, options);
   } else if (csv instanceof ReadableStream) {
-    yield* parseStream(csv, options);
+    const [branch1, branch2] = csv.tee();
+    const reader1 = branch1.getReader();
+    const { value: firstChunk } = await reader1.read();
+    reader1.releaseLock();
+    if (typeof firstChunk === "string") {
+      yield* parseStringStream(branch2, options);
+    } else if (firstChunk instanceof Uint8Array) {
+      yield* parseUint8ArrayStream(branch2, options);
+    }
   } else if (csv instanceof Response) {
     yield* parseResponse(csv, options);
   }
@@ -417,14 +564,12 @@ export {
   Field,
   FieldDelimiter,
   LexerTransformer,
-  RecordAssemblerTransformar,
+  RecordAssemblerTransformer,
   RecordDelimiter,
   parse,
-  parseArrayBuffer,
+  parseBinary,
   parseResponse,
-  parseStream,
   parseString,
   parseStringStream,
-  parseUint8Array,
   parseUint8ArrayStream,
 };