web-csv-toolbox 0.13.0-next-bd865d6ddb1cf9691d7b9a83d0790651f074dd47 → 0.13.0-next-b21b6d89a7a3f18dcbf79ec04ffefde0d7ff4c4c

package/dist/CSVRecordAssemblerTransformer.js

@@ -2,28 +2,65 @@ import { CSVRecordAssembler } from './CSVRecordAssembler.js';
 
 class CSVRecordAssemblerTransformer extends TransformStream {
   assembler;
-  constructor(options = {}) {
+  /**
+   * Yields to the event loop to allow backpressure handling.
+   * Can be overridden for testing purposes.
+   * @internal
+   */
+  async yieldToEventLoop() {
+    await new Promise((resolve) => setTimeout(resolve, 0));
+  }
+  constructor(options = {}, writableStrategy = {
+    highWaterMark: 1024,
+    // 1024 tokens
+    size: (tokens) => tokens.length,
+    // Count by number of tokens in array
+    checkInterval: 10
+    // Check backpressure every 10 records
+  }, readableStrategy = {
+    highWaterMark: 256,
+    // 256 records
+    size: () => 1,
+    // Each record counts as 1
+    checkInterval: 10
+    // Check backpressure every 10 records
+  }) {
     const assembler = new CSVRecordAssembler(options);
-    super({
-      transform: (tokens, controller) => {
-        try {
-          for (const token of assembler.assemble(tokens, { stream: true })) {
-            controller.enqueue(token);
+    const checkInterval = writableStrategy.checkInterval ?? readableStrategy.checkInterval ?? 10;
+    super(
+      {
+        transform: async (tokens, controller) => {
+          try {
+            let recordCount = 0;
+            for (const record of assembler.assemble(tokens, { stream: true })) {
+              controller.enqueue(record);
+              recordCount++;
+              if (recordCount % checkInterval === 0 && controller.desiredSize !== null && controller.desiredSize <= 0) {
+                await this.yieldToEventLoop();
+              }
+            }
+          } catch (error) {
+            controller.error(error);
           }
-        } catch (error) {
-          controller.error(error);
-        }
-      },
-      flush: (controller) => {
-        try {
-          for (const token of assembler.assemble()) {
-            controller.enqueue(token);
+        },
+        flush: async (controller) => {
+          try {
+            let recordCount = 0;
+            for (const record of assembler.assemble()) {
+              controller.enqueue(record);
+              recordCount++;
+              if (recordCount % checkInterval === 0 && controller.desiredSize !== null && controller.desiredSize <= 0) {
+                await this.yieldToEventLoop();
+              }
+            }
+          } catch (error) {
+            controller.error(error);
           }
-        } catch (error) {
-          controller.error(error);
         }
-      }
-    });
+      },
+      writableStrategy,
+      readableStrategy
+    );
     this.assembler = assembler;
   }
 }

package/dist/CSVRecordAssemblerTransformer.js.map

@@ -1 +1 @@
-{"version":3,"file":"CSVRecordAssemblerTransformer.js","sources":["../src/CSVRecordAssemblerTransformer.ts"],"sourcesContent":["import { CSVRecordAssembler } from \"./CSVRecordAssembler.ts\";\nimport type {\n  CSVRecord,\n  CSVRecordAssemblerOptions,\n  Token,\n} from \"./common/types.ts\";\n\n/**\n * A transform stream that converts a stream of tokens into a stream of CSV records.\n * @template Header The type of the header row.\n * @param options The options for the parser.\n *\n * @category Low-level API\n *\n * @example Parse a CSV with headers by data\n *  ```ts\n * new ReadableStream({\n *   start(controller) {\n *     controller.enqueue(\"name,age\\r\\n\");\n *     controller.enqueue(\"Alice,20\\r\\n\");\n *     controller.enqueue(\"Bob,25\\r\\n\");\n *     controller.enqueue(\"Charlie,30\\r\\n\");\n *     controller.close();\n *   })\n *   .pipeThrough(new CSVLexerTransformer())\n *   .pipeThrough(new CSVRecordAssemblerTransformer())\n *   .pipeTo(new WritableStream({ write(row) { console.log(row); }}));\n * // { name: \"Alice\", age: \"20\" }\n * // { name: \"Bob\", age: \"25\" }\n * // { name: \"Charlie\", age: \"30\" }\n * ```\n *\n * @example Parse a CSV with headers by options\n * ```ts\n * new ReadableStream({\n *   start(controller) {\n *     controller.enqueue(\"Alice,20\\r\\n\");\n *     controller.enqueue(\"Bob,25\\r\\n\");\n *     controller.enqueue(\"Charlie,30\\r\\n\");\n *     controller.close();\n *   }\n * })\n * .pipeThrough(new CSVLexerTransformer())\n * .pipeThrough(new CSVRecordAssemblerTransformer({ header: [\"name\", \"age\"] }))\n * .pipeTo(new WritableStream({ write(row) { console.log(row); }}));\n * // { name: \"Alice\", age: \"20\" }\n * // { name: \"Bob\", age: \"25\" }\n * // { name: \"Charlie\", age: \"30\" }\n * ```\n */\nexport class CSVRecordAssemblerTransformer<\n  Header extends ReadonlyArray<string>,\n> extends TransformStream<Token[], CSVRecord<Header>> {\n  public readonly assembler: CSVRecordAssembler<Header>;\n\n  constructor(options: CSVRecordAssemblerOptions<Header> = {}) {\n    const assembler = new CSVRecordAssembler(options);\n    super({\n      transform: (tokens, controller) => {\n        try {\n          for (const token of assembler.assemble(tokens, { stream: true })) {\n            controller.enqueue(token);\n          }\n        } catch (error) {\n          controller.error(error);\n        }\n      },\n      flush: (controller) => {\n        try {\n          for (const token of assembler.assemble()) {\n            controller.enqueue(token);\n          }\n        } catch (error) {\n          controller.error(error);\n        }\n      },\n    });\n    this.assembler = assembler;\n  }\n}\n"],"names":[],"mappings":";;AAkDO,MAAM,sCAEH,eAA4C,CAAA;AAAA,EACpC,SAAA;AAAA,EAEhB,WAAA,CAAY,OAA6C,GAAA,EAAI,EAAA;AAC3D,IAAM,MAAA,SAAA,GAAY,IAAI,kBAAA,CAAmB,OAAO,CAAA;AAChD,IAAM,KAAA,CAAA;AAAA,MACJ,SAAA,EAAW,CAAC,MAAA,EAAQ,UAAe,KAAA;AACjC,QAAI,IAAA;AACF,UAAW,KAAA,MAAA,KAAA,IAAS,UAAU,QAAS,CAAA,MAAA,EAAQ,EAAE,MAAQ,EAAA,IAAA,EAAM,CAAG,EAAA;AAChE,YAAA,UAAA,CAAW,QAAQ,KAAK,CAAA;AAAA;AAC1B,iBACO,KAAO,EAAA;AACd,UAAA,UAAA,CAAW,MAAM,KAAK,CAAA;AAAA;AACxB,OACF;AAAA,MACA,KAAA,EAAO,CAAC,UAAe,KAAA;AACrB,QAAI,IAAA;AACF,UAAW,KAAA,MAAA,KAAA,IAAS,SAAU,CAAA,QAAA,EAAY,EAAA;AACxC,YAAA,UAAA,CAAW,QAAQ,KAAK,CAAA;AAAA;AAC1B,iBACO,KAAO,EAAA;AACd,UAAA,UAAA,CAAW,MAAM,KAAK,CAAA;AAAA;AACxB;AACF,KACD,CAAA;AACD,IAAA,IAAA,CAAK,SAAY,GAAA,SAAA;AAAA;AAErB;;;;"}
+{"version":3,"file":"CSVRecordAssemblerTransformer.js","sources":["../src/CSVRecordAssemblerTransformer.ts"],"sourcesContent":["import { CSVRecordAssembler } from \"./CSVRecordAssembler.ts\";\nimport type {\n  CSVRecord,\n  CSVRecordAssemblerOptions,\n  ExtendedQueuingStrategy,\n  Token,\n} from \"./common/types.ts\";\n\n/**\n * A transform stream that converts a stream of tokens into a stream of CSV records.\n *\n * @template Header The type of the header row.\n * @param options - CSV-specific options (header, maxFieldCount, etc.)\n * @param writableStrategy - Strategy for the writable side (default: `{ highWaterMark: 1024, size: tokens => tokens.length, checkInterval: 10 }`)\n * @param readableStrategy - Strategy for the readable side (default: `{ highWaterMark: 256, size: () => 1, checkInterval: 10 }`)\n *\n * @category Low-level API\n *\n * @remarks\n * Follows the Web Streams API pattern where queuing strategies are passed as\n * constructor arguments, similar to the standard `TransformStream`.\n *\n * **Default Queuing Strategy:**\n * - Writable side: Counts by number of tokens in each array. Default highWaterMark is 1024 tokens.\n * - Readable side: Counts each record as 1. Default highWaterMark is 256 records.\n *\n * **Backpressure Handling:**\n * The transformer monitors `controller.desiredSize` and yields to the event loop when backpressure\n * is detected (desiredSize ≤ 0). This prevents blocking the main thread during heavy processing\n * and allows the downstream consumer to catch up.\n *\n * These defaults are starting points based on data flow characteristics, not empirical benchmarks.\n * Optimal values depend on your runtime environment, data size, and performance requirements.\n *\n * @example Parse a CSV with headers by data\n *  ```ts\n * new ReadableStream({\n *   start(controller) {\n *     controller.enqueue(\"name,age\\r\\n\");\n *     controller.enqueue(\"Alice,20\\r\\n\");\n *     controller.enqueue(\"Bob,25\\r\\n\");\n *     controller.enqueue(\"Charlie,30\\r\\n\");\n *     controller.close();\n *   })\n *   .pipeThrough(new CSVLexerTransformer())\n *   .pipeThrough(new CSVRecordAssemblerTransformer())\n *   .pipeTo(new WritableStream({ write(row) { console.log(row); }}));\n * // { name: \"Alice\", age: \"20\" }\n * // { name: \"Bob\", age: \"25\" }\n * // { name: \"Charlie\", age: \"30\" }\n * ```\n *\n * @example Parse a CSV with headers by options\n * ```ts\n * new ReadableStream({\n *   start(controller) {\n *     controller.enqueue(\"Alice,20\\r\\n\");\n *     controller.enqueue(\"Bob,25\\r\\n\");\n *     controller.enqueue(\"Charlie,30\\r\\n\");\n *     controller.close();\n *   }\n * })\n * .pipeThrough(new CSVLexerTransformer())\n * .pipeThrough(new CSVRecordAssemblerTransformer({ header: [\"name\", \"age\"] }))\n * .pipeTo(new WritableStream({ write(row) { console.log(row); }}));\n * // { name: \"Alice\", age: \"20\" }\n * // { name: \"Bob\", age: \"25\" }\n * // { name: \"Charlie\", age: \"30\" }\n * ```\n *\n * @example Custom queuing strategies with backpressure tuning\n * ```ts\n * const transformer = new CSVRecordAssemblerTransformer(\n *   {},\n *   {\n *     highWaterMark: 2048,             // 2048 tokens\n *     size: (tokens) => tokens.length, // Count by token count\n *     checkInterval: 20                // Check backpressure every 20 records\n *   },\n *   {\n *     highWaterMark: 512,              // 512 records\n *     size: () => 1,                   // Each record counts as 1\n *     checkInterval: 5                 // Check backpressure every 5 records\n *   }\n * );\n *\n * await tokenStream\n *   .pipeThrough(transformer)\n *   .pipeTo(yourRecordProcessor);\n * ```\n */\nexport class CSVRecordAssemblerTransformer<\n  Header extends ReadonlyArray<string>,\n> extends TransformStream<Token[], CSVRecord<Header>> {\n  public readonly assembler: CSVRecordAssembler<Header>;\n\n  /**\n   * Yields to the event loop to allow backpressure handling.\n   * Can be overridden for testing purposes.\n   * @internal\n   */\n  protected async yieldToEventLoop(): Promise<void> {\n    await new Promise((resolve) => setTimeout(resolve, 0));\n  }\n\n  constructor(\n    options: CSVRecordAssemblerOptions<Header> = {},\n    writableStrategy: ExtendedQueuingStrategy<Token[]> = {\n      highWaterMark: 1024, // 1024 tokens\n      size: (tokens) => tokens.length, // Count by number of tokens in array\n      checkInterval: 10, // Check backpressure every 10 records\n    },\n    readableStrategy: ExtendedQueuingStrategy<CSVRecord<Header>> = {\n      highWaterMark: 256, // 256 records\n      size: () => 1, // Each record counts as 1\n      checkInterval: 10, // Check backpressure every 10 records\n    },\n  ) {\n    const assembler = new CSVRecordAssembler(options);\n    const checkInterval =\n      writableStrategy.checkInterval ?? readableStrategy.checkInterval ?? 10;\n\n    super(\n      {\n        transform: async (tokens, controller) => {\n          try {\n            let recordCount = 0;\n            for (const record of assembler.assemble(tokens, { stream: true })) {\n              controller.enqueue(record);\n              recordCount++;\n\n              // Check backpressure periodically based on checkInterval\n              if (\n                recordCount % checkInterval === 0 &&\n                controller.desiredSize !== null &&\n                controller.desiredSize <= 0\n              ) {\n                // Yield to event loop when backpressure is detected\n                await this.yieldToEventLoop();\n              }\n            }\n          } catch (error) {\n            controller.error(error);\n          }\n        },\n        flush: async (controller) => {\n          try {\n            let recordCount = 0;\n            for (const record of assembler.assemble()) {\n              controller.enqueue(record);\n              recordCount++;\n\n              // Check backpressure periodically based on checkInterval\n              if (\n                recordCount % checkInterval === 0 &&\n                controller.desiredSize !== null &&\n                controller.desiredSize <= 0\n              ) {\n                await this.yieldToEventLoop();\n              }\n            }\n          } catch (error) {\n            controller.error(error);\n          }\n        },\n      },\n      writableStrategy,\n      readableStrategy,\n    );\n    this.assembler = assembler;\n  }\n}\n"],"names":[],"mappings":";;AA2FO,MAAM,sCAEH,eAAA,CAA4C;AAAA,EACpC,SAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOhB,MAAgB,gBAAA,GAAkC;AAChD,IAAA,MAAM,IAAI,OAAA,CAAQ,CAAC,YAAY,UAAA,CAAW,OAAA,EAAS,CAAC,CAAC,CAAA;AAAA,EACvD;AAAA,EAEA,WAAA,CACE,OAAA,GAA6C,EAAC,EAC9C,gBAAA,GAAqD;AAAA,IACnD,aAAA,EAAe,IAAA;AAAA;AAAA,IACf,IAAA,EAAM,CAAC,MAAA,KAAW,MAAA,CAAO,MAAA;AAAA;AAAA,IACzB,aAAA,EAAe;AAAA;AAAA,KAEjB,gBAAA,GAA+D;AAAA,IAC7D,aAAA,EAAe,GAAA;AAAA;AAAA,IACf,MAAM,MAAM,CAAA;AAAA;AAAA,IACZ,aAAA,EAAe;AAAA;AAAA,GACjB,EACA;AACA,IAAA,MAAM,SAAA,GAAY,IAAI,kBAAA,CAAmB,OAAO,CAAA;AAChD,IAAA,MAAM,aAAA,GACJ,gBAAA,CAAiB,aAAA,IAAiB,gBAAA,CAAiB,aAAA,IAAiB,EAAA;AAEtE,IAAA,KAAA;AAAA,MACE;AAAA,QACE,SAAA,EAAW,OAAO,MAAA,EAAQ,UAAA,KAAe;AACvC,UAAA,IAAI;AACF,YAAA,IAAI,WAAA,GAAc,CAAA;AAClB,YAAA,KAAA,MAAW,MAAA,IAAU,UAAU,QAAA,CAAS,MAAA,EAAQ,EAAE,MAAA,EAAQ,IAAA,EAAM,CAAA,EAAG;AACjE,cAAA,UAAA,CAAW,QAAQ,MAAM,CAAA;AACzB,cAAA,WAAA,EAAA;AAGA,cAAA,IACE,WAAA,GAAc,kBAAkB,CAAA,IAChC,UAAA,CAAW,gBAAgB,IAAA,IAC3B,UAAA,CAAW,eAAe,CAAA,EAC1B;AAEA,gBAAA,MAAM,KAAK,gBAAA,EAAiB;AAAA,cAC9B;AAAA,YACF;AAAA,UACF,SAAS,KAAA,EAAO;AACd,YAAA,UAAA,CAAW,MAAM,KAAK,CAAA;AAAA,UACxB;AAAA,QACF,CAAA;AAAA,QACA,KAAA,EAAO,OAAO,UAAA,KAAe;AAC3B,UAAA,IAAI;AACF,YAAA,IAAI,WAAA,GAAc,CAAA;AAClB,YAAA,KAAA,MAAW,MAAA,IAAU,SAAA,CAAU,QAAA,EAAS,EAAG;AACzC,cAAA,UAAA,CAAW,QAAQ,MAAM,CAAA;AACzB,cAAA,WAAA,EAAA;AAGA,cAAA,IACE,WAAA,GAAc,kBAAkB,CAAA,IAChC,UAAA,CAAW,gBAAgB,IAAA,IAC3B,UAAA,CAAW,eAAe,CAAA,EAC1B;AACA,gBAAA,MAAM,KAAK,gBAAA,EAAiB;AAAA,cAC9B;AAAA,YACF;AAAA,UACF,SAAS,KAAA,EAAO;AACd,YAAA,UAAA,CAAW,MAAM,KAAK,CAAA;AAAA,UACxB;AAAA,QACF;AAAA,OACF;AAAA,MACA,gBAAA;AAAA,MACA;AAAA,KACF;AACA,IAAA,IAAA,CAAK,SAAA,GAAY,SAAA;AAAA,EACnB;AACF;;;;"}