web-csv-toolbox 0.13.0-next-bd865d6ddb1cf9691d7b9a83d0790651f074dd47 → 0.13.0-next-7d51d5285be9cffa5103de58469d8de0c98959d7

package/README.md

@@ -357,8 +357,134 @@ ideal for developers looking for in-depth control and flexibility.
 
 - **`class CSVLexerTransformer`**: [📑](https://kamiazya.github.io/web-csv-toolbox/classes/CSVLexerTransformer.html)
   - A TransformStream class for lexical analysis of CSV data.
+  - Supports custom queuing strategies for controlling backpressure and memory usage.
 - **`class CSVRecordAssemblerTransformer`**: [📑](https://kamiazya.github.io/web-csv-toolbox/classes/CSVRecordAssemblerTransformer.html)
   - Handles the assembly of parsed data into records.
+  - Supports custom queuing strategies for controlling backpressure and memory usage.
+
+#### Customizing Queuing Strategies
+
+Both `CSVLexerTransformer` and `CSVRecordAssemblerTransformer` support custom queuing strategies following the Web Streams API pattern. Strategies are passed as constructor arguments with **data-type-aware size counting** and **configurable backpressure handling**.
+
+**Constructor signature:**
+```typescript
+new CSVLexerTransformer(options?, writableStrategy?, readableStrategy?)
+new CSVRecordAssemblerTransformer(options?, writableStrategy?, readableStrategy?)
+```
+
+**Default queuing strategies (starting points, not benchmarked):**
+```typescript
+// CSVLexerTransformer defaults
+writableStrategy: {
+  highWaterMark: 65536,           // 64KB of characters
+  size: (chunk) => chunk.length,  // Count by string length
+  checkInterval: 100              // Check backpressure every 100 tokens
+}
+readableStrategy: {
+  highWaterMark: 1024,              // 1024 tokens
+  size: (tokens) => tokens.length,  // Count by number of tokens
+  checkInterval: 100                // Check backpressure every 100 tokens
+}
+
+// CSVRecordAssemblerTransformer defaults
+writableStrategy: {
+  highWaterMark: 1024,              // 1024 tokens
+  size: (tokens) => tokens.length,  // Count by number of tokens
+  checkInterval: 10                 // Check backpressure every 10 records
+}
+readableStrategy: {
+  highWaterMark: 256,     // 256 records
+  size: () => 1,          // Each record counts as 1
+  checkInterval: 10       // Check backpressure every 10 records
+}
+```
+
+**Key Features:**
+
+🎯 **Smart Size Counting:**
+- Character-based counting for string inputs (accurate memory tracking)
+- Token-based counting between transformers (smooth pipeline flow)
+- Record-based counting for output (intuitive and predictable)
+
+⚡ **Cooperative Backpressure:**
+- Monitors `controller.desiredSize` during processing
+- Yields to event loop when backpressure detected
+- Prevents blocking the main thread
+- Critical for browser UI responsiveness
+
+🔧 **Tunable Check Interval:**
+- `checkInterval`: How often to check for backpressure
+- Lower values (5-25): More responsive, slight overhead
+- Higher values (100-500): Less overhead, slower response
+- Customize based on downstream consumer speed
+
+> ⚠️ **Important**: These defaults are theoretical starting points based on data flow characteristics, **not empirical benchmarks**. Optimal values vary by runtime (browser/Node.js/Deno), file size, memory constraints, and CPU performance. **Profile your specific use case** to find the best values.
+
+**When to customize:**
+- 🚀 **High-throughput servers**: Higher `highWaterMark` (128KB+, 2048+ tokens), higher `checkInterval` (200-500)
+- 📱 **Memory-constrained environments**: Lower `highWaterMark` (16KB, 256 tokens), lower `checkInterval` (10-25)
+- 🐌 **Slow consumers** (DB writes, API calls): Lower `highWaterMark`, lower `checkInterval` for responsive backpressure
+- 🏃 **Fast processing**: Higher values to reduce overhead
+
+**Example - High-throughput server:**
+```typescript
+import { CSVLexerTransformer, CSVRecordAssemblerTransformer } from 'web-csv-toolbox';
+
+const response = await fetch('large-dataset.csv');
+await response.body
+  .pipeThrough(new TextDecoderStream())
+  .pipeThrough(new CSVLexerTransformer(
+    {},
+    {
+      highWaterMark: 131072,          // 128KB
+      size: (chunk) => chunk.length,
+      checkInterval: 200              // Less frequent checks
+    },
+    {
+      highWaterMark: 2048,            // 2048 tokens
+      size: (tokens) => tokens.length,
+      checkInterval: 100
+    }
+  ))
+  .pipeThrough(new CSVRecordAssemblerTransformer(
+    {},
+    {
+      highWaterMark: 2048,            // 2048 tokens
+      size: (tokens) => tokens.length,
+      checkInterval: 20
+    },
+    {
+      highWaterMark: 512,             // 512 records
+      size: () => 1,
+      checkInterval: 10
+    }
+  ))
+  .pipeTo(yourRecordProcessor);
+```
+
+**Example - Slow consumer (API writes):**
+```typescript
+await csvStream
+  .pipeThrough(new CSVLexerTransformer())  // Use defaults
+  .pipeThrough(new CSVRecordAssemblerTransformer(
+    {},
+    { highWaterMark: 512, size: (t) => t.length, checkInterval: 5 },
+    { highWaterMark: 64, size: () => 1, checkInterval: 2 }  // Very responsive
+  ))
+  .pipeTo(new WritableStream({
+    async write(record) {
+      await fetch('/api/save', { method: 'POST', body: JSON.stringify(record) });
+    }
+  }));
+```
+
+**Benchmarking:**
+Use the provided benchmark tool to find optimal values for your use case:
+```bash
+pnpm --filter web-csv-toolbox-benchmark queuing-strategy
+```
+
+See `benchmark/queuing-strategy.bench.ts` for implementation details.
 
 ### Experimental APIs 🧪
 

package/dist/CSVLexerTransformer.d.ts

@@ -1,12 +1,32 @@
 import { CSVLexer } from './CSVLexer.ts';
-import { CSVLexerTransformerOptions, Token } from './common/types.ts';
+import { CSVLexerTransformerOptions, ExtendedQueuingStrategy, Token } from './common/types.ts';
 import { DEFAULT_DELIMITER, DEFAULT_QUOTATION } from './constants.ts';
 /**
  * A transform stream that converts a stream of strings into a stream of tokens.
  *
  * @category Low-level API
  *
- * @example Parse a CSV with headers by data
+ * @param options - CSV-specific options (delimiter, quotation, etc.)
+ * @param writableStrategy - Strategy for the writable side (default: `{ highWaterMark: 65536, size: chunk => chunk.length, checkInterval: 100 }`)
+ * @param readableStrategy - Strategy for the readable side (default: `{ highWaterMark: 1024, size: tokens => tokens.length, checkInterval: 100 }`)
+ *
+ * @remarks
+ * Follows the Web Streams API pattern where queuing strategies are passed as
+ * constructor arguments, similar to the standard `TransformStream`.
+ *
+ * **Default Queuing Strategy:**
+ * - Writable side: Counts by string length (characters). Default highWaterMark is 65536 characters (≈64KB).
+ * - Readable side: Counts by number of tokens in each array. Default highWaterMark is 1024 tokens.
+ *
+ * **Backpressure Handling:**
+ * The transformer monitors `controller.desiredSize` and yields to the event loop when backpressure
+ * is detected (desiredSize ≤ 0). This prevents blocking the main thread during heavy processing
+ * and allows the downstream consumer to catch up.
+ *
+ * These defaults are starting points based on data flow characteristics, not empirical benchmarks.
+ * Optimal values depend on your runtime environment, data size, and performance requirements.
+ *
+ * @example Basic usage
  * ```ts
  * new ReadableStream({
  *   start(controller) {
@@ -30,8 +50,37 @@ import { DEFAULT_DELIMITER, DEFAULT_QUOTATION } from './constants.ts';
  * // { type: Field, value: "20" }
  * // { type: RecordDelimiter, value: "\r\n", location: {...} }
  * ```
+ *
+ * @example Custom queuing strategies with backpressure tuning
+ * ```ts
+ * const transformer = new CSVLexerTransformer(
+ *   { delimiter: ',' },
+ *   {
+ *     highWaterMark: 131072,           // 128KB of characters
+ *     size: (chunk) => chunk.length,   // Count by character length
+ *     checkInterval: 200               // Check backpressure every 200 tokens
+ *   },
+ *   {
+ *     highWaterMark: 2048,             // 2048 tokens
+ *     size: (tokens) => tokens.length, // Count by token count
+ *     checkInterval: 50                // Check backpressure every 50 tokens
+ *   }
+ * );
+ *
+ * await fetch('large-file.csv')
+ *   .then(res => res.body)
+ *   .pipeThrough(new TextDecoderStream())
+ *   .pipeThrough(transformer)
+ *   .pipeTo(yourProcessor);
+ * ```
  */
 export declare class CSVLexerTransformer<Delimiter extends string = DEFAULT_DELIMITER, Quotation extends string = DEFAULT_QUOTATION> extends TransformStream<string, Token[]> {
     readonly lexer: CSVLexer<Delimiter, Quotation>;
-    constructor(options?: CSVLexerTransformerOptions<Delimiter, Quotation>);
+    /**
+     * Yields to the event loop to allow backpressure handling.
+     * Can be overridden for testing purposes.
+     * @internal
+     */
+    protected yieldToEventLoop(): Promise<void>;
+    constructor(options?: CSVLexerTransformerOptions<Delimiter, Quotation>, writableStrategy?: ExtendedQueuingStrategy<string>, readableStrategy?: ExtendedQueuingStrategy<Token[]>);
 }

package/dist/CSVLexerTransformer.js

@@ -2,26 +2,71 @@ import { CSVLexer } from './CSVLexer.js';
 
 class CSVLexerTransformer extends TransformStream {
   lexer;
-  constructor(options = {}) {
+  /**
+   * Yields to the event loop to allow backpressure handling.
+   * Can be overridden for testing purposes.
+   * @internal
+   */
+  async yieldToEventLoop() {
+    await new Promise((resolve) => setTimeout(resolve, 0));
+  }
+  constructor(options = {}, writableStrategy = {
+    highWaterMark: 65536,
+    // 64KB worth of characters
+    size: (chunk) => chunk.length,
+    // Count by string length (character count)
+    checkInterval: 100
+    // Check backpressure every 100 tokens
+  }, readableStrategy = {
+    highWaterMark: 1024,
+    // 1024 tokens
+    size: (tokens) => tokens.length,
+    // Count by number of tokens in array
+    checkInterval: 100
+    // Check backpressure every 100 tokens
+  }) {
     const lexer = new CSVLexer(options);
-    super({
-      transform: (chunk, controller) => {
-        if (chunk.length !== 0) {
+    const checkInterval = writableStrategy.checkInterval ?? readableStrategy.checkInterval ?? 100;
+    super(
+      {
+        transform: async (chunk, controller) => {
+          if (chunk.length !== 0) {
+            try {
+              const tokens = [];
+              for (const token of lexer.lex(chunk, { stream: true })) {
+                tokens.push(token);
+                if (tokens.length % checkInterval === 0 && controller.desiredSize !== null && controller.desiredSize <= 0) {
+                  await this.yieldToEventLoop();
+                }
+              }
+              if (tokens.length > 0) {
+                controller.enqueue(tokens);
+              }
+            } catch (error) {
+              controller.error(error);
+            }
+          }
+        },
+        flush: async (controller) => {
           try {
-            controller.enqueue([...lexer.lex(chunk, { stream: true })]);
+            const tokens = [];
+            for (const token of lexer.lex()) {
+              tokens.push(token);
+              if (tokens.length % checkInterval === 0 && controller.desiredSize !== null && controller.desiredSize <= 0) {
+                await this.yieldToEventLoop();
+              }
+            }
+            if (tokens.length > 0) {
+              controller.enqueue(tokens);
+            }
           } catch (error) {
             controller.error(error);
           }
         }
       },
-      flush: (controller) => {
-        try {
-          controller.enqueue([...lexer.lex()]);
-        } catch (error) {
-          controller.error(error);
-        }
-      }
-    });
+      writableStrategy,
+      readableStrategy
+    );
     this.lexer = lexer;
   }
 }

package/dist/CSVLexerTransformer.js.map

@@ -1 +1 @@
-{"version":3,"file":"CSVLexerTransformer.js","sources":["../src/CSVLexerTransformer.ts"],"sourcesContent":["import { CSVLexer } from \"./CSVLexer.ts\";\nimport type { CSVLexerTransformerOptions, Token } from \"./common/types.ts\";\nimport type { DEFAULT_DELIMITER, DEFAULT_QUOTATION } from \"./constants.ts\";\n\n/**\n * A transform stream that converts a stream of strings into a stream of tokens.\n *\n * @category Low-level API\n *\n * @example Parse a CSV with headers by data\n * ```ts\n * new ReadableStream({\n *   start(controller) {\n *     controller.enqueue(\"name,age\\r\\n\");\n *     controller.enqueue(\"Alice,20\\r\\n\");\n *     controller.close();\n *   }\n * })\n *   .pipeThrough(new CSVLexerTransformer())\n *   .pipeTo(new WritableStream({ write(tokens) {\n *     for (const token of tokens) {\n *       console.log(token);\n *     }\n *   }}));\n * // { type: Field, value: \"name\", location: {...} }\n * // { type: FieldDelimiter, value: \",\", location: {...} }\n * // { type: Field, value: \"age\", location: {...} }\n * // { type: RecordDelimiter, value: \"\\r\\n\", location: {...} }\n * // { type: Field, value: \"Alice\", location: {...} }\n * // { type: FieldDelimiter, value: \",\", location: {...} }\n * // { type: Field, value: \"20\" }\n * // { type: RecordDelimiter, value: \"\\r\\n\", location: {...} }\n * ```\n */\nexport class CSVLexerTransformer<\n  Delimiter extends string = DEFAULT_DELIMITER,\n  Quotation extends string = DEFAULT_QUOTATION,\n> extends TransformStream<string, Token[]> {\n  public readonly lexer: CSVLexer<Delimiter, Quotation>;\n  constructor(options: CSVLexerTransformerOptions<Delimiter, Quotation> = {}) {\n    const lexer = new CSVLexer(options);\n    super({\n      transform: (chunk, controller) => {\n        if (chunk.length !== 0) {\n          try {\n            controller.enqueue([...lexer.lex(chunk, { stream: true })]);\n          } catch (error) {\n            controller.error(error);\n          }\n        }\n      },\n      flush: (controller) => {\n        try {\n          controller.enqueue([...lexer.lex()]);\n        } catch (error) {\n          controller.error(error);\n        }\n      },\n    });\n    this.lexer = lexer;\n  }\n}\n"],"names":[],"mappings":";;AAkCO,MAAM,4BAGH,eAAiC,CAAA;AAAA,EACzB,KAAA;AAAA,EAChB,WAAA,CAAY,OAA4D,GAAA,EAAI,EAAA;AAC1E,IAAM,MAAA,KAAA,GAAQ,IAAI,QAAA,CAAS,OAAO,CAAA;AAClC,IAAM,KAAA,CAAA;AAAA,MACJ,SAAA,EAAW,CAAC,KAAA,EAAO,UAAe,KAAA;AAChC,QAAI,IAAA,KAAA,CAAM,WAAW,CAAG,EAAA;AACtB,UAAI,IAAA;AACF,YAAW,UAAA,CAAA,OAAA,CAAQ,CAAC,GAAG,KAAM,CAAA,GAAA,CAAI,KAAO,EAAA,EAAE,MAAQ,EAAA,IAAA,EAAM,CAAC,CAAC,CAAA;AAAA,mBACnD,KAAO,EAAA;AACd,YAAA,UAAA,CAAW,MAAM,KAAK,CAAA;AAAA;AACxB;AACF,OACF;AAAA,MACA,KAAA,EAAO,CAAC,UAAe,KAAA;AACrB,QAAI,IAAA;AACF,UAAA,UAAA,CAAW,QAAQ,CAAC,GAAG,KAAM,CAAA,GAAA,EAAK,CAAC,CAAA;AAAA,iBAC5B,KAAO,EAAA;AACd,UAAA,UAAA,CAAW,MAAM,KAAK,CAAA;AAAA;AACxB;AACF,KACD,CAAA;AACD,IAAA,IAAA,CAAK,KAAQ,GAAA,KAAA;AAAA;AAEjB;;;;"}
+{"version":3,"file":"CSVLexerTransformer.js","sources":["../src/CSVLexerTransformer.ts"],"sourcesContent":["import { CSVLexer } from \"./CSVLexer.ts\";\nimport type {\n  CSVLexerTransformerOptions,\n  ExtendedQueuingStrategy,\n  Token,\n} from \"./common/types.ts\";\nimport type { DEFAULT_DELIMITER, DEFAULT_QUOTATION } from \"./constants.ts\";\n\n/**\n * A transform stream that converts a stream of strings into a stream of tokens.\n *\n * @category Low-level API\n *\n * @param options - CSV-specific options (delimiter, quotation, etc.)\n * @param writableStrategy - Strategy for the writable side (default: `{ highWaterMark: 65536, size: chunk => chunk.length, checkInterval: 100 }`)\n * @param readableStrategy - Strategy for the readable side (default: `{ highWaterMark: 1024, size: tokens => tokens.length, checkInterval: 100 }`)\n *\n * @remarks\n * Follows the Web Streams API pattern where queuing strategies are passed as\n * constructor arguments, similar to the standard `TransformStream`.\n *\n * **Default Queuing Strategy:**\n * - Writable side: Counts by string length (characters). Default highWaterMark is 65536 characters (≈64KB).\n * - Readable side: Counts by number of tokens in each array. Default highWaterMark is 1024 tokens.\n *\n * **Backpressure Handling:**\n * The transformer monitors `controller.desiredSize` and yields to the event loop when backpressure\n * is detected (desiredSize ≤ 0). This prevents blocking the main thread during heavy processing\n * and allows the downstream consumer to catch up.\n *\n * These defaults are starting points based on data flow characteristics, not empirical benchmarks.\n * Optimal values depend on your runtime environment, data size, and performance requirements.\n *\n * @example Basic usage\n * ```ts\n * new ReadableStream({\n *   start(controller) {\n *     controller.enqueue(\"name,age\\r\\n\");\n *     controller.enqueue(\"Alice,20\\r\\n\");\n *     controller.close();\n *   }\n * })\n *   .pipeThrough(new CSVLexerTransformer())\n *   .pipeTo(new WritableStream({ write(tokens) {\n *     for (const token of tokens) {\n *       console.log(token);\n *     }\n *   }}));\n * // { type: Field, value: \"name\", location: {...} }\n * // { type: FieldDelimiter, value: \",\", location: {...} }\n * // { type: Field, value: \"age\", location: {...} }\n * // { type: RecordDelimiter, value: \"\\r\\n\", location: {...} }\n * // { type: Field, value: \"Alice\", location: {...} }\n * // { type: FieldDelimiter, value: \",\", location: {...} }\n * // { type: Field, value: \"20\" }\n * // { type: RecordDelimiter, value: \"\\r\\n\", location: {...} }\n * ```\n *\n * @example Custom queuing strategies with backpressure tuning\n * ```ts\n * const transformer = new CSVLexerTransformer(\n *   { delimiter: ',' },\n *   {\n *     highWaterMark: 131072,           // 128KB of characters\n *     size: (chunk) => chunk.length,   // Count by character length\n *     checkInterval: 200               // Check backpressure every 200 tokens\n *   },\n *   {\n *     highWaterMark: 2048,             // 2048 tokens\n *     size: (tokens) => tokens.length, // Count by token count\n *     checkInterval: 50                // Check backpressure every 50 tokens\n *   }\n * );\n *\n * await fetch('large-file.csv')\n *   .then(res => res.body)\n *   .pipeThrough(new TextDecoderStream())\n *   .pipeThrough(transformer)\n *   .pipeTo(yourProcessor);\n * ```\n */\nexport class CSVLexerTransformer<\n  Delimiter extends string = DEFAULT_DELIMITER,\n  Quotation extends string = DEFAULT_QUOTATION,\n> extends TransformStream<string, Token[]> {\n  public readonly lexer: CSVLexer<Delimiter, Quotation>;\n\n  /**\n   * Yields to the event loop to allow backpressure handling.\n   * Can be overridden for testing purposes.\n   * @internal\n   */\n  protected async yieldToEventLoop(): Promise<void> {\n    await new Promise((resolve) => setTimeout(resolve, 0));\n  }\n\n  constructor(\n    options: CSVLexerTransformerOptions<Delimiter, Quotation> = {},\n    writableStrategy: ExtendedQueuingStrategy<string> = {\n      highWaterMark: 65536, // 64KB worth of characters\n      size: (chunk) => chunk.length, // Count by string length (character count)\n      checkInterval: 100, // Check backpressure every 100 tokens\n    },\n    readableStrategy: ExtendedQueuingStrategy<Token[]> = {\n      highWaterMark: 1024, // 1024 tokens\n      size: (tokens) => tokens.length, // Count by number of tokens in array\n      checkInterval: 100, // Check backpressure every 100 tokens\n    },\n  ) {\n    const lexer = new CSVLexer(options);\n    const checkInterval =\n      writableStrategy.checkInterval ?? readableStrategy.checkInterval ?? 100;\n\n    super(\n      {\n        transform: async (chunk, controller) => {\n          if (chunk.length !== 0) {\n            try {\n              const tokens: Token[] = [];\n              for (const token of lexer.lex(chunk, { stream: true })) {\n                tokens.push(token);\n\n                // Check backpressure periodically based on checkInterval\n                if (\n                  tokens.length % checkInterval === 0 &&\n                  controller.desiredSize !== null &&\n                  controller.desiredSize <= 0\n                ) {\n                  // Yield to event loop when backpressure is detected\n                  await this.yieldToEventLoop();\n                }\n              }\n\n              if (tokens.length > 0) {\n                controller.enqueue(tokens);\n              }\n            } catch (error) {\n              controller.error(error);\n            }\n          }\n        },\n        flush: async (controller) => {\n          try {\n            const tokens: Token[] = [];\n            for (const token of lexer.lex()) {\n              tokens.push(token);\n\n              // Check backpressure periodically based on checkInterval\n              if (\n                tokens.length % checkInterval === 0 &&\n                controller.desiredSize !== null &&\n                controller.desiredSize <= 0\n              ) {\n                await this.yieldToEventLoop();\n              }\n            }\n\n            if (tokens.length > 0) {\n              controller.enqueue(tokens);\n            }\n          } catch (error) {\n            controller.error(error);\n          }\n        },\n      },\n      writableStrategy,\n      readableStrategy,\n    );\n    this.lexer = lexer;\n  }\n}\n"],"names":[],"mappings":";;AAiFO,MAAM,4BAGH,eAAiC,CAAA;AAAA,EACzB,KAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOhB,MAAgB,gBAAkC,GAAA;AAChD,IAAA,MAAM,IAAI,OAAQ,CAAA,CAAC,YAAY,UAAW,CAAA,OAAA,EAAS,CAAC,CAAC,CAAA;AAAA;AACvD,EAEA,WACE,CAAA,OAAA,GAA4D,EAAC,EAC7D,gBAAoD,GAAA;AAAA,IAClD,aAAe,EAAA,KAAA;AAAA;AAAA,IACf,IAAA,EAAM,CAAC,KAAA,KAAU,KAAM,CAAA,MAAA;AAAA;AAAA,IACvB,aAAe,EAAA;AAAA;AAAA,KAEjB,gBAAqD,GAAA;AAAA,IACnD,aAAe,EAAA,IAAA;AAAA;AAAA,IACf,IAAA,EAAM,CAAC,MAAA,KAAW,MAAO,CAAA,MAAA;AAAA;AAAA,IACzB,aAAe,EAAA;AAAA;AAAA,GAEjB,EAAA;AACA,IAAM,MAAA,KAAA,GAAQ,IAAI,QAAA,CAAS,OAAO,CAAA;AAClC,IAAA,MAAM,aACJ,GAAA,gBAAA,CAAiB,aAAiB,IAAA,gBAAA,CAAiB,aAAiB,IAAA,GAAA;AAEtE,IAAA,KAAA;AAAA,MACE;AAAA,QACE,SAAA,EAAW,OAAO,KAAA,EAAO,UAAe,KAAA;AACtC,UAAI,IAAA,KAAA,CAAM,WAAW,CAAG,EAAA;AACtB,YAAI,IAAA;AACF,cAAA,MAAM,SAAkB,EAAC;AACzB,cAAW,KAAA,MAAA,KAAA,IAAS,MAAM,GAAI,CAAA,KAAA,EAAO,EAAE,MAAQ,EAAA,IAAA,EAAM,CAAG,EAAA;AACtD,gBAAA,MAAA,CAAO,KAAK,KAAK,CAAA;AAGjB,gBACE,IAAA,MAAA,CAAO,SAAS,aAAkB,KAAA,CAAA,IAClC,WAAW,WAAgB,KAAA,IAAA,IAC3B,UAAW,CAAA,WAAA,IAAe,CAC1B,EAAA;AAEA,kBAAA,MAAM,KAAK,gBAAiB,EAAA;AAAA;AAC9B;AAGF,cAAI,IAAA,MAAA,CAAO,SAAS,CAAG,EAAA;AACrB,gBAAA,UAAA,CAAW,QAAQ,MAAM,CAAA;AAAA;AAC3B,qBACO,KAAO,EAAA;AACd,cAAA,UAAA,CAAW,MAAM,KAAK,CAAA;AAAA;AACxB;AACF,SACF;AAAA,QACA,KAAA,EAAO,OAAO,UAAe,KAAA;AAC3B,UAAI,IAAA;AACF,YAAA,MAAM,SAAkB,EAAC;AACzB,YAAW,KAAA,MAAA,KAAA,IAAS,KAAM,CAAA,GAAA,EAAO,EAAA;AAC/B,cAAA,MAAA,CAAO,KAAK,KAAK,CAAA;AAGjB,cACE,IAAA,MAAA,CAAO,SAAS,aAAkB,KAAA,CAAA,IAClC,WAAW,WAAgB,KAAA,IAAA,IAC3B,UAAW,CAAA,WAAA,IAAe,CAC1B,EAAA;AACA,gBAAA,MAAM,KAAK,gBAAiB,EAAA;AAAA;AAC9B;AAGF,YAAI,IAAA,MAAA,CAAO,SAAS,CAAG,EAAA;AACrB,cAAA,UAAA,CAAW,QAAQ,MAAM,CAAA;AAAA;AAC3B,mBACO,KAAO,EAAA;AACd,YAAA,UAAA,CAAW,MAAM,KAAK,CAAA;AAAA;AACxB;AACF,OACF;AAAA,MACA,gBAAA;AAAA,MACA;AAAA,KACF;AACA,IAAA,IAAA,CAAK,KAAQ,GAAA,KAAA;AAAA;AAEjB;;;;"}

package/dist/CSVRecordAssemblerTransformer.d.ts

@@ -1,12 +1,31 @@
 import { CSVRecordAssembler } from './CSVRecordAssembler.ts';
-import { CSVRecord, CSVRecordAssemblerOptions, Token } from './common/types.ts';
+import { CSVRecord, CSVRecordAssemblerOptions, ExtendedQueuingStrategy, Token } from './common/types.ts';
 /**
  * A transform stream that converts a stream of tokens into a stream of CSV records.
+ *
  * @template Header The type of the header row.
- * @param options The options for the parser.
+ * @param options - CSV-specific options (header, maxFieldCount, etc.)
+ * @param writableStrategy - Strategy for the writable side (default: `{ highWaterMark: 1024, size: tokens => tokens.length, checkInterval: 10 }`)
+ * @param readableStrategy - Strategy for the readable side (default: `{ highWaterMark: 256, size: () => 1, checkInterval: 10 }`)
  *
  * @category Low-level API
  *
+ * @remarks
+ * Follows the Web Streams API pattern where queuing strategies are passed as
+ * constructor arguments, similar to the standard `TransformStream`.
+ *
+ * **Default Queuing Strategy:**
+ * - Writable side: Counts by number of tokens in each array. Default highWaterMark is 1024 tokens.
+ * - Readable side: Counts each record as 1. Default highWaterMark is 256 records.
+ *
+ * **Backpressure Handling:**
+ * The transformer monitors `controller.desiredSize` and yields to the event loop when backpressure
+ * is detected (desiredSize ≤ 0). This prevents blocking the main thread during heavy processing
+ * and allows the downstream consumer to catch up.
+ *
+ * These defaults are starting points based on data flow characteristics, not empirical benchmarks.
+ * Optimal values depend on your runtime environment, data size, and performance requirements.
+ *
  * @example Parse a CSV with headers by data
  *  ```ts
  * new ReadableStream({
@@ -42,8 +61,35 @@ import { CSVRecord, CSVRecordAssemblerOptions, Token } from './common/types.ts';
  * // { name: "Bob", age: "25" }
  * // { name: "Charlie", age: "30" }
  * ```
+ *
+ * @example Custom queuing strategies with backpressure tuning
+ * ```ts
+ * const transformer = new CSVRecordAssemblerTransformer(
+ *   {},
+ *   {
+ *     highWaterMark: 2048,             // 2048 tokens
+ *     size: (tokens) => tokens.length, // Count by token count
+ *     checkInterval: 20                // Check backpressure every 20 records
+ *   },
+ *   {
+ *     highWaterMark: 512,              // 512 records
+ *     size: () => 1,                   // Each record counts as 1
+ *     checkInterval: 5                 // Check backpressure every 5 records
+ *   }
+ * );
+ *
+ * await tokenStream
+ *   .pipeThrough(transformer)
+ *   .pipeTo(yourRecordProcessor);
+ * ```
  */
 export declare class CSVRecordAssemblerTransformer<Header extends ReadonlyArray<string>> extends TransformStream<Token[], CSVRecord<Header>> {
     readonly assembler: CSVRecordAssembler<Header>;
-    constructor(options?: CSVRecordAssemblerOptions<Header>);
+    /**
+     * Yields to the event loop to allow backpressure handling.
+     * Can be overridden for testing purposes.
+     * @internal
+     */
+    protected yieldToEventLoop(): Promise<void>;
+    constructor(options?: CSVRecordAssemblerOptions<Header>, writableStrategy?: ExtendedQueuingStrategy<Token[]>, readableStrategy?: ExtendedQueuingStrategy<CSVRecord<Header>>);
 }

package/dist/CSVRecordAssemblerTransformer.js

@@ -2,28 +2,65 @@ import { CSVRecordAssembler } from './CSVRecordAssembler.js';
 
 class CSVRecordAssemblerTransformer extends TransformStream {
   assembler;
-  constructor(options = {}) {
+  /**
+   * Yields to the event loop to allow backpressure handling.
+   * Can be overridden for testing purposes.
+   * @internal
+   */
+  async yieldToEventLoop() {
+    await new Promise((resolve) => setTimeout(resolve, 0));
+  }
+  constructor(options = {}, writableStrategy = {
+    highWaterMark: 1024,
+    // 1024 tokens
+    size: (tokens) => tokens.length,
+    // Count by number of tokens in array
+    checkInterval: 10
+    // Check backpressure every 10 records
+  }, readableStrategy = {
+    highWaterMark: 256,
+    // 256 records
+    size: () => 1,
+    // Each record counts as 1
+    checkInterval: 10
+    // Check backpressure every 10 records
+  }) {
     const assembler = new CSVRecordAssembler(options);
-    super({
-      transform: (tokens, controller) => {
-        try {
-          for (const token of assembler.assemble(tokens, { stream: true })) {
-            controller.enqueue(token);
+    const checkInterval = writableStrategy.checkInterval ?? readableStrategy.checkInterval ?? 10;
+    super(
+      {
+        transform: async (tokens, controller) => {
+          try {
+            let recordCount = 0;
+            for (const record of assembler.assemble(tokens, { stream: true })) {
+              controller.enqueue(record);
+              recordCount++;
+              if (recordCount % checkInterval === 0 && controller.desiredSize !== null && controller.desiredSize <= 0) {
+                await this.yieldToEventLoop();
+              }
+            }
+          } catch (error) {
+            controller.error(error);
           }
-        } catch (error) {
-          controller.error(error);
-        }
-      },
-      flush: (controller) => {
-        try {
-          for (const token of assembler.assemble()) {
-            controller.enqueue(token);
+        },
+        flush: async (controller) => {
+          try {
+            let recordCount = 0;
+            for (const record of assembler.assemble()) {
+              controller.enqueue(record);
+              recordCount++;
+              if (recordCount % checkInterval === 0 && controller.desiredSize !== null && controller.desiredSize <= 0) {
+                await this.yieldToEventLoop();
+              }
+            }
+          } catch (error) {
+            controller.error(error);
           }
-        } catch (error) {
-          controller.error(error);
         }
-      }
-    });
+      },
+      writableStrategy,
+      readableStrategy
+    );
     this.assembler = assembler;
   }
 }

package/dist/CSVRecordAssemblerTransformer.js.map

@@ -1 +1 @@
-{"version":3,"file":"CSVRecordAssemblerTransformer.js","sources":["../src/CSVRecordAssemblerTransformer.ts"],"sourcesContent":["import { CSVRecordAssembler } from \"./CSVRecordAssembler.ts\";\nimport type {\n  CSVRecord,\n  CSVRecordAssemblerOptions,\n  Token,\n} from \"./common/types.ts\";\n\n/**\n * A transform stream that converts a stream of tokens into a stream of CSV records.\n * @template Header The type of the header row.\n * @param options The options for the parser.\n *\n * @category Low-level API\n *\n * @example Parse a CSV with headers by data\n *  ```ts\n * new ReadableStream({\n *   start(controller) {\n *     controller.enqueue(\"name,age\\r\\n\");\n *     controller.enqueue(\"Alice,20\\r\\n\");\n *     controller.enqueue(\"Bob,25\\r\\n\");\n *     controller.enqueue(\"Charlie,30\\r\\n\");\n *     controller.close();\n *   })\n *   .pipeThrough(new CSVLexerTransformer())\n *   .pipeThrough(new CSVRecordAssemblerTransformer())\n *   .pipeTo(new WritableStream({ write(row) { console.log(row); }}));\n * // { name: \"Alice\", age: \"20\" }\n * // { name: \"Bob\", age: \"25\" }\n * // { name: \"Charlie\", age: \"30\" }\n * ```\n *\n * @example Parse a CSV with headers by options\n * ```ts\n * new ReadableStream({\n *   start(controller) {\n *     controller.enqueue(\"Alice,20\\r\\n\");\n *     controller.enqueue(\"Bob,25\\r\\n\");\n *     controller.enqueue(\"Charlie,30\\r\\n\");\n *     controller.close();\n *   }\n * })\n * .pipeThrough(new CSVLexerTransformer())\n * .pipeThrough(new CSVRecordAssemblerTransformer({ header: [\"name\", \"age\"] }))\n * .pipeTo(new WritableStream({ write(row) { console.log(row); }}));\n * // { name: \"Alice\", age: \"20\" }\n * // { name: \"Bob\", age: \"25\" }\n * // { name: \"Charlie\", age: \"30\" }\n * ```\n */\nexport class CSVRecordAssemblerTransformer<\n  Header extends ReadonlyArray<string>,\n> extends TransformStream<Token[], CSVRecord<Header>> {\n  public readonly assembler: CSVRecordAssembler<Header>;\n\n  constructor(options: CSVRecordAssemblerOptions<Header> = {}) {\n    const assembler = new CSVRecordAssembler(options);\n    super({\n      transform: (tokens, controller) => {\n        try {\n          for (const token of assembler.assemble(tokens, { stream: true })) {\n            controller.enqueue(token);\n          }\n        } catch (error) {\n          controller.error(error);\n        }\n      },\n      flush: (controller) => {\n        try {\n          for (const token of assembler.assemble()) {\n            controller.enqueue(token);\n          }\n        } catch (error) {\n          controller.error(error);\n        }\n      },\n    });\n    this.assembler = assembler;\n  }\n}\n"],"names":[],"mappings":";;AAkDO,MAAM,sCAEH,eAA4C,CAAA;AAAA,EACpC,SAAA;AAAA,EAEhB,WAAA,CAAY,OAA6C,GAAA,EAAI,EAAA;AAC3D,IAAM,MAAA,SAAA,GAAY,IAAI,kBAAA,CAAmB,OAAO,CAAA;AAChD,IAAM,KAAA,CAAA;AAAA,MACJ,SAAA,EAAW,CAAC,MAAA,EAAQ,UAAe,KAAA;AACjC,QAAI,IAAA;AACF,UAAW,KAAA,MAAA,KAAA,IAAS,UAAU,QAAS,CAAA,MAAA,EAAQ,EAAE,MAAQ,EAAA,IAAA,EAAM,CAAG,EAAA;AAChE,YAAA,UAAA,CAAW,QAAQ,KAAK,CAAA;AAAA;AAC1B,iBACO,KAAO,EAAA;AACd,UAAA,UAAA,CAAW,MAAM,KAAK,CAAA;AAAA;AACxB,OACF;AAAA,MACA,KAAA,EAAO,CAAC,UAAe,KAAA;AACrB,QAAI,IAAA;AACF,UAAW,KAAA,MAAA,KAAA,IAAS,SAAU,CAAA,QAAA,EAAY,EAAA;AACxC,YAAA,UAAA,CAAW,QAAQ,KAAK,CAAA;AAAA;AAC1B,iBACO,KAAO,EAAA;AACd,UAAA,UAAA,CAAW,MAAM,KAAK,CAAA;AAAA;AACxB;AACF,KACD,CAAA;AACD,IAAA,IAAA,CAAK,SAAY,GAAA,SAAA;AAAA;AAErB;;;;"}
+{"version":3,"file":"CSVRecordAssemblerTransformer.js","sources":["../src/CSVRecordAssemblerTransformer.ts"],"sourcesContent":["import { CSVRecordAssembler } from \"./CSVRecordAssembler.ts\";\nimport type {\n  CSVRecord,\n  CSVRecordAssemblerOptions,\n  ExtendedQueuingStrategy,\n  Token,\n} from \"./common/types.ts\";\n\n/**\n * A transform stream that converts a stream of tokens into a stream of CSV records.\n *\n * @template Header The type of the header row.\n * @param options - CSV-specific options (header, maxFieldCount, etc.)\n * @param writableStrategy - Strategy for the writable side (default: `{ highWaterMark: 1024, size: tokens => tokens.length, checkInterval: 10 }`)\n * @param readableStrategy - Strategy for the readable side (default: `{ highWaterMark: 256, size: () => 1, checkInterval: 10 }`)\n *\n * @category Low-level API\n *\n * @remarks\n * Follows the Web Streams API pattern where queuing strategies are passed as\n * constructor arguments, similar to the standard `TransformStream`.\n *\n * **Default Queuing Strategy:**\n * - Writable side: Counts by number of tokens in each array. Default highWaterMark is 1024 tokens.\n * - Readable side: Counts each record as 1. Default highWaterMark is 256 records.\n *\n * **Backpressure Handling:**\n * The transformer monitors `controller.desiredSize` and yields to the event loop when backpressure\n * is detected (desiredSize ≤ 0). This prevents blocking the main thread during heavy processing\n * and allows the downstream consumer to catch up.\n *\n * These defaults are starting points based on data flow characteristics, not empirical benchmarks.\n * Optimal values depend on your runtime environment, data size, and performance requirements.\n *\n * @example Parse a CSV with headers by data\n *  ```ts\n * new ReadableStream({\n *   start(controller) {\n *     controller.enqueue(\"name,age\\r\\n\");\n *     controller.enqueue(\"Alice,20\\r\\n\");\n *     controller.enqueue(\"Bob,25\\r\\n\");\n *     controller.enqueue(\"Charlie,30\\r\\n\");\n *     controller.close();\n *   })\n *   .pipeThrough(new CSVLexerTransformer())\n *   .pipeThrough(new CSVRecordAssemblerTransformer())\n *   .pipeTo(new WritableStream({ write(row) { console.log(row); }}));\n * // { name: \"Alice\", age: \"20\" }\n * // { name: \"Bob\", age: \"25\" }\n * // { name: \"Charlie\", age: \"30\" }\n * ```\n *\n * @example Parse a CSV with headers by options\n * ```ts\n * new ReadableStream({\n *   start(controller) {\n *     controller.enqueue(\"Alice,20\\r\\n\");\n *     controller.enqueue(\"Bob,25\\r\\n\");\n *     controller.enqueue(\"Charlie,30\\r\\n\");\n *     controller.close();\n *   }\n * })\n * .pipeThrough(new CSVLexerTransformer())\n * .pipeThrough(new CSVRecordAssemblerTransformer({ header: [\"name\", \"age\"] }))\n * .pipeTo(new WritableStream({ write(row) { console.log(row); }}));\n * // { name: \"Alice\", age: \"20\" }\n * // { name: \"Bob\", age: \"25\" }\n * // { name: \"Charlie\", age: \"30\" }\n * ```\n *\n * @example Custom queuing strategies with backpressure tuning\n * ```ts\n * const transformer = new CSVRecordAssemblerTransformer(\n *   {},\n *   {\n *     highWaterMark: 2048,             // 2048 tokens\n *     size: (tokens) => tokens.length, // Count by token count\n *     checkInterval: 20                // Check backpressure every 20 records\n *   },\n *   {\n *     highWaterMark: 512,              // 512 records\n *     size: () => 1,                   // Each record counts as 1\n *     checkInterval: 5                 // Check backpressure every 5 records\n *   }\n * );\n *\n * await tokenStream\n *   .pipeThrough(transformer)\n *   .pipeTo(yourRecordProcessor);\n * ```\n */\nexport class CSVRecordAssemblerTransformer<\n  Header extends ReadonlyArray<string>,\n> extends TransformStream<Token[], CSVRecord<Header>> {\n  public readonly assembler: CSVRecordAssembler<Header>;\n\n  /**\n   * Yields to the event loop to allow backpressure handling.\n   * Can be overridden for testing purposes.\n   * @internal\n   */\n  protected async yieldToEventLoop(): Promise<void> {\n    await new Promise((resolve) => setTimeout(resolve, 0));\n  }\n\n  constructor(\n    options: CSVRecordAssemblerOptions<Header> = {},\n    writableStrategy: ExtendedQueuingStrategy<Token[]> = {\n      highWaterMark: 1024, // 1024 tokens\n      size: (tokens) => tokens.length, // Count by number of tokens in array\n      checkInterval: 10, // Check backpressure every 10 records\n    },\n    readableStrategy: ExtendedQueuingStrategy<CSVRecord<Header>> = {\n      highWaterMark: 256, // 256 records\n      size: () => 1, // Each record counts as 1\n      checkInterval: 10, // Check backpressure every 10 records\n    },\n  ) {\n    const assembler = new CSVRecordAssembler(options);\n    const checkInterval =\n      writableStrategy.checkInterval ?? readableStrategy.checkInterval ?? 10;\n\n    super(\n      {\n        transform: async (tokens, controller) => {\n          try {\n            let recordCount = 0;\n            for (const record of assembler.assemble(tokens, { stream: true })) {\n              controller.enqueue(record);\n              recordCount++;\n\n              // Check backpressure periodically based on checkInterval\n              if (\n                recordCount % checkInterval === 0 &&\n                controller.desiredSize !== null &&\n                controller.desiredSize <= 0\n              ) {\n                // Yield to event loop when backpressure is detected\n                await this.yieldToEventLoop();\n              }\n            }\n          } catch (error) {\n            controller.error(error);\n          }\n        },\n        flush: async (controller) => {\n          try {\n            let recordCount = 0;\n            for (const record of assembler.assemble()) {\n              controller.enqueue(record);\n              recordCount++;\n\n              // Check backpressure periodically based on checkInterval\n              if (\n                recordCount % checkInterval === 0 &&\n                controller.desiredSize !== null &&\n                controller.desiredSize <= 0\n              ) {\n                await this.yieldToEventLoop();\n              }\n            }\n          } catch (error) {\n            controller.error(error);\n          }\n        },\n      },\n      writableStrategy,\n      readableStrategy,\n    );\n    this.assembler = assembler;\n  }\n}\n"],"names":[],"mappings":";;AA2FO,MAAM,sCAEH,eAA4C,CAAA;AAAA,EACpC,SAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOhB,MAAgB,gBAAkC,GAAA;AAChD,IAAA,MAAM,IAAI,OAAQ,CAAA,CAAC,YAAY,UAAW,CAAA,OAAA,EAAS,CAAC,CAAC,CAAA;AAAA;AACvD,EAEA,WACE,CAAA,OAAA,GAA6C,EAAC,EAC9C,gBAAqD,GAAA;AAAA,IACnD,aAAe,EAAA,IAAA;AAAA;AAAA,IACf,IAAA,EAAM,CAAC,MAAA,KAAW,MAAO,CAAA,MAAA;AAAA;AAAA,IACzB,aAAe,EAAA;AAAA;AAAA,KAEjB,gBAA+D,GAAA;AAAA,IAC7D,aAAe,EAAA,GAAA;AAAA;AAAA,IACf,MAAM,MAAM,CAAA;AAAA;AAAA,IACZ,aAAe,EAAA;AAAA;AAAA,GAEjB,EAAA;AACA,IAAM,MAAA,SAAA,GAAY,IAAI,kBAAA,CAAmB,OAAO,CAAA;AAChD,IAAA,MAAM,aACJ,GAAA,gBAAA,CAAiB,aAAiB,IAAA,gBAAA,CAAiB,aAAiB,IAAA,EAAA;AAEtE,IAAA,KAAA;AAAA,MACE;AAAA,QACE,SAAA,EAAW,OAAO,MAAA,EAAQ,UAAe,KAAA;AACvC,UAAI,IAAA;AACF,YAAA,IAAI,WAAc,GAAA,CAAA;AAClB,YAAW,KAAA,MAAA,MAAA,IAAU,UAAU,QAAS,CAAA,MAAA,EAAQ,EAAE,MAAQ,EAAA,IAAA,EAAM,CAAG,EAAA;AACjE,cAAA,UAAA,CAAW,QAAQ,MAAM,CAAA;AACzB,cAAA,WAAA,EAAA;AAGA,cACE,IAAA,WAAA,GAAc,kBAAkB,CAChC,IAAA,UAAA,CAAW,gBAAgB,IAC3B,IAAA,UAAA,CAAW,eAAe,CAC1B,EAAA;AAEA,gBAAA,MAAM,KAAK,gBAAiB,EAAA;AAAA;AAC9B;AACF,mBACO,KAAO,EAAA;AACd,YAAA,UAAA,CAAW,MAAM,KAAK,CAAA;AAAA;AACxB,SACF;AAAA,QACA,KAAA,EAAO,OAAO,UAAe,KAAA;AAC3B,UAAI,IAAA;AACF,YAAA,IAAI,WAAc,GAAA,CAAA;AAClB,YAAW,KAAA,MAAA,MAAA,IAAU,SAAU,CAAA,QAAA,EAAY,EAAA;AACzC,cAAA,UAAA,CAAW,QAAQ,MAAM,CAAA;AACzB,cAAA,WAAA,EAAA;AAGA,cACE,IAAA,WAAA,GAAc,kBAAkB,CAChC,IAAA,UAAA,CAAW,gBAAgB,IAC3B,IAAA,UAAA,CAAW,eAAe,CAC1B,EAAA;AACA,gBAAA,MAAM,KAAK,gBAAiB,EAAA;AAAA;AAC9B;AACF,mBACO,KAAO,EAAA;AACd,YAAA,UAAA,CAAW,MAAM,KAAK,CAAA;AAAA;AACxB;AACF,OACF;AAAA,MACA,gBAAA;AAAA,MACA;AAAA,KACF;AACA,IAAA,IAAA,CAAK,SAAY,GAAA,SAAA;AAAA;AAErB;;;;"}

package/dist/common/types.d.ts

@@ -356,6 +356,31 @@ export type CSVString<Header extends ReadonlyArray<string> = [], Delimiter exten
  * @category Types
  */
 export type CSVBinary = ReadableStream<Uint8Array> | Response | ArrayBuffer | Uint8Array;
+/**
+ * Backpressure monitoring options.
+ *
+ * @category Types
+ */
+export interface BackpressureOptions {
+    /**
+     * How often to check for backpressure (in number of items processed).
+     *
+     * Lower values = more responsive to backpressure but slight performance overhead.
+     * Higher values = less overhead but slower backpressure response.
+     *
+     * Default:
+     * - CSVLexerTransformer: 100 tokens
+     * - CSVRecordAssemblerTransformer: 10 records
+     */
+    checkInterval?: number;
+}
+/**
+ * Extended queuing strategy with backpressure monitoring options.
+ *
+ * @category Types
+ */
+export interface ExtendedQueuingStrategy<T> extends QueuingStrategy<T>, BackpressureOptions {
+}
 /**
  * CSV.
  *

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "web-csv-toolbox",
-  "version": "0.13.0-next-bd865d6ddb1cf9691d7b9a83d0790651f074dd47",
+  "version": "0.13.0-next-7d51d5285be9cffa5103de58469d8de0c98959d7",
   "description": "A CSV Toolbox utilizing Web Standard APIs.",
   "type": "module",
   "module": "dist/web-csv-toolbox.js",