web-csv-toolbox 0.13.0-next-b21b6d89a7a3f18dcbf79ec04ffefde0d7ff4c4c → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/CSVLexerTransformer.d.ts +12 -14
- package/dist/CSVLexerTransformer.js +10 -14
- package/dist/CSVLexerTransformer.js.map +1 -1
- package/dist/CSVRecordAssembler.d.ts +2 -2
- package/dist/CSVRecordAssembler.js +62 -44
- package/dist/CSVRecordAssembler.js.map +1 -1
- package/dist/CSVRecordAssemblerTransformer.d.ts +10 -10
- package/dist/CSVRecordAssemblerTransformer.js +4 -4
- package/dist/CSVRecordAssemblerTransformer.js.map +1 -1
- package/package.json +1 -1
|
@@ -8,7 +8,7 @@ import { DEFAULT_DELIMITER, DEFAULT_QUOTATION } from './constants.ts';
|
|
|
8
8
|
*
|
|
9
9
|
* @param options - CSV-specific options (delimiter, quotation, etc.)
|
|
10
10
|
* @param writableStrategy - Strategy for the writable side (default: `{ highWaterMark: 65536, size: chunk => chunk.length, checkInterval: 100 }`)
|
|
11
|
-
* @param readableStrategy - Strategy for the readable side (default: `{ highWaterMark: 1024, size:
|
|
11
|
+
* @param readableStrategy - Strategy for the readable side (default: `{ highWaterMark: 1024, size: () => 1, checkInterval: 100 }`)
|
|
12
12
|
*
|
|
13
13
|
* @remarks
|
|
14
14
|
* Follows the Web Streams API pattern where queuing strategies are passed as
|
|
@@ -16,7 +16,7 @@ import { DEFAULT_DELIMITER, DEFAULT_QUOTATION } from './constants.ts';
|
|
|
16
16
|
*
|
|
17
17
|
* **Default Queuing Strategy:**
|
|
18
18
|
* - Writable side: Counts by string length (characters). Default highWaterMark is 65536 characters (≈64KB).
|
|
19
|
-
* - Readable side: Counts
|
|
19
|
+
* - Readable side: Counts each token as 1. Default highWaterMark is 1024 tokens.
|
|
20
20
|
*
|
|
21
21
|
* **Backpressure Handling:**
|
|
22
22
|
* The transformer monitors `controller.desiredSize` and yields to the event loop when backpressure
|
|
@@ -36,10 +36,8 @@ import { DEFAULT_DELIMITER, DEFAULT_QUOTATION } from './constants.ts';
|
|
|
36
36
|
* }
|
|
37
37
|
* })
|
|
38
38
|
* .pipeThrough(new CSVLexerTransformer())
|
|
39
|
-
* .pipeTo(new WritableStream({ write(
|
|
40
|
-
*
|
|
41
|
-
* console.log(token);
|
|
42
|
-
* }
|
|
39
|
+
* .pipeTo(new WritableStream({ write(token) {
|
|
40
|
+
* console.log(token);
|
|
43
41
|
* }}));
|
|
44
42
|
* // { type: Field, value: "name", location: {...} }
|
|
45
43
|
* // { type: FieldDelimiter, value: ",", location: {...} }
|
|
@@ -56,14 +54,14 @@ import { DEFAULT_DELIMITER, DEFAULT_QUOTATION } from './constants.ts';
|
|
|
56
54
|
* const transformer = new CSVLexerTransformer(
|
|
57
55
|
* { delimiter: ',' },
|
|
58
56
|
* {
|
|
59
|
-
* highWaterMark: 131072,
|
|
60
|
-
* size: (chunk) => chunk.length,
|
|
61
|
-
* checkInterval: 200
|
|
57
|
+
* highWaterMark: 131072, // 128KB of characters
|
|
58
|
+
* size: (chunk) => chunk.length, // Count by character length
|
|
59
|
+
* checkInterval: 200 // Check backpressure every 200 tokens
|
|
62
60
|
* },
|
|
63
61
|
* {
|
|
64
|
-
* highWaterMark: 2048,
|
|
65
|
-
* size: (
|
|
66
|
-
* checkInterval: 50
|
|
62
|
+
* highWaterMark: 2048, // 2048 tokens
|
|
63
|
+
* size: () => 1, // Each token counts as 1
|
|
64
|
+
* checkInterval: 50 // Check backpressure every 50 tokens
|
|
67
65
|
* }
|
|
68
66
|
* );
|
|
69
67
|
*
|
|
@@ -74,7 +72,7 @@ import { DEFAULT_DELIMITER, DEFAULT_QUOTATION } from './constants.ts';
|
|
|
74
72
|
* .pipeTo(yourProcessor);
|
|
75
73
|
* ```
|
|
76
74
|
*/
|
|
77
|
-
export declare class CSVLexerTransformer<Delimiter extends string = DEFAULT_DELIMITER, Quotation extends string = DEFAULT_QUOTATION> extends TransformStream<string, Token
|
|
75
|
+
export declare class CSVLexerTransformer<Delimiter extends string = DEFAULT_DELIMITER, Quotation extends string = DEFAULT_QUOTATION> extends TransformStream<string, Token> {
|
|
78
76
|
readonly lexer: CSVLexer<Delimiter, Quotation>;
|
|
79
77
|
/**
|
|
80
78
|
* Yields to the event loop to allow backpressure handling.
|
|
@@ -82,5 +80,5 @@ export declare class CSVLexerTransformer<Delimiter extends string = DEFAULT_DELI
|
|
|
82
80
|
* @internal
|
|
83
81
|
*/
|
|
84
82
|
protected yieldToEventLoop(): Promise<void>;
|
|
85
|
-
constructor(options?: CSVLexerTransformerOptions<Delimiter, Quotation>, writableStrategy?: ExtendedQueuingStrategy<string>, readableStrategy?: ExtendedQueuingStrategy<Token
|
|
83
|
+
constructor(options?: CSVLexerTransformerOptions<Delimiter, Quotation>, writableStrategy?: ExtendedQueuingStrategy<string>, readableStrategy?: ExtendedQueuingStrategy<Token>);
|
|
86
84
|
}
|
|
@@ -20,8 +20,8 @@ class CSVLexerTransformer extends TransformStream {
|
|
|
20
20
|
}, readableStrategy = {
|
|
21
21
|
highWaterMark: 1024,
|
|
22
22
|
// 1024 tokens
|
|
23
|
-
size: (
|
|
24
|
-
//
|
|
23
|
+
size: () => 1,
|
|
24
|
+
// Each token counts as 1
|
|
25
25
|
checkInterval: 100
|
|
26
26
|
// Check backpressure every 100 tokens
|
|
27
27
|
}) {
|
|
@@ -32,16 +32,14 @@ class CSVLexerTransformer extends TransformStream {
|
|
|
32
32
|
transform: async (chunk, controller) => {
|
|
33
33
|
if (chunk.length !== 0) {
|
|
34
34
|
try {
|
|
35
|
-
|
|
35
|
+
let tokenCount = 0;
|
|
36
36
|
for (const token of lexer.lex(chunk, { stream: true })) {
|
|
37
|
-
|
|
38
|
-
|
|
37
|
+
controller.enqueue(token);
|
|
38
|
+
tokenCount++;
|
|
39
|
+
if (tokenCount % checkInterval === 0 && controller.desiredSize !== null && controller.desiredSize <= 0) {
|
|
39
40
|
await this.yieldToEventLoop();
|
|
40
41
|
}
|
|
41
42
|
}
|
|
42
|
-
if (tokens.length > 0) {
|
|
43
|
-
controller.enqueue(tokens);
|
|
44
|
-
}
|
|
45
43
|
} catch (error) {
|
|
46
44
|
controller.error(error);
|
|
47
45
|
}
|
|
@@ -49,16 +47,14 @@ class CSVLexerTransformer extends TransformStream {
|
|
|
49
47
|
},
|
|
50
48
|
flush: async (controller) => {
|
|
51
49
|
try {
|
|
52
|
-
|
|
50
|
+
let tokenCount = 0;
|
|
53
51
|
for (const token of lexer.lex()) {
|
|
54
|
-
|
|
55
|
-
|
|
52
|
+
controller.enqueue(token);
|
|
53
|
+
tokenCount++;
|
|
54
|
+
if (tokenCount % checkInterval === 0 && controller.desiredSize !== null && controller.desiredSize <= 0) {
|
|
56
55
|
await this.yieldToEventLoop();
|
|
57
56
|
}
|
|
58
57
|
}
|
|
59
|
-
if (tokens.length > 0) {
|
|
60
|
-
controller.enqueue(tokens);
|
|
61
|
-
}
|
|
62
58
|
} catch (error) {
|
|
63
59
|
controller.error(error);
|
|
64
60
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"CSVLexerTransformer.js","sources":["../src/CSVLexerTransformer.ts"],"sourcesContent":["import { CSVLexer } from \"./CSVLexer.ts\";\nimport type {\n CSVLexerTransformerOptions,\n ExtendedQueuingStrategy,\n Token,\n} from \"./common/types.ts\";\nimport type { DEFAULT_DELIMITER, DEFAULT_QUOTATION } from \"./constants.ts\";\n\n/**\n * A transform stream that converts a stream of strings into a stream of tokens.\n *\n * @category Low-level API\n *\n * @param options - CSV-specific options (delimiter, quotation, etc.)\n * @param writableStrategy - Strategy for the writable side (default: `{ highWaterMark: 65536, size: chunk => chunk.length, checkInterval: 100 }`)\n * @param readableStrategy - Strategy for the readable side (default: `{ highWaterMark: 1024, size:
|
|
1
|
+
{"version":3,"file":"CSVLexerTransformer.js","sources":["../src/CSVLexerTransformer.ts"],"sourcesContent":["import { CSVLexer } from \"./CSVLexer.ts\";\nimport type {\n CSVLexerTransformerOptions,\n ExtendedQueuingStrategy,\n Token,\n} from \"./common/types.ts\";\nimport type { DEFAULT_DELIMITER, DEFAULT_QUOTATION } from \"./constants.ts\";\n\n/**\n * A transform stream that converts a stream of strings into a stream of tokens.\n *\n * @category Low-level API\n *\n * @param options - CSV-specific options (delimiter, quotation, etc.)\n * @param writableStrategy - Strategy for the writable side (default: `{ highWaterMark: 65536, size: chunk => chunk.length, checkInterval: 100 }`)\n * @param readableStrategy - Strategy for the readable side (default: `{ highWaterMark: 1024, size: () => 1, checkInterval: 100 }`)\n *\n * @remarks\n * Follows the Web Streams API pattern where queuing strategies are passed as\n * constructor arguments, similar to the standard `TransformStream`.\n *\n * **Default Queuing Strategy:**\n * - Writable side: Counts by string length (characters). Default highWaterMark is 65536 characters (≈64KB).\n * - Readable side: Counts each token as 1. Default highWaterMark is 1024 tokens.\n *\n * **Backpressure Handling:**\n * The transformer monitors `controller.desiredSize` and yields to the event loop when backpressure\n * is detected (desiredSize ≤ 0). This prevents blocking the main thread during heavy processing\n * and allows the downstream consumer to catch up.\n *\n * These defaults are starting points based on data flow characteristics, not empirical benchmarks.\n * Optimal values depend on your runtime environment, data size, and performance requirements.\n *\n * @example Basic usage\n * ```ts\n * new ReadableStream({\n * start(controller) {\n * controller.enqueue(\"name,age\\r\\n\");\n * controller.enqueue(\"Alice,20\\r\\n\");\n * controller.close();\n * }\n * })\n * .pipeThrough(new CSVLexerTransformer())\n * .pipeTo(new WritableStream({ write(token) {\n * console.log(token);\n * }}));\n * // { type: Field, value: \"name\", location: {...} }\n * // { type: FieldDelimiter, value: \",\", location: {...} }\n * // { type: Field, value: \"age\", location: {...} }\n * // { type: RecordDelimiter, value: \"\\r\\n\", location: {...} }\n * // { type: Field, value: \"Alice\", location: {...} }\n * // { type: FieldDelimiter, value: \",\", location: {...} }\n * // { type: Field, value: \"20\" }\n * // { type: RecordDelimiter, value: \"\\r\\n\", location: {...} }\n * ```\n *\n * @example Custom queuing strategies with backpressure tuning\n * ```ts\n * const transformer = new CSVLexerTransformer(\n * { delimiter: ',' },\n * {\n * highWaterMark: 131072, // 128KB of characters\n * size: (chunk) => chunk.length, // Count by character length\n * checkInterval: 200 // Check backpressure every 200 tokens\n * },\n * {\n * highWaterMark: 2048, // 2048 tokens\n * size: () => 1, // Each token counts as 1\n * checkInterval: 50 // Check backpressure every 50 tokens\n * }\n * );\n *\n * await fetch('large-file.csv')\n * .then(res => res.body)\n * .pipeThrough(new TextDecoderStream())\n * .pipeThrough(transformer)\n * .pipeTo(yourProcessor);\n * ```\n */\nexport class CSVLexerTransformer<\n Delimiter extends string = DEFAULT_DELIMITER,\n Quotation extends string = DEFAULT_QUOTATION,\n> extends TransformStream<string, Token> {\n public readonly lexer: CSVLexer<Delimiter, Quotation>;\n\n /**\n * Yields to the event loop to allow backpressure handling.\n * Can be overridden for testing purposes.\n * @internal\n */\n protected async yieldToEventLoop(): Promise<void> {\n await new Promise((resolve) => setTimeout(resolve, 0));\n }\n\n constructor(\n options: CSVLexerTransformerOptions<Delimiter, Quotation> = {},\n writableStrategy: ExtendedQueuingStrategy<string> = {\n highWaterMark: 65536, // 64KB worth of characters\n size: (chunk) => chunk.length, // Count by string length (character count)\n checkInterval: 100, // Check backpressure every 100 tokens\n },\n readableStrategy: ExtendedQueuingStrategy<Token> = {\n highWaterMark: 1024, // 1024 tokens\n size: () => 1, // Each token counts as 1\n checkInterval: 100, // Check backpressure every 100 tokens\n },\n ) {\n const lexer = new CSVLexer(options);\n const checkInterval =\n writableStrategy.checkInterval ?? readableStrategy.checkInterval ?? 100;\n\n super(\n {\n transform: async (chunk, controller) => {\n if (chunk.length !== 0) {\n try {\n let tokenCount = 0;\n for (const token of lexer.lex(chunk, { stream: true })) {\n controller.enqueue(token);\n tokenCount++;\n\n // Check backpressure periodically based on checkInterval\n if (\n tokenCount % checkInterval === 0 &&\n controller.desiredSize !== null &&\n controller.desiredSize <= 0\n ) {\n // Yield to event loop when backpressure is detected\n await this.yieldToEventLoop();\n }\n }\n } catch (error) {\n controller.error(error);\n }\n }\n },\n flush: async (controller) => {\n try {\n let tokenCount = 0;\n for (const token of lexer.lex()) {\n controller.enqueue(token);\n tokenCount++;\n\n // Check backpressure periodically based on checkInterval\n if (\n tokenCount % checkInterval === 0 &&\n controller.desiredSize !== null &&\n controller.desiredSize <= 0\n ) {\n await this.yieldToEventLoop();\n }\n }\n } catch (error) {\n controller.error(error);\n }\n },\n },\n writableStrategy,\n readableStrategy,\n );\n this.lexer = lexer;\n }\n}\n"],"names":[],"mappings":";;AA+EO,MAAM,4BAGH,eAAA,CAA+B;AAAA,EACvB,KAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOhB,MAAgB,gBAAA,GAAkC;AAChD,IAAA,MAAM,IAAI,OAAA,CAAQ,CAAC,YAAY,UAAA,CAAW,OAAA,EAAS,CAAC,CAAC,CAAA;AAAA,EACvD;AAAA,EAEA,WAAA,CACE,OAAA,GAA4D,EAAC,EAC7D,gBAAA,GAAoD;AAAA,IAClD,aAAA,EAAe,KAAA;AAAA;AAAA,IACf,IAAA,EAAM,CAAC,KAAA,KAAU,KAAA,CAAM,MAAA;AAAA;AAAA,IACvB,aAAA,EAAe;AAAA;AAAA,KAEjB,gBAAA,GAAmD;AAAA,IACjD,aAAA,EAAe,IAAA;AAAA;AAAA,IACf,MAAM,MAAM,CAAA;AAAA;AAAA,IACZ,aAAA,EAAe;AAAA;AAAA,GACjB,EACA;AACA,IAAA,MAAM,KAAA,GAAQ,IAAI,QAAA,CAAS,OAAO,CAAA;AAClC,IAAA,MAAM,aAAA,GACJ,gBAAA,CAAiB,aAAA,IAAiB,gBAAA,CAAiB,aAAA,IAAiB,GAAA;AAEtE,IAAA,KAAA;AAAA,MACE;AAAA,QACE,SAAA,EAAW,OAAO,KAAA,EAAO,UAAA,KAAe;AACtC,UAAA,IAAI,KAAA,CAAM,WAAW,CAAA,EAAG;AACtB,YAAA,IAAI;AACF,cAAA,IAAI,UAAA,GAAa,CAAA;AACjB,cAAA,KAAA,MAAW,KAAA,IAAS,MAAM,GAAA,CAAI,KAAA,EAAO,EAAE,MAAA,EAAQ,IAAA,EAAM,CAAA,EAAG;AACtD,gBAAA,UAAA,CAAW,QAAQ,KAAK,CAAA;AACxB,gBAAA,UAAA,EAAA;AAGA,gBAAA,IACE,UAAA,GAAa,kBAAkB,CAAA,IAC/B,UAAA,CAAW,gBAAgB,IAAA,IAC3B,UAAA,CAAW,eAAe,CAAA,EAC1B;AAEA,kBAAA,MAAM,KAAK,gBAAA,EAAiB;AAAA,gBAC9B;AAAA,cACF;AAAA,YACF,SAAS,KAAA,EAAO;AACd,cAAA,UAAA,CAAW,MAAM,KAAK,CAAA;AAAA,YACxB;AAAA,UACF;AAAA,QACF,CAAA;AAAA,QACA,KAAA,EAAO,OAAO,UAAA,KAAe;AAC3B,UAAA,IAAI;AACF,YAAA,IAAI,UAAA,GAAa,CAAA;AACjB,YAAA,KAAA,MAAW,KAAA,IAAS,KAAA,CAAM,GAAA,EAAI,EAAG;AAC/B,cAAA,UAAA,CAAW,QAAQ,KAAK,CAAA;AACxB,cAAA,UAAA,EAAA;AAGA,cAAA,IACE,UAAA,GAAa,kBAAkB,CAAA,IAC/B,UAAA,CAAW,gBAAgB,IAAA,IAC3B,UAAA,CAAW,eAAe,CAAA,EAC1B;AACA,gBAAA,MAAM,KAAK,gBAAA,EAAiB;AAAA,cAC9B;AAAA,YACF;AAAA,UACF,SAAS,KAAA,EAAO;AACd,YAAA,UAAA,CAAW,MAAM,KAAK,CAAA;AAAA,UACxB;AAAA,QACF;AAAA,OACF;AAAA,MACA,gBAAA;AAAA,MACA;AAAA,KACF;AACA,IAAA,IAAA,CAAK,KAAA,GAAQ,KAAA;AAAA,EACf;AACF;;;;"}
|
|
@@ -19,9 +19,9 @@ export declare class CSVRecordAssembler<Header extends ReadonlyArray<string>> {
|
|
|
19
19
|
constructor(options?: CSVRecordAssemblerOptions<Header>);
|
|
20
20
|
/**
|
|
21
21
|
* Assembles tokens into CSV records.
|
|
22
|
-
* @param
|
|
22
|
+
* @param input - A single token or an iterable of tokens. Omit to flush remaining data.
|
|
23
23
|
* @param options - Assembler options.
|
|
24
24
|
* @returns An iterable iterator of CSV records.
|
|
25
25
|
*/
|
|
26
|
-
assemble(
|
|
26
|
+
assemble(input?: Token | Iterable<Token>, options?: CSVRecordAssemblerAssembleOptions): IterableIterator<CSVRecord<Header>>;
|
|
27
27
|
}
|
|
@@ -28,59 +28,77 @@ class CSVRecordAssembler {
|
|
|
28
28
|
}
|
|
29
29
|
/**
|
|
30
30
|
* Assembles tokens into CSV records.
|
|
31
|
-
* @param
|
|
31
|
+
* @param input - A single token or an iterable of tokens. Omit to flush remaining data.
|
|
32
32
|
* @param options - Assembler options.
|
|
33
33
|
* @returns An iterable iterator of CSV records.
|
|
34
34
|
*/
|
|
35
|
-
*assemble(
|
|
35
|
+
*assemble(input, options) {
|
|
36
36
|
const stream = options?.stream ?? false;
|
|
37
|
-
if (
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
case FieldDelimiter:
|
|
42
|
-
this.#fieldIndex++;
|
|
43
|
-
this.#checkFieldCount();
|
|
44
|
-
this.#dirty = true;
|
|
45
|
-
break;
|
|
46
|
-
case RecordDelimiter:
|
|
47
|
-
if (this.#header === void 0) {
|
|
48
|
-
this.#setHeader(this.#row);
|
|
49
|
-
} else {
|
|
50
|
-
if (this.#dirty) {
|
|
51
|
-
yield Object.fromEntries(
|
|
52
|
-
this.#header.map((header, index) => [
|
|
53
|
-
header,
|
|
54
|
-
this.#row.at(index)
|
|
55
|
-
])
|
|
56
|
-
);
|
|
57
|
-
} else {
|
|
58
|
-
if (this.#skipEmptyLines) {
|
|
59
|
-
continue;
|
|
60
|
-
}
|
|
61
|
-
yield Object.fromEntries(
|
|
62
|
-
this.#header.map((header) => [header, ""])
|
|
63
|
-
);
|
|
64
|
-
}
|
|
65
|
-
}
|
|
66
|
-
this.#fieldIndex = 0;
|
|
67
|
-
this.#row = new Array(this.#header?.length).fill("");
|
|
68
|
-
this.#dirty = false;
|
|
69
|
-
break;
|
|
70
|
-
default:
|
|
71
|
-
this.#dirty = true;
|
|
72
|
-
this.#row[this.#fieldIndex] = token.value;
|
|
73
|
-
break;
|
|
37
|
+
if (input !== void 0) {
|
|
38
|
+
if (this.#isIterable(input)) {
|
|
39
|
+
for (const token of input) {
|
|
40
|
+
yield* this.#processToken(token);
|
|
74
41
|
}
|
|
42
|
+
} else {
|
|
43
|
+
yield* this.#processToken(input);
|
|
75
44
|
}
|
|
76
45
|
}
|
|
77
46
|
if (!stream) {
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
47
|
+
yield* this.#flush();
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
/**
|
|
51
|
+
* Checks if a value is iterable.
|
|
52
|
+
*/
|
|
53
|
+
#isIterable(value) {
|
|
54
|
+
return value != null && typeof value[Symbol.iterator] === "function";
|
|
55
|
+
}
|
|
56
|
+
/**
|
|
57
|
+
* Processes a single token and yields a record if one is completed.
|
|
58
|
+
*/
|
|
59
|
+
*#processToken(token) {
|
|
60
|
+
this.#signal?.throwIfAborted();
|
|
61
|
+
switch (token.type) {
|
|
62
|
+
case FieldDelimiter:
|
|
63
|
+
this.#fieldIndex++;
|
|
64
|
+
this.#checkFieldCount();
|
|
65
|
+
this.#dirty = true;
|
|
66
|
+
break;
|
|
67
|
+
case RecordDelimiter:
|
|
68
|
+
if (this.#header === void 0) {
|
|
69
|
+
this.#setHeader(this.#row);
|
|
70
|
+
} else {
|
|
71
|
+
if (this.#dirty) {
|
|
72
|
+
yield Object.fromEntries(
|
|
73
|
+
this.#header.map((header, index) => [header, index]).filter(([header]) => header).map(([header, index]) => [header, this.#row.at(index)])
|
|
74
|
+
);
|
|
75
|
+
} else {
|
|
76
|
+
if (!this.#skipEmptyLines) {
|
|
77
|
+
yield Object.fromEntries(
|
|
78
|
+
this.#header.filter((header) => header).map((header) => [header, ""])
|
|
79
|
+
);
|
|
80
|
+
}
|
|
81
|
+
}
|
|
83
82
|
}
|
|
83
|
+
this.#fieldIndex = 0;
|
|
84
|
+
this.#row = new Array(this.#header?.length).fill("");
|
|
85
|
+
this.#dirty = false;
|
|
86
|
+
break;
|
|
87
|
+
default:
|
|
88
|
+
this.#dirty = true;
|
|
89
|
+
this.#row[this.#fieldIndex] = token.value;
|
|
90
|
+
break;
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
/**
|
|
94
|
+
* Flushes any remaining buffered data as a final record.
|
|
95
|
+
*/
|
|
96
|
+
*#flush() {
|
|
97
|
+
if (this.#header !== void 0) {
|
|
98
|
+
if (this.#dirty) {
|
|
99
|
+
yield Object.fromEntries(
|
|
100
|
+
this.#header.map((header, index) => [header, index]).filter(([header]) => header).map(([header, index]) => [header, this.#row.at(index)])
|
|
101
|
+
);
|
|
84
102
|
}
|
|
85
103
|
}
|
|
86
104
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"CSVRecordAssembler.js","sources":["../src/CSVRecordAssembler.ts"],"sourcesContent":["import { FieldDelimiter, RecordDelimiter } from \"./common/constants.ts\";\nimport { ParseError } from \"./common/errors.ts\";\nimport type {\n CSVRecord,\n CSVRecordAssemblerOptions,\n Token,\n} from \"./common/types.ts\";\n\n/**\n * Default maximum field count per record (100,000 fields).\n */\nconst DEFAULT_MAX_FIELD_COUNT = 100_000;\n\n/**\n * Options for the CSVRecordAssembler.assemble method.\n */\nexport interface CSVRecordAssemblerAssembleOptions {\n /**\n * If true, indicates that more tokens are expected.\n * If false or omitted, flushes remaining data.\n */\n stream?: boolean;\n}\n\n/**\n * CSV Record Assembler.\n *\n * CSVRecordAssembler assembles tokens into CSV records.\n */\nexport class CSVRecordAssembler<Header extends ReadonlyArray<string>> {\n #fieldIndex = 0;\n #row: string[] = [];\n #header: Header | undefined;\n #dirty = false;\n #signal?: AbortSignal;\n #maxFieldCount: number;\n #skipEmptyLines: boolean;\n\n constructor(options: CSVRecordAssemblerOptions<Header> = {}) {\n const mfc = options.maxFieldCount ?? DEFAULT_MAX_FIELD_COUNT;\n // Validate maxFieldCount\n if (\n !(Number.isFinite(mfc) || mfc === Number.POSITIVE_INFINITY) ||\n (Number.isFinite(mfc) && (mfc < 1 || !Number.isInteger(mfc)))\n ) {\n throw new RangeError(\n \"maxFieldCount must be a positive integer or Number.POSITIVE_INFINITY\",\n );\n }\n this.#maxFieldCount = mfc;\n this.#skipEmptyLines = options.skipEmptyLines ?? false;\n if (options.header !== undefined && Array.isArray(options.header)) {\n this.#setHeader(options.header);\n }\n if (options.signal) {\n this.#signal = options.signal;\n }\n }\n\n /**\n * Assembles tokens into CSV records.\n * @param
|
|
1
|
+
{"version":3,"file":"CSVRecordAssembler.js","sources":["../src/CSVRecordAssembler.ts"],"sourcesContent":["import { FieldDelimiter, RecordDelimiter } from \"./common/constants.ts\";\nimport { ParseError } from \"./common/errors.ts\";\nimport type {\n CSVRecord,\n CSVRecordAssemblerOptions,\n Token,\n} from \"./common/types.ts\";\n\n/**\n * Default maximum field count per record (100,000 fields).\n */\nconst DEFAULT_MAX_FIELD_COUNT = 100_000;\n\n/**\n * Options for the CSVRecordAssembler.assemble method.\n */\nexport interface CSVRecordAssemblerAssembleOptions {\n /**\n * If true, indicates that more tokens are expected.\n * If false or omitted, flushes remaining data.\n */\n stream?: boolean;\n}\n\n/**\n * CSV Record Assembler.\n *\n * CSVRecordAssembler assembles tokens into CSV records.\n */\nexport class CSVRecordAssembler<Header extends ReadonlyArray<string>> {\n #fieldIndex = 0;\n #row: string[] = [];\n #header: Header | undefined;\n #dirty = false;\n #signal?: AbortSignal;\n #maxFieldCount: number;\n #skipEmptyLines: boolean;\n\n constructor(options: CSVRecordAssemblerOptions<Header> = {}) {\n const mfc = options.maxFieldCount ?? DEFAULT_MAX_FIELD_COUNT;\n // Validate maxFieldCount\n if (\n !(Number.isFinite(mfc) || mfc === Number.POSITIVE_INFINITY) ||\n (Number.isFinite(mfc) && (mfc < 1 || !Number.isInteger(mfc)))\n ) {\n throw new RangeError(\n \"maxFieldCount must be a positive integer or Number.POSITIVE_INFINITY\",\n );\n }\n this.#maxFieldCount = mfc;\n this.#skipEmptyLines = options.skipEmptyLines ?? false;\n if (options.header !== undefined && Array.isArray(options.header)) {\n this.#setHeader(options.header);\n }\n if (options.signal) {\n this.#signal = options.signal;\n }\n }\n\n /**\n * Assembles tokens into CSV records.\n * @param input - A single token or an iterable of tokens. Omit to flush remaining data.\n * @param options - Assembler options.\n * @returns An iterable iterator of CSV records.\n */\n public *assemble(\n input?: Token | Iterable<Token>,\n options?: CSVRecordAssemblerAssembleOptions,\n ): IterableIterator<CSVRecord<Header>> {\n const stream = options?.stream ?? false;\n\n if (input !== undefined) {\n // Check if input is iterable (has Symbol.iterator)\n if (this.#isIterable(input)) {\n for (const token of input) {\n yield* this.#processToken(token);\n }\n } else {\n // Single token\n yield* this.#processToken(input);\n }\n }\n\n if (!stream) {\n yield* this.#flush();\n }\n }\n\n /**\n * Checks if a value is iterable.\n */\n #isIterable(value: any): value is Iterable<Token> {\n return value != null && typeof value[Symbol.iterator] === \"function\";\n }\n\n /**\n * Processes a single token and yields a record if one is completed.\n */\n *#processToken(token: Token): IterableIterator<CSVRecord<Header>> {\n this.#signal?.throwIfAborted();\n\n switch (token.type) {\n case FieldDelimiter:\n this.#fieldIndex++;\n this.#checkFieldCount();\n this.#dirty = true;\n break;\n case RecordDelimiter:\n if (this.#header === undefined) {\n this.#setHeader(this.#row as unknown as Header);\n } else {\n if (this.#dirty) {\n yield Object.fromEntries(\n this.#header\n .map((header, index) => [header, index] as const)\n .filter(([header]) => header)\n .map(([header, index]) => [header, this.#row.at(index)]),\n ) as unknown as CSVRecord<Header>;\n } else {\n if (!this.#skipEmptyLines) {\n yield Object.fromEntries(\n this.#header\n .filter((header) => header)\n .map((header) => [header, \"\"]),\n ) as CSVRecord<Header>;\n }\n }\n }\n // Reset the row fields buffer.\n this.#fieldIndex = 0;\n this.#row = new Array(this.#header?.length).fill(\"\");\n this.#dirty = false;\n break;\n default:\n this.#dirty = true;\n this.#row[this.#fieldIndex] = token.value;\n break;\n }\n }\n\n /**\n * Flushes any remaining buffered data as a final record.\n */\n *#flush(): IterableIterator<CSVRecord<Header>> {\n if (this.#header !== undefined) {\n if (this.#dirty) {\n yield Object.fromEntries(\n this.#header\n .map((header, index) => [header, index] as const)\n .filter(([header]) => header)\n .map(([header, index]) => [header, this.#row.at(index)]),\n ) as unknown as CSVRecord<Header>;\n }\n }\n }\n\n #checkFieldCount(): void {\n if (this.#fieldIndex + 1 > this.#maxFieldCount) {\n throw new RangeError(\n `Field count (${this.#fieldIndex + 1}) exceeded maximum allowed count of ${this.#maxFieldCount}`,\n );\n }\n }\n\n #setHeader(header: Header) {\n if (header.length > this.#maxFieldCount) {\n throw new RangeError(\n `Header field count (${header.length}) exceeded maximum allowed count of ${this.#maxFieldCount}`,\n );\n }\n this.#header = header;\n if (this.#header.length === 0) {\n throw new ParseError(\"The header must not be empty.\");\n }\n if (new Set(this.#header).size !== this.#header.length) {\n throw new ParseError(\"The header must not contain duplicate fields.\");\n }\n }\n}\n"],"names":[],"mappings":";;;AAWA,MAAM,uBAAA,GAA0B,GAAA;AAkBzB,MAAM,kBAAA,CAAyD;AAAA,EACpE,WAAA,GAAc,CAAA;AAAA,EACd,OAAiB,EAAC;AAAA,EAClB,OAAA;AAAA,EACA,MAAA,GAAS,KAAA;AAAA,EACT,OAAA;AAAA,EACA,cAAA;AAAA,EACA,eAAA;AAAA,EAEA,WAAA,CAAY,OAAA,GAA6C,EAAC,EAAG;AAC3D,IAAA,MAAM,GAAA,GAAM,QAAQ,aAAA,IAAiB,uBAAA;AAErC,IAAA,IACE,EAAE,MAAA,CAAO,QAAA,CAAS,GAAG,CAAA,IAAK,GAAA,KAAQ,OAAO,iBAAA,CAAA,IACxC,MAAA,CAAO,QAAA,CAAS,GAAG,MAAM,GAAA,GAAM,CAAA,IAAK,CAAC,MAAA,CAAO,SAAA,CAAU,GAAG,CAAA,CAAA,EAC1D;AACA,MAAA,MAAM,IAAI,UAAA;AAAA,QACR;AAAA,OACF;AAAA,IACF;AACA,IAAA,IAAA,CAAK,cAAA,GAAiB,GAAA;AACtB,IAAA,IAAA,CAAK,eAAA,GAAkB,QAAQ,cAAA,IAAkB,KAAA;AACjD,IAAA,IAAI,QAAQ,MAAA,KAAW,MAAA,IAAa,MAAM,OAAA,CAAQ,OAAA,CAAQ,MAAM,CAAA,EAAG;AACjE,MAAA,IAAA,CAAK,UAAA,CAAW,QAAQ,MAAM,CAAA;AAAA,IAChC;AACA,IAAA,IAAI,QAAQ,MAAA,EAAQ;AAClB,MAAA,IAAA,CAAK,UAAU,OAAA,CAAQ,MAAA;AAAA,IACzB;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAQA,CAAQ,QAAA,CACN,KAAA,EACA,OAAA,EACqC;AACrC,IAAA,MAAM,MAAA,GAAS,SAAS,MAAA,IAAU,KAAA;AAElC,IAAA,IAAI,UAAU,MAAA,EAAW;AAEvB,MAAA,IAAI,IAAA,CAAK,WAAA,CAAY,KAAK,CAAA,EAAG;AAC3B,QAAA,KAAA,MAAW,SAAS,KAAA,EAAO;AACzB,UAAA,OAAO,IAAA,CAAK,cAAc,KAAK,CAAA;AAAA,QACjC;AAAA,MACF,CAAA,MAAO;AAEL,QAAA,OAAO,IAAA,CAAK,cAAc,KAAK,CAAA;AAAA,MACjC;AAAA,IACF;AAEA,IAAA,IAAI,CAAC,MAAA,EAAQ;AACX,MAAA,OAAO,KAAK,MAAA,EAAO;AAAA,IACrB;AAAA,EACF;AAAA;AAAA;AAAA;AAAA,EAKA,YAAY,KAAA,EAAsC;AAChD,IAAA,OAAO,SAAS,IAAA,IAAQ,OAAO,KAAA,CAAM,MAAA,CAAO,QAAQ,CAAA,KAAM,UAAA;AAAA,EAC5D;AAAA;AAAA;AAAA;AAAA,EAKA,CAAC,cAAc,KAAA,EAAmD;AAChE,IAAA,IAAA,CAAK,SAAS,cAAA,EAAe;AAE7B,IAAA,QAAQ,MAAM,IAAA;AAAM,MAClB,KAAK,cAAA;AACH,QAAA,IAAA,CAAK,WAAA,EAAA;AACL,QAAA,IAAA,CAAK,gBAAA,EAAiB;AACtB,QAAA,IAAA,CAAK,MAAA,GAAS,IAAA;AACd,QAAA;AAAA,MACF,KAAK,eAAA;AACH,QAAA,IAAI,IAAA,CAAK,YAAY,MAAA,EAAW;AAC9B,UAAA,IAAA,CAAK,UAAA,CAAW,KAAK,IAAyB,CAAA;AAAA,QAChD,CAAA,MAAO;AACL,UAAA,IAAI,KAAK,MAAA,EAAQ;AACf,YAAA,MAAM,MAAA,CAAO,WAAA;AAAA,cACX,IAAA,CAAK,OAAA,CACF,GAAA,CAAI,CAAC,MAAA,EAAQ,KAAA,KAAU,CAAC,MAAA,EAAQ,KAAK,CAAU,CAAA,CAC/C,MAAA,CAAO,CAAC,CAAC,MAAM,CAAA,KAAM,MAAM,CAAA,CAC3B,GAAA,CAAI,CAAC,CAAC,QAAQ,KAAK,CAAA,KAAM,CAAC,MAAA,EAAQ,IAAA,CAAK,IAAA,CAAK,EAAA,CAAG,KAAK,CAAC,CAAC;AAAA,aAC3D;AAAA,UACF,CAAA,MAAO;AACL,YAAA,IAAI,CAAC,KAAK,eAAA,EAAiB;AACzB,cAAA,MAAM,MAAA,CAAO,WAAA;AAAA,gBACX,IAAA,CAAK,OAAA,CACF,MAAA,CAAO,CAAC,MAAA,KAAW,MAAM,CAAA,CACzB,GAAA,CAAI,CAAC,MAAA,KAAW,CAAC,MAAA,EAAQ,EAAE,CAAC;AAAA,eACjC;AAAA,YACF;AAAA,UACF;AAAA,QACF;AAEA,QAAA,IAAA,CAAK,WAAA,GAAc,CAAA;AACnB,QAAA,IAAA,CAAK,IAAA,GAAO,IAAI,KAAA,CAAM,IAAA,CAAK,SAAS,MAAM,CAAA,CAAE,KAAK,EAAE,CAAA;AACnD,QAAA,IAAA,CAAK,MAAA,GAAS,KAAA;AACd,QAAA;AAAA,MACF;AACE,QAAA,IAAA,CAAK,MAAA,GAAS,IAAA;AACd,QAAA,IAAA,CAAK,IAAA,CAAK,IAAA,CAAK,WAAW,CAAA,GAAI,KAAA,CAAM,KAAA;AACpC,QAAA;AAAA;AACJ,EACF;AAAA;AAAA;AAAA;AAAA,EAKA,CAAC,MAAA,GAA8C;AAC7C,IAAA,IAAI,IAAA,CAAK,YAAY,MAAA,EAAW;AAC9B,MAAA,IAAI,KAAK,MAAA,EAAQ;AACf,QAAA,MAAM,MAAA,CAAO,WAAA;AAAA,UACX,IAAA,CAAK,OAAA,CACF,GAAA,CAAI,CAAC,MAAA,EAAQ,KAAA,KAAU,CAAC,MAAA,EAAQ,KAAK,CAAU,CAAA,CAC/C,MAAA,CAAO,CAAC,CAAC,MAAM,CAAA,KAAM,MAAM,CAAA,CAC3B,GAAA,CAAI,CAAC,CAAC,QAAQ,KAAK,CAAA,KAAM,CAAC,MAAA,EAAQ,IAAA,CAAK,IAAA,CAAK,EAAA,CAAG,KAAK,CAAC,CAAC;AAAA,SAC3D;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA,EAEA,gBAAA,GAAyB;AACvB,IAAA,IAAI,IAAA,CAAK,WAAA,GAAc,CAAA,GAAI,IAAA,CAAK,cAAA,EAAgB;AAC9C,MAAA,MAAM,IAAI,UAAA;AAAA,QACR,gBAAgB,IAAA,CAAK,WAAA,GAAc,CAAC,CAAA,oCAAA,EAAuC,KAAK,cAAc,CAAA;AAAA,OAChG;AAAA,IACF;AAAA,EACF;AAAA,EAEA,WAAW,MAAA,EAAgB;AACzB,IAAA,IAAI,MAAA,CAAO,MAAA,GAAS,IAAA,CAAK,cAAA,EAAgB;AACvC,MAAA,MAAM,IAAI,UAAA;AAAA,QACR,CAAA,oBAAA,EAAuB,MAAA,CAAO,MAAM,CAAA,oCAAA,EAAuC,KAAK,cAAc,CAAA;AAAA,OAChG;AAAA,IACF;AACA,IAAA,IAAA,CAAK,OAAA,GAAU,MAAA;AACf,IAAA,IAAI,IAAA,CAAK,OAAA,CAAQ,MAAA,KAAW,CAAA,EAAG;AAC7B,MAAA,MAAM,IAAI,WAAW,+BAA+B,CAAA;AAAA,IACtD;AACA,IAAA,IAAI,IAAI,IAAI,IAAA,CAAK,OAAO,EAAE,IAAA,KAAS,IAAA,CAAK,QAAQ,MAAA,EAAQ;AACtD,MAAA,MAAM,IAAI,WAAW,+CAA+C,CAAA;AAAA,IACtE;AAAA,EACF;AACF;;;;"}
|
|
@@ -5,7 +5,7 @@ import { CSVRecord, CSVRecordAssemblerOptions, ExtendedQueuingStrategy, Token }
|
|
|
5
5
|
*
|
|
6
6
|
* @template Header The type of the header row.
|
|
7
7
|
* @param options - CSV-specific options (header, maxFieldCount, etc.)
|
|
8
|
-
* @param writableStrategy - Strategy for the writable side (default: `{ highWaterMark: 1024, size:
|
|
8
|
+
* @param writableStrategy - Strategy for the writable side (default: `{ highWaterMark: 1024, size: () => 1, checkInterval: 10 }`)
|
|
9
9
|
* @param readableStrategy - Strategy for the readable side (default: `{ highWaterMark: 256, size: () => 1, checkInterval: 10 }`)
|
|
10
10
|
*
|
|
11
11
|
* @category Low-level API
|
|
@@ -15,7 +15,7 @@ import { CSVRecord, CSVRecordAssemblerOptions, ExtendedQueuingStrategy, Token }
|
|
|
15
15
|
* constructor arguments, similar to the standard `TransformStream`.
|
|
16
16
|
*
|
|
17
17
|
* **Default Queuing Strategy:**
|
|
18
|
-
* - Writable side: Counts
|
|
18
|
+
* - Writable side: Counts each token as 1. Default highWaterMark is 1024 tokens.
|
|
19
19
|
* - Readable side: Counts each record as 1. Default highWaterMark is 256 records.
|
|
20
20
|
*
|
|
21
21
|
* **Backpressure Handling:**
|
|
@@ -67,14 +67,14 @@ import { CSVRecord, CSVRecordAssemblerOptions, ExtendedQueuingStrategy, Token }
|
|
|
67
67
|
* const transformer = new CSVRecordAssemblerTransformer(
|
|
68
68
|
* {},
|
|
69
69
|
* {
|
|
70
|
-
* highWaterMark: 2048,
|
|
71
|
-
* size: (
|
|
72
|
-
* checkInterval: 20
|
|
70
|
+
* highWaterMark: 2048, // 2048 tokens
|
|
71
|
+
* size: () => 1, // Each token counts as 1
|
|
72
|
+
* checkInterval: 20 // Check backpressure every 20 records
|
|
73
73
|
* },
|
|
74
74
|
* {
|
|
75
|
-
* highWaterMark: 512,
|
|
76
|
-
* size: () => 1,
|
|
77
|
-
* checkInterval: 5
|
|
75
|
+
* highWaterMark: 512, // 512 records
|
|
76
|
+
* size: () => 1, // Each record counts as 1
|
|
77
|
+
* checkInterval: 5 // Check backpressure every 5 records
|
|
78
78
|
* }
|
|
79
79
|
* );
|
|
80
80
|
*
|
|
@@ -83,7 +83,7 @@ import { CSVRecord, CSVRecordAssemblerOptions, ExtendedQueuingStrategy, Token }
|
|
|
83
83
|
* .pipeTo(yourRecordProcessor);
|
|
84
84
|
* ```
|
|
85
85
|
*/
|
|
86
|
-
export declare class CSVRecordAssemblerTransformer<Header extends ReadonlyArray<string>> extends TransformStream<Token
|
|
86
|
+
export declare class CSVRecordAssemblerTransformer<Header extends ReadonlyArray<string>> extends TransformStream<Token, CSVRecord<Header>> {
|
|
87
87
|
readonly assembler: CSVRecordAssembler<Header>;
|
|
88
88
|
/**
|
|
89
89
|
* Yields to the event loop to allow backpressure handling.
|
|
@@ -91,5 +91,5 @@ export declare class CSVRecordAssemblerTransformer<Header extends ReadonlyArray<
|
|
|
91
91
|
* @internal
|
|
92
92
|
*/
|
|
93
93
|
protected yieldToEventLoop(): Promise<void>;
|
|
94
|
-
constructor(options?: CSVRecordAssemblerOptions<Header>, writableStrategy?: ExtendedQueuingStrategy<Token
|
|
94
|
+
constructor(options?: CSVRecordAssemblerOptions<Header>, writableStrategy?: ExtendedQueuingStrategy<Token>, readableStrategy?: ExtendedQueuingStrategy<CSVRecord<Header>>);
|
|
95
95
|
}
|
|
@@ -13,8 +13,8 @@ class CSVRecordAssemblerTransformer extends TransformStream {
|
|
|
13
13
|
constructor(options = {}, writableStrategy = {
|
|
14
14
|
highWaterMark: 1024,
|
|
15
15
|
// 1024 tokens
|
|
16
|
-
size: (
|
|
17
|
-
//
|
|
16
|
+
size: () => 1,
|
|
17
|
+
// Each token counts as 1
|
|
18
18
|
checkInterval: 10
|
|
19
19
|
// Check backpressure every 10 records
|
|
20
20
|
}, readableStrategy = {
|
|
@@ -29,10 +29,10 @@ class CSVRecordAssemblerTransformer extends TransformStream {
|
|
|
29
29
|
const checkInterval = writableStrategy.checkInterval ?? readableStrategy.checkInterval ?? 10;
|
|
30
30
|
super(
|
|
31
31
|
{
|
|
32
|
-
transform: async (
|
|
32
|
+
transform: async (token, controller) => {
|
|
33
33
|
try {
|
|
34
34
|
let recordCount = 0;
|
|
35
|
-
for (const record of assembler.assemble(
|
|
35
|
+
for (const record of assembler.assemble(token, { stream: true })) {
|
|
36
36
|
controller.enqueue(record);
|
|
37
37
|
recordCount++;
|
|
38
38
|
if (recordCount % checkInterval === 0 && controller.desiredSize !== null && controller.desiredSize <= 0) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"CSVRecordAssemblerTransformer.js","sources":["../src/CSVRecordAssemblerTransformer.ts"],"sourcesContent":["import { CSVRecordAssembler } from \"./CSVRecordAssembler.ts\";\nimport type {\n CSVRecord,\n CSVRecordAssemblerOptions,\n ExtendedQueuingStrategy,\n Token,\n} from \"./common/types.ts\";\n\n/**\n * A transform stream that converts a stream of tokens into a stream of CSV records.\n *\n * @template Header The type of the header row.\n * @param options - CSV-specific options (header, maxFieldCount, etc.)\n * @param writableStrategy - Strategy for the writable side (default: `{ highWaterMark: 1024, size:
|
|
1
|
+
{"version":3,"file":"CSVRecordAssemblerTransformer.js","sources":["../src/CSVRecordAssemblerTransformer.ts"],"sourcesContent":["import { CSVRecordAssembler } from \"./CSVRecordAssembler.ts\";\nimport type {\n CSVRecord,\n CSVRecordAssemblerOptions,\n ExtendedQueuingStrategy,\n Token,\n} from \"./common/types.ts\";\n\n/**\n * A transform stream that converts a stream of tokens into a stream of CSV records.\n *\n * @template Header The type of the header row.\n * @param options - CSV-specific options (header, maxFieldCount, etc.)\n * @param writableStrategy - Strategy for the writable side (default: `{ highWaterMark: 1024, size: () => 1, checkInterval: 10 }`)\n * @param readableStrategy - Strategy for the readable side (default: `{ highWaterMark: 256, size: () => 1, checkInterval: 10 }`)\n *\n * @category Low-level API\n *\n * @remarks\n * Follows the Web Streams API pattern where queuing strategies are passed as\n * constructor arguments, similar to the standard `TransformStream`.\n *\n * **Default Queuing Strategy:**\n * - Writable side: Counts each token as 1. Default highWaterMark is 1024 tokens.\n * - Readable side: Counts each record as 1. Default highWaterMark is 256 records.\n *\n * **Backpressure Handling:**\n * The transformer monitors `controller.desiredSize` and yields to the event loop when backpressure\n * is detected (desiredSize ≤ 0). This prevents blocking the main thread during heavy processing\n * and allows the downstream consumer to catch up.\n *\n * These defaults are starting points based on data flow characteristics, not empirical benchmarks.\n * Optimal values depend on your runtime environment, data size, and performance requirements.\n *\n * @example Parse a CSV with headers by data\n * ```ts\n * new ReadableStream({\n * start(controller) {\n * controller.enqueue(\"name,age\\r\\n\");\n * controller.enqueue(\"Alice,20\\r\\n\");\n * controller.enqueue(\"Bob,25\\r\\n\");\n * controller.enqueue(\"Charlie,30\\r\\n\");\n * controller.close();\n * })\n * .pipeThrough(new CSVLexerTransformer())\n * .pipeThrough(new CSVRecordAssemblerTransformer())\n * .pipeTo(new WritableStream({ write(row) { console.log(row); }}));\n * // { name: \"Alice\", age: \"20\" }\n * // { name: \"Bob\", age: \"25\" }\n * // { name: \"Charlie\", age: \"30\" }\n * ```\n *\n * @example Parse a CSV with headers by options\n * ```ts\n * new ReadableStream({\n * start(controller) {\n * controller.enqueue(\"Alice,20\\r\\n\");\n * controller.enqueue(\"Bob,25\\r\\n\");\n * controller.enqueue(\"Charlie,30\\r\\n\");\n * controller.close();\n * }\n * })\n * .pipeThrough(new CSVLexerTransformer())\n * .pipeThrough(new CSVRecordAssemblerTransformer({ header: [\"name\", \"age\"] }))\n * .pipeTo(new WritableStream({ write(row) { console.log(row); }}));\n * // { name: \"Alice\", age: \"20\" }\n * // { name: \"Bob\", age: \"25\" }\n * // { name: \"Charlie\", age: \"30\" }\n * ```\n *\n * @example Custom queuing strategies with backpressure tuning\n * ```ts\n * const transformer = new CSVRecordAssemblerTransformer(\n * {},\n * {\n * highWaterMark: 2048, // 2048 tokens\n * size: () => 1, // Each token counts as 1\n * checkInterval: 20 // Check backpressure every 20 records\n * },\n * {\n * highWaterMark: 512, // 512 records\n * size: () => 1, // Each record counts as 1\n * checkInterval: 5 // Check backpressure every 5 records\n * }\n * );\n *\n * await tokenStream\n * .pipeThrough(transformer)\n * .pipeTo(yourRecordProcessor);\n * ```\n */\nexport class CSVRecordAssemblerTransformer<\n Header extends ReadonlyArray<string>,\n> extends TransformStream<Token, CSVRecord<Header>> {\n public readonly assembler: CSVRecordAssembler<Header>;\n\n /**\n * Yields to the event loop to allow backpressure handling.\n * Can be overridden for testing purposes.\n * @internal\n */\n protected async yieldToEventLoop(): Promise<void> {\n await new Promise((resolve) => setTimeout(resolve, 0));\n }\n\n constructor(\n options: CSVRecordAssemblerOptions<Header> = {},\n writableStrategy: ExtendedQueuingStrategy<Token> = {\n highWaterMark: 1024, // 1024 tokens\n size: () => 1, // Each token counts as 1\n checkInterval: 10, // Check backpressure every 10 records\n },\n readableStrategy: ExtendedQueuingStrategy<CSVRecord<Header>> = {\n highWaterMark: 256, // 256 records\n size: () => 1, // Each record counts as 1\n checkInterval: 10, // Check backpressure every 10 records\n },\n ) {\n const assembler = new CSVRecordAssembler(options);\n const checkInterval =\n writableStrategy.checkInterval ?? readableStrategy.checkInterval ?? 10;\n\n super(\n {\n transform: async (token, controller) => {\n try {\n let recordCount = 0;\n // Pass single token directly to assemble (no array creation)\n for (const record of assembler.assemble(token, { stream: true })) {\n controller.enqueue(record);\n recordCount++;\n\n // Check backpressure periodically based on checkInterval\n if (\n recordCount % checkInterval === 0 &&\n controller.desiredSize !== null &&\n controller.desiredSize <= 0\n ) {\n // Yield to event loop when backpressure is detected\n await this.yieldToEventLoop();\n }\n }\n } catch (error) {\n controller.error(error);\n }\n },\n flush: async (controller) => {\n try {\n let recordCount = 0;\n // Call assemble without arguments to flush\n for (const record of assembler.assemble()) {\n controller.enqueue(record);\n recordCount++;\n\n // Check backpressure periodically based on checkInterval\n if (\n recordCount % checkInterval === 0 &&\n controller.desiredSize !== null &&\n controller.desiredSize <= 0\n ) {\n await this.yieldToEventLoop();\n }\n }\n } catch (error) {\n controller.error(error);\n }\n },\n },\n writableStrategy,\n readableStrategy,\n );\n this.assembler = assembler;\n }\n}\n"],"names":[],"mappings":";;AA2FO,MAAM,sCAEH,eAAA,CAA0C;AAAA,EAClC,SAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOhB,MAAgB,gBAAA,GAAkC;AAChD,IAAA,MAAM,IAAI,OAAA,CAAQ,CAAC,YAAY,UAAA,CAAW,OAAA,EAAS,CAAC,CAAC,CAAA;AAAA,EACvD;AAAA,EAEA,WAAA,CACE,OAAA,GAA6C,EAAC,EAC9C,gBAAA,GAAmD;AAAA,IACjD,aAAA,EAAe,IAAA;AAAA;AAAA,IACf,MAAM,MAAM,CAAA;AAAA;AAAA,IACZ,aAAA,EAAe;AAAA;AAAA,KAEjB,gBAAA,GAA+D;AAAA,IAC7D,aAAA,EAAe,GAAA;AAAA;AAAA,IACf,MAAM,MAAM,CAAA;AAAA;AAAA,IACZ,aAAA,EAAe;AAAA;AAAA,GACjB,EACA;AACA,IAAA,MAAM,SAAA,GAAY,IAAI,kBAAA,CAAmB,OAAO,CAAA;AAChD,IAAA,MAAM,aAAA,GACJ,gBAAA,CAAiB,aAAA,IAAiB,gBAAA,CAAiB,aAAA,IAAiB,EAAA;AAEtE,IAAA,KAAA;AAAA,MACE;AAAA,QACE,SAAA,EAAW,OAAO,KAAA,EAAO,UAAA,KAAe;AACtC,UAAA,IAAI;AACF,YAAA,IAAI,WAAA,GAAc,CAAA;AAElB,YAAA,KAAA,MAAW,MAAA,IAAU,UAAU,QAAA,CAAS,KAAA,EAAO,EAAE,MAAA,EAAQ,IAAA,EAAM,CAAA,EAAG;AAChE,cAAA,UAAA,CAAW,QAAQ,MAAM,CAAA;AACzB,cAAA,WAAA,EAAA;AAGA,cAAA,IACE,WAAA,GAAc,kBAAkB,CAAA,IAChC,UAAA,CAAW,gBAAgB,IAAA,IAC3B,UAAA,CAAW,eAAe,CAAA,EAC1B;AAEA,gBAAA,MAAM,KAAK,gBAAA,EAAiB;AAAA,cAC9B;AAAA,YACF;AAAA,UACF,SAAS,KAAA,EAAO;AACd,YAAA,UAAA,CAAW,MAAM,KAAK,CAAA;AAAA,UACxB;AAAA,QACF,CAAA;AAAA,QACA,KAAA,EAAO,OAAO,UAAA,KAAe;AAC3B,UAAA,IAAI;AACF,YAAA,IAAI,WAAA,GAAc,CAAA;AAElB,YAAA,KAAA,MAAW,MAAA,IAAU,SAAA,CAAU,QAAA,EAAS,EAAG;AACzC,cAAA,UAAA,CAAW,QAAQ,MAAM,CAAA;AACzB,cAAA,WAAA,EAAA;AAGA,cAAA,IACE,WAAA,GAAc,kBAAkB,CAAA,IAChC,UAAA,CAAW,gBAAgB,IAAA,IAC3B,UAAA,CAAW,eAAe,CAAA,EAC1B;AACA,gBAAA,MAAM,KAAK,gBAAA,EAAiB;AAAA,cAC9B;AAAA,YACF;AAAA,UACF,SAAS,KAAA,EAAO;AACd,YAAA,UAAA,CAAW,MAAM,KAAK,CAAA;AAAA,UACxB;AAAA,QACF;AAAA,OACF;AAAA,MACA,gBAAA;AAAA,MACA;AAAA,KACF;AACA,IAAA,IAAA,CAAK,SAAA,GAAY,SAAA;AAAA,EACnB;AACF;;;;"}
|
package/package.json
CHANGED