rowpipe 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +401 -0
- package/benchmarks/memory-bench.ts +109 -0
- package/dist/analytics/reduce.d.ts +33 -0
- package/dist/analytics/reduce.d.ts.map +1 -0
- package/dist/analytics/reduce.js +318 -0
- package/dist/analytics/reduce.js.map +1 -0
- package/dist/analytics/schema-inference.d.ts +31 -0
- package/dist/analytics/schema-inference.d.ts.map +1 -0
- package/dist/analytics/schema-inference.js +166 -0
- package/dist/analytics/schema-inference.js.map +1 -0
- package/dist/analytics/semantic-types.d.ts +6 -0
- package/dist/analytics/semantic-types.d.ts.map +1 -0
- package/dist/analytics/semantic-types.js +40 -0
- package/dist/analytics/semantic-types.js.map +1 -0
- package/dist/analytics/stats.d.ts +102 -0
- package/dist/analytics/stats.d.ts.map +1 -0
- package/dist/analytics/stats.js +307 -0
- package/dist/analytics/stats.js.map +1 -0
- package/dist/analytics/validator.d.ts +32 -0
- package/dist/analytics/validator.d.ts.map +1 -0
- package/dist/analytics/validator.js +96 -0
- package/dist/analytics/validator.js.map +1 -0
- package/dist/cli/commands/cast.d.ts +13 -0
- package/dist/cli/commands/cast.d.ts.map +1 -0
- package/dist/cli/commands/cast.js +39 -0
- package/dist/cli/commands/cast.js.map +1 -0
- package/dist/cli/commands/convert.d.ts +15 -0
- package/dist/cli/commands/convert.d.ts.map +1 -0
- package/dist/cli/commands/convert.js +89 -0
- package/dist/cli/commands/convert.js.map +1 -0
- package/dist/cli/commands/filter.d.ts +11 -0
- package/dist/cli/commands/filter.d.ts.map +1 -0
- package/dist/cli/commands/filter.js +38 -0
- package/dist/cli/commands/filter.js.map +1 -0
- package/dist/cli/commands/inspect.d.ts +10 -0
- package/dist/cli/commands/inspect.d.ts.map +1 -0
- package/dist/cli/commands/inspect.js +134 -0
- package/dist/cli/commands/inspect.js.map +1 -0
- package/dist/cli/commands/map.d.ts +11 -0
- package/dist/cli/commands/map.d.ts.map +1 -0
- package/dist/cli/commands/map.js +43 -0
- package/dist/cli/commands/map.js.map +1 -0
- package/dist/cli/commands/reduce.d.ts +12 -0
- package/dist/cli/commands/reduce.d.ts.map +1 -0
- package/dist/cli/commands/reduce.js +49 -0
- package/dist/cli/commands/reduce.js.map +1 -0
- package/dist/cli/commands/rename.d.ts +11 -0
- package/dist/cli/commands/rename.d.ts.map +1 -0
- package/dist/cli/commands/rename.js +39 -0
- package/dist/cli/commands/rename.js.map +1 -0
- package/dist/cli/commands/sample.d.ts +13 -0
- package/dist/cli/commands/sample.d.ts.map +1 -0
- package/dist/cli/commands/sample.js +40 -0
- package/dist/cli/commands/sample.js.map +1 -0
- package/dist/cli/commands/schema.d.ts +13 -0
- package/dist/cli/commands/schema.d.ts.map +1 -0
- package/dist/cli/commands/schema.js +57 -0
- package/dist/cli/commands/schema.js.map +1 -0
- package/dist/cli/commands/select.d.ts +11 -0
- package/dist/cli/commands/select.d.ts.map +1 -0
- package/dist/cli/commands/select.js +39 -0
- package/dist/cli/commands/select.js.map +1 -0
- package/dist/cli/commands/stats.d.ts +15 -0
- package/dist/cli/commands/stats.d.ts.map +1 -0
- package/dist/cli/commands/stats.js +75 -0
- package/dist/cli/commands/stats.js.map +1 -0
- package/dist/cli/commands/validate.d.ts +12 -0
- package/dist/cli/commands/validate.d.ts.map +1 -0
- package/dist/cli/commands/validate.js +86 -0
- package/dist/cli/commands/validate.js.map +1 -0
- package/dist/cli/index.d.ts +3 -0
- package/dist/cli/index.d.ts.map +1 -0
- package/dist/cli/index.js +262 -0
- package/dist/cli/index.js.map +1 -0
- package/dist/core/batch.d.ts +15 -0
- package/dist/core/batch.d.ts.map +1 -0
- package/dist/core/batch.js +67 -0
- package/dist/core/batch.js.map +1 -0
- package/dist/core/errors.d.ts +35 -0
- package/dist/core/errors.d.ts.map +1 -0
- package/dist/core/errors.js +55 -0
- package/dist/core/errors.js.map +1 -0
- package/dist/core/pipeline.d.ts +40 -0
- package/dist/core/pipeline.d.ts.map +1 -0
- package/dist/core/pipeline.js +120 -0
- package/dist/core/pipeline.js.map +1 -0
- package/dist/core/types.d.ts +73 -0
- package/dist/core/types.d.ts.map +1 -0
- package/dist/core/types.js +2 -0
- package/dist/core/types.js.map +1 -0
- package/dist/index.d.ts +22 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +28 -0
- package/dist/index.js.map +1 -0
- package/dist/readers/csv.d.ts +24 -0
- package/dist/readers/csv.d.ts.map +1 -0
- package/dist/readers/csv.js +251 -0
- package/dist/readers/csv.js.map +1 -0
- package/dist/readers/index.d.ts +17 -0
- package/dist/readers/index.d.ts.map +1 -0
- package/dist/readers/index.js +74 -0
- package/dist/readers/index.js.map +1 -0
- package/dist/readers/json.d.ts +20 -0
- package/dist/readers/json.d.ts.map +1 -0
- package/dist/readers/json.js +219 -0
- package/dist/readers/json.js.map +1 -0
- package/dist/readers/jsonl.d.ts +14 -0
- package/dist/readers/jsonl.d.ts.map +1 -0
- package/dist/readers/jsonl.js +95 -0
- package/dist/readers/jsonl.js.map +1 -0
- package/dist/readers/xlsx.d.ts +18 -0
- package/dist/readers/xlsx.d.ts.map +1 -0
- package/dist/readers/xlsx.js +138 -0
- package/dist/readers/xlsx.js.map +1 -0
- package/dist/transforms/cast.d.ts +21 -0
- package/dist/transforms/cast.d.ts.map +1 -0
- package/dist/transforms/cast.js +182 -0
- package/dist/transforms/cast.js.map +1 -0
- package/dist/transforms/expression.d.ts +67 -0
- package/dist/transforms/expression.d.ts.map +1 -0
- package/dist/transforms/expression.js +943 -0
- package/dist/transforms/expression.js.map +1 -0
- package/dist/transforms/filter.d.ts +6 -0
- package/dist/transforms/filter.d.ts.map +1 -0
- package/dist/transforms/filter.js +31 -0
- package/dist/transforms/filter.js.map +1 -0
- package/dist/transforms/map.d.ts +12 -0
- package/dist/transforms/map.d.ts.map +1 -0
- package/dist/transforms/map.js +76 -0
- package/dist/transforms/map.js.map +1 -0
- package/dist/transforms/rename.d.ts +10 -0
- package/dist/transforms/rename.d.ts.map +1 -0
- package/dist/transforms/rename.js +39 -0
- package/dist/transforms/rename.js.map +1 -0
- package/dist/transforms/sample.d.ts +10 -0
- package/dist/transforms/sample.d.ts.map +1 -0
- package/dist/transforms/sample.js +47 -0
- package/dist/transforms/sample.js.map +1 -0
- package/dist/transforms/select.d.ts +6 -0
- package/dist/transforms/select.d.ts.map +1 -0
- package/dist/transforms/select.js +63 -0
- package/dist/transforms/select.js.map +1 -0
- package/dist/utils/compression.d.ts +14 -0
- package/dist/utils/compression.d.ts.map +1 -0
- package/dist/utils/compression.js +38 -0
- package/dist/utils/compression.js.map +1 -0
- package/dist/utils/formatting.d.ts +6 -0
- package/dist/utils/formatting.d.ts.map +1 -0
- package/dist/utils/formatting.js +56 -0
- package/dist/utils/formatting.js.map +1 -0
- package/dist/utils/progress.d.ts +13 -0
- package/dist/utils/progress.d.ts.map +1 -0
- package/dist/utils/progress.js +27 -0
- package/dist/utils/progress.js.map +1 -0
- package/dist/writers/csv.d.ts +18 -0
- package/dist/writers/csv.d.ts.map +1 -0
- package/dist/writers/csv.js +84 -0
- package/dist/writers/csv.js.map +1 -0
- package/dist/writers/index.d.ts +17 -0
- package/dist/writers/index.d.ts.map +1 -0
- package/dist/writers/index.js +73 -0
- package/dist/writers/index.js.map +1 -0
- package/dist/writers/json.d.ts +12 -0
- package/dist/writers/json.d.ts.map +1 -0
- package/dist/writers/json.js +42 -0
- package/dist/writers/json.js.map +1 -0
- package/dist/writers/jsonl.d.ts +12 -0
- package/dist/writers/jsonl.d.ts.map +1 -0
- package/dist/writers/jsonl.js +41 -0
- package/dist/writers/jsonl.js.map +1 -0
- package/dist/writers/xlsx.d.ts +13 -0
- package/dist/writers/xlsx.d.ts.map +1 -0
- package/dist/writers/xlsx.js +65 -0
- package/dist/writers/xlsx.js.map +1 -0
- package/package.json +44 -0
- package/src/analytics/reduce.ts +388 -0
- package/src/analytics/schema-inference.ts +217 -0
- package/src/analytics/semantic-types.ts +36 -0
- package/src/analytics/stats.ts +383 -0
- package/src/analytics/validator.ts +134 -0
- package/src/cli/commands/cast.ts +60 -0
- package/src/cli/commands/convert.ts +128 -0
- package/src/cli/commands/filter.ts +58 -0
- package/src/cli/commands/inspect.ts +193 -0
- package/src/cli/commands/map.ts +66 -0
- package/src/cli/commands/reduce.ts +74 -0
- package/src/cli/commands/rename.ts +59 -0
- package/src/cli/commands/sample.ts +61 -0
- package/src/cli/commands/schema.ts +86 -0
- package/src/cli/commands/select.ts +59 -0
- package/src/cli/commands/stats.ts +100 -0
- package/src/cli/commands/validate.ts +123 -0
- package/src/cli/index.ts +285 -0
- package/src/core/batch.ts +81 -0
- package/src/core/errors.ts +81 -0
- package/src/core/pipeline.ts +150 -0
- package/src/core/types.ts +110 -0
- package/src/index.ts +52 -0
- package/src/readers/csv.ts +302 -0
- package/src/readers/index.ts +88 -0
- package/src/readers/json.ts +256 -0
- package/src/readers/jsonl.ts +126 -0
- package/src/readers/xlsx.ts +182 -0
- package/src/transforms/cast.ts +212 -0
- package/src/transforms/expression.ts +1114 -0
- package/src/transforms/filter.ts +38 -0
- package/src/transforms/map.ts +99 -0
- package/src/transforms/rename.ts +47 -0
- package/src/transforms/sample.ts +57 -0
- package/src/transforms/select.ts +64 -0
- package/src/utils/compression.ts +43 -0
- package/src/utils/formatting.ts +75 -0
- package/src/utils/progress.ts +37 -0
- package/src/writers/csv.ts +106 -0
- package/src/writers/index.ts +87 -0
- package/src/writers/json.ts +52 -0
- package/src/writers/jsonl.ts +49 -0
- package/src/writers/xlsx.ts +87 -0
- package/tests/analytics.test.ts +119 -0
- package/tests/cli-integration.test.ts +187 -0
- package/tests/map-reduce.test.ts +223 -0
- package/tests/readers-writers.test.ts +189 -0
- package/tests/transforms.test.ts +199 -0
- package/tsconfig.json +21 -0
- package/vitest.config.ts +9 -0
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import type { DataBatch, DataStream, Row } from "./types.js";
|
|
2
|
+
/**
|
|
3
|
+
* Packs an AsyncIterable of individual rows into DataBatch chunks with a given batchSize.
|
|
4
|
+
*/
|
|
5
|
+
export declare function rowsToBatches(rows: AsyncIterable<Row>, batchSize?: number): AsyncGenerator<DataBatch>;
|
|
6
|
+
/**
|
|
7
|
+
* Unpacks DataBatches into individual rows.
|
|
8
|
+
*/
|
|
9
|
+
export declare function batchesToRows(stream: DataStream): AsyncGenerator<Row>;
|
|
10
|
+
/**
|
|
11
|
+
* Applies a mapping function to each row in batches, maintaining batch structure
|
|
12
|
+
* and filtering out null/undefined results.
|
|
13
|
+
*/
|
|
14
|
+
export declare function batchMap(stream: DataStream, fn: (row: Row, globalIndex: number) => Row | null | undefined, targetBatchSize?: number): AsyncGenerator<DataBatch>;
|
|
15
|
+
//# sourceMappingURL=batch.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"batch.d.ts","sourceRoot":"","sources":["../../src/core/batch.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,UAAU,EAAE,GAAG,EAAE,MAAM,YAAY,CAAC;AAE7D;;GAEG;AACH,wBAAuB,aAAa,CAClC,IAAI,EAAE,aAAa,CAAC,GAAG,CAAC,EACxB,SAAS,SAAO,GACf,cAAc,CAAC,SAAS,CAAC,CAuB3B;AAED;;GAEG;AACH,wBAAuB,aAAa,CAAC,MAAM,EAAE,UAAU,GAAG,cAAc,CAAC,GAAG,CAAC,CAM5E;AAED;;;GAGG;AACH,wBAAuB,QAAQ,CAC7B,MAAM,EAAE,UAAU,EAClB,EAAE,EAAE,CAAC,GAAG,EAAE,GAAG,EAAE,WAAW,EAAE,MAAM,KAAK,GAAG,GAAG,IAAI,GAAG,SAAS,EAC7D,eAAe,SAAO,GACrB,cAAc,CAAC,SAAS,CAAC,CA4B3B"}
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Packs an AsyncIterable of individual rows into DataBatch chunks with a given batchSize.
|
|
3
|
+
*/
|
|
4
|
+
export async function* rowsToBatches(rows, batchSize = 1000) {
|
|
5
|
+
const effectiveBatchSize = Math.max(1, batchSize);
|
|
6
|
+
let currentRows = [];
|
|
7
|
+
let totalOffset = 0;
|
|
8
|
+
for await (const row of rows) {
|
|
9
|
+
currentRows.push(row);
|
|
10
|
+
if (currentRows.length >= effectiveBatchSize) {
|
|
11
|
+
yield {
|
|
12
|
+
rows: currentRows,
|
|
13
|
+
offset: totalOffset,
|
|
14
|
+
};
|
|
15
|
+
totalOffset += currentRows.length;
|
|
16
|
+
currentRows = [];
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
if (currentRows.length > 0) {
|
|
20
|
+
yield {
|
|
21
|
+
rows: currentRows,
|
|
22
|
+
offset: totalOffset,
|
|
23
|
+
};
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
/**
|
|
27
|
+
* Unpacks DataBatches into individual rows.
|
|
28
|
+
*/
|
|
29
|
+
export async function* batchesToRows(stream) {
|
|
30
|
+
for await (const batch of stream) {
|
|
31
|
+
for (const row of batch.rows) {
|
|
32
|
+
yield row;
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
/**
|
|
37
|
+
* Applies a mapping function to each row in batches, maintaining batch structure
|
|
38
|
+
* and filtering out null/undefined results.
|
|
39
|
+
*/
|
|
40
|
+
export async function* batchMap(stream, fn, targetBatchSize = 1000) {
|
|
41
|
+
let pendingRows = [];
|
|
42
|
+
let currentOffset = 0;
|
|
43
|
+
let globalIndex = 0;
|
|
44
|
+
for await (const batch of stream) {
|
|
45
|
+
for (const row of batch.rows) {
|
|
46
|
+
const transformed = fn(row, globalIndex++);
|
|
47
|
+
if (transformed !== null && transformed !== undefined) {
|
|
48
|
+
pendingRows.push(transformed);
|
|
49
|
+
if (pendingRows.length >= targetBatchSize) {
|
|
50
|
+
yield {
|
|
51
|
+
rows: pendingRows,
|
|
52
|
+
offset: currentOffset,
|
|
53
|
+
};
|
|
54
|
+
currentOffset += pendingRows.length;
|
|
55
|
+
pendingRows = [];
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
if (pendingRows.length > 0) {
|
|
61
|
+
yield {
|
|
62
|
+
rows: pendingRows,
|
|
63
|
+
offset: currentOffset,
|
|
64
|
+
};
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
//# sourceMappingURL=batch.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"batch.js","sourceRoot":"","sources":["../../src/core/batch.ts"],"names":[],"mappings":"AAEA;;GAEG;AACH,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,aAAa,CAClC,IAAwB,EACxB,SAAS,GAAG,IAAI;IAEhB,MAAM,kBAAkB,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;IAClD,IAAI,WAAW,GAAU,EAAE,CAAC;IAC5B,IAAI,WAAW,GAAG,CAAC,CAAC;IAEpB,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QAC7B,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACtB,IAAI,WAAW,CAAC,MAAM,IAAI,kBAAkB,EAAE,CAAC;YAC7C,MAAM;gBACJ,IAAI,EAAE,WAAW;gBACjB,MAAM,EAAE,WAAW;aACpB,CAAC;YACF,WAAW,IAAI,WAAW,CAAC,MAAM,CAAC;YAClC,WAAW,GAAG,EAAE,CAAC;QACnB,CAAC;IACH,CAAC;IAED,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC3B,MAAM;YACJ,IAAI,EAAE,WAAW;YACjB,MAAM,EAAE,WAAW;SACpB,CAAC;IACJ,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,aAAa,CAAC,MAAkB;IACrD,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QACjC,KAAK,MAAM,GAAG,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC;YAC7B,MAAM,GAAG,CAAC;QACZ,CAAC;IACH,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,QAAQ,CAC7B,MAAkB,EAClB,EAA6D,EAC7D,eAAe,GAAG,IAAI;IAEtB,IAAI,WAAW,GAAU,EAAE,CAAC;IAC5B,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,IAAI,WAAW,GAAG,CAAC,CAAC;IAEpB,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QACjC,KAAK,MAAM,GAAG,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC;YAC7B,MAAM,WAAW,GAAG,EAAE,CAAC,GAAG,EAAE,WAAW,EAAE,CAAC,CAAC;YAC3C,IAAI,WAAW,KAAK,IAAI,IAAI,WAAW,KAAK,SAAS,EAAE,CAAC;gBACtD,WAAW,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;gBAC9B,IAAI,WAAW,CAAC,MAAM,IAAI,eAAe,EAAE,CAAC;oBAC1C,MAAM;wBACJ,IAAI,EAAE,WAAW;wBACjB,MAAM,EAAE,aAAa;qBACtB,CAAC;oBACF,aAAa,IAAI,WAAW,CAAC,MAAM,CAAC;oBACpC,WAAW,GAAG,EAAE,CAAC;gBACnB,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC3B,MAAM;YACJ,IAAI,EAAE,WAAW;YACjB,MAAM,EAAE,aAAa;SACtB,CAAC;IACJ,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
export declare class RowpipeError extends Error {
|
|
2
|
+
readonly exitCode: number;
|
|
3
|
+
constructor(message: string, exitCode?: number);
|
|
4
|
+
}
|
|
5
|
+
export declare class InvalidArgumentError extends RowpipeError {
|
|
6
|
+
constructor(message: string);
|
|
7
|
+
}
|
|
8
|
+
export interface ParseErrorContext {
|
|
9
|
+
file?: string;
|
|
10
|
+
sheet?: string;
|
|
11
|
+
row?: number;
|
|
12
|
+
column?: string | number;
|
|
13
|
+
byteOffset?: number;
|
|
14
|
+
line?: number;
|
|
15
|
+
cell?: string;
|
|
16
|
+
}
|
|
17
|
+
export declare class ParseError extends RowpipeError {
|
|
18
|
+
readonly context: ParseErrorContext;
|
|
19
|
+
constructor(message: string, context?: ParseErrorContext);
|
|
20
|
+
}
|
|
21
|
+
export interface ValidationViolation {
|
|
22
|
+
column: string;
|
|
23
|
+
expected: string;
|
|
24
|
+
actual?: string;
|
|
25
|
+
count: number;
|
|
26
|
+
examples?: unknown[];
|
|
27
|
+
}
|
|
28
|
+
export declare class ValidationError extends RowpipeError {
|
|
29
|
+
readonly violations: ValidationViolation[];
|
|
30
|
+
readonly totalRows: number;
|
|
31
|
+
readonly validRows: number;
|
|
32
|
+
readonly invalidRows: number;
|
|
33
|
+
constructor(message: string, violations?: ValidationViolation[], totalRows?: number, validRows?: number, invalidRows?: number);
|
|
34
|
+
}
|
|
35
|
+
//# sourceMappingURL=errors.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"errors.d.ts","sourceRoot":"","sources":["../../src/core/errors.ts"],"names":[],"mappings":"AAAA,qBAAa,YAAa,SAAQ,KAAK;IACrC,SAAgB,QAAQ,EAAE,MAAM,CAAC;gBAErB,OAAO,EAAE,MAAM,EAAE,QAAQ,SAAI;CAM1C;AAED,qBAAa,oBAAqB,SAAQ,YAAY;gBACxC,OAAO,EAAE,MAAM;CAI5B;AAED,MAAM,WAAW,iBAAiB;IAChC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IACzB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED,qBAAa,UAAW,SAAQ,YAAY;IAC1C,SAAgB,OAAO,EAAE,iBAAiB,CAAC;gBAE/B,OAAO,EAAE,MAAM,EAAE,OAAO,GAAE,iBAAsB;CAmB7D;AAED,MAAM,WAAW,mBAAmB;IAClC,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,CAAC,EAAE,OAAO,EAAE,CAAC;CACtB;AAED,qBAAa,eAAgB,SAAQ,YAAY;IAC/C,SAAgB,UAAU,EAAE,mBAAmB,EAAE,CAAC;IAClD,SAAgB,SAAS,EAAE,MAAM,CAAC;IAClC,SAAgB,SAAS,EAAE,MAAM,CAAC;IAClC,SAAgB,WAAW,EAAE,MAAM,CAAC;gBAGlC,OAAO,EAAE,MAAM,EACf,UAAU,GAAE,mBAAmB,EAAO,EACtC,SAAS,SAAI,EACb,SAAS,SAAI,EACb,WAAW,SAAI;CASlB"}
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
export class RowpipeError extends Error {
|
|
2
|
+
exitCode;
|
|
3
|
+
constructor(message, exitCode = 1) {
|
|
4
|
+
super(message);
|
|
5
|
+
this.name = "RowpipeError";
|
|
6
|
+
this.exitCode = exitCode;
|
|
7
|
+
Object.setPrototypeOf(this, new.target.prototype);
|
|
8
|
+
}
|
|
9
|
+
}
|
|
10
|
+
export class InvalidArgumentError extends RowpipeError {
|
|
11
|
+
constructor(message) {
|
|
12
|
+
super(message, 2);
|
|
13
|
+
this.name = "InvalidArgumentError";
|
|
14
|
+
}
|
|
15
|
+
}
|
|
16
|
+
export class ParseError extends RowpipeError {
|
|
17
|
+
context;
|
|
18
|
+
constructor(message, context = {}) {
|
|
19
|
+
let detailedMsg = `Parse error: ${message}`;
|
|
20
|
+
const parts = [];
|
|
21
|
+
if (context.file)
|
|
22
|
+
parts.push(`File: ${context.file}`);
|
|
23
|
+
if (context.sheet)
|
|
24
|
+
parts.push(`Sheet: ${context.sheet}`);
|
|
25
|
+
if (context.row !== undefined)
|
|
26
|
+
parts.push(`Row: ${context.row}`);
|
|
27
|
+
if (context.column !== undefined)
|
|
28
|
+
parts.push(`Column: ${context.column}`);
|
|
29
|
+
if (context.cell)
|
|
30
|
+
parts.push(`Cell: ${context.cell}`);
|
|
31
|
+
if (context.byteOffset !== undefined)
|
|
32
|
+
parts.push(`Byte offset: ${context.byteOffset}`);
|
|
33
|
+
if (parts.length > 0) {
|
|
34
|
+
detailedMsg += `\n Context: ${parts.join(", ")}`;
|
|
35
|
+
}
|
|
36
|
+
super(detailedMsg, 3);
|
|
37
|
+
this.name = "ParseError";
|
|
38
|
+
this.context = context;
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
export class ValidationError extends RowpipeError {
|
|
42
|
+
violations;
|
|
43
|
+
totalRows;
|
|
44
|
+
validRows;
|
|
45
|
+
invalidRows;
|
|
46
|
+
constructor(message, violations = [], totalRows = 0, validRows = 0, invalidRows = 0) {
|
|
47
|
+
super(message, 4);
|
|
48
|
+
this.name = "ValidationError";
|
|
49
|
+
this.violations = violations;
|
|
50
|
+
this.totalRows = totalRows;
|
|
51
|
+
this.validRows = validRows;
|
|
52
|
+
this.invalidRows = invalidRows;
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
//# sourceMappingURL=errors.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"errors.js","sourceRoot":"","sources":["../../src/core/errors.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,YAAa,SAAQ,KAAK;IACrB,QAAQ,CAAS;IAEjC,YAAY,OAAe,EAAE,QAAQ,GAAG,CAAC;QACvC,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,cAAc,CAAC;QAC3B,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,MAAM,CAAC,cAAc,CAAC,IAAI,EAAE,GAAG,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IACpD,CAAC;CACF;AAED,MAAM,OAAO,oBAAqB,SAAQ,YAAY;IACpD,YAAY,OAAe;QACzB,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;QAClB,IAAI,CAAC,IAAI,GAAG,sBAAsB,CAAC;IACrC,CAAC;CACF;AAYD,MAAM,OAAO,UAAW,SAAQ,YAAY;IAC1B,OAAO,CAAoB;IAE3C,YAAY,OAAe,EAAE,UAA6B,EAAE;QAC1D,IAAI,WAAW,GAAG,gBAAgB,OAAO,EAAE,CAAC;QAC5C,MAAM,KAAK,GAAa,EAAE,CAAC;QAE3B,IAAI,OAAO,CAAC,IAAI;YAAE,KAAK,CAAC,IAAI,CAAC,SAAS,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;QACtD,IAAI,OAAO,CAAC,KAAK;YAAE,KAAK,CAAC,IAAI,CAAC,UAAU,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC;QACzD,IAAI,OAAO,CAAC,GAAG,KAAK,SAAS;YAAE,KAAK,CAAC,IAAI,CAAC,QAAQ,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;QACjE,IAAI,OAAO,CAAC,MAAM,KAAK,SAAS;YAAE,KAAK,CAAC,IAAI,CAAC,WAAW,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;QAC1E,IAAI,OAAO,CAAC,IAAI;YAAE,KAAK,CAAC,IAAI,CAAC,SAAS,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;QACtD,IAAI,OAAO,CAAC,UAAU,KAAK,SAAS;YAAE,KAAK,CAAC,IAAI,CAAC,gBAAgB,OAAO,CAAC,UAAU,EAAE,CAAC,CAAC;QAEvF,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACrB,WAAW,IAAI,gBAAgB,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QACpD,CAAC;QAED,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;QACtB,IAAI,CAAC,IAAI,GAAG,YAAY,CAAC;QACzB,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;IACzB,CAAC;CACF;AAUD,MAAM,OAAO,eAAgB,SAAQ,YAAY;IAC/B,UAAU,CAAwB;IAClC,SAAS,CAAS;IAClB,SAAS,CAAS;IAClB,WAAW,CAAS;IAEpC,YACE,OAAe,EACf,aAAoC,EAAE,EACtC,SAAS,GAAG,CAAC,EACb,SAAS,GAAG,CAAC,EACb,WAAW,GAAG,CAAC;QAEf,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;QAClB,IAAI,CAAC,IAAI,GAAG,iBAAiB,CAAC;QAC9B,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAC7B,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;IACjC,CAAC;CACF"}
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
import type { Aggregator, DataStream, ProgressCallback, ReaderOptions, Row, TabularReader, TabularWriter, TransformFunction, WriterOptions } from "./types.js";
|
|
2
|
+
export declare class Pipeline {
|
|
3
|
+
private stream;
|
|
4
|
+
private progressCallbacks;
|
|
5
|
+
constructor(source: DataStream | TabularReader, options?: ReaderOptions);
|
|
6
|
+
/**
|
|
7
|
+
* Chains a transform function to the pipeline.
|
|
8
|
+
*/
|
|
9
|
+
pipe(transform: TransformFunction): this;
|
|
10
|
+
/**
|
|
11
|
+
* Adds a progress listener.
|
|
12
|
+
*/
|
|
13
|
+
onProgress(callback: ProgressCallback): this;
|
|
14
|
+
/**
|
|
15
|
+
* Wraps the current stream with a progress tracking generator if callbacks are registered.
|
|
16
|
+
*/
|
|
17
|
+
private getTrackedStream;
|
|
18
|
+
/**
|
|
19
|
+
* Pipes the stream into a TabularWriter.
|
|
20
|
+
*/
|
|
21
|
+
to(writer: TabularWriter, options?: WriterOptions): Promise<void>;
|
|
22
|
+
/**
|
|
23
|
+
* Consumes the stream using an Aggregator.
|
|
24
|
+
*/
|
|
25
|
+
reduce<T>(aggregator: Aggregator<T>): Promise<T>;
|
|
26
|
+
/**
|
|
27
|
+
* Returns an AsyncIterable of DataBatches.
|
|
28
|
+
*/
|
|
29
|
+
batches(): DataStream;
|
|
30
|
+
/**
|
|
31
|
+
* Returns an AsyncIterable of individual rows.
|
|
32
|
+
*/
|
|
33
|
+
rows(): AsyncIterable<Row>;
|
|
34
|
+
/**
|
|
35
|
+
* Collects all rows up to a maximum limit (default 10,000 to prevent unbounded memory).
|
|
36
|
+
*/
|
|
37
|
+
toArray(maxRows?: number): Promise<Row[]>;
|
|
38
|
+
}
|
|
39
|
+
export declare function createPipeline(source: DataStream | TabularReader, options?: ReaderOptions): Pipeline;
|
|
40
|
+
//# sourceMappingURL=pipeline.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../../src/core/pipeline.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,UAAU,EAEV,UAAU,EACV,gBAAgB,EAChB,aAAa,EACb,GAAG,EACH,aAAa,EACb,aAAa,EACb,iBAAiB,EACjB,aAAa,EACd,MAAM,YAAY,CAAC;AAGpB,qBAAa,QAAQ;IACnB,OAAO,CAAC,MAAM,CAAa;IAC3B,OAAO,CAAC,iBAAiB,CAA0B;gBAEvC,MAAM,EAAE,UAAU,GAAG,aAAa,EAAE,OAAO,CAAC,EAAE,aAAa;IAQvE;;OAEG;IACH,IAAI,CAAC,SAAS,EAAE,iBAAiB,GAAG,IAAI;IAKxC;;OAEG;IACH,UAAU,CAAC,QAAQ,EAAE,gBAAgB,GAAG,IAAI;IAK5C;;OAEG;IACH,OAAO,CAAC,gBAAgB;IAgDxB;;OAEG;IACG,EAAE,CAAC,MAAM,EAAE,aAAa,EAAE,OAAO,CAAC,EAAE,aAAa,GAAG,OAAO,CAAC,IAAI,CAAC;IAQvE;;OAEG;IACG,MAAM,CAAC,CAAC,EAAE,UAAU,EAAE,UAAU,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC;IAQtD;;OAEG;IACH,OAAO,IAAI,UAAU;IAIrB;;OAEG;IACH,IAAI,IAAI,aAAa,CAAC,GAAG,CAAC;IAI1B;;OAEG;IACG,OAAO,CAAC,OAAO,SAAQ,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;CAU/C;AAED,wBAAgB,cAAc,CAC5B,MAAM,EAAE,UAAU,GAAG,aAAa,EAClC,OAAO,CAAC,EAAE,aAAa,GACtB,QAAQ,CAEV"}
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
import { batchesToRows } from "./batch.js";
|
|
2
|
+
export class Pipeline {
|
|
3
|
+
stream;
|
|
4
|
+
progressCallbacks = [];
|
|
5
|
+
constructor(source, options) {
|
|
6
|
+
if ("read" in source && typeof source.read === "function") {
|
|
7
|
+
this.stream = source.read(options);
|
|
8
|
+
}
|
|
9
|
+
else {
|
|
10
|
+
this.stream = source;
|
|
11
|
+
}
|
|
12
|
+
}
|
|
13
|
+
/**
|
|
14
|
+
* Chains a transform function to the pipeline.
|
|
15
|
+
*/
|
|
16
|
+
pipe(transform) {
|
|
17
|
+
this.stream = transform(this.stream);
|
|
18
|
+
return this;
|
|
19
|
+
}
|
|
20
|
+
/**
|
|
21
|
+
* Adds a progress listener.
|
|
22
|
+
*/
|
|
23
|
+
onProgress(callback) {
|
|
24
|
+
this.progressCallbacks.push(callback);
|
|
25
|
+
return this;
|
|
26
|
+
}
|
|
27
|
+
/**
|
|
28
|
+
* Wraps the current stream with a progress tracking generator if callbacks are registered.
|
|
29
|
+
*/
|
|
30
|
+
getTrackedStream() {
|
|
31
|
+
if (this.progressCallbacks.length === 0) {
|
|
32
|
+
return this.stream;
|
|
33
|
+
}
|
|
34
|
+
const callbacks = this.progressCallbacks;
|
|
35
|
+
const stream = this.stream;
|
|
36
|
+
return (async function* () {
|
|
37
|
+
let rowsProcessed = 0;
|
|
38
|
+
const startTime = Date.now();
|
|
39
|
+
let lastReport = startTime;
|
|
40
|
+
for await (const batch of stream) {
|
|
41
|
+
rowsProcessed += batch.rows.length;
|
|
42
|
+
const now = Date.now();
|
|
43
|
+
// Throttle progress notifications to ~100ms intervals
|
|
44
|
+
if (now - lastReport >= 100) {
|
|
45
|
+
const elapsedSec = (now - startTime) / 1000 || 0.001;
|
|
46
|
+
const info = {
|
|
47
|
+
rowsProcessed,
|
|
48
|
+
elapsedMs: now - startTime,
|
|
49
|
+
rowsPerSecond: Math.round(rowsProcessed / elapsedSec),
|
|
50
|
+
};
|
|
51
|
+
for (const cb of callbacks) {
|
|
52
|
+
cb(info);
|
|
53
|
+
}
|
|
54
|
+
lastReport = now;
|
|
55
|
+
}
|
|
56
|
+
yield batch;
|
|
57
|
+
}
|
|
58
|
+
// Final progress callback
|
|
59
|
+
const finalNow = Date.now();
|
|
60
|
+
const finalElapsedSec = (finalNow - startTime) / 1000 || 0.001;
|
|
61
|
+
const finalInfo = {
|
|
62
|
+
rowsProcessed,
|
|
63
|
+
elapsedMs: finalNow - startTime,
|
|
64
|
+
rowsPerSecond: Math.round(rowsProcessed / finalElapsedSec),
|
|
65
|
+
};
|
|
66
|
+
for (const cb of callbacks) {
|
|
67
|
+
cb(finalInfo);
|
|
68
|
+
}
|
|
69
|
+
})();
|
|
70
|
+
}
|
|
71
|
+
/**
|
|
72
|
+
* Pipes the stream into a TabularWriter.
|
|
73
|
+
*/
|
|
74
|
+
async to(writer, options) {
|
|
75
|
+
const tracked = this.getTrackedStream();
|
|
76
|
+
await writer.write(tracked, options);
|
|
77
|
+
if (writer.close) {
|
|
78
|
+
await writer.close();
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
/**
|
|
82
|
+
* Consumes the stream using an Aggregator.
|
|
83
|
+
*/
|
|
84
|
+
async reduce(aggregator) {
|
|
85
|
+
const tracked = this.getTrackedStream();
|
|
86
|
+
for await (const row of batchesToRows(tracked)) {
|
|
87
|
+
aggregator.add(row);
|
|
88
|
+
}
|
|
89
|
+
return aggregator.result();
|
|
90
|
+
}
|
|
91
|
+
/**
|
|
92
|
+
* Returns an AsyncIterable of DataBatches.
|
|
93
|
+
*/
|
|
94
|
+
batches() {
|
|
95
|
+
return this.getTrackedStream();
|
|
96
|
+
}
|
|
97
|
+
/**
|
|
98
|
+
* Returns an AsyncIterable of individual rows.
|
|
99
|
+
*/
|
|
100
|
+
rows() {
|
|
101
|
+
return batchesToRows(this.getTrackedStream());
|
|
102
|
+
}
|
|
103
|
+
/**
|
|
104
|
+
* Collects all rows up to a maximum limit (default 10,000 to prevent unbounded memory).
|
|
105
|
+
*/
|
|
106
|
+
async toArray(maxRows = 10000) {
|
|
107
|
+
const results = [];
|
|
108
|
+
for await (const row of this.rows()) {
|
|
109
|
+
results.push(row);
|
|
110
|
+
if (results.length >= maxRows) {
|
|
111
|
+
break;
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
return results;
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
export function createPipeline(source, options) {
|
|
118
|
+
return new Pipeline(source, options);
|
|
119
|
+
}
|
|
120
|
+
//# sourceMappingURL=pipeline.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"pipeline.js","sourceRoot":"","sources":["../../src/core/pipeline.ts"],"names":[],"mappings":"AAYA,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAE3C,MAAM,OAAO,QAAQ;IACX,MAAM,CAAa;IACnB,iBAAiB,GAAuB,EAAE,CAAC;IAEnD,YAAY,MAAkC,EAAE,OAAuB;QACrE,IAAI,MAAM,IAAI,MAAM,IAAI,OAAO,MAAM,CAAC,IAAI,KAAK,UAAU,EAAE,CAAC;YAC1D,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACrC,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,MAAM,GAAG,MAAoB,CAAC;QACrC,CAAC;IACH,CAAC;IAED;;OAEG;IACH,IAAI,CAAC,SAA4B;QAC/B,IAAI,CAAC,MAAM,GAAG,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACrC,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;OAEG;IACH,UAAU,CAAC,QAA0B;QACnC,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACtC,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;OAEG;IACK,gBAAgB;QACtB,IAAI,IAAI,CAAC,iBAAiB,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACxC,OAAO,IAAI,CAAC,MAAM,CAAC;QACrB,CAAC;QAED,MAAM,SAAS,GAAG,IAAI,CAAC,iBAAiB,CAAC;QACzC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QAE3B,OAAO,CAAC,KAAK,SAAS,CAAC;YACrB,IAAI,aAAa,GAAG,CAAC,CAAC;YACtB,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAC7B,IAAI,UAAU,GAAG,SAAS,CAAC;YAE3B,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;gBACjC,aAAa,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC;gBACnC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;gBAEvB,sDAAsD;gBACtD,IAAI,GAAG,GAAG,UAAU,IAAI,GAAG,EAAE,CAAC;oBAC5B,MAAM,UAAU,GAAG,CAAC,GAAG,GAAG,SAAS,CAAC,GAAG,IAAI,IAAI,KAAK,CAAC;oBACrD,MAAM,IAAI,GAAG;wBACX,aAAa;wBACb,SAAS,EAAE,GAAG,GAAG,SAAS;wBAC1B,aAAa,EAAE,IAAI,CAAC,KAAK,CAAC,aAAa,GAAG,UAAU,CAAC;qBACtD,CAAC;oBACF,KAAK,MAAM,EAAE,IAAI,SAAS,EAAE,CAAC;wBAC3B,EAAE,CAAC,IAAI,CAAC,CAAC;oBACX,CAAC;oBACD,UAAU,GAAG,GAAG,CAAC;gBACnB,CAAC;gBAED,MAAM,KAAK,CAAC;YACd,CAAC;YAED,0BAA0B;YAC1B,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAC5B,MAAM,eAAe,GAAG,CAAC,QAAQ,GAAG,SAAS,CAAC,GAAG,IAAI,IAAI,KAAK,CAAC;YAC/D,MAAM,SAAS,GAAG;gBAChB,aAAa;gBACb,SAAS,EAAE,QAAQ,GAAG,SAAS;gBAC/B,aAAa,EAAE,IAAI,CAAC,KAAK,CAAC,aAAa,GAAG,eAAe,CAAC;aAC3D,CAAC;YACF,KAAK,MAAM,EAAE,IAAI,SAAS,EAAE,CAAC;gBAC3B,EAAE,CAAC,SAAS,CAAC,CAAC;YAChB,CAAC;QACH,CAAC,CAAC,EAAE,CAAC;IACP,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,EAAE,CAAC,MAAqB,EAAE,OAAuB;QACrD,MAAM,OAAO,GAAG,IAAI,CAAC,gBAAgB,EAAE,CAAC;QACxC,MAAM,MAAM,CAAC,KAAK,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QACrC,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;YACjB,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;QACvB,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,MAAM,CAAI,UAAyB;QACvC,MAAM,OAAO,GAAG,IAAI,CAAC,gBAAgB,EAAE,CAAC;QACxC,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,aAAa,CAAC,OAAO,CAAC,EAAE,CAAC;YAC/C,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACtB,CAAC;QACD,OAAO,UAAU,CAAC,MAAM,EAAE,CAAC;IAC7B,CAAC;IAED;;OAEG;IACH,OAAO;QACL,OAAO,IAAI,CAAC,gBAAgB,EAAE,CAAC;IACjC,CAAC;IAED;;OAEG;IACH,IAAI;QACF,OAAO,aAAa,CAAC,IAAI,CAAC,gBAAgB,EAAE,CAAC,CAAC;IAChD,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,OAAO,CAAC,OAAO,GAAG,KAAK;QAC3B,MAAM,OAAO,GAAU,EAAE,CAAC;QAC1B,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC;YACpC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAClB,IAAI,OAAO,CAAC,MAAM,IAAI,OAAO,EAAE,CAAC;gBAC9B,MAAM;YACR,CAAC;QACH,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;CACF;AAED,MAAM,UAAU,cAAc,CAC5B,MAAkC,EAClC,OAAuB;IAEvB,OAAO,IAAI,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;AACvC,CAAC"}
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
import type { Readable, Writable } from "node:stream";
|
|
2
|
+
export type Row = Record<string, unknown>;
|
|
3
|
+
export interface DataBatch {
|
|
4
|
+
rows: Row[];
|
|
5
|
+
offset: number;
|
|
6
|
+
}
|
|
7
|
+
export type DataStream = AsyncIterable<DataBatch>;
|
|
8
|
+
export type ColumnType = "string" | "integer" | "number" | "boolean" | "date" | "datetime" | "null" | "mixed";
|
|
9
|
+
export type SemanticType = "email" | "url" | "uuid" | "ipv4" | "ipv6" | "country-code" | "currency" | "phone";
|
|
10
|
+
export interface ReaderOptions {
|
|
11
|
+
batchSize?: number;
|
|
12
|
+
header?: boolean;
|
|
13
|
+
delimiter?: string;
|
|
14
|
+
path?: string;
|
|
15
|
+
sheet?: string | number;
|
|
16
|
+
maxRows?: number;
|
|
17
|
+
filePath?: string;
|
|
18
|
+
}
|
|
19
|
+
export interface WriterOptions {
|
|
20
|
+
delimiter?: string;
|
|
21
|
+
header?: boolean;
|
|
22
|
+
sheet?: string;
|
|
23
|
+
}
|
|
24
|
+
export interface SheetMetadata {
|
|
25
|
+
name: string;
|
|
26
|
+
rowCount: number;
|
|
27
|
+
columnCount?: number;
|
|
28
|
+
columns?: string[];
|
|
29
|
+
}
|
|
30
|
+
export interface InspectionMetadata {
|
|
31
|
+
format: string;
|
|
32
|
+
sizeBytes?: number;
|
|
33
|
+
rowCount?: number;
|
|
34
|
+
columnCount?: number;
|
|
35
|
+
columns?: Array<{
|
|
36
|
+
name: string;
|
|
37
|
+
type: ColumnType;
|
|
38
|
+
nullPercentage: number;
|
|
39
|
+
approxUniquePercentage?: number;
|
|
40
|
+
}>;
|
|
41
|
+
sheetsCount?: number;
|
|
42
|
+
sheets?: SheetMetadata[];
|
|
43
|
+
}
|
|
44
|
+
export interface TabularReader {
|
|
45
|
+
read(options?: ReaderOptions): DataStream;
|
|
46
|
+
inspect?(options?: ReaderOptions): Promise<InspectionMetadata>;
|
|
47
|
+
close?(): Promise<void>;
|
|
48
|
+
}
|
|
49
|
+
export interface TabularWriter {
|
|
50
|
+
write(stream: DataStream, options?: WriterOptions): Promise<void>;
|
|
51
|
+
close?(): Promise<void>;
|
|
52
|
+
}
|
|
53
|
+
export interface FormatAdapter {
|
|
54
|
+
name: string;
|
|
55
|
+
extensions: string[];
|
|
56
|
+
createReader(input: Readable | string, options?: ReaderOptions): TabularReader;
|
|
57
|
+
createWriter(output: Writable | string, options?: WriterOptions): TabularWriter;
|
|
58
|
+
}
|
|
59
|
+
export type TransformFunction = (stream: DataStream) => DataStream;
|
|
60
|
+
export interface Aggregator<T> {
|
|
61
|
+
add(row: Row): void;
|
|
62
|
+
merge?(other: Aggregator<T>): void;
|
|
63
|
+
result(): T;
|
|
64
|
+
}
|
|
65
|
+
export interface ProgressCallbackInfo {
|
|
66
|
+
rowsProcessed: number;
|
|
67
|
+
bytesRead?: number;
|
|
68
|
+
elapsedMs: number;
|
|
69
|
+
rowsPerSecond: number;
|
|
70
|
+
megabytesPerSecond?: number;
|
|
71
|
+
}
|
|
72
|
+
export type ProgressCallback = (info: ProgressCallbackInfo) => void;
|
|
73
|
+
//# sourceMappingURL=types.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/core/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAEtD,MAAM,MAAM,GAAG,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;AAE1C,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,GAAG,EAAE,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,MAAM,UAAU,GAAG,aAAa,CAAC,SAAS,CAAC,CAAC;AAElD,MAAM,MAAM,UAAU,GAClB,QAAQ,GACR,SAAS,GACT,QAAQ,GACR,SAAS,GACT,MAAM,GACN,UAAU,GACV,MAAM,GACN,OAAO,CAAC;AAEZ,MAAM,MAAM,YAAY,GACpB,OAAO,GACP,KAAK,GACL,MAAM,GACN,MAAM,GACN,MAAM,GACN,cAAc,GACd,UAAU,GACV,OAAO,CAAC;AAEZ,MAAM,WAAW,aAAa;IAC5B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IACxB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,aAAa;IAC5B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;CACpB;AAED,MAAM,WAAW,kBAAkB;IACjC,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,KAAK,CAAC;QACd,IAAI,EAAE,MAAM,CAAC;QACb,IAAI,EAAE,UAAU,CAAC;QACjB,cAAc,EAAE,MAAM,CAAC;QACvB,sBAAsB,CAAC,EAAE,MAAM,CAAC;KACjC,CAAC,CAAC;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,MAAM,CAAC,EAAE,aAAa,EAAE,CAAC;CAC1B;AAED,MAAM,WAAW,aAAa;IAC5B,IAAI,CAAC,OAAO,CAAC,EAAE,aAAa,GAAG,UAAU,CAAC;IAC1C,OAAO,CAAC,CAAC,OAAO,CAAC,EAAE,aAAa,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAAC;IAC/D,KAAK,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACzB;AAED,MAAM,WAAW,aAAa;IAC5B,KAAK,CAAC,MAAM,EAAE,UAAU,EAAE,OAAO,CAAC,EAAE,aAAa,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAClE,KAAK,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACzB;AAED,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,YAAY,CACV,KAAK,EAAE,QAAQ,GAAG,MAAM,EACxB,OAAO,CAAC,EAAE,aAAa,GACtB,aAAa,CAAC;IACjB,YAAY,CACV,MAAM,EAAE,QAAQ,GAAG,MAAM,EACzB,OAAO,CAAC,EAAE,aAAa,GACtB,aAAa,CAAC;CAClB;AAED,MAAM,MAAM,iBAAiB,GAAG,CAAC,MAAM,EAAE,UAAU,KAAK,UAAU,CAAC;AAEnE,MAAM,WAAW,UAAU,CAAC,CAAC;IAC3B,GAAG,CAAC,GAAG,EAAE,GAAG,GAAG,IAAI,CAAC;IACpB,KAAK,CAAC,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC;IACnC,MAAM,IAAI,CAAC,CAAC;CACb;AAED,MAAM,WAAW,oBAAoB;IACnC,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,aAAa,EAAE,MAAM,CAAC;IACtB,kBAAkB,CAAC,EAAE,MAAM,CAAC;CAC7B;AAED,MAAM,MAAM,gBAAgB,GAAG,CAAC,IAAI,EAAE,oBAAoB,KAAK,IAAI,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/core/types.ts"],"names":[],"mappings":""}
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
export * from "./core/types.js";
|
|
2
|
+
export * from "./core/errors.js";
|
|
3
|
+
export * from "./core/batch.js";
|
|
4
|
+
export * from "./core/pipeline.js";
|
|
5
|
+
export { CSVReader, type CSVReaderOptions, JSONReader, type JSONReaderOptions, JSONLReader, type JSONLReaderOptions, XLSXReader, type XLSXReaderOptions, createReader, inferFormatFromPath as inferReaderFormatFromPath, } from "./readers/index.js";
|
|
6
|
+
export { CSVWriter, type CSVWriterOptions, JSONWriter, JSONLWriter, XLSXWriter, type XLSXWriterOptions, createWriter, inferFormatFromPath as inferWriterFormatFromPath, } from "./writers/index.js";
|
|
7
|
+
export * from "./transforms/select.js";
|
|
8
|
+
export * from "./transforms/rename.js";
|
|
9
|
+
export * from "./transforms/cast.js";
|
|
10
|
+
export * from "./transforms/sample.js";
|
|
11
|
+
export * from "./transforms/filter.js";
|
|
12
|
+
export * from "./transforms/expression.js";
|
|
13
|
+
export * from "./transforms/map.js";
|
|
14
|
+
export * from "./analytics/stats.js";
|
|
15
|
+
export * from "./analytics/reduce.js";
|
|
16
|
+
export * from "./analytics/schema-inference.js";
|
|
17
|
+
export * from "./analytics/semantic-types.js";
|
|
18
|
+
export * from "./analytics/validator.js";
|
|
19
|
+
export * from "./utils/compression.js";
|
|
20
|
+
export * from "./utils/formatting.js";
|
|
21
|
+
export * from "./utils/progress.js";
|
|
22
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,cAAc,iBAAiB,CAAC;AAChC,cAAc,kBAAkB,CAAC;AACjC,cAAc,iBAAiB,CAAC;AAChC,cAAc,oBAAoB,CAAC;AAGnC,OAAO,EACL,SAAS,EACT,KAAK,gBAAgB,EACrB,UAAU,EACV,KAAK,iBAAiB,EACtB,WAAW,EACX,KAAK,kBAAkB,EACvB,UAAU,EACV,KAAK,iBAAiB,EACtB,YAAY,EACZ,mBAAmB,IAAI,yBAAyB,GACjD,MAAM,oBAAoB,CAAC;AAG5B,OAAO,EACL,SAAS,EACT,KAAK,gBAAgB,EACrB,UAAU,EACV,WAAW,EACX,UAAU,EACV,KAAK,iBAAiB,EACtB,YAAY,EACZ,mBAAmB,IAAI,yBAAyB,GACjD,MAAM,oBAAoB,CAAC;AAG5B,cAAc,wBAAwB,CAAC;AACvC,cAAc,wBAAwB,CAAC;AACvC,cAAc,sBAAsB,CAAC;AACrC,cAAc,wBAAwB,CAAC;AACvC,cAAc,wBAAwB,CAAC;AACvC,cAAc,4BAA4B,CAAC;AAC3C,cAAc,qBAAqB,CAAC;AAGpC,cAAc,sBAAsB,CAAC;AACrC,cAAc,uBAAuB,CAAC;AACtC,cAAc,iCAAiC,CAAC;AAChD,cAAc,+BAA+B,CAAC;AAC9C,cAAc,0BAA0B,CAAC;AAGzC,cAAc,wBAAwB,CAAC;AACvC,cAAc,uBAAuB,CAAC;AACtC,cAAc,qBAAqB,CAAC"}
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
// Core
|
|
2
|
+
export * from "./core/types.js";
|
|
3
|
+
export * from "./core/errors.js";
|
|
4
|
+
export * from "./core/batch.js";
|
|
5
|
+
export * from "./core/pipeline.js";
|
|
6
|
+
// Readers
|
|
7
|
+
export { CSVReader, JSONReader, JSONLReader, XLSXReader, createReader, inferFormatFromPath as inferReaderFormatFromPath, } from "./readers/index.js";
|
|
8
|
+
// Writers
|
|
9
|
+
export { CSVWriter, JSONWriter, JSONLWriter, XLSXWriter, createWriter, inferFormatFromPath as inferWriterFormatFromPath, } from "./writers/index.js";
|
|
10
|
+
// Transforms
|
|
11
|
+
export * from "./transforms/select.js";
|
|
12
|
+
export * from "./transforms/rename.js";
|
|
13
|
+
export * from "./transforms/cast.js";
|
|
14
|
+
export * from "./transforms/sample.js";
|
|
15
|
+
export * from "./transforms/filter.js";
|
|
16
|
+
export * from "./transforms/expression.js";
|
|
17
|
+
export * from "./transforms/map.js";
|
|
18
|
+
// Analytics
|
|
19
|
+
export * from "./analytics/stats.js";
|
|
20
|
+
export * from "./analytics/reduce.js";
|
|
21
|
+
export * from "./analytics/schema-inference.js";
|
|
22
|
+
export * from "./analytics/semantic-types.js";
|
|
23
|
+
export * from "./analytics/validator.js";
|
|
24
|
+
// Utilities
|
|
25
|
+
export * from "./utils/compression.js";
|
|
26
|
+
export * from "./utils/formatting.js";
|
|
27
|
+
export * from "./utils/progress.js";
|
|
28
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO;AACP,cAAc,iBAAiB,CAAC;AAChC,cAAc,kBAAkB,CAAC;AACjC,cAAc,iBAAiB,CAAC;AAChC,cAAc,oBAAoB,CAAC;AAEnC,UAAU;AACV,OAAO,EACL,SAAS,EAET,UAAU,EAEV,WAAW,EAEX,UAAU,EAEV,YAAY,EACZ,mBAAmB,IAAI,yBAAyB,GACjD,MAAM,oBAAoB,CAAC;AAE5B,UAAU;AACV,OAAO,EACL,SAAS,EAET,UAAU,EACV,WAAW,EACX,UAAU,EAEV,YAAY,EACZ,mBAAmB,IAAI,yBAAyB,GACjD,MAAM,oBAAoB,CAAC;AAE5B,aAAa;AACb,cAAc,wBAAwB,CAAC;AACvC,cAAc,wBAAwB,CAAC;AACvC,cAAc,sBAAsB,CAAC;AACrC,cAAc,wBAAwB,CAAC;AACvC,cAAc,wBAAwB,CAAC;AACvC,cAAc,4BAA4B,CAAC;AAC3C,cAAc,qBAAqB,CAAC;AAEpC,YAAY;AACZ,cAAc,sBAAsB,CAAC;AACrC,cAAc,uBAAuB,CAAC;AACtC,cAAc,iCAAiC,CAAC;AAChD,cAAc,+BAA+B,CAAC;AAC9C,cAAc,0BAA0B,CAAC;AAEzC,YAAY;AACZ,cAAc,wBAAwB,CAAC;AACvC,cAAc,uBAAuB,CAAC;AACtC,cAAc,qBAAqB,CAAC"}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import type { Readable } from "node:stream";
|
|
2
|
+
import type { DataStream, InspectionMetadata, ReaderOptions, TabularReader } from "../core/types.js";
|
|
3
|
+
export interface CSVReaderOptions extends ReaderOptions {
|
|
4
|
+
delimiter?: string;
|
|
5
|
+
header?: boolean;
|
|
6
|
+
filePath?: string;
|
|
7
|
+
}
|
|
8
|
+
/**
|
|
9
|
+
* Streaming CSV Reader supporting RFC 4180 quotes, multiline cells,
|
|
10
|
+
* custom & auto-detected delimiters, BOM removal, and exact error reporting.
|
|
11
|
+
*/
|
|
12
|
+
export declare class CSVReader implements TabularReader {
|
|
13
|
+
private input;
|
|
14
|
+
private options;
|
|
15
|
+
constructor(input: Readable | string, options?: CSVReaderOptions);
|
|
16
|
+
private getInputStream;
|
|
17
|
+
/**
|
|
18
|
+
* Auto-detects delimiter by scanning candidate delimiters in a sample string.
|
|
19
|
+
*/
|
|
20
|
+
static detectDelimiter(sample: string): string;
|
|
21
|
+
read(options?: ReaderOptions): DataStream;
|
|
22
|
+
inspect(options?: ReaderOptions): Promise<InspectionMetadata>;
|
|
23
|
+
}
|
|
24
|
+
//# sourceMappingURL=csv.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"csv.d.ts","sourceRoot":"","sources":["../../src/readers/csv.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAE5C,OAAO,KAAK,EAEV,UAAU,EACV,kBAAkB,EAClB,aAAa,EAEb,aAAa,EACd,MAAM,kBAAkB,CAAC;AAE1B,MAAM,WAAW,gBAAiB,SAAQ,aAAa;IACrD,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED;;;GAGG;AACH,qBAAa,SAAU,YAAW,aAAa;IAC7C,OAAO,CAAC,KAAK,CAAoB;IACjC,OAAO,CAAC,OAAO,CAAmB;gBAEtB,KAAK,EAAE,QAAQ,GAAG,MAAM,EAAE,OAAO,GAAE,gBAAqB;IAQpE,OAAO,CAAC,cAAc;IAOtB;;OAEG;WACW,eAAe,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM;IA2B9C,IAAI,CAAC,OAAO,CAAC,EAAE,aAAa,GAAG,UAAU;IAgN1C,OAAO,CAAC,OAAO,CAAC,EAAE,aAAa,GAAG,OAAO,CAAC,kBAAkB,CAAC;CAsBpE"}
|