rowpipe 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +401 -0
- package/benchmarks/memory-bench.ts +109 -0
- package/dist/analytics/reduce.d.ts +33 -0
- package/dist/analytics/reduce.d.ts.map +1 -0
- package/dist/analytics/reduce.js +318 -0
- package/dist/analytics/reduce.js.map +1 -0
- package/dist/analytics/schema-inference.d.ts +31 -0
- package/dist/analytics/schema-inference.d.ts.map +1 -0
- package/dist/analytics/schema-inference.js +166 -0
- package/dist/analytics/schema-inference.js.map +1 -0
- package/dist/analytics/semantic-types.d.ts +6 -0
- package/dist/analytics/semantic-types.d.ts.map +1 -0
- package/dist/analytics/semantic-types.js +40 -0
- package/dist/analytics/semantic-types.js.map +1 -0
- package/dist/analytics/stats.d.ts +102 -0
- package/dist/analytics/stats.d.ts.map +1 -0
- package/dist/analytics/stats.js +307 -0
- package/dist/analytics/stats.js.map +1 -0
- package/dist/analytics/validator.d.ts +32 -0
- package/dist/analytics/validator.d.ts.map +1 -0
- package/dist/analytics/validator.js +96 -0
- package/dist/analytics/validator.js.map +1 -0
- package/dist/cli/commands/cast.d.ts +13 -0
- package/dist/cli/commands/cast.d.ts.map +1 -0
- package/dist/cli/commands/cast.js +39 -0
- package/dist/cli/commands/cast.js.map +1 -0
- package/dist/cli/commands/convert.d.ts +15 -0
- package/dist/cli/commands/convert.d.ts.map +1 -0
- package/dist/cli/commands/convert.js +89 -0
- package/dist/cli/commands/convert.js.map +1 -0
- package/dist/cli/commands/filter.d.ts +11 -0
- package/dist/cli/commands/filter.d.ts.map +1 -0
- package/dist/cli/commands/filter.js +38 -0
- package/dist/cli/commands/filter.js.map +1 -0
- package/dist/cli/commands/inspect.d.ts +10 -0
- package/dist/cli/commands/inspect.d.ts.map +1 -0
- package/dist/cli/commands/inspect.js +134 -0
- package/dist/cli/commands/inspect.js.map +1 -0
- package/dist/cli/commands/map.d.ts +11 -0
- package/dist/cli/commands/map.d.ts.map +1 -0
- package/dist/cli/commands/map.js +43 -0
- package/dist/cli/commands/map.js.map +1 -0
- package/dist/cli/commands/reduce.d.ts +12 -0
- package/dist/cli/commands/reduce.d.ts.map +1 -0
- package/dist/cli/commands/reduce.js +49 -0
- package/dist/cli/commands/reduce.js.map +1 -0
- package/dist/cli/commands/rename.d.ts +11 -0
- package/dist/cli/commands/rename.d.ts.map +1 -0
- package/dist/cli/commands/rename.js +39 -0
- package/dist/cli/commands/rename.js.map +1 -0
- package/dist/cli/commands/sample.d.ts +13 -0
- package/dist/cli/commands/sample.d.ts.map +1 -0
- package/dist/cli/commands/sample.js +40 -0
- package/dist/cli/commands/sample.js.map +1 -0
- package/dist/cli/commands/schema.d.ts +13 -0
- package/dist/cli/commands/schema.d.ts.map +1 -0
- package/dist/cli/commands/schema.js +57 -0
- package/dist/cli/commands/schema.js.map +1 -0
- package/dist/cli/commands/select.d.ts +11 -0
- package/dist/cli/commands/select.d.ts.map +1 -0
- package/dist/cli/commands/select.js +39 -0
- package/dist/cli/commands/select.js.map +1 -0
- package/dist/cli/commands/stats.d.ts +15 -0
- package/dist/cli/commands/stats.d.ts.map +1 -0
- package/dist/cli/commands/stats.js +75 -0
- package/dist/cli/commands/stats.js.map +1 -0
- package/dist/cli/commands/validate.d.ts +12 -0
- package/dist/cli/commands/validate.d.ts.map +1 -0
- package/dist/cli/commands/validate.js +86 -0
- package/dist/cli/commands/validate.js.map +1 -0
- package/dist/cli/index.d.ts +3 -0
- package/dist/cli/index.d.ts.map +1 -0
- package/dist/cli/index.js +262 -0
- package/dist/cli/index.js.map +1 -0
- package/dist/core/batch.d.ts +15 -0
- package/dist/core/batch.d.ts.map +1 -0
- package/dist/core/batch.js +67 -0
- package/dist/core/batch.js.map +1 -0
- package/dist/core/errors.d.ts +35 -0
- package/dist/core/errors.d.ts.map +1 -0
- package/dist/core/errors.js +55 -0
- package/dist/core/errors.js.map +1 -0
- package/dist/core/pipeline.d.ts +40 -0
- package/dist/core/pipeline.d.ts.map +1 -0
- package/dist/core/pipeline.js +120 -0
- package/dist/core/pipeline.js.map +1 -0
- package/dist/core/types.d.ts +73 -0
- package/dist/core/types.d.ts.map +1 -0
- package/dist/core/types.js +2 -0
- package/dist/core/types.js.map +1 -0
- package/dist/index.d.ts +22 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +28 -0
- package/dist/index.js.map +1 -0
- package/dist/readers/csv.d.ts +24 -0
- package/dist/readers/csv.d.ts.map +1 -0
- package/dist/readers/csv.js +251 -0
- package/dist/readers/csv.js.map +1 -0
- package/dist/readers/index.d.ts +17 -0
- package/dist/readers/index.d.ts.map +1 -0
- package/dist/readers/index.js +74 -0
- package/dist/readers/index.js.map +1 -0
- package/dist/readers/json.d.ts +20 -0
- package/dist/readers/json.d.ts.map +1 -0
- package/dist/readers/json.js +219 -0
- package/dist/readers/json.js.map +1 -0
- package/dist/readers/jsonl.d.ts +14 -0
- package/dist/readers/jsonl.d.ts.map +1 -0
- package/dist/readers/jsonl.js +95 -0
- package/dist/readers/jsonl.js.map +1 -0
- package/dist/readers/xlsx.d.ts +18 -0
- package/dist/readers/xlsx.d.ts.map +1 -0
- package/dist/readers/xlsx.js +138 -0
- package/dist/readers/xlsx.js.map +1 -0
- package/dist/transforms/cast.d.ts +21 -0
- package/dist/transforms/cast.d.ts.map +1 -0
- package/dist/transforms/cast.js +182 -0
- package/dist/transforms/cast.js.map +1 -0
- package/dist/transforms/expression.d.ts +67 -0
- package/dist/transforms/expression.d.ts.map +1 -0
- package/dist/transforms/expression.js +943 -0
- package/dist/transforms/expression.js.map +1 -0
- package/dist/transforms/filter.d.ts +6 -0
- package/dist/transforms/filter.d.ts.map +1 -0
- package/dist/transforms/filter.js +31 -0
- package/dist/transforms/filter.js.map +1 -0
- package/dist/transforms/map.d.ts +12 -0
- package/dist/transforms/map.d.ts.map +1 -0
- package/dist/transforms/map.js +76 -0
- package/dist/transforms/map.js.map +1 -0
- package/dist/transforms/rename.d.ts +10 -0
- package/dist/transforms/rename.d.ts.map +1 -0
- package/dist/transforms/rename.js +39 -0
- package/dist/transforms/rename.js.map +1 -0
- package/dist/transforms/sample.d.ts +10 -0
- package/dist/transforms/sample.d.ts.map +1 -0
- package/dist/transforms/sample.js +47 -0
- package/dist/transforms/sample.js.map +1 -0
- package/dist/transforms/select.d.ts +6 -0
- package/dist/transforms/select.d.ts.map +1 -0
- package/dist/transforms/select.js +63 -0
- package/dist/transforms/select.js.map +1 -0
- package/dist/utils/compression.d.ts +14 -0
- package/dist/utils/compression.d.ts.map +1 -0
- package/dist/utils/compression.js +38 -0
- package/dist/utils/compression.js.map +1 -0
- package/dist/utils/formatting.d.ts +6 -0
- package/dist/utils/formatting.d.ts.map +1 -0
- package/dist/utils/formatting.js +56 -0
- package/dist/utils/formatting.js.map +1 -0
- package/dist/utils/progress.d.ts +13 -0
- package/dist/utils/progress.d.ts.map +1 -0
- package/dist/utils/progress.js +27 -0
- package/dist/utils/progress.js.map +1 -0
- package/dist/writers/csv.d.ts +18 -0
- package/dist/writers/csv.d.ts.map +1 -0
- package/dist/writers/csv.js +84 -0
- package/dist/writers/csv.js.map +1 -0
- package/dist/writers/index.d.ts +17 -0
- package/dist/writers/index.d.ts.map +1 -0
- package/dist/writers/index.js +73 -0
- package/dist/writers/index.js.map +1 -0
- package/dist/writers/json.d.ts +12 -0
- package/dist/writers/json.d.ts.map +1 -0
- package/dist/writers/json.js +42 -0
- package/dist/writers/json.js.map +1 -0
- package/dist/writers/jsonl.d.ts +12 -0
- package/dist/writers/jsonl.d.ts.map +1 -0
- package/dist/writers/jsonl.js +41 -0
- package/dist/writers/jsonl.js.map +1 -0
- package/dist/writers/xlsx.d.ts +13 -0
- package/dist/writers/xlsx.d.ts.map +1 -0
- package/dist/writers/xlsx.js +65 -0
- package/dist/writers/xlsx.js.map +1 -0
- package/package.json +44 -0
- package/src/analytics/reduce.ts +388 -0
- package/src/analytics/schema-inference.ts +217 -0
- package/src/analytics/semantic-types.ts +36 -0
- package/src/analytics/stats.ts +383 -0
- package/src/analytics/validator.ts +134 -0
- package/src/cli/commands/cast.ts +60 -0
- package/src/cli/commands/convert.ts +128 -0
- package/src/cli/commands/filter.ts +58 -0
- package/src/cli/commands/inspect.ts +193 -0
- package/src/cli/commands/map.ts +66 -0
- package/src/cli/commands/reduce.ts +74 -0
- package/src/cli/commands/rename.ts +59 -0
- package/src/cli/commands/sample.ts +61 -0
- package/src/cli/commands/schema.ts +86 -0
- package/src/cli/commands/select.ts +59 -0
- package/src/cli/commands/stats.ts +100 -0
- package/src/cli/commands/validate.ts +123 -0
- package/src/cli/index.ts +285 -0
- package/src/core/batch.ts +81 -0
- package/src/core/errors.ts +81 -0
- package/src/core/pipeline.ts +150 -0
- package/src/core/types.ts +110 -0
- package/src/index.ts +52 -0
- package/src/readers/csv.ts +302 -0
- package/src/readers/index.ts +88 -0
- package/src/readers/json.ts +256 -0
- package/src/readers/jsonl.ts +126 -0
- package/src/readers/xlsx.ts +182 -0
- package/src/transforms/cast.ts +212 -0
- package/src/transforms/expression.ts +1114 -0
- package/src/transforms/filter.ts +38 -0
- package/src/transforms/map.ts +99 -0
- package/src/transforms/rename.ts +47 -0
- package/src/transforms/sample.ts +57 -0
- package/src/transforms/select.ts +64 -0
- package/src/utils/compression.ts +43 -0
- package/src/utils/formatting.ts +75 -0
- package/src/utils/progress.ts +37 -0
- package/src/writers/csv.ts +106 -0
- package/src/writers/index.ts +87 -0
- package/src/writers/json.ts +52 -0
- package/src/writers/jsonl.ts +49 -0
- package/src/writers/xlsx.ts +87 -0
- package/tests/analytics.test.ts +119 -0
- package/tests/cli-integration.test.ts +187 -0
- package/tests/map-reduce.test.ts +223 -0
- package/tests/readers-writers.test.ts +189 -0
- package/tests/transforms.test.ts +199 -0
- package/tsconfig.json +21 -0
- package/vitest.config.ts +9 -0
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
import { mkdir } from "node:fs/promises";
|
|
2
|
+
import { join } from "node:path";
|
|
3
|
+
import { createPipeline } from "../../core/pipeline.js";
|
|
4
|
+
import { createReader, inferFormatFromPath as inferReaderFormat } from "../../readers/index.js";
|
|
5
|
+
import { XLSXReader } from "../../readers/xlsx.js";
|
|
6
|
+
import { createWriter, inferFormatFromPath as inferWriterFormat } from "../../writers/index.js";
|
|
7
|
+
import { openReadableStream, openWritableStream } from "../../utils/compression.js";
|
|
8
|
+
import { formatNumber, logMemoryDebug } from "../../utils/formatting.js";
|
|
9
|
+
import { ProgressReporter } from "../../utils/progress.js";
|
|
10
|
+
export async function convertCommand(inputPath = "-", outputPath, options = {}) {
|
|
11
|
+
const effectiveBatchSize = Number(options.batchSize) || 1000;
|
|
12
|
+
// Determine input format
|
|
13
|
+
let fromFormat = options.from?.toLowerCase();
|
|
14
|
+
if (!fromFormat && inputPath !== "-") {
|
|
15
|
+
fromFormat = inferReaderFormat(inputPath) ?? undefined;
|
|
16
|
+
}
|
|
17
|
+
if (!fromFormat) {
|
|
18
|
+
fromFormat = "csv";
|
|
19
|
+
}
|
|
20
|
+
// Handle multi-sheet export if --all-sheets is specified
|
|
21
|
+
if (options.allSheets && fromFormat === "xlsx" && inputPath !== "-") {
|
|
22
|
+
const outDir = options.outDir || outputPath || "./";
|
|
23
|
+
const toFormat = (options.to?.toLowerCase() || "csv").replace(/^\./, "");
|
|
24
|
+
await mkdir(outDir, { recursive: true });
|
|
25
|
+
const xlsxReader = new XLSXReader(inputPath);
|
|
26
|
+
const sheets = await xlsxReader.getParsedSheets();
|
|
27
|
+
if (sheets.length === 0) {
|
|
28
|
+
process.stderr.write("No sheets found in workbook.\n");
|
|
29
|
+
return;
|
|
30
|
+
}
|
|
31
|
+
process.stderr.write(`Exporting ${sheets.length} sheets to "${outDir}"...\n`);
|
|
32
|
+
for (const sheet of sheets) {
|
|
33
|
+
const sheetName = sheet.sheet;
|
|
34
|
+
const targetFileName = `${sheetName}.${toFormat}`;
|
|
35
|
+
const targetFilePath = join(outDir, targetFileName);
|
|
36
|
+
const sheetReader = createReader(inputPath, {
|
|
37
|
+
format: "xlsx",
|
|
38
|
+
sheet: sheetName,
|
|
39
|
+
batchSize: effectiveBatchSize,
|
|
40
|
+
filePath: inputPath,
|
|
41
|
+
});
|
|
42
|
+
const sheetWriter = createWriter(targetFilePath, {
|
|
43
|
+
format: toFormat,
|
|
44
|
+
delimiter: options.delimiter,
|
|
45
|
+
header: options.header,
|
|
46
|
+
});
|
|
47
|
+
const pipeline = createPipeline(sheetReader, { batchSize: effectiveBatchSize });
|
|
48
|
+
await pipeline.to(sheetWriter);
|
|
49
|
+
const rowCount = Math.max(0, sheet.data.length - 1);
|
|
50
|
+
process.stderr.write(` ✓ Sheet "${sheetName}" -> ${targetFilePath} (${formatNumber(rowCount)} rows)\n`);
|
|
51
|
+
}
|
|
52
|
+
process.stderr.write("Done.\n");
|
|
53
|
+
logMemoryDebug();
|
|
54
|
+
return;
|
|
55
|
+
}
|
|
56
|
+
// Determine output format
|
|
57
|
+
let toFormat = options.to?.toLowerCase();
|
|
58
|
+
if (!toFormat && outputPath && outputPath !== "-") {
|
|
59
|
+
toFormat = inferWriterFormat(outputPath) ?? undefined;
|
|
60
|
+
}
|
|
61
|
+
if (!toFormat) {
|
|
62
|
+
toFormat = "csv";
|
|
63
|
+
}
|
|
64
|
+
const progress = new ProgressReporter(options);
|
|
65
|
+
const effectiveOutput = outputPath || "-";
|
|
66
|
+
const inputStream = openReadableStream(inputPath);
|
|
67
|
+
const outputStream = openWritableStream(effectiveOutput);
|
|
68
|
+
const reader = createReader(inputStream, {
|
|
69
|
+
format: fromFormat,
|
|
70
|
+
sheet: options.sheet,
|
|
71
|
+
path: options.path,
|
|
72
|
+
delimiter: options.delimiter,
|
|
73
|
+
header: options.header,
|
|
74
|
+
batchSize: effectiveBatchSize,
|
|
75
|
+
filePath: inputPath,
|
|
76
|
+
});
|
|
77
|
+
const writer = createWriter(outputStream, {
|
|
78
|
+
format: toFormat,
|
|
79
|
+
delimiter: options.delimiter,
|
|
80
|
+
header: options.header,
|
|
81
|
+
sheet: options.sheet,
|
|
82
|
+
});
|
|
83
|
+
const pipeline = createPipeline(reader, { batchSize: effectiveBatchSize });
|
|
84
|
+
pipeline.onProgress((info) => progress.update(info));
|
|
85
|
+
await pipeline.to(writer);
|
|
86
|
+
progress.done();
|
|
87
|
+
logMemoryDebug();
|
|
88
|
+
}
|
|
89
|
+
//# sourceMappingURL=convert.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"convert.js","sourceRoot":"","sources":["../../../src/cli/commands/convert.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AACzC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAC;AACxD,OAAO,EAAE,YAAY,EAAE,mBAAmB,IAAI,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAChG,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AACnD,OAAO,EAAE,YAAY,EAAE,mBAAmB,IAAI,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAChG,OAAO,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,MAAM,4BAA4B,CAAC;AACpF,OAAO,EAAE,YAAY,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AACzE,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAgB3D,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,SAAS,GAAG,GAAG,EACf,UAAmB,EACnB,UAAiC,EAAE;IAEnC,MAAM,kBAAkB,GAAG,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,IAAI,IAAI,CAAC;IAE7D,yBAAyB;IACzB,IAAI,UAAU,GAAG,OAAO,CAAC,IAAI,EAAE,WAAW,EAAE,CAAC;IAC7C,IAAI,CAAC,UAAU,IAAI,SAAS,KAAK,GAAG,EAAE,CAAC;QACrC,UAAU,GAAG,iBAAiB,CAAC,SAAS,CAAC,IAAI,SAAS,CAAC;IACzD,CAAC;IACD,IAAI,CAAC,UAAU,EAAE,CAAC;QAChB,UAAU,GAAG,KAAK,CAAC;IACrB,CAAC;IAED,yDAAyD;IACzD,IAAI,OAAO,CAAC,SAAS,IAAI,UAAU,KAAK,MAAM,IAAI,SAAS,KAAK,GAAG,EAAE,CAAC;QACpE,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,UAAU,IAAI,IAAI,CAAC;QACpD,MAAM,QAAQ,GAAG,CAAC,OAAO,CAAC,EAAE,EAAE,WAAW,EAAE,IAAI,KAAK,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QAEzE,MAAM,KAAK,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAEzC,MAAM,UAAU,GAAG,IAAI,UAAU,CAAC,SAAS,CAAC,CAAC;QAC7C,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,eAAe,EAAE,CAAC;QAElD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACxB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,gCAAgC,CAAC,CAAC;YACvD,OAAO;QACT,CAAC;QAED,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,aAAa,MAAM,CAAC,MAAM,eAAe,MAAM,QAAQ,CAAC,CAAC;QAE9E,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,MAAM,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC;YAC9B,MAAM,cAAc,GAAG,GAAG,SAAS,IAAI,QAAQ,EAAE,CAAC;YAClD,MAAM,cAAc,GAAG,IAAI,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC;YAEpD,MAAM,WAAW,GAAG,YAAY,CAAC,SAAS,EAAE;gBAC1C,MAAM,EAAE,MAAM;gBACd,KAAK,EAAE,SAAS;gBAChB,SAAS,EAAE,kBAAkB;gBAC7B,QAAQ,EAAE,SAAS;aACpB,CAAC,CAAC;YAEH,MAAM,WAAW,GAAG,YAAY,CAAC,cAAc,EAAE;gBAC/C,MAAM,EAAE,QAAQ;gBAChB,SAAS,EAAE,OAAO,CAAC,SAAS;gBAC5B,MAAM,EAAE,OAAO,CAAC,MAAM;aACvB,CAAC,CAAC;YAEH,MAAM,QAAQ,GAAG,cAAc,CAAC,WAAW,EAAE,EAAE,SAAS,EAAE,kBAAkB,EAAE,CAAC,CAAC;YAChF,MAAM,QAAQ,CAAC,EAAE,CAAC,WAAW,CAAC,CAAC;YAE/B,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;YACpD,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,cAAc,SAAS,QAAQ,cAAc,KAAK,YAAY,CAAC,QAAQ,CAAC,UAAU,CACnF,CAAC;QACJ,CAAC;QAED,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;QAChC,cAAc,EAAE,CAAC;QACjB,OAAO;IACT,CAAC;IAED,0BAA0B;IAC1B,IAAI,QAAQ,GAAG,OAAO,CAAC,EAAE,EAAE,WAAW,EAAE,CAAC;IACzC,IAAI,CAAC,QAAQ,IAAI,UAAU,IAAI,UAAU,KAAK,GAAG,EAAE,CAAC;QAClD,QAAQ,GAAG,iBAAiB,CAAC,UAAU,CAAC,IAAI,SAAS,CAAC;IACxD,CAAC;IACD,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,QAAQ,GAAG,KAAK,CAAC;IACnB,CAAC;IAED,MAAM,QAAQ,GAAG,IAAI,gBAAgB,CAAC,OAAO,CAAC,CAAC;IAC/C,MAAM,eAAe,GAAG,UAAU,IAAI,GAAG,CAAC;IAE1C,MAAM,WAAW,GAAG,kBAAkB,CAAC,SAAS,CAAC,CAAC;IAClD,MAAM,YAAY,GAAG,kBAAkB,CAAC,eAAe,CAAC,CAAC;IAEzD,MAAM,MAAM,GAAG,YAAY,CAAC,WAAW,EAAE;QACvC,MAAM,EAAE,UAAU;QAClB,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,IAAI,EAAE,OAAO,CAAC,IAAI;QAClB,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,SAAS,EAAE,kBAAkB;QAC7B,QAAQ,EAAE,SAAS;KACpB,CAAC,CAAC;IAEH,MAAM,MAAM,GAAG,YAAY,CAAC,YAAY,EAAE;QACxC,MAAM,EAAE,QAAQ;QAChB,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,KAAK,EAAE,OAAO,CAAC,KAAK;KACrB,CAAC,CAAC;IAEH,MAAM,QAAQ,GAAG,cAAc,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,kBAAkB,EAAE,CAAC,CAAC;IAC3E,QAAQ,CAAC,UAAU,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;IAErD,MAAM,QAAQ,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC;IAC1B,QAAQ,CAAC,IAAI,EAAE,CAAC;IAChB,cAAc,EAAE,CAAC;AACnB,CAAC"}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
export interface FilterCommandOptions {
|
|
2
|
+
from?: string;
|
|
3
|
+
to?: string;
|
|
4
|
+
sheet?: string;
|
|
5
|
+
delimiter?: string;
|
|
6
|
+
batchSize?: string | number;
|
|
7
|
+
quiet?: boolean;
|
|
8
|
+
noProgress?: boolean;
|
|
9
|
+
}
|
|
10
|
+
export declare function filterCommand(inputPath: string | undefined, expression: string, options?: FilterCommandOptions): Promise<void>;
|
|
11
|
+
//# sourceMappingURL=filter.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"filter.d.ts","sourceRoot":"","sources":["../../../src/cli/commands/filter.ts"],"names":[],"mappings":"AAQA,MAAM,WAAW,oBAAoB;IACnC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IAC5B,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,UAAU,CAAC,EAAE,OAAO,CAAC;CACtB;AAED,wBAAsB,aAAa,CACjC,SAAS,oBAAM,EACf,UAAU,EAAE,MAAM,EAClB,OAAO,GAAE,oBAAyB,GACjC,OAAO,CAAC,IAAI,CAAC,CAmCf"}
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
import { createPipeline } from "../../core/pipeline.js";
|
|
2
|
+
import { createReader, inferFormatFromPath as inferReaderFormat } from "../../readers/index.js";
|
|
3
|
+
import { createWriter } from "../../writers/index.js";
|
|
4
|
+
import { filterRows } from "../../transforms/filter.js";
|
|
5
|
+
import { openReadableStream } from "../../utils/compression.js";
|
|
6
|
+
import { logMemoryDebug } from "../../utils/formatting.js";
|
|
7
|
+
import { ProgressReporter } from "../../utils/progress.js";
|
|
8
|
+
export async function filterCommand(inputPath = "-", expression, options = {}) {
|
|
9
|
+
const effectiveBatchSize = Number(options.batchSize) || 1000;
|
|
10
|
+
const progress = new ProgressReporter(options);
|
|
11
|
+
let fromFormat = options.from?.toLowerCase();
|
|
12
|
+
if (!fromFormat && inputPath !== "-") {
|
|
13
|
+
fromFormat = inferReaderFormat(inputPath) ?? undefined;
|
|
14
|
+
}
|
|
15
|
+
if (!fromFormat) {
|
|
16
|
+
fromFormat = "csv";
|
|
17
|
+
}
|
|
18
|
+
const toFormat = options.to?.toLowerCase() || (fromFormat === "xlsx" ? "csv" : fromFormat);
|
|
19
|
+
const inputStream = openReadableStream(inputPath);
|
|
20
|
+
const reader = createReader(inputStream, {
|
|
21
|
+
format: fromFormat,
|
|
22
|
+
sheet: options.sheet,
|
|
23
|
+
delimiter: options.delimiter,
|
|
24
|
+
batchSize: effectiveBatchSize,
|
|
25
|
+
filePath: inputPath,
|
|
26
|
+
});
|
|
27
|
+
const writer = createWriter(process.stdout, {
|
|
28
|
+
format: toFormat,
|
|
29
|
+
delimiter: options.delimiter,
|
|
30
|
+
});
|
|
31
|
+
const pipeline = createPipeline(reader, { batchSize: effectiveBatchSize });
|
|
32
|
+
pipeline.pipe(filterRows(expression));
|
|
33
|
+
pipeline.onProgress((info) => progress.update(info));
|
|
34
|
+
await pipeline.to(writer);
|
|
35
|
+
progress.done();
|
|
36
|
+
logMemoryDebug();
|
|
37
|
+
}
|
|
38
|
+
//# sourceMappingURL=filter.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"filter.js","sourceRoot":"","sources":["../../../src/cli/commands/filter.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAC;AACxD,OAAO,EAAE,YAAY,EAAE,mBAAmB,IAAI,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAChG,OAAO,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AACtD,OAAO,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAC;AACxD,OAAO,EAAE,kBAAkB,EAAE,MAAM,4BAA4B,CAAC;AAChE,OAAO,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAC3D,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAY3D,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,SAAS,GAAG,GAAG,EACf,UAAkB,EAClB,UAAgC,EAAE;IAElC,MAAM,kBAAkB,GAAG,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,IAAI,IAAI,CAAC;IAC7D,MAAM,QAAQ,GAAG,IAAI,gBAAgB,CAAC,OAAO,CAAC,CAAC;IAE/C,IAAI,UAAU,GAAG,OAAO,CAAC,IAAI,EAAE,WAAW,EAAE,CAAC;IAC7C,IAAI,CAAC,UAAU,IAAI,SAAS,KAAK,GAAG,EAAE,CAAC;QACrC,UAAU,GAAG,iBAAiB,CAAC,SAAS,CAAC,IAAI,SAAS,CAAC;IACzD,CAAC;IACD,IAAI,CAAC,UAAU,EAAE,CAAC;QAChB,UAAU,GAAG,KAAK,CAAC;IACrB,CAAC;IAED,MAAM,QAAQ,GAAG,OAAO,CAAC,EAAE,EAAE,WAAW,EAAE,IAAI,CAAC,UAAU,KAAK,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC;IAE3F,MAAM,WAAW,GAAG,kBAAkB,CAAC,SAAS,CAAC,CAAC;IAClD,MAAM,MAAM,GAAG,YAAY,CAAC,WAAW,EAAE;QACvC,MAAM,EAAE,UAAU;QAClB,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,SAAS,EAAE,kBAAkB;QAC7B,QAAQ,EAAE,SAAS;KACpB,CAAC,CAAC;IAEH,MAAM,MAAM,GAAG,YAAY,CAAC,OAAO,CAAC,MAAM,EAAE;QAC1C,MAAM,EAAE,QAAQ;QAChB,SAAS,EAAE,OAAO,CAAC,SAAS;KAC7B,CAAC,CAAC;IAEH,MAAM,QAAQ,GAAG,cAAc,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,kBAAkB,EAAE,CAAC,CAAC;IAC3E,QAAQ,CAAC,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC,CAAC;IACtC,QAAQ,CAAC,UAAU,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;IAErD,MAAM,QAAQ,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC;IAC1B,QAAQ,CAAC,IAAI,EAAE,CAAC;IAChB,cAAc,EAAE,CAAC;AACnB,CAAC"}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
export interface InspectCommandOptions {
|
|
2
|
+
sheet?: string;
|
|
3
|
+
path?: string;
|
|
4
|
+
delimiter?: string;
|
|
5
|
+
json?: boolean;
|
|
6
|
+
quiet?: boolean;
|
|
7
|
+
noProgress?: boolean;
|
|
8
|
+
}
|
|
9
|
+
export declare function inspectCommand(inputPath?: string, options?: InspectCommandOptions): Promise<void>;
|
|
10
|
+
//# sourceMappingURL=inspect.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"inspect.d.ts","sourceRoot":"","sources":["../../../src/cli/commands/inspect.ts"],"names":[],"mappings":"AAgBA,MAAM,WAAW,qBAAqB;IACpC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,UAAU,CAAC,EAAE,OAAO,CAAC;CACtB;AAED,wBAAsB,cAAc,CAClC,SAAS,SAAM,EACf,OAAO,GAAE,qBAA0B,GAClC,OAAO,CAAC,IAAI,CAAC,CAoKf"}
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
import { stat } from "node:fs/promises";
|
|
2
|
+
import { DatasetStatsAggregator } from "../../analytics/stats.js";
|
|
3
|
+
import { SchemaInferenceAggregator } from "../../analytics/schema-inference.js";
|
|
4
|
+
import { createPipeline } from "../../core/pipeline.js";
|
|
5
|
+
import { createReader, inferFormatFromPath } from "../../readers/index.js";
|
|
6
|
+
import { openReadableStream } from "../../utils/compression.js";
|
|
7
|
+
import { formatBytes, formatDecimal, formatNumber, formatTable, logMemoryDebug, } from "../../utils/formatting.js";
|
|
8
|
+
import { ProgressReporter } from "../../utils/progress.js";
|
|
9
|
+
export async function inspectCommand(inputPath = "-", options = {}) {
|
|
10
|
+
const progress = new ProgressReporter(options);
|
|
11
|
+
let fileSize;
|
|
12
|
+
if (inputPath !== "-") {
|
|
13
|
+
try {
|
|
14
|
+
const fileStat = await stat(inputPath);
|
|
15
|
+
fileSize = fileStat.size;
|
|
16
|
+
}
|
|
17
|
+
catch {
|
|
18
|
+
// Ignore if cannot stat
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
const format = inputPath !== "-" ? inferFormatFromPath(inputPath) || "CSV" : "CSV";
|
|
22
|
+
const stream = openReadableStream(inputPath);
|
|
23
|
+
const reader = createReader(stream, {
|
|
24
|
+
format,
|
|
25
|
+
sheet: options.sheet,
|
|
26
|
+
path: options.path,
|
|
27
|
+
delimiter: options.delimiter,
|
|
28
|
+
filePath: inputPath,
|
|
29
|
+
});
|
|
30
|
+
// If XLSX without a specific --sheet requested, show rich workbook & sheet breakdown
|
|
31
|
+
if (format === "xlsx" && !options.sheet && reader.inspect) {
|
|
32
|
+
const meta = await reader.inspect();
|
|
33
|
+
if (options.json) {
|
|
34
|
+
process.stdout.write(JSON.stringify({
|
|
35
|
+
file: inputPath === "-" ? "stdin" : inputPath,
|
|
36
|
+
format: "XLSX",
|
|
37
|
+
sizeBytes: fileSize,
|
|
38
|
+
sizeFormatted: fileSize !== undefined ? formatBytes(fileSize) : undefined,
|
|
39
|
+
sheetsCount: meta.sheets?.length ?? 0,
|
|
40
|
+
sheets: meta.sheets,
|
|
41
|
+
}, null, 2) + "\n");
|
|
42
|
+
logMemoryDebug();
|
|
43
|
+
return;
|
|
44
|
+
}
|
|
45
|
+
process.stdout.write(`File: ${inputPath === "-" ? "stdin" : inputPath}\n`);
|
|
46
|
+
process.stdout.write(`Format: XLSX\n`);
|
|
47
|
+
if (fileSize !== undefined) {
|
|
48
|
+
process.stdout.write(`Size: ${formatBytes(fileSize)}\n`);
|
|
49
|
+
}
|
|
50
|
+
if (meta.sheets && meta.sheets.length > 0) {
|
|
51
|
+
process.stdout.write(`Sheets: ${meta.sheets.length}\n\n`);
|
|
52
|
+
const sheetRows = meta.sheets.map((s) => {
|
|
53
|
+
const previewCols = (s.columns || []).slice(0, 5).join(", ") + (s.columns && s.columns.length > 5 ? ", ..." : "");
|
|
54
|
+
return [
|
|
55
|
+
s.name,
|
|
56
|
+
formatNumber(s.rowCount),
|
|
57
|
+
s.columnCount !== undefined ? String(s.columnCount) : "-",
|
|
58
|
+
previewCols,
|
|
59
|
+
];
|
|
60
|
+
});
|
|
61
|
+
process.stdout.write(formatTable(["SHEET", "ROWS", "COLUMNS", "PREVIEW COLUMNS"], sheetRows, ["left", "right", "right", "left"]) + "\n\nTip: Run with --sheet <name> to inspect a specific sheet in full detail.\n");
|
|
62
|
+
}
|
|
63
|
+
logMemoryDebug();
|
|
64
|
+
return;
|
|
65
|
+
}
|
|
66
|
+
// General inspection across stream
|
|
67
|
+
const schemaAgg = new SchemaInferenceAggregator();
|
|
68
|
+
const statsAgg = new DatasetStatsAggregator();
|
|
69
|
+
const pipeline = createPipeline(reader);
|
|
70
|
+
pipeline.onProgress((info) => progress.update(info));
|
|
71
|
+
for await (const row of pipeline.rows()) {
|
|
72
|
+
schemaAgg.add(row);
|
|
73
|
+
statsAgg.add(row);
|
|
74
|
+
}
|
|
75
|
+
progress.done();
|
|
76
|
+
const schemaRes = schemaAgg.result();
|
|
77
|
+
const statsRes = statsAgg.result();
|
|
78
|
+
const totalRows = schemaRes.totalRowsScanned;
|
|
79
|
+
const columnCount = schemaRes.columns.length;
|
|
80
|
+
const columnDetails = schemaRes.columns.map((col) => {
|
|
81
|
+
const nullPct = col.sampleCount > 0 ? (col.nullCount / col.sampleCount) * 100 : 0;
|
|
82
|
+
const colStats = statsRes.columns[col.name];
|
|
83
|
+
const distinctCount = colStats?.numeric?.approxDistinct ?? colStats?.string?.approxDistinct ?? 0;
|
|
84
|
+
const uniquePct = totalRows > 0 ? Math.min(100, (distinctCount / totalRows) * 100) : 0;
|
|
85
|
+
return {
|
|
86
|
+
name: col.name,
|
|
87
|
+
type: col.type,
|
|
88
|
+
nullPct,
|
|
89
|
+
uniquePct,
|
|
90
|
+
semantic: col.semantic,
|
|
91
|
+
};
|
|
92
|
+
});
|
|
93
|
+
if (options.json) {
|
|
94
|
+
const resultJson = {
|
|
95
|
+
file: inputPath === "-" ? "stdin" : inputPath,
|
|
96
|
+
format: format.toUpperCase(),
|
|
97
|
+
sheet: options.sheet,
|
|
98
|
+
sizeBytes: fileSize,
|
|
99
|
+
sizeFormatted: fileSize !== undefined ? formatBytes(fileSize) : undefined,
|
|
100
|
+
rows: totalRows,
|
|
101
|
+
columnsCount: columnCount,
|
|
102
|
+
columns: columnDetails.map((c) => ({
|
|
103
|
+
name: c.name,
|
|
104
|
+
type: c.type,
|
|
105
|
+
nullPercentage: Math.round(c.nullPct * 10) / 10,
|
|
106
|
+
approxUniquePercentage: Math.round(c.uniquePct * 10) / 10,
|
|
107
|
+
semantic: c.semantic,
|
|
108
|
+
})),
|
|
109
|
+
};
|
|
110
|
+
process.stdout.write(JSON.stringify(resultJson, null, 2) + "\n");
|
|
111
|
+
}
|
|
112
|
+
else {
|
|
113
|
+
process.stdout.write(`File: ${inputPath === "-" ? "stdin" : inputPath}\n`);
|
|
114
|
+
process.stdout.write(`Format: ${format.toUpperCase()}`);
|
|
115
|
+
if (options.sheet) {
|
|
116
|
+
process.stdout.write(` (Sheet: ${options.sheet})`);
|
|
117
|
+
}
|
|
118
|
+
process.stdout.write("\n");
|
|
119
|
+
if (fileSize !== undefined) {
|
|
120
|
+
process.stdout.write(`Size: ${formatBytes(fileSize)}\n`);
|
|
121
|
+
}
|
|
122
|
+
process.stdout.write(`Rows: ${formatNumber(totalRows)}\n`);
|
|
123
|
+
process.stdout.write(`Columns: ${columnCount}\n\n`);
|
|
124
|
+
const tableRows = columnDetails.map((c) => [
|
|
125
|
+
c.name,
|
|
126
|
+
c.type,
|
|
127
|
+
`${formatDecimal(c.nullPct, 1)}%`,
|
|
128
|
+
`${formatDecimal(c.uniquePct, 1)}%`,
|
|
129
|
+
]);
|
|
130
|
+
process.stdout.write(formatTable(["COLUMN", "TYPE", "NULL", "UNIQUE*"], tableRows, ["left", "left", "right", "right"]) + "\n\n* approximate\n");
|
|
131
|
+
}
|
|
132
|
+
logMemoryDebug();
|
|
133
|
+
}
|
|
134
|
+
//# sourceMappingURL=inspect.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"inspect.js","sourceRoot":"","sources":["../../../src/cli/commands/inspect.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AACxC,OAAO,EAAE,sBAAsB,EAAE,MAAM,0BAA0B,CAAC;AAClE,OAAO,EAAE,yBAAyB,EAAE,MAAM,qCAAqC,CAAC;AAEhF,OAAO,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAC;AACxD,OAAO,EAAE,YAAY,EAAE,mBAAmB,EAAE,MAAM,wBAAwB,CAAC;AAC3E,OAAO,EAAE,kBAAkB,EAAE,MAAM,4BAA4B,CAAC;AAChE,OAAO,EACL,WAAW,EACX,aAAa,EACb,YAAY,EACZ,WAAW,EACX,cAAc,GACf,MAAM,2BAA2B,CAAC;AACnC,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAW3D,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,SAAS,GAAG,GAAG,EACf,UAAiC,EAAE;IAEnC,MAAM,QAAQ,GAAG,IAAI,gBAAgB,CAAC,OAAO,CAAC,CAAC;IAE/C,IAAI,QAA4B,CAAC;IACjC,IAAI,SAAS,KAAK,GAAG,EAAE,CAAC;QACtB,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,CAAC;YACvC,QAAQ,GAAG,QAAQ,CAAC,IAAI,CAAC;QAC3B,CAAC;QAAC,MAAM,CAAC;YACP,wBAAwB;QAC1B,CAAC;IACH,CAAC;IAED,MAAM,MAAM,GACV,SAAS,KAAK,GAAG,CAAC,CAAC,CAAC,mBAAmB,CAAC,SAAS,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC;IAEtE,MAAM,MAAM,GAAG,kBAAkB,CAAC,SAAS,CAAC,CAAC;IAC7C,MAAM,MAAM,GAAG,YAAY,CAAC,MAAM,EAAE;QAClC,MAAM;QACN,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,IAAI,EAAE,OAAO,CAAC,IAAI;QAClB,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,QAAQ,EAAE,SAAS;KACpB,CAAC,CAAC;IAEH,qFAAqF;IACrF,IAAI,MAAM,KAAK,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;QAC1D,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,OAAO,EAAE,CAAC;QACpC,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;YACjB,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,IAAI,CAAC,SAAS,CACZ;gBACE,IAAI,EAAE,SAAS,KAAK,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS;gBAC7C,MAAM,EAAE,MAAM;gBACd,SAAS,EAAE,QAAQ;gBACnB,aAAa,EAAE,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,SAAS;gBACzE,WAAW,EAAE,IAAI,CAAC,MAAM,EAAE,MAAM,IAAI,CAAC;gBACrC,MAAM,EAAE,IAAI,CAAC,MAAM;aACpB,EACD,IAAI,EACJ,CAAC,CACF,GAAG,IAAI,CACT,CAAC;YACF,cAAc,EAAE,CAAC;YACjB,OAAO;QACT,CAAC;QAED,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,SAAS,SAAS,KAAK,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,IAAI,CAAC,CAAC;QAC3E,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC;QACvC,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC3B,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,SAAS,WAAW,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QAC3D,CAAC;QAED,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC1C,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,WAAW,IAAI,CAAC,MAAM,CAAC,MAAM,MAAM,CAAC,CAAC;YAC1D,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;gBACtC,MAAM,WAAW,GAAG,CAAC,CAAC,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;gBAClH,OAAO;oBACL,CAAC,CAAC,IAAI;oBACN,YAAY,CAAC,CAAC,CAAC,QAAQ,CAAC;oBACxB,CAAC,CAAC,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,GAAG;oBACzD,WAAW;iBACZ,CAAC;YACJ,CAAC,CAAC,CAAC;YAEH,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,WAAW,CACT,CAAC,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,iBAAiB,CAAC,EAC/C,SAAS,EACT,CAAC,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,CAAC,CACnC,GAAG,gFAAgF,CACrF,CAAC;QACJ,CAAC;QACD,cAAc,EAAE,CAAC;QACjB,OAAO;IACT,CAAC;IAED,mCAAmC;IACnC,MAAM,SAAS,GAAG,IAAI,yBAAyB,EAAE,CAAC;IAClD,MAAM,QAAQ,GAAG,IAAI,sBAAsB,EAAE,CAAC;IAE9C,MAAM,QAAQ,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC;IACxC,QAAQ,CAAC,UAAU,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;IAErD,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC;QACxC,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACnB,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IACpB,CAAC;IAED,QAAQ,CAAC,IAAI,EAAE,CAAC;IAEhB,MAAM,SAAS,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC;IACrC,MAAM,QAAQ,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC;IAEnC,MAAM,SAAS,GAAG,SAAS,CAAC,gBAAgB,CAAC;IAC7C,MAAM,WAAW,GAAG,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC;IAE7C,MAAM,aAAa,GAAG,SAAS,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;QAClD,MAAM,OAAO,GACX,GAAG,CAAC,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,SAAS,GAAG,GAAG,CAAC,WAAW,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACpE,MAAM,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAC5C,MAAM,aAAa,GACjB,QAAQ,EAAE,OAAO,EAAE,cAAc,IAAI,QAAQ,EAAE,MAAM,EAAE,cAAc,IAAI,CAAC,CAAC;QAC7E,MAAM,SAAS,GACb,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,aAAa,GAAG,SAAS,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAEvE,OAAO;YACL,IAAI,EAAE,GAAG,CAAC,IAAI;YACd,IAAI,EAAE,GAAG,CAAC,IAAI;YACd,OAAO;YACP,SAAS;YACT,QAAQ,EAAE,GAAG,CAAC,QAAQ;SACvB,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;QACjB,MAAM,UAAU,GAAG;YACjB,IAAI,EAAE,SAAS,KAAK,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS;YAC7C,MAAM,EAAE,MAAM,CAAC,WAAW,EAAE;YAC5B,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,SAAS,EAAE,QAAQ;YACnB,aAAa,EAAE,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,SAAS;YACzE,IAAI,EAAE,SAAS;YACf,YAAY,EAAE,WAAW;YACzB,OAAO,EAAE,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBACjC,IAAI,EAAE,CAAC,CAAC,IAAI;gBACZ,IAAI,EAAE,CAAC,CAAC,IAAI;gBACZ,cAAc,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,GAAG,EAAE,CAAC,GAAG,EAAE;gBAC/C,sBAAsB,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,GAAG,EAAE,CAAC,GAAG,EAAE;gBACzD,QAAQ,EAAE,CAAC,CAAC,QAAQ;aACrB,CAAC,CAAC;SACJ,CAAC;QACF,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC;IACnE,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,SAAS,SAAS,KAAK,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,IAAI,CAAC,CAAC;QAC3E,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,WAAW,MAAM,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC;QACxD,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;YAClB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,YAAY,OAAO,CAAC,KAAK,GAAG,CAAC,CAAC;QACrD,CAAC;QACD,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAE3B,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC3B,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,SAAS,WAAW,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QAC3D,CAAC;QACD,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,SAAS,YAAY,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;QAC3D,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,YAAY,WAAW,MAAM,CAAC,CAAC;QAEpD,MAAM,SAAS,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;YACzC,CAAC,CAAC,IAAI;YACN,CAAC,CAAC,IAAI;YACN,GAAG,aAAa,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,GAAG;YACjC,GAAG,aAAa,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,GAAG;SACpC,CAAC,CAAC;QAEH,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,WAAW,CACT,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,CAAC,EACrC,SAAS,EACT,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,CAAC,CACnC,GAAG,qBAAqB,CAC1B,CAAC;IACJ,CAAC;IAED,cAAc,EAAE,CAAC;AACnB,CAAC"}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
export interface MapCommandOptions {
|
|
2
|
+
from?: string;
|
|
3
|
+
to?: string;
|
|
4
|
+
sheet?: string;
|
|
5
|
+
delimiter?: string;
|
|
6
|
+
batchSize?: string | number;
|
|
7
|
+
quiet?: boolean;
|
|
8
|
+
noProgress?: boolean;
|
|
9
|
+
}
|
|
10
|
+
export declare function mapCommand(inputPath: string | undefined, specs: string[], options?: MapCommandOptions): Promise<void>;
|
|
11
|
+
//# sourceMappingURL=map.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"map.d.ts","sourceRoot":"","sources":["../../../src/cli/commands/map.ts"],"names":[],"mappings":"AASA,MAAM,WAAW,iBAAiB;IAChC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IAC5B,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,UAAU,CAAC,EAAE,OAAO,CAAC;CACtB;AAED,wBAAsB,UAAU,CAC9B,SAAS,oBAAM,EACf,KAAK,EAAE,MAAM,EAAE,EACf,OAAO,GAAE,iBAAsB,GAC9B,OAAO,CAAC,IAAI,CAAC,CA0Cf"}
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import { InvalidArgumentError } from "../../core/errors.js";
|
|
2
|
+
import { createPipeline } from "../../core/pipeline.js";
|
|
3
|
+
import { createReader, inferFormatFromPath as inferReaderFormat } from "../../readers/index.js";
|
|
4
|
+
import { createWriter } from "../../writers/index.js";
|
|
5
|
+
import { mapRows, parseMapSpecs } from "../../transforms/map.js";
|
|
6
|
+
import { openReadableStream } from "../../utils/compression.js";
|
|
7
|
+
import { logMemoryDebug } from "../../utils/formatting.js";
|
|
8
|
+
import { ProgressReporter } from "../../utils/progress.js";
|
|
9
|
+
export async function mapCommand(inputPath = "-", specs, options = {}) {
|
|
10
|
+
if (!specs || specs.length === 0) {
|
|
11
|
+
throw new InvalidArgumentError("At least one map specification is required (e.g. rowpipe map users.csv 'full_name=concat(first, \" \", last)')");
|
|
12
|
+
}
|
|
13
|
+
const effectiveBatchSize = Number(options.batchSize) || 1000;
|
|
14
|
+
const progress = new ProgressReporter(options);
|
|
15
|
+
const parsedSpecs = parseMapSpecs(specs);
|
|
16
|
+
let fromFormat = options.from?.toLowerCase();
|
|
17
|
+
if (!fromFormat && inputPath !== "-") {
|
|
18
|
+
fromFormat = inferReaderFormat(inputPath) ?? undefined;
|
|
19
|
+
}
|
|
20
|
+
if (!fromFormat) {
|
|
21
|
+
fromFormat = "csv";
|
|
22
|
+
}
|
|
23
|
+
const toFormat = options.to?.toLowerCase() || (fromFormat === "xlsx" ? "csv" : fromFormat);
|
|
24
|
+
const inputStream = openReadableStream(inputPath);
|
|
25
|
+
const reader = createReader(inputStream, {
|
|
26
|
+
format: fromFormat,
|
|
27
|
+
sheet: options.sheet,
|
|
28
|
+
delimiter: options.delimiter,
|
|
29
|
+
batchSize: effectiveBatchSize,
|
|
30
|
+
filePath: inputPath,
|
|
31
|
+
});
|
|
32
|
+
const writer = createWriter(process.stdout, {
|
|
33
|
+
format: toFormat,
|
|
34
|
+
delimiter: options.delimiter,
|
|
35
|
+
});
|
|
36
|
+
const pipeline = createPipeline(reader, { batchSize: effectiveBatchSize });
|
|
37
|
+
pipeline.pipe(mapRows(parsedSpecs));
|
|
38
|
+
pipeline.onProgress((info) => progress.update(info));
|
|
39
|
+
await pipeline.to(writer);
|
|
40
|
+
progress.done();
|
|
41
|
+
logMemoryDebug();
|
|
42
|
+
}
|
|
43
|
+
//# sourceMappingURL=map.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"map.js","sourceRoot":"","sources":["../../../src/cli/commands/map.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,oBAAoB,EAAE,MAAM,sBAAsB,CAAC;AAC5D,OAAO,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAC;AACxD,OAAO,EAAE,YAAY,EAAE,mBAAmB,IAAI,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAChG,OAAO,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AACtD,OAAO,EAAE,OAAO,EAAE,aAAa,EAAE,MAAM,yBAAyB,CAAC;AACjE,OAAO,EAAE,kBAAkB,EAAE,MAAM,4BAA4B,CAAC;AAChE,OAAO,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAC3D,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAY3D,MAAM,CAAC,KAAK,UAAU,UAAU,CAC9B,SAAS,GAAG,GAAG,EACf,KAAe,EACf,UAA6B,EAAE;IAE/B,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACjC,MAAM,IAAI,oBAAoB,CAC5B,gHAAgH,CACjH,CAAC;IACJ,CAAC;IAED,MAAM,kBAAkB,GAAG,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,IAAI,IAAI,CAAC;IAC7D,MAAM,QAAQ,GAAG,IAAI,gBAAgB,CAAC,OAAO,CAAC,CAAC;IAC/C,MAAM,WAAW,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;IAEzC,IAAI,UAAU,GAAG,OAAO,CAAC,IAAI,EAAE,WAAW,EAAE,CAAC;IAC7C,IAAI,CAAC,UAAU,IAAI,SAAS,KAAK,GAAG,EAAE,CAAC;QACrC,UAAU,GAAG,iBAAiB,CAAC,SAAS,CAAC,IAAI,SAAS,CAAC;IACzD,CAAC;IACD,IAAI,CAAC,UAAU,EAAE,CAAC;QAChB,UAAU,GAAG,KAAK,CAAC;IACrB,CAAC;IAED,MAAM,QAAQ,GAAG,OAAO,CAAC,EAAE,EAAE,WAAW,EAAE,IAAI,CAAC,UAAU,KAAK,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC;IAE3F,MAAM,WAAW,GAAG,kBAAkB,CAAC,SAAS,CAAC,CAAC;IAClD,MAAM,MAAM,GAAG,YAAY,CAAC,WAAW,EAAE;QACvC,MAAM,EAAE,UAAU;QAClB,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,SAAS,EAAE,kBAAkB;QAC7B,QAAQ,EAAE,SAAS;KACpB,CAAC,CAAC;IAEH,MAAM,MAAM,GAAG,YAAY,CAAC,OAAO,CAAC,MAAM,EAAE;QAC1C,MAAM,EAAE,QAAQ;QAChB,SAAS,EAAE,OAAO,CAAC,SAAS;KAC7B,CAAC,CAAC;IAEH,MAAM,QAAQ,GAAG,cAAc,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,kBAAkB,EAAE,CAAC,CAAC;IAC3E,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC;IACpC,QAAQ,CAAC,UAAU,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;IAErD,MAAM,QAAQ,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC;IAC1B,QAAQ,CAAC,IAAI,EAAE,CAAC;IAChB,cAAc,EAAE,CAAC;AACnB,CAAC"}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
export interface ReduceCommandOptions {
|
|
2
|
+
by?: string;
|
|
3
|
+
from?: string;
|
|
4
|
+
to?: string;
|
|
5
|
+
sheet?: string;
|
|
6
|
+
delimiter?: string;
|
|
7
|
+
batchSize?: string | number;
|
|
8
|
+
quiet?: boolean;
|
|
9
|
+
noProgress?: boolean;
|
|
10
|
+
}
|
|
11
|
+
export declare function reduceCommand(inputPath: string | undefined, specs: string[], options?: ReduceCommandOptions): Promise<void>;
|
|
12
|
+
//# sourceMappingURL=reduce.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"reduce.d.ts","sourceRoot":"","sources":["../../../src/cli/commands/reduce.ts"],"names":[],"mappings":"AASA,MAAM,WAAW,oBAAoB;IACnC,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IAC5B,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,UAAU,CAAC,EAAE,OAAO,CAAC;CACtB;AAED,wBAAsB,aAAa,CACjC,SAAS,oBAAM,EACf,KAAK,EAAE,MAAM,EAAE,EACf,OAAO,GAAE,oBAAyB,GACjC,OAAO,CAAC,IAAI,CAAC,CAiDf"}
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import { InvalidArgumentError } from "../../core/errors.js";
|
|
2
|
+
import { createPipeline } from "../../core/pipeline.js";
|
|
3
|
+
import { createReader, inferFormatFromPath as inferReaderFormat } from "../../readers/index.js";
|
|
4
|
+
import { createWriter } from "../../writers/index.js";
|
|
5
|
+
import { parseReduceSpecs, reduceRows } from "../../analytics/reduce.js";
|
|
6
|
+
import { openReadableStream } from "../../utils/compression.js";
|
|
7
|
+
import { logMemoryDebug } from "../../utils/formatting.js";
|
|
8
|
+
import { ProgressReporter } from "../../utils/progress.js";
|
|
9
|
+
export async function reduceCommand(inputPath = "-", specs, options = {}) {
|
|
10
|
+
if (!specs || specs.length === 0) {
|
|
11
|
+
throw new InvalidArgumentError("At least one aggregation specification is required (e.g. rowpipe reduce sales.csv 'total=sum(revenue)' --by country)");
|
|
12
|
+
}
|
|
13
|
+
const effectiveBatchSize = Number(options.batchSize) || 1000;
|
|
14
|
+
const progress = new ProgressReporter(options);
|
|
15
|
+
const parsedSpecs = parseReduceSpecs(specs);
|
|
16
|
+
const byCols = options.by
|
|
17
|
+
? options.by
|
|
18
|
+
.split(",")
|
|
19
|
+
.map((c) => c.trim())
|
|
20
|
+
.filter(Boolean)
|
|
21
|
+
: undefined;
|
|
22
|
+
let fromFormat = options.from?.toLowerCase();
|
|
23
|
+
if (!fromFormat && inputPath !== "-") {
|
|
24
|
+
fromFormat = inferReaderFormat(inputPath) ?? undefined;
|
|
25
|
+
}
|
|
26
|
+
if (!fromFormat) {
|
|
27
|
+
fromFormat = "csv";
|
|
28
|
+
}
|
|
29
|
+
const toFormat = options.to?.toLowerCase() || (fromFormat === "xlsx" ? "csv" : fromFormat);
|
|
30
|
+
const inputStream = openReadableStream(inputPath);
|
|
31
|
+
const reader = createReader(inputStream, {
|
|
32
|
+
format: fromFormat,
|
|
33
|
+
sheet: options.sheet,
|
|
34
|
+
delimiter: options.delimiter,
|
|
35
|
+
batchSize: effectiveBatchSize,
|
|
36
|
+
filePath: inputPath,
|
|
37
|
+
});
|
|
38
|
+
const writer = createWriter(process.stdout, {
|
|
39
|
+
format: toFormat,
|
|
40
|
+
delimiter: options.delimiter,
|
|
41
|
+
});
|
|
42
|
+
const pipeline = createPipeline(reader, { batchSize: effectiveBatchSize });
|
|
43
|
+
pipeline.pipe(reduceRows({ by: byCols, aggregations: parsedSpecs }));
|
|
44
|
+
pipeline.onProgress((info) => progress.update(info));
|
|
45
|
+
await pipeline.to(writer);
|
|
46
|
+
progress.done();
|
|
47
|
+
logMemoryDebug();
|
|
48
|
+
}
|
|
49
|
+
//# sourceMappingURL=reduce.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"reduce.js","sourceRoot":"","sources":["../../../src/cli/commands/reduce.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,oBAAoB,EAAE,MAAM,sBAAsB,CAAC;AAC5D,OAAO,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAC;AACxD,OAAO,EAAE,YAAY,EAAE,mBAAmB,IAAI,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAChG,OAAO,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AACtD,OAAO,EAAE,gBAAgB,EAAE,UAAU,EAAE,MAAM,2BAA2B,CAAC;AACzE,OAAO,EAAE,kBAAkB,EAAE,MAAM,4BAA4B,CAAC;AAChE,OAAO,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAC3D,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAa3D,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,SAAS,GAAG,GAAG,EACf,KAAe,EACf,UAAgC,EAAE;IAElC,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACjC,MAAM,IAAI,oBAAoB,CAC5B,sHAAsH,CACvH,CAAC;IACJ,CAAC;IAED,MAAM,kBAAkB,GAAG,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,IAAI,IAAI,CAAC;IAC7D,MAAM,QAAQ,GAAG,IAAI,gBAAgB,CAAC,OAAO,CAAC,CAAC;IAC/C,MAAM,WAAW,GAAG,gBAAgB,CAAC,KAAK,CAAC,CAAC;IAE5C,MAAM,MAAM,GAAG,OAAO,CAAC,EAAE;QACvB,CAAC,CAAC,OAAO,CAAC,EAAE;aACP,KAAK,CAAC,GAAG,CAAC;aACV,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;aACpB,MAAM,CAAC,OAAO,CAAC;QACpB,CAAC,CAAC,SAAS,CAAC;IAEd,IAAI,UAAU,GAAG,OAAO,CAAC,IAAI,EAAE,WAAW,EAAE,CAAC;IAC7C,IAAI,CAAC,UAAU,IAAI,SAAS,KAAK,GAAG,EAAE,CAAC;QACrC,UAAU,GAAG,iBAAiB,CAAC,SAAS,CAAC,IAAI,SAAS,CAAC;IACzD,CAAC;IACD,IAAI,CAAC,UAAU,EAAE,CAAC;QAChB,UAAU,GAAG,KAAK,CAAC;IACrB,CAAC;IAED,MAAM,QAAQ,GAAG,OAAO,CAAC,EAAE,EAAE,WAAW,EAAE,IAAI,CAAC,UAAU,KAAK,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC;IAE3F,MAAM,WAAW,GAAG,kBAAkB,CAAC,SAAS,CAAC,CAAC;IAClD,MAAM,MAAM,GAAG,YAAY,CAAC,WAAW,EAAE;QACvC,MAAM,EAAE,UAAU;QAClB,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,SAAS,EAAE,kBAAkB;QAC7B,QAAQ,EAAE,SAAS;KACpB,CAAC,CAAC;IAEH,MAAM,MAAM,GAAG,YAAY,CAAC,OAAO,CAAC,MAAM,EAAE;QAC1C,MAAM,EAAE,QAAQ;QAChB,SAAS,EAAE,OAAO,CAAC,SAAS;KAC7B,CAAC,CAAC;IAEH,MAAM,QAAQ,GAAG,cAAc,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,kBAAkB,EAAE,CAAC,CAAC;IAC3E,QAAQ,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,YAAY,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC;IACrE,QAAQ,CAAC,UAAU,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;IAErD,MAAM,QAAQ,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC;IAC1B,QAAQ,CAAC,IAAI,EAAE,CAAC;IAChB,cAAc,EAAE,CAAC;AACnB,CAAC"}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
export interface RenameCommandOptions {
|
|
2
|
+
from?: string;
|
|
3
|
+
to?: string;
|
|
4
|
+
sheet?: string;
|
|
5
|
+
delimiter?: string;
|
|
6
|
+
batchSize?: string | number;
|
|
7
|
+
quiet?: boolean;
|
|
8
|
+
noProgress?: boolean;
|
|
9
|
+
}
|
|
10
|
+
export declare function renameCommand(inputPath: string | undefined, specs: string[], options?: RenameCommandOptions): Promise<void>;
|
|
11
|
+
//# sourceMappingURL=rename.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"rename.d.ts","sourceRoot":"","sources":["../../../src/cli/commands/rename.ts"],"names":[],"mappings":"AAQA,MAAM,WAAW,oBAAoB;IACnC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IAC5B,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,UAAU,CAAC,EAAE,OAAO,CAAC;CACtB;AAED,wBAAsB,aAAa,CACjC,SAAS,oBAAM,EACf,KAAK,EAAE,MAAM,EAAE,EACf,OAAO,GAAE,oBAAyB,GACjC,OAAO,CAAC,IAAI,CAAC,CAoCf"}
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import { createPipeline } from "../../core/pipeline.js";
|
|
2
|
+
import { createReader, inferFormatFromPath as inferReaderFormat } from "../../readers/index.js";
|
|
3
|
+
import { createWriter } from "../../writers/index.js";
|
|
4
|
+
import { parseRenameSpecs, renameColumns } from "../../transforms/rename.js";
|
|
5
|
+
import { openReadableStream } from "../../utils/compression.js";
|
|
6
|
+
import { logMemoryDebug } from "../../utils/formatting.js";
|
|
7
|
+
import { ProgressReporter } from "../../utils/progress.js";
|
|
8
|
+
export async function renameCommand(inputPath = "-", specs, options = {}) {
|
|
9
|
+
const effectiveBatchSize = Number(options.batchSize) || 1000;
|
|
10
|
+
const progress = new ProgressReporter(options);
|
|
11
|
+
let fromFormat = options.from?.toLowerCase();
|
|
12
|
+
if (!fromFormat && inputPath !== "-") {
|
|
13
|
+
fromFormat = inferReaderFormat(inputPath) ?? undefined;
|
|
14
|
+
}
|
|
15
|
+
if (!fromFormat) {
|
|
16
|
+
fromFormat = "csv";
|
|
17
|
+
}
|
|
18
|
+
const toFormat = options.to?.toLowerCase() || (fromFormat === "xlsx" ? "csv" : fromFormat);
|
|
19
|
+
const mapping = parseRenameSpecs(specs);
|
|
20
|
+
const inputStream = openReadableStream(inputPath);
|
|
21
|
+
const reader = createReader(inputStream, {
|
|
22
|
+
format: fromFormat,
|
|
23
|
+
sheet: options.sheet,
|
|
24
|
+
delimiter: options.delimiter,
|
|
25
|
+
batchSize: effectiveBatchSize,
|
|
26
|
+
filePath: inputPath,
|
|
27
|
+
});
|
|
28
|
+
const writer = createWriter(process.stdout, {
|
|
29
|
+
format: toFormat,
|
|
30
|
+
delimiter: options.delimiter,
|
|
31
|
+
});
|
|
32
|
+
const pipeline = createPipeline(reader, { batchSize: effectiveBatchSize });
|
|
33
|
+
pipeline.pipe(renameColumns(mapping));
|
|
34
|
+
pipeline.onProgress((info) => progress.update(info));
|
|
35
|
+
await pipeline.to(writer);
|
|
36
|
+
progress.done();
|
|
37
|
+
logMemoryDebug();
|
|
38
|
+
}
|
|
39
|
+
//# sourceMappingURL=rename.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"rename.js","sourceRoot":"","sources":["../../../src/cli/commands/rename.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAC;AACxD,OAAO,EAAE,YAAY,EAAE,mBAAmB,IAAI,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAChG,OAAO,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AACtD,OAAO,EAAE,gBAAgB,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAC7E,OAAO,EAAE,kBAAkB,EAAE,MAAM,4BAA4B,CAAC;AAChE,OAAO,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAC3D,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAY3D,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,SAAS,GAAG,GAAG,EACf,KAAe,EACf,UAAgC,EAAE;IAElC,MAAM,kBAAkB,GAAG,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,IAAI,IAAI,CAAC;IAC7D,MAAM,QAAQ,GAAG,IAAI,gBAAgB,CAAC,OAAO,CAAC,CAAC;IAE/C,IAAI,UAAU,GAAG,OAAO,CAAC,IAAI,EAAE,WAAW,EAAE,CAAC;IAC7C,IAAI,CAAC,UAAU,IAAI,SAAS,KAAK,GAAG,EAAE,CAAC;QACrC,UAAU,GAAG,iBAAiB,CAAC,SAAS,CAAC,IAAI,SAAS,CAAC;IACzD,CAAC;IACD,IAAI,CAAC,UAAU,EAAE,CAAC;QAChB,UAAU,GAAG,KAAK,CAAC;IACrB,CAAC;IAED,MAAM,QAAQ,GAAG,OAAO,CAAC,EAAE,EAAE,WAAW,EAAE,IAAI,CAAC,UAAU,KAAK,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC;IAC3F,MAAM,OAAO,GAAG,gBAAgB,CAAC,KAAK,CAAC,CAAC;IAExC,MAAM,WAAW,GAAG,kBAAkB,CAAC,SAAS,CAAC,CAAC;IAClD,MAAM,MAAM,GAAG,YAAY,CAAC,WAAW,EAAE;QACvC,MAAM,EAAE,UAAU;QAClB,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,SAAS,EAAE,kBAAkB;QAC7B,QAAQ,EAAE,SAAS;KACpB,CAAC,CAAC;IAEH,MAAM,MAAM,GAAG,YAAY,CAAC,OAAO,CAAC,MAAM,EAAE;QAC1C,MAAM,EAAE,QAAQ;QAChB,SAAS,EAAE,OAAO,CAAC,SAAS;KAC7B,CAAC,CAAC;IAEH,MAAM,QAAQ,GAAG,cAAc,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,kBAAkB,EAAE,CAAC,CAAC;IAC3E,QAAQ,CAAC,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC;IACtC,QAAQ,CAAC,UAAU,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;IAErD,MAAM,QAAQ,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC;IAC1B,QAAQ,CAAC,IAAI,EAAE,CAAC;IAChB,cAAc,EAAE,CAAC;AACnB,CAAC"}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
export interface SampleCommandOptions {
|
|
2
|
+
rows?: string | number;
|
|
3
|
+
seed?: string | number;
|
|
4
|
+
from?: string;
|
|
5
|
+
to?: string;
|
|
6
|
+
sheet?: string;
|
|
7
|
+
delimiter?: string;
|
|
8
|
+
batchSize?: string | number;
|
|
9
|
+
quiet?: boolean;
|
|
10
|
+
noProgress?: boolean;
|
|
11
|
+
}
|
|
12
|
+
export declare function sampleCommand(inputPath?: string, options?: SampleCommandOptions): Promise<void>;
|
|
13
|
+
//# sourceMappingURL=sample.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"sample.d.ts","sourceRoot":"","sources":["../../../src/cli/commands/sample.ts"],"names":[],"mappings":"AAQA,MAAM,WAAW,oBAAoB;IACnC,IAAI,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IACvB,IAAI,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IACvB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IAC5B,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,UAAU,CAAC,EAAE,OAAO,CAAC;CACtB;AAED,wBAAsB,aAAa,CACjC,SAAS,SAAM,EACf,OAAO,GAAE,oBAAyB,GACjC,OAAO,CAAC,IAAI,CAAC,CAqCf"}
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
import { createPipeline } from "../../core/pipeline.js";
|
|
2
|
+
import { createReader, inferFormatFromPath as inferReaderFormat } from "../../readers/index.js";
|
|
3
|
+
import { createWriter } from "../../writers/index.js";
|
|
4
|
+
import { sampleRows } from "../../transforms/sample.js";
|
|
5
|
+
import { openReadableStream } from "../../utils/compression.js";
|
|
6
|
+
import { logMemoryDebug } from "../../utils/formatting.js";
|
|
7
|
+
import { ProgressReporter } from "../../utils/progress.js";
|
|
8
|
+
export async function sampleCommand(inputPath = "-", options = {}) {
|
|
9
|
+
const sampleCount = options.rows !== undefined ? Number(options.rows) : 1000;
|
|
10
|
+
const seed = options.seed !== undefined ? Number(options.seed) : undefined;
|
|
11
|
+
const effectiveBatchSize = Number(options.batchSize) || 1000;
|
|
12
|
+
const progress = new ProgressReporter(options);
|
|
13
|
+
let fromFormat = options.from?.toLowerCase();
|
|
14
|
+
if (!fromFormat && inputPath !== "-") {
|
|
15
|
+
fromFormat = inferReaderFormat(inputPath) ?? undefined;
|
|
16
|
+
}
|
|
17
|
+
if (!fromFormat) {
|
|
18
|
+
fromFormat = "csv";
|
|
19
|
+
}
|
|
20
|
+
const toFormat = options.to?.toLowerCase() || (fromFormat === "xlsx" ? "csv" : fromFormat);
|
|
21
|
+
const inputStream = openReadableStream(inputPath);
|
|
22
|
+
const reader = createReader(inputStream, {
|
|
23
|
+
format: fromFormat,
|
|
24
|
+
sheet: options.sheet,
|
|
25
|
+
delimiter: options.delimiter,
|
|
26
|
+
batchSize: effectiveBatchSize,
|
|
27
|
+
filePath: inputPath,
|
|
28
|
+
});
|
|
29
|
+
const writer = createWriter(process.stdout, {
|
|
30
|
+
format: toFormat,
|
|
31
|
+
delimiter: options.delimiter,
|
|
32
|
+
});
|
|
33
|
+
const pipeline = createPipeline(reader, { batchSize: effectiveBatchSize });
|
|
34
|
+
pipeline.pipe(sampleRows({ rows: sampleCount, seed }));
|
|
35
|
+
pipeline.onProgress((info) => progress.update(info));
|
|
36
|
+
await pipeline.to(writer);
|
|
37
|
+
progress.done();
|
|
38
|
+
logMemoryDebug();
|
|
39
|
+
}
|
|
40
|
+
//# sourceMappingURL=sample.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"sample.js","sourceRoot":"","sources":["../../../src/cli/commands/sample.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAC;AACxD,OAAO,EAAE,YAAY,EAAE,mBAAmB,IAAI,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAChG,OAAO,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AACtD,OAAO,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAC;AACxD,OAAO,EAAE,kBAAkB,EAAE,MAAM,4BAA4B,CAAC;AAChE,OAAO,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAC3D,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAc3D,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,SAAS,GAAG,GAAG,EACf,UAAgC,EAAE;IAElC,MAAM,WAAW,GAAG,OAAO,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAC7E,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAC3E,MAAM,kBAAkB,GAAG,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,IAAI,IAAI,CAAC;IAC7D,MAAM,QAAQ,GAAG,IAAI,gBAAgB,CAAC,OAAO,CAAC,CAAC;IAE/C,IAAI,UAAU,GAAG,OAAO,CAAC,IAAI,EAAE,WAAW,EAAE,CAAC;IAC7C,IAAI,CAAC,UAAU,IAAI,SAAS,KAAK,GAAG,EAAE,CAAC;QACrC,UAAU,GAAG,iBAAiB,CAAC,SAAS,CAAC,IAAI,SAAS,CAAC;IACzD,CAAC;IACD,IAAI,CAAC,UAAU,EAAE,CAAC;QAChB,UAAU,GAAG,KAAK,CAAC;IACrB,CAAC;IAED,MAAM,QAAQ,GAAG,OAAO,CAAC,EAAE,EAAE,WAAW,EAAE,IAAI,CAAC,UAAU,KAAK,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC;IAE3F,MAAM,WAAW,GAAG,kBAAkB,CAAC,SAAS,CAAC,CAAC;IAClD,MAAM,MAAM,GAAG,YAAY,CAAC,WAAW,EAAE;QACvC,MAAM,EAAE,UAAU;QAClB,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,SAAS,EAAE,kBAAkB;QAC7B,QAAQ,EAAE,SAAS;KACpB,CAAC,CAAC;IAEH,MAAM,MAAM,GAAG,YAAY,CAAC,OAAO,CAAC,MAAM,EAAE;QAC1C,MAAM,EAAE,QAAQ;QAChB,SAAS,EAAE,OAAO,CAAC,SAAS;KAC7B,CAAC,CAAC;IAEH,MAAM,QAAQ,GAAG,cAAc,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,kBAAkB,EAAE,CAAC,CAAC;IAC3E,QAAQ,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;IACvD,QAAQ,CAAC,UAAU,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;IAErD,MAAM,QAAQ,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC;IAC1B,QAAQ,CAAC,IAAI,EAAE,CAAC;IAChB,cAAc,EAAE,CAAC;AACnB,CAAC"}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
export interface SchemaCommandOptions {
|
|
2
|
+
from?: string;
|
|
3
|
+
sheet?: string;
|
|
4
|
+
path?: string;
|
|
5
|
+
delimiter?: string;
|
|
6
|
+
sample?: string | number;
|
|
7
|
+
full?: boolean;
|
|
8
|
+
json?: boolean;
|
|
9
|
+
quiet?: boolean;
|
|
10
|
+
noProgress?: boolean;
|
|
11
|
+
}
|
|
12
|
+
export declare function schemaCommand(inputPath?: string, options?: SchemaCommandOptions): Promise<void>;
|
|
13
|
+
//# sourceMappingURL=schema.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../../../src/cli/commands/schema.ts"],"names":[],"mappings":"AAOA,MAAM,WAAW,oBAAoB;IACnC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IACzB,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,UAAU,CAAC,EAAE,OAAO,CAAC;CACtB;AAED,wBAAsB,aAAa,CACjC,SAAS,SAAM,EACf,OAAO,GAAE,oBAAyB,GACjC,OAAO,CAAC,IAAI,CAAC,CA+Df"}
|