rowpipe 1.2.0 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +238 -55
- package/dist/cli/commands/count.d.ts +12 -0
- package/dist/cli/commands/count.d.ts.map +1 -0
- package/dist/cli/commands/count.js +49 -0
- package/dist/cli/commands/count.js.map +1 -0
- package/dist/cli/commands/explain.d.ts +3 -0
- package/dist/cli/commands/explain.d.ts.map +1 -0
- package/dist/cli/commands/explain.js +11 -0
- package/dist/cli/commands/explain.js.map +1 -0
- package/dist/cli/commands/files.d.ts +3 -1
- package/dist/cli/commands/files.d.ts.map +1 -1
- package/dist/cli/commands/files.js +6 -2
- package/dist/cli/commands/files.js.map +1 -1
- package/dist/cli/commands/group.d.ts +12 -0
- package/dist/cli/commands/group.d.ts.map +1 -0
- package/dist/cli/commands/group.js +51 -0
- package/dist/cli/commands/group.js.map +1 -0
- package/dist/cli/commands/head.d.ts +7 -0
- package/dist/cli/commands/head.d.ts.map +1 -0
- package/dist/cli/commands/head.js +6 -0
- package/dist/cli/commands/head.js.map +1 -0
- package/dist/cli/commands/limit.d.ts +11 -0
- package/dist/cli/commands/limit.d.ts.map +1 -0
- package/dist/cli/commands/limit.js +40 -0
- package/dist/cli/commands/limit.js.map +1 -0
- package/dist/cli/commands/offset.d.ts +11 -0
- package/dist/cli/commands/offset.d.ts.map +1 -0
- package/dist/cli/commands/offset.js +39 -0
- package/dist/cli/commands/offset.js.map +1 -0
- package/dist/cli/commands/pipeline.d.ts +43 -0
- package/dist/cli/commands/pipeline.d.ts.map +1 -0
- package/dist/cli/commands/pipeline.js +198 -0
- package/dist/cli/commands/pipeline.js.map +1 -0
- package/dist/cli/commands/sort.d.ts +17 -0
- package/dist/cli/commands/sort.d.ts.map +1 -0
- package/dist/cli/commands/sort.js +55 -0
- package/dist/cli/commands/sort.js.map +1 -0
- package/dist/cli/commands/tail.d.ts +13 -0
- package/dist/cli/commands/tail.d.ts.map +1 -0
- package/dist/cli/commands/tail.js +39 -0
- package/dist/cli/commands/tail.js.map +1 -0
- package/dist/cli/commands/top.d.ts +19 -0
- package/dist/cli/commands/top.d.ts.map +1 -0
- package/dist/cli/commands/top.js +51 -0
- package/dist/cli/commands/top.js.map +1 -0
- package/dist/cli/commands/unique.d.ts +15 -0
- package/dist/cli/commands/unique.d.ts.map +1 -0
- package/dist/cli/commands/unique.js +44 -0
- package/dist/cli/commands/unique.js.map +1 -0
- package/dist/cli/index.js +253 -3
- package/dist/cli/index.js.map +1 -1
- package/dist/core/pipeline.d.ts +40 -0
- package/dist/core/pipeline.d.ts.map +1 -1
- package/dist/core/pipeline.js +65 -0
- package/dist/core/pipeline.js.map +1 -1
- package/dist/files/reader.d.ts.map +1 -1
- package/dist/files/reader.js +4 -0
- package/dist/files/reader.js.map +1 -1
- package/dist/files/types.d.ts +2 -0
- package/dist/files/types.d.ts.map +1 -1
- package/dist/index.d.ts +12 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +13 -0
- package/dist/index.js.map +1 -1
- package/dist/planner/executor.d.ts +8 -0
- package/dist/planner/executor.d.ts.map +1 -0
- package/dist/planner/executor.js +86 -0
- package/dist/planner/executor.js.map +1 -0
- package/dist/planner/explain.d.ts +6 -0
- package/dist/planner/explain.d.ts.map +1 -0
- package/dist/planner/explain.js +91 -0
- package/dist/planner/explain.js.map +1 -0
- package/dist/planner/index.d.ts +5 -0
- package/dist/planner/index.d.ts.map +1 -0
- package/dist/planner/index.js +5 -0
- package/dist/planner/index.js.map +1 -0
- package/dist/planner/optimizer.d.ts +9 -0
- package/dist/planner/optimizer.d.ts.map +1 -0
- package/dist/planner/optimizer.js +78 -0
- package/dist/planner/optimizer.js.map +1 -0
- package/dist/planner/types.d.ts +63 -0
- package/dist/planner/types.d.ts.map +1 -0
- package/dist/planner/types.js +2 -0
- package/dist/planner/types.js.map +1 -0
- package/dist/transforms/count.d.ts +21 -0
- package/dist/transforms/count.d.ts.map +1 -0
- package/dist/transforms/count.js +65 -0
- package/dist/transforms/count.js.map +1 -0
- package/dist/transforms/expression.d.ts.map +1 -1
- package/dist/transforms/expression.js +15 -0
- package/dist/transforms/expression.js.map +1 -1
- package/dist/transforms/group.d.ts +25 -0
- package/dist/transforms/group.d.ts.map +1 -0
- package/dist/transforms/group.js +71 -0
- package/dist/transforms/group.js.map +1 -0
- package/dist/transforms/limit.d.ts +7 -0
- package/dist/transforms/limit.d.ts.map +1 -0
- package/dist/transforms/limit.js +40 -0
- package/dist/transforms/limit.js.map +1 -0
- package/dist/transforms/offset.d.ts +7 -0
- package/dist/transforms/offset.d.ts.map +1 -0
- package/dist/transforms/offset.js +38 -0
- package/dist/transforms/offset.js.map +1 -0
- package/dist/transforms/sort/comparator.d.ts +43 -0
- package/dist/transforms/sort/comparator.d.ts.map +1 -0
- package/dist/transforms/sort/comparator.js +131 -0
- package/dist/transforms/sort/comparator.js.map +1 -0
- package/dist/transforms/sort/external-sort.d.ts +8 -0
- package/dist/transforms/sort/external-sort.d.ts.map +1 -0
- package/dist/transforms/sort/external-sort.js +159 -0
- package/dist/transforms/sort/external-sort.js.map +1 -0
- package/dist/transforms/sort/index.d.ts +10 -0
- package/dist/transforms/sort/index.d.ts.map +1 -0
- package/dist/transforms/sort/index.js +11 -0
- package/dist/transforms/sort/index.js.map +1 -0
- package/dist/transforms/tail.d.ts +6 -0
- package/dist/transforms/tail.d.ts.map +1 -0
- package/dist/transforms/tail.js +27 -0
- package/dist/transforms/tail.js.map +1 -0
- package/dist/transforms/top.d.ts +17 -0
- package/dist/transforms/top.d.ts.map +1 -0
- package/dist/transforms/top.js +52 -0
- package/dist/transforms/top.js.map +1 -0
- package/dist/transforms/unique.d.ts +13 -0
- package/dist/transforms/unique.d.ts.map +1 -0
- package/dist/transforms/unique.js +91 -0
- package/dist/transforms/unique.js.map +1 -0
- package/dist/utils/formatting.d.ts +1 -0
- package/dist/utils/formatting.d.ts.map +1 -1
- package/dist/utils/formatting.js +34 -4
- package/dist/utils/formatting.js.map +1 -1
- package/dist/utils/heap.d.ts +25 -0
- package/dist/utils/heap.d.ts.map +1 -0
- package/dist/utils/heap.js +90 -0
- package/dist/utils/heap.js.map +1 -0
- package/dist/utils/keystore.d.ts +80 -0
- package/dist/utils/keystore.d.ts.map +1 -0
- package/dist/utils/keystore.js +217 -0
- package/dist/utils/keystore.js.map +1 -0
- package/dist/utils/ring-buffer.d.ts +15 -0
- package/dist/utils/ring-buffer.d.ts.map +1 -0
- package/dist/utils/ring-buffer.js +40 -0
- package/dist/utils/ring-buffer.js.map +1 -0
- package/dist/writers/markdown.d.ts.map +1 -1
- package/dist/writers/markdown.js +41 -12
- package/dist/writers/markdown.js.map +1 -1
- package/package.json +3 -2
- package/skills/rowpipe/SKILL.md +27 -9
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
import { createPipeline } from "../../core/pipeline.js";
|
|
2
|
+
import { createReader, inferFormatFromPath as inferReaderFormat } from "../../readers/index.js";
|
|
3
|
+
import { createWriter } from "../../writers/index.js";
|
|
4
|
+
import { limitRows } from "../../transforms/limit.js";
|
|
5
|
+
import { openReadableStream } from "../../utils/compression.js";
|
|
6
|
+
import { logMemoryDebug } from "../../utils/formatting.js";
|
|
7
|
+
import { ProgressReporter } from "../../utils/progress.js";
|
|
8
|
+
export async function limitCommand(inputPath = "-", count = 10, options = {}) {
|
|
9
|
+
const effectiveLimit = Math.max(0, Number(count) || 10);
|
|
10
|
+
const effectiveBatchSize = Number(options.batchSize) || 1000;
|
|
11
|
+
const progress = new ProgressReporter(options);
|
|
12
|
+
let fromFormat = options.from?.toLowerCase();
|
|
13
|
+
if (!fromFormat && inputPath !== "-") {
|
|
14
|
+
fromFormat = inferReaderFormat(inputPath) ?? undefined;
|
|
15
|
+
}
|
|
16
|
+
if (!fromFormat) {
|
|
17
|
+
fromFormat = "csv";
|
|
18
|
+
}
|
|
19
|
+
const toFormat = options.to?.toLowerCase() || (fromFormat === "xlsx" ? "csv" : fromFormat);
|
|
20
|
+
const inputStream = openReadableStream(inputPath);
|
|
21
|
+
const reader = createReader(inputStream, {
|
|
22
|
+
format: fromFormat,
|
|
23
|
+
sheet: options.sheet,
|
|
24
|
+
delimiter: options.delimiter,
|
|
25
|
+
batchSize: effectiveBatchSize,
|
|
26
|
+
maxRows: effectiveLimit,
|
|
27
|
+
filePath: inputPath,
|
|
28
|
+
});
|
|
29
|
+
const writer = createWriter(process.stdout, {
|
|
30
|
+
format: toFormat,
|
|
31
|
+
delimiter: options.delimiter,
|
|
32
|
+
});
|
|
33
|
+
const pipeline = createPipeline(reader, { batchSize: effectiveBatchSize, maxRows: effectiveLimit });
|
|
34
|
+
pipeline.pipe(limitRows(effectiveLimit));
|
|
35
|
+
pipeline.onProgress((info) => progress.update(info));
|
|
36
|
+
await pipeline.to(writer);
|
|
37
|
+
progress.done();
|
|
38
|
+
logMemoryDebug();
|
|
39
|
+
}
|
|
40
|
+
//# sourceMappingURL=limit.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"limit.js","sourceRoot":"","sources":["../../../src/cli/commands/limit.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAC;AACxD,OAAO,EAAE,YAAY,EAAE,mBAAmB,IAAI,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAChG,OAAO,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AACtD,OAAO,EAAE,SAAS,EAAE,MAAM,2BAA2B,CAAC;AACtD,OAAO,EAAE,kBAAkB,EAAE,MAAM,4BAA4B,CAAC;AAChE,OAAO,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAC3D,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAY3D,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,SAAS,GAAG,GAAG,EACf,KAAK,GAAG,EAAE,EACV,UAA+B,EAAE;IAEjC,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;IACxD,MAAM,kBAAkB,GAAG,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,IAAI,IAAI,CAAC;IAC7D,MAAM,QAAQ,GAAG,IAAI,gBAAgB,CAAC,OAAO,CAAC,CAAC;IAE/C,IAAI,UAAU,GAAG,OAAO,CAAC,IAAI,EAAE,WAAW,EAAE,CAAC;IAC7C,IAAI,CAAC,UAAU,IAAI,SAAS,KAAK,GAAG,EAAE,CAAC;QACrC,UAAU,GAAG,iBAAiB,CAAC,SAAS,CAAC,IAAI,SAAS,CAAC;IACzD,CAAC;IACD,IAAI,CAAC,UAAU,EAAE,CAAC;QAChB,UAAU,GAAG,KAAK,CAAC;IACrB,CAAC;IAED,MAAM,QAAQ,GAAG,OAAO,CAAC,EAAE,EAAE,WAAW,EAAE,IAAI,CAAC,UAAU,KAAK,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC;IAE3F,MAAM,WAAW,GAAG,kBAAkB,CAAC,SAAS,CAAC,CAAC;IAClD,MAAM,MAAM,GAAG,YAAY,CAAC,WAAW,EAAE;QACvC,MAAM,EAAE,UAAU;QAClB,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,SAAS,EAAE,kBAAkB;QAC7B,OAAO,EAAE,cAAc;QACvB,QAAQ,EAAE,SAAS;KACpB,CAAC,CAAC;IAEH,MAAM,MAAM,GAAG,YAAY,CAAC,OAAO,CAAC,MAAM,EAAE;QAC1C,MAAM,EAAE,QAAQ;QAChB,SAAS,EAAE,OAAO,CAAC,SAAS;KAC7B,CAAC,CAAC;IAEH,MAAM,QAAQ,GAAG,cAAc,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,kBAAkB,EAAE,OAAO,EAAE,cAAc,EAAE,CAAC,CAAC;IACpG,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,CAAC,CAAC;IACzC,QAAQ,CAAC,UAAU,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;IAErD,MAAM,QAAQ,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC;IAC1B,QAAQ,CAAC,IAAI,EAAE,CAAC;IAChB,cAAc,EAAE,CAAC;AACnB,CAAC"}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
export interface OffsetCommandOptions {
|
|
2
|
+
from?: string;
|
|
3
|
+
to?: string;
|
|
4
|
+
sheet?: string;
|
|
5
|
+
delimiter?: string;
|
|
6
|
+
batchSize?: string | number;
|
|
7
|
+
quiet?: boolean;
|
|
8
|
+
noProgress?: boolean;
|
|
9
|
+
}
|
|
10
|
+
export declare function offsetCommand(inputPath?: string, count?: number, options?: OffsetCommandOptions): Promise<void>;
|
|
11
|
+
//# sourceMappingURL=offset.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"offset.d.ts","sourceRoot":"","sources":["../../../src/cli/commands/offset.ts"],"names":[],"mappings":"AAQA,MAAM,WAAW,oBAAoB;IACnC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IAC5B,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,UAAU,CAAC,EAAE,OAAO,CAAC;CACtB;AAED,wBAAsB,aAAa,CACjC,SAAS,SAAM,EACf,KAAK,SAAI,EACT,OAAO,GAAE,oBAAyB,GACjC,OAAO,CAAC,IAAI,CAAC,CAoCf"}
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import { createPipeline } from "../../core/pipeline.js";
|
|
2
|
+
import { createReader, inferFormatFromPath as inferReaderFormat } from "../../readers/index.js";
|
|
3
|
+
import { createWriter } from "../../writers/index.js";
|
|
4
|
+
import { offsetRows } from "../../transforms/offset.js";
|
|
5
|
+
import { openReadableStream } from "../../utils/compression.js";
|
|
6
|
+
import { logMemoryDebug } from "../../utils/formatting.js";
|
|
7
|
+
import { ProgressReporter } from "../../utils/progress.js";
|
|
8
|
+
export async function offsetCommand(inputPath = "-", count = 0, options = {}) {
|
|
9
|
+
const effectiveOffset = Math.max(0, Number(count) || 0);
|
|
10
|
+
const effectiveBatchSize = Number(options.batchSize) || 1000;
|
|
11
|
+
const progress = new ProgressReporter(options);
|
|
12
|
+
let fromFormat = options.from?.toLowerCase();
|
|
13
|
+
if (!fromFormat && inputPath !== "-") {
|
|
14
|
+
fromFormat = inferReaderFormat(inputPath) ?? undefined;
|
|
15
|
+
}
|
|
16
|
+
if (!fromFormat) {
|
|
17
|
+
fromFormat = "csv";
|
|
18
|
+
}
|
|
19
|
+
const toFormat = options.to?.toLowerCase() || (fromFormat === "xlsx" ? "csv" : fromFormat);
|
|
20
|
+
const inputStream = openReadableStream(inputPath);
|
|
21
|
+
const reader = createReader(inputStream, {
|
|
22
|
+
format: fromFormat,
|
|
23
|
+
sheet: options.sheet,
|
|
24
|
+
delimiter: options.delimiter,
|
|
25
|
+
batchSize: effectiveBatchSize,
|
|
26
|
+
filePath: inputPath,
|
|
27
|
+
});
|
|
28
|
+
const writer = createWriter(process.stdout, {
|
|
29
|
+
format: toFormat,
|
|
30
|
+
delimiter: options.delimiter,
|
|
31
|
+
});
|
|
32
|
+
const pipeline = createPipeline(reader, { batchSize: effectiveBatchSize });
|
|
33
|
+
pipeline.pipe(offsetRows(effectiveOffset));
|
|
34
|
+
pipeline.onProgress((info) => progress.update(info));
|
|
35
|
+
await pipeline.to(writer);
|
|
36
|
+
progress.done();
|
|
37
|
+
logMemoryDebug();
|
|
38
|
+
}
|
|
39
|
+
//# sourceMappingURL=offset.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"offset.js","sourceRoot":"","sources":["../../../src/cli/commands/offset.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAC;AACxD,OAAO,EAAE,YAAY,EAAE,mBAAmB,IAAI,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAChG,OAAO,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AACtD,OAAO,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAC;AACxD,OAAO,EAAE,kBAAkB,EAAE,MAAM,4BAA4B,CAAC;AAChE,OAAO,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAC3D,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAY3D,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,SAAS,GAAG,GAAG,EACf,KAAK,GAAG,CAAC,EACT,UAAgC,EAAE;IAElC,MAAM,eAAe,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;IACxD,MAAM,kBAAkB,GAAG,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,IAAI,IAAI,CAAC;IAC7D,MAAM,QAAQ,GAAG,IAAI,gBAAgB,CAAC,OAAO,CAAC,CAAC;IAE/C,IAAI,UAAU,GAAG,OAAO,CAAC,IAAI,EAAE,WAAW,EAAE,CAAC;IAC7C,IAAI,CAAC,UAAU,IAAI,SAAS,KAAK,GAAG,EAAE,CAAC;QACrC,UAAU,GAAG,iBAAiB,CAAC,SAAS,CAAC,IAAI,SAAS,CAAC;IACzD,CAAC;IACD,IAAI,CAAC,UAAU,EAAE,CAAC;QAChB,UAAU,GAAG,KAAK,CAAC;IACrB,CAAC;IAED,MAAM,QAAQ,GAAG,OAAO,CAAC,EAAE,EAAE,WAAW,EAAE,IAAI,CAAC,UAAU,KAAK,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC;IAE3F,MAAM,WAAW,GAAG,kBAAkB,CAAC,SAAS,CAAC,CAAC;IAClD,MAAM,MAAM,GAAG,YAAY,CAAC,WAAW,EAAE;QACvC,MAAM,EAAE,UAAU;QAClB,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,SAAS,EAAE,kBAAkB;QAC7B,QAAQ,EAAE,SAAS;KACpB,CAAC,CAAC;IAEH,MAAM,MAAM,GAAG,YAAY,CAAC,OAAO,CAAC,MAAM,EAAE;QAC1C,MAAM,EAAE,QAAQ;QAChB,SAAS,EAAE,OAAO,CAAC,SAAS;KAC7B,CAAC,CAAC;IAEH,MAAM,QAAQ,GAAG,cAAc,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,kBAAkB,EAAE,CAAC,CAAC;IAC3E,QAAQ,CAAC,IAAI,CAAC,UAAU,CAAC,eAAe,CAAC,CAAC,CAAC;IAC3C,QAAQ,CAAC,UAAU,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;IAErD,MAAM,QAAQ,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC;IAC1B,QAAQ,CAAC,IAAI,EAAE,CAAC;IAChB,cAAc,EAAE,CAAC;AACnB,CAAC"}
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import { PipelineOperation } from "../../planner/index.js";
|
|
2
|
+
export interface UnifiedPipelineCliOptions {
|
|
3
|
+
filter?: string | string[];
|
|
4
|
+
select?: string | string[];
|
|
5
|
+
rename?: string | string[];
|
|
6
|
+
cast?: string | string[];
|
|
7
|
+
map?: string | string[];
|
|
8
|
+
sort?: string | string[];
|
|
9
|
+
top?: string | number;
|
|
10
|
+
limit?: string | number;
|
|
11
|
+
offset?: string | number;
|
|
12
|
+
tail?: string | number;
|
|
13
|
+
unique?: string | boolean;
|
|
14
|
+
groupBy?: string | string[];
|
|
15
|
+
count?: boolean;
|
|
16
|
+
sum?: string | string[];
|
|
17
|
+
avg?: string | string[];
|
|
18
|
+
min?: string | string[];
|
|
19
|
+
max?: string | string[];
|
|
20
|
+
first?: string | string[];
|
|
21
|
+
last?: string | string[];
|
|
22
|
+
agg?: string | string[];
|
|
23
|
+
nulls?: "first" | "last";
|
|
24
|
+
ignoreCase?: boolean;
|
|
25
|
+
natural?: boolean;
|
|
26
|
+
memoryLimit?: string;
|
|
27
|
+
tempDir?: string;
|
|
28
|
+
from?: string;
|
|
29
|
+
to?: string;
|
|
30
|
+
sheet?: string;
|
|
31
|
+
delimiter?: string;
|
|
32
|
+
json?: boolean;
|
|
33
|
+
output?: string;
|
|
34
|
+
batchSize?: string | number;
|
|
35
|
+
quiet?: boolean;
|
|
36
|
+
noProgress?: boolean;
|
|
37
|
+
}
|
|
38
|
+
/**
|
|
39
|
+
* Builds an AST of PipelineOperations from CLI flags.
|
|
40
|
+
*/
|
|
41
|
+
export declare function buildOperationsFromOptions(options: UnifiedPipelineCliOptions): PipelineOperation[];
|
|
42
|
+
export declare function unifiedPipelineCommand(inputPath?: string, options?: UnifiedPipelineCliOptions): Promise<void>;
|
|
43
|
+
//# sourceMappingURL=pipeline.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../../../src/cli/commands/pipeline.ts"],"names":[],"mappings":"AAEA,OAAO,EAA4C,iBAAiB,EAAmB,MAAM,wBAAwB,CAAC;AAOtH,MAAM,WAAW,yBAAyB;IACxC,MAAM,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IAC3B,MAAM,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IAC3B,MAAM,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IAC3B,IAAI,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IACzB,GAAG,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IACxB,IAAI,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IACzB,GAAG,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IACtB,KAAK,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IACxB,MAAM,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IACzB,IAAI,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IACvB,MAAM,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC;IAC1B,OAAO,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IAC5B,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,GAAG,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IACxB,GAAG,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IACxB,GAAG,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IACxB,GAAG,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IACxB,KAAK,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IAC1B,IAAI,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IACzB,GAAG,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IACxB,KAAK,CAAC,EAAE,OAAO,GAAG,MAAM,CAAC;IACzB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IAC5B,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,UAAU,CAAC,EAAE,OAAO,CAAC;CACtB;AAED;;GAEG;AACH,wBAAgB,0BAA0B,CAAC,OAAO,EAAE,yBAAyB,GAAG,iBAAiB,EAAE,CA2JlG;AAED,wBAAsB,sBAAsB,CAC1C,SAAS,SAAM,EACf,OAAO,GAAE,yBAA8B,GACtC,OAAO,CAAC,IAAI,CAAC,CAgDf"}
|
|
@@ -0,0 +1,198 @@
|
|
|
1
|
+
import { buildGroupReduceSpecs } from "../../transforms/group.js";
|
|
2
|
+
import { parseSortSpecs } from "../../transforms/sort/comparator.js";
|
|
3
|
+
import { executePlannedPipeline, optimizePipeline } from "../../planner/index.js";
|
|
4
|
+
import { createReader, inferFormatFromPath as inferReaderFormat } from "../../readers/index.js";
|
|
5
|
+
import { createWriter, inferFormatFromPath as inferWriterFormat } from "../../writers/index.js";
|
|
6
|
+
import { openReadableStream, openWritableStream } from "../../utils/compression.js";
|
|
7
|
+
import { logMemoryDebug } from "../../utils/formatting.js";
|
|
8
|
+
import { ProgressReporter } from "../../utils/progress.js";
|
|
9
|
+
/**
|
|
10
|
+
* Builds an AST of PipelineOperations from CLI flags.
|
|
11
|
+
*/
|
|
12
|
+
export function buildOperationsFromOptions(options) {
|
|
13
|
+
const ops = [];
|
|
14
|
+
// 1. Filter
|
|
15
|
+
if (options.filter) {
|
|
16
|
+
const filters = Array.isArray(options.filter) ? options.filter : [options.filter];
|
|
17
|
+
for (const f of filters) {
|
|
18
|
+
if (f && f.trim().length > 0) {
|
|
19
|
+
ops.push({ type: "filter", expression: f });
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
// 2. Map
|
|
24
|
+
if (options.map) {
|
|
25
|
+
const maps = Array.isArray(options.map) ? options.map : [options.map];
|
|
26
|
+
const mapSpecs = {};
|
|
27
|
+
for (const m of maps) {
|
|
28
|
+
const idx = m.indexOf("=");
|
|
29
|
+
if (idx > 0) {
|
|
30
|
+
mapSpecs[m.slice(0, idx).trim()] = m.slice(idx + 1).trim();
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
if (Object.keys(mapSpecs).length > 0) {
|
|
34
|
+
ops.push({ type: "map", specs: mapSpecs });
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
// 3. Rename
|
|
38
|
+
if (options.rename) {
|
|
39
|
+
const renames = Array.isArray(options.rename) ? options.rename : [options.rename];
|
|
40
|
+
const renameSpecs = {};
|
|
41
|
+
for (const r of renames) {
|
|
42
|
+
const idx = r.indexOf("=");
|
|
43
|
+
if (idx > 0) {
|
|
44
|
+
renameSpecs[r.slice(0, idx).trim()] = r.slice(idx + 1).trim();
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
if (Object.keys(renameSpecs).length > 0) {
|
|
48
|
+
ops.push({ type: "rename", specs: renameSpecs });
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
// 4. Cast
|
|
52
|
+
if (options.cast) {
|
|
53
|
+
const casts = Array.isArray(options.cast) ? options.cast : [options.cast];
|
|
54
|
+
const castSpecs = {};
|
|
55
|
+
for (const c of casts) {
|
|
56
|
+
const idx = c.indexOf(":");
|
|
57
|
+
if (idx > 0) {
|
|
58
|
+
castSpecs[c.slice(0, idx).trim()] = c.slice(idx + 1).trim();
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
if (Object.keys(castSpecs).length > 0) {
|
|
62
|
+
ops.push({ type: "cast", specs: castSpecs });
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
// 5. Select
|
|
66
|
+
if (options.select) {
|
|
67
|
+
const selects = Array.isArray(options.select) ? options.select : options.select.split(",");
|
|
68
|
+
const cols = selects.map((s) => s.trim()).filter(Boolean);
|
|
69
|
+
if (cols.length > 0) {
|
|
70
|
+
ops.push({ type: "select", columns: cols });
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
// 6. Unique
|
|
74
|
+
if (options.unique !== undefined && options.unique !== false) {
|
|
75
|
+
const by = typeof options.unique === "string" ? options.unique.split(",").map((s) => s.trim()).filter(Boolean) : undefined;
|
|
76
|
+
ops.push({
|
|
77
|
+
type: "unique",
|
|
78
|
+
by,
|
|
79
|
+
memoryLimit: options.memoryLimit,
|
|
80
|
+
});
|
|
81
|
+
}
|
|
82
|
+
// 7. Group By / Aggregations
|
|
83
|
+
if (options.groupBy || options.count || options.sum || options.avg || options.min || options.max || options.first || options.last || options.agg) {
|
|
84
|
+
const by = options.groupBy ? (Array.isArray(options.groupBy) ? options.groupBy : options.groupBy.split(",").map((s) => s.trim()).filter(Boolean)) : undefined;
|
|
85
|
+
const aggregations = buildGroupReduceSpecs({
|
|
86
|
+
by,
|
|
87
|
+
count: options.count,
|
|
88
|
+
sum: options.sum,
|
|
89
|
+
avg: options.avg,
|
|
90
|
+
min: options.min,
|
|
91
|
+
max: options.max,
|
|
92
|
+
first: options.first,
|
|
93
|
+
last: options.last,
|
|
94
|
+
agg: options.agg,
|
|
95
|
+
});
|
|
96
|
+
ops.push({
|
|
97
|
+
type: "group",
|
|
98
|
+
by,
|
|
99
|
+
aggregations,
|
|
100
|
+
memoryLimit: options.memoryLimit,
|
|
101
|
+
});
|
|
102
|
+
}
|
|
103
|
+
// 8. Sort
|
|
104
|
+
if (options.sort) {
|
|
105
|
+
const specs = parseSortSpecs(options.sort, {
|
|
106
|
+
nulls: options.nulls,
|
|
107
|
+
ignoreCase: options.ignoreCase,
|
|
108
|
+
natural: options.natural,
|
|
109
|
+
});
|
|
110
|
+
ops.push({
|
|
111
|
+
type: "sort",
|
|
112
|
+
specs,
|
|
113
|
+
nulls: options.nulls,
|
|
114
|
+
ignoreCase: options.ignoreCase,
|
|
115
|
+
natural: options.natural,
|
|
116
|
+
memoryLimit: options.memoryLimit,
|
|
117
|
+
});
|
|
118
|
+
}
|
|
119
|
+
// 9. Top
|
|
120
|
+
if (options.top !== undefined) {
|
|
121
|
+
const count = Number(options.top) || 10;
|
|
122
|
+
const sortSpecs = options.sort
|
|
123
|
+
? parseSortSpecs(options.sort)
|
|
124
|
+
: [{ column: "value", direction: "desc" }];
|
|
125
|
+
ops.push({
|
|
126
|
+
type: "top",
|
|
127
|
+
specs: sortSpecs,
|
|
128
|
+
count,
|
|
129
|
+
order: "desc",
|
|
130
|
+
smallest: false,
|
|
131
|
+
nulls: options.nulls,
|
|
132
|
+
ignoreCase: options.ignoreCase,
|
|
133
|
+
natural: options.natural,
|
|
134
|
+
});
|
|
135
|
+
}
|
|
136
|
+
// 10. Offset
|
|
137
|
+
if (options.offset !== undefined) {
|
|
138
|
+
const count = Number(options.offset) || 0;
|
|
139
|
+
if (count > 0) {
|
|
140
|
+
ops.push({ type: "offset", count });
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
// 11. Limit
|
|
144
|
+
if (options.limit !== undefined) {
|
|
145
|
+
const count = Number(options.limit) || 0;
|
|
146
|
+
ops.push({ type: "limit", count });
|
|
147
|
+
}
|
|
148
|
+
// 12. Tail
|
|
149
|
+
if (options.tail !== undefined) {
|
|
150
|
+
const count = Number(options.tail) || 10;
|
|
151
|
+
ops.push({ type: "tail", count });
|
|
152
|
+
}
|
|
153
|
+
return ops;
|
|
154
|
+
}
|
|
155
|
+
export async function unifiedPipelineCommand(inputPath = "-", options = {}) {
|
|
156
|
+
const operations = buildOperationsFromOptions(options);
|
|
157
|
+
const plan = optimizePipeline(operations);
|
|
158
|
+
const effectiveBatchSize = Number(options.batchSize) || 1000;
|
|
159
|
+
const progress = new ProgressReporter(options);
|
|
160
|
+
let fromFormat = options.from?.toLowerCase();
|
|
161
|
+
if (!fromFormat && inputPath !== "-") {
|
|
162
|
+
fromFormat = inferReaderFormat(inputPath) ?? undefined;
|
|
163
|
+
}
|
|
164
|
+
if (!fromFormat) {
|
|
165
|
+
fromFormat = "csv";
|
|
166
|
+
}
|
|
167
|
+
let toFormat = options.to?.toLowerCase();
|
|
168
|
+
if (options.json) {
|
|
169
|
+
toFormat = "json";
|
|
170
|
+
}
|
|
171
|
+
else if (!toFormat && options.output && options.output !== "-") {
|
|
172
|
+
toFormat = inferWriterFormat(options.output) ?? undefined;
|
|
173
|
+
}
|
|
174
|
+
if (!toFormat) {
|
|
175
|
+
toFormat = fromFormat === "xlsx" ? "csv" : fromFormat;
|
|
176
|
+
}
|
|
177
|
+
const inputStream = openReadableStream(inputPath);
|
|
178
|
+
const reader = createReader(inputStream, {
|
|
179
|
+
format: fromFormat,
|
|
180
|
+
sheet: options.sheet,
|
|
181
|
+
delimiter: options.delimiter,
|
|
182
|
+
batchSize: effectiveBatchSize,
|
|
183
|
+
maxRows: plan.effectiveReadLimit,
|
|
184
|
+
filePath: inputPath,
|
|
185
|
+
});
|
|
186
|
+
const effectiveOutput = options.output || "-";
|
|
187
|
+
const outputStream = openWritableStream(effectiveOutput);
|
|
188
|
+
const writer = createWriter(outputStream, {
|
|
189
|
+
format: toFormat,
|
|
190
|
+
delimiter: options.delimiter,
|
|
191
|
+
});
|
|
192
|
+
const pipeline = executePlannedPipeline(reader, plan, { batchSize: effectiveBatchSize });
|
|
193
|
+
pipeline.onProgress((info) => progress.update(info));
|
|
194
|
+
await pipeline.to(writer);
|
|
195
|
+
progress.done();
|
|
196
|
+
logMemoryDebug();
|
|
197
|
+
}
|
|
198
|
+
//# sourceMappingURL=pipeline.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"pipeline.js","sourceRoot":"","sources":["../../../src/cli/commands/pipeline.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,qBAAqB,EAAE,MAAM,2BAA2B,CAAC;AAClE,OAAO,EAAE,cAAc,EAAoB,MAAM,qCAAqC,CAAC;AACvF,OAAO,EAAE,sBAAsB,EAAE,gBAAgB,EAAsC,MAAM,wBAAwB,CAAC;AACtH,OAAO,EAAE,YAAY,EAAE,mBAAmB,IAAI,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAChG,OAAO,EAAE,YAAY,EAAE,mBAAmB,IAAI,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAChG,OAAO,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,MAAM,4BAA4B,CAAC;AACpF,OAAO,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAC3D,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAuC3D;;GAEG;AACH,MAAM,UAAU,0BAA0B,CAAC,OAAkC;IAC3E,MAAM,GAAG,GAAwB,EAAE,CAAC;IAEpC,YAAY;IACZ,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;QACnB,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAClF,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;YACxB,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC7B,GAAG,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,UAAU,EAAE,CAAC,EAAE,CAAC,CAAC;YAC9C,CAAC;QACH,CAAC;IACH,CAAC;IAED,SAAS;IACT,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC;QAChB,MAAM,IAAI,GAAG,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QACtE,MAAM,QAAQ,GAA2B,EAAE,CAAC;QAC5C,KAAK,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;YACrB,MAAM,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;YAC3B,IAAI,GAAG,GAAG,CAAC,EAAE,CAAC;gBACZ,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YAC7D,CAAC;QACH,CAAC;QACD,IAAI,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACrC,GAAG,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC,CAAC;QAC7C,CAAC;IACH,CAAC;IAED,YAAY;IACZ,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;QACnB,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAClF,MAAM,WAAW,GAA2B,EAAE,CAAC;QAC/C,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;YACxB,MAAM,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;YAC3B,IAAI,GAAG,GAAG,CAAC,EAAE,CAAC;gBACZ,WAAW,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YAChE,CAAC;QACH,CAAC;QACD,IAAI,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACxC,GAAG,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,WAAW,EAAE,CAAC,CAAC;QACnD,CAAC;IACH,CAAC;IAED,UAAU;IACV,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;QACjB,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QAC1E,MAAM,SAAS,GAA2B,EAAE,CAAC;QAC7C,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;YACtB,MAAM,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;YAC3B,IAAI,GAAG,GAAG,CAAC,EAAE,CAAC;gBACZ,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YAC9D,CAAC;QACH,CAAC;QACD,IAAI,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACtC,GAAG,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC,CAAC;QAC/C,CAAC;IACH,CAAC;IAED,YAAY;IACZ,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;QACnB,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC3F,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAC1D,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACpB,GAAG,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;QAC9C,CAAC;IACH,CAAC;IAED,YAAY;IACZ,IAAI,OAAO,CAAC,MAAM,KAAK,SAAS,IAAI,OAAO,CAAC,MAAM,KAAK,KAAK,EAAE,CAAC;QAC7D,MAAM,EAAE,GAAG,OAAO,OAAO,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QAC3H,GAAG,CAAC,IAAI,CAAC;YACP,IAAI,EAAE,QAAQ;YACd,EAAE;YACF,WAAW,EAAE,OAAO,CAAC,WAAW;SACjC,CAAC,CAAC;IACL,CAAC;IAED,6BAA6B;IAC7B,IAAI,OAAO,CAAC,OAAO,IAAI,OAAO,CAAC,KAAK,IAAI,OAAO,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG,IAAI,OAAO,CAAC,KAAK,IAAI,OAAO,CAAC,IAAI,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC;QACjJ,MAAM,EAAE,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QAC9J,MAAM,YAAY,GAAG,qBAAqB,CAAC;YACzC,EAAE;YACF,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,GAAG,EAAE,OAAO,CAAC,GAAG;YAChB,GAAG,EAAE,OAAO,CAAC,GAAG;YAChB,GAAG,EAAE,OAAO,CAAC,GAAG;YAChB,GAAG,EAAE,OAAO,CAAC,GAAG;YAChB,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,IAAI,EAAE,OAAO,CAAC,IAAI;YAClB,GAAG,EAAE,OAAO,CAAC,GAAG;SACjB,CAAC,CAAC;QACH,GAAG,CAAC,IAAI,CAAC;YACP,IAAI,EAAE,OAAO;YACb,EAAE;YACF,YAAY;YACZ,WAAW,EAAE,OAAO,CAAC,WAAW;SACjC,CAAC,CAAC;IACL,CAAC;IAED,UAAU;IACV,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;QACjB,MAAM,KAAK,GAAG,cAAc,CAAC,OAAO,CAAC,IAAI,EAAE;YACzC,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,UAAU,EAAE,OAAO,CAAC,UAAU;YAC9B,OAAO,EAAE,OAAO,CAAC,OAAO;SACzB,CAAC,CAAC;QACH,GAAG,CAAC,IAAI,CAAC;YACP,IAAI,EAAE,MAAM;YACZ,KAAK;YACL,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,UAAU,EAAE,OAAO,CAAC,UAAU;YAC9B,OAAO,EAAE,OAAO,CAAC,OAAO;YACxB,WAAW,EAAE,OAAO,CAAC,WAAW;SACjC,CAAC,CAAC;IACL,CAAC;IAED,SAAS;IACT,IAAI,OAAO,CAAC,GAAG,KAAK,SAAS,EAAE,CAAC;QAC9B,MAAM,KAAK,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;QACxC,MAAM,SAAS,GAAkB,OAAO,CAAC,IAAI;YAC3C,CAAC,CAAC,cAAc,CAAC,OAAO,CAAC,IAAI,CAAC;YAC9B,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC;QAC7C,GAAG,CAAC,IAAI,CAAC;YACP,IAAI,EAAE,KAAK;YACX,KAAK,EAAE,SAAS;YAChB,KAAK;YACL,KAAK,EAAE,MAAM;YACb,QAAQ,EAAE,KAAK;YACf,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,UAAU,EAAE,OAAO,CAAC,UAAU;YAC9B,OAAO,EAAE,OAAO,CAAC,OAAO;SACzB,CAAC,CAAC;IACL,CAAC;IAED,aAAa;IACb,IAAI,OAAO,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;QACjC,MAAM,KAAK,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAC1C,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;YACd,GAAG,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;QACtC,CAAC;IACH,CAAC;IAED,YAAY;IACZ,IAAI,OAAO,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;QAChC,MAAM,KAAK,GAAG,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACzC,GAAG,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;IACrC,CAAC;IAED,WAAW;IACX,IAAI,OAAO,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;QAC/B,MAAM,KAAK,GAAG,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;QACzC,GAAG,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;IACpC,CAAC;IAED,OAAO,GAAG,CAAC;AACb,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAC1C,SAAS,GAAG,GAAG,EACf,UAAqC,EAAE;IAEvC,MAAM,UAAU,GAAG,0BAA0B,CAAC,OAAO,CAAC,CAAC;IACvD,MAAM,IAAI,GAAG,gBAAgB,CAAC,UAAU,CAAC,CAAC;IAE1C,MAAM,kBAAkB,GAAG,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,IAAI,IAAI,CAAC;IAC7D,MAAM,QAAQ,GAAG,IAAI,gBAAgB,CAAC,OAAO,CAAC,CAAC;IAE/C,IAAI,UAAU,GAAG,OAAO,CAAC,IAAI,EAAE,WAAW,EAAE,CAAC;IAC7C,IAAI,CAAC,UAAU,IAAI,SAAS,KAAK,GAAG,EAAE,CAAC;QACrC,UAAU,GAAG,iBAAiB,CAAC,SAAS,CAAC,IAAI,SAAS,CAAC;IACzD,CAAC;IACD,IAAI,CAAC,UAAU,EAAE,CAAC;QAChB,UAAU,GAAG,KAAK,CAAC;IACrB,CAAC;IAED,IAAI,QAAQ,GAAG,OAAO,CAAC,EAAE,EAAE,WAAW,EAAE,CAAC;IACzC,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;QACjB,QAAQ,GAAG,MAAM,CAAC;IACpB,CAAC;SAAM,IAAI,CAAC,QAAQ,IAAI,OAAO,CAAC,MAAM,IAAI,OAAO,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;QACjE,QAAQ,GAAG,iBAAiB,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,SAAS,CAAC;IAC5D,CAAC;IACD,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,QAAQ,GAAG,UAAU,KAAK,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,UAAU,CAAC;IACxD,CAAC;IAED,MAAM,WAAW,GAAG,kBAAkB,CAAC,SAAS,CAAC,CAAC;IAClD,MAAM,MAAM,GAAG,YAAY,CAAC,WAAW,EAAE;QACvC,MAAM,EAAE,UAAU;QAClB,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,SAAS,EAAE,kBAAkB;QAC7B,OAAO,EAAE,IAAI,CAAC,kBAAkB;QAChC,QAAQ,EAAE,SAAS;KACpB,CAAC,CAAC;IAEH,MAAM,eAAe,GAAG,OAAO,CAAC,MAAM,IAAI,GAAG,CAAC;IAC9C,MAAM,YAAY,GAAG,kBAAkB,CAAC,eAAe,CAAC,CAAC;IACzD,MAAM,MAAM,GAAG,YAAY,CAAC,YAAY,EAAE;QACxC,MAAM,EAAE,QAAQ;QAChB,SAAS,EAAE,OAAO,CAAC,SAAS;KAC7B,CAAC,CAAC;IAEH,MAAM,QAAQ,GAAG,sBAAsB,CAAC,MAAM,EAAE,IAAI,EAAE,EAAE,SAAS,EAAE,kBAAkB,EAAE,CAAC,CAAC;IACzF,QAAQ,CAAC,UAAU,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;IAErD,MAAM,QAAQ,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC;IAC1B,QAAQ,CAAC,IAAI,EAAE,CAAC;IAChB,cAAc,EAAE,CAAC;AACnB,CAAC"}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
export interface SortCommandOptions {
|
|
2
|
+
by?: string | string[];
|
|
3
|
+
nulls?: "first" | "last";
|
|
4
|
+
ignoreCase?: boolean;
|
|
5
|
+
natural?: boolean;
|
|
6
|
+
memoryLimit?: string;
|
|
7
|
+
tempDir?: string;
|
|
8
|
+
from?: string;
|
|
9
|
+
to?: string;
|
|
10
|
+
sheet?: string;
|
|
11
|
+
delimiter?: string;
|
|
12
|
+
batchSize?: string | number;
|
|
13
|
+
quiet?: boolean;
|
|
14
|
+
noProgress?: boolean;
|
|
15
|
+
}
|
|
16
|
+
export declare function sortCommand(inputPath?: string, options?: SortCommandOptions): Promise<void>;
|
|
17
|
+
//# sourceMappingURL=sort.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"sort.d.ts","sourceRoot":"","sources":["../../../src/cli/commands/sort.ts"],"names":[],"mappings":"AASA,MAAM,WAAW,kBAAkB;IACjC,EAAE,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IACvB,KAAK,CAAC,EAAE,OAAO,GAAG,MAAM,CAAC;IACzB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IAC5B,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,UAAU,CAAC,EAAE,OAAO,CAAC;CACtB;AAED,wBAAsB,WAAW,CAC/B,SAAS,SAAM,EACf,OAAO,GAAE,kBAAuB,GAC/B,OAAO,CAAC,IAAI,CAAC,CAyDf"}
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
import { InvalidArgumentError } from "../../core/errors.js";
|
|
2
|
+
import { createPipeline } from "../../core/pipeline.js";
|
|
3
|
+
import { createReader, inferFormatFromPath as inferReaderFormat } from "../../readers/index.js";
|
|
4
|
+
import { createWriter } from "../../writers/index.js";
|
|
5
|
+
import { parseSortSpecs, sortRows } from "../../transforms/sort/index.js";
|
|
6
|
+
import { openReadableStream } from "../../utils/compression.js";
|
|
7
|
+
import { logMemoryDebug } from "../../utils/formatting.js";
|
|
8
|
+
import { ProgressReporter } from "../../utils/progress.js";
|
|
9
|
+
export async function sortCommand(inputPath = "-", options = {}) {
|
|
10
|
+
if (!options.by) {
|
|
11
|
+
throw new InvalidArgumentError("The --by option is required for sort (e.g. rowpipe sort users.csv --by age:desc)");
|
|
12
|
+
}
|
|
13
|
+
const effectiveBatchSize = Number(options.batchSize) || 1000;
|
|
14
|
+
const progress = new ProgressReporter(options);
|
|
15
|
+
const specs = parseSortSpecs(options.by, {
|
|
16
|
+
nulls: options.nulls,
|
|
17
|
+
ignoreCase: options.ignoreCase,
|
|
18
|
+
natural: options.natural,
|
|
19
|
+
});
|
|
20
|
+
let fromFormat = options.from?.toLowerCase();
|
|
21
|
+
if (!fromFormat && inputPath !== "-") {
|
|
22
|
+
fromFormat = inferReaderFormat(inputPath) ?? undefined;
|
|
23
|
+
}
|
|
24
|
+
if (!fromFormat) {
|
|
25
|
+
fromFormat = "csv";
|
|
26
|
+
}
|
|
27
|
+
const toFormat = options.to?.toLowerCase() || (fromFormat === "xlsx" ? "csv" : fromFormat);
|
|
28
|
+
const inputStream = openReadableStream(inputPath);
|
|
29
|
+
const reader = createReader(inputStream, {
|
|
30
|
+
format: fromFormat,
|
|
31
|
+
sheet: options.sheet,
|
|
32
|
+
delimiter: options.delimiter,
|
|
33
|
+
batchSize: effectiveBatchSize,
|
|
34
|
+
filePath: inputPath,
|
|
35
|
+
});
|
|
36
|
+
const writer = createWriter(process.stdout, {
|
|
37
|
+
format: toFormat,
|
|
38
|
+
delimiter: options.delimiter,
|
|
39
|
+
});
|
|
40
|
+
const pipeline = createPipeline(reader, { batchSize: effectiveBatchSize });
|
|
41
|
+
pipeline.pipe(sortRows({
|
|
42
|
+
by: specs,
|
|
43
|
+
nulls: options.nulls,
|
|
44
|
+
ignoreCase: options.ignoreCase,
|
|
45
|
+
natural: options.natural,
|
|
46
|
+
memoryLimit: options.memoryLimit,
|
|
47
|
+
tempDir: options.tempDir,
|
|
48
|
+
batchSize: effectiveBatchSize,
|
|
49
|
+
}));
|
|
50
|
+
pipeline.onProgress((info) => progress.update(info));
|
|
51
|
+
await pipeline.to(writer);
|
|
52
|
+
progress.done();
|
|
53
|
+
logMemoryDebug();
|
|
54
|
+
}
|
|
55
|
+
//# sourceMappingURL=sort.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"sort.js","sourceRoot":"","sources":["../../../src/cli/commands/sort.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,oBAAoB,EAAE,MAAM,sBAAsB,CAAC;AAC5D,OAAO,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAC;AACxD,OAAO,EAAE,YAAY,EAAE,mBAAmB,IAAI,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAChG,OAAO,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AACtD,OAAO,EAAE,cAAc,EAAE,QAAQ,EAAE,MAAM,gCAAgC,CAAC;AAC1E,OAAO,EAAE,kBAAkB,EAAE,MAAM,4BAA4B,CAAC;AAChE,OAAO,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAC3D,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAkB3D,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,SAAS,GAAG,GAAG,EACf,UAA8B,EAAE;IAEhC,IAAI,CAAC,OAAO,CAAC,EAAE,EAAE,CAAC;QAChB,MAAM,IAAI,oBAAoB,CAC5B,kFAAkF,CACnF,CAAC;IACJ,CAAC;IAED,MAAM,kBAAkB,GAAG,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,IAAI,IAAI,CAAC;IAC7D,MAAM,QAAQ,GAAG,IAAI,gBAAgB,CAAC,OAAO,CAAC,CAAC;IAE/C,MAAM,KAAK,GAAG,cAAc,CAAC,OAAO,CAAC,EAAE,EAAE;QACvC,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,UAAU,EAAE,OAAO,CAAC,UAAU;QAC9B,OAAO,EAAE,OAAO,CAAC,OAAO;KACzB,CAAC,CAAC;IAEH,IAAI,UAAU,GAAG,OAAO,CAAC,IAAI,EAAE,WAAW,EAAE,CAAC;IAC7C,IAAI,CAAC,UAAU,IAAI,SAAS,KAAK,GAAG,EAAE,CAAC;QACrC,UAAU,GAAG,iBAAiB,CAAC,SAAS,CAAC,IAAI,SAAS,CAAC;IACzD,CAAC;IACD,IAAI,CAAC,UAAU,EAAE,CAAC;QAChB,UAAU,GAAG,KAAK,CAAC;IACrB,CAAC;IAED,MAAM,QAAQ,GAAG,OAAO,CAAC,EAAE,EAAE,WAAW,EAAE,IAAI,CAAC,UAAU,KAAK,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC;IAE3F,MAAM,WAAW,GAAG,kBAAkB,CAAC,SAAS,CAAC,CAAC;IAClD,MAAM,MAAM,GAAG,YAAY,CAAC,WAAW,EAAE;QACvC,MAAM,EAAE,UAAU;QAClB,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,SAAS,EAAE,kBAAkB;QAC7B,QAAQ,EAAE,SAAS;KACpB,CAAC,CAAC;IAEH,MAAM,MAAM,GAAG,YAAY,CAAC,OAAO,CAAC,MAAM,EAAE;QAC1C,MAAM,EAAE,QAAQ;QAChB,SAAS,EAAE,OAAO,CAAC,SAAS;KAC7B,CAAC,CAAC;IAEH,MAAM,QAAQ,GAAG,cAAc,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,kBAAkB,EAAE,CAAC,CAAC;IAC3E,QAAQ,CAAC,IAAI,CACX,QAAQ,CAAC;QACP,EAAE,EAAE,KAAK;QACT,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,UAAU,EAAE,OAAO,CAAC,UAAU;QAC9B,OAAO,EAAE,OAAO,CAAC,OAAO;QACxB,WAAW,EAAE,OAAO,CAAC,WAAW;QAChC,OAAO,EAAE,OAAO,CAAC,OAAO;QACxB,SAAS,EAAE,kBAAkB;KAC9B,CAAC,CACH,CAAC;IACF,QAAQ,CAAC,UAAU,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;IAErD,MAAM,QAAQ,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC;IAC1B,QAAQ,CAAC,IAAI,EAAE,CAAC;IAChB,cAAc,EAAE,CAAC;AACnB,CAAC"}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
export interface TailCommandOptions {
|
|
2
|
+
lines?: string | number;
|
|
3
|
+
n?: string | number;
|
|
4
|
+
from?: string;
|
|
5
|
+
to?: string;
|
|
6
|
+
sheet?: string;
|
|
7
|
+
delimiter?: string;
|
|
8
|
+
batchSize?: string | number;
|
|
9
|
+
quiet?: boolean;
|
|
10
|
+
noProgress?: boolean;
|
|
11
|
+
}
|
|
12
|
+
export declare function tailCommand(inputPath?: string, options?: TailCommandOptions): Promise<void>;
|
|
13
|
+
//# sourceMappingURL=tail.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"tail.d.ts","sourceRoot":"","sources":["../../../src/cli/commands/tail.ts"],"names":[],"mappings":"AAQA,MAAM,WAAW,kBAAkB;IACjC,KAAK,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IACxB,CAAC,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IACpB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IAC5B,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,UAAU,CAAC,EAAE,OAAO,CAAC;CACtB;AAED,wBAAsB,WAAW,CAC/B,SAAS,SAAM,EACf,OAAO,GAAE,kBAAuB,GAC/B,OAAO,CAAC,IAAI,CAAC,CAoCf"}
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import { createPipeline } from "../../core/pipeline.js";
|
|
2
|
+
import { createReader, inferFormatFromPath as inferReaderFormat } from "../../readers/index.js";
|
|
3
|
+
import { createWriter } from "../../writers/index.js";
|
|
4
|
+
import { tailRows } from "../../transforms/tail.js";
|
|
5
|
+
import { openReadableStream } from "../../utils/compression.js";
|
|
6
|
+
import { logMemoryDebug } from "../../utils/formatting.js";
|
|
7
|
+
import { ProgressReporter } from "../../utils/progress.js";
|
|
8
|
+
export async function tailCommand(inputPath = "-", options = {}) {
|
|
9
|
+
const count = Number(options.n ?? options.lines ?? 10) || 10;
|
|
10
|
+
const effectiveBatchSize = Number(options.batchSize) || 1000;
|
|
11
|
+
const progress = new ProgressReporter(options);
|
|
12
|
+
let fromFormat = options.from?.toLowerCase();
|
|
13
|
+
if (!fromFormat && inputPath !== "-") {
|
|
14
|
+
fromFormat = inferReaderFormat(inputPath) ?? undefined;
|
|
15
|
+
}
|
|
16
|
+
if (!fromFormat) {
|
|
17
|
+
fromFormat = "csv";
|
|
18
|
+
}
|
|
19
|
+
const toFormat = options.to?.toLowerCase() || (fromFormat === "xlsx" ? "csv" : fromFormat);
|
|
20
|
+
const inputStream = openReadableStream(inputPath);
|
|
21
|
+
const reader = createReader(inputStream, {
|
|
22
|
+
format: fromFormat,
|
|
23
|
+
sheet: options.sheet,
|
|
24
|
+
delimiter: options.delimiter,
|
|
25
|
+
batchSize: effectiveBatchSize,
|
|
26
|
+
filePath: inputPath,
|
|
27
|
+
});
|
|
28
|
+
const writer = createWriter(process.stdout, {
|
|
29
|
+
format: toFormat,
|
|
30
|
+
delimiter: options.delimiter,
|
|
31
|
+
});
|
|
32
|
+
const pipeline = createPipeline(reader, { batchSize: effectiveBatchSize });
|
|
33
|
+
pipeline.pipe(tailRows(count));
|
|
34
|
+
pipeline.onProgress((info) => progress.update(info));
|
|
35
|
+
await pipeline.to(writer);
|
|
36
|
+
progress.done();
|
|
37
|
+
logMemoryDebug();
|
|
38
|
+
}
|
|
39
|
+
//# sourceMappingURL=tail.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"tail.js","sourceRoot":"","sources":["../../../src/cli/commands/tail.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAC;AACxD,OAAO,EAAE,YAAY,EAAE,mBAAmB,IAAI,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAChG,OAAO,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AACtD,OAAO,EAAE,QAAQ,EAAE,MAAM,0BAA0B,CAAC;AACpD,OAAO,EAAE,kBAAkB,EAAE,MAAM,4BAA4B,CAAC;AAChE,OAAO,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAC3D,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAc3D,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,SAAS,GAAG,GAAG,EACf,UAA8B,EAAE;IAEhC,MAAM,KAAK,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,OAAO,CAAC,KAAK,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;IAC7D,MAAM,kBAAkB,GAAG,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,IAAI,IAAI,CAAC;IAC7D,MAAM,QAAQ,GAAG,IAAI,gBAAgB,CAAC,OAAO,CAAC,CAAC;IAE/C,IAAI,UAAU,GAAG,OAAO,CAAC,IAAI,EAAE,WAAW,EAAE,CAAC;IAC7C,IAAI,CAAC,UAAU,IAAI,SAAS,KAAK,GAAG,EAAE,CAAC;QACrC,UAAU,GAAG,iBAAiB,CAAC,SAAS,CAAC,IAAI,SAAS,CAAC;IACzD,CAAC;IACD,IAAI,CAAC,UAAU,EAAE,CAAC;QAChB,UAAU,GAAG,KAAK,CAAC;IACrB,CAAC;IAED,MAAM,QAAQ,GAAG,OAAO,CAAC,EAAE,EAAE,WAAW,EAAE,IAAI,CAAC,UAAU,KAAK,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC;IAE3F,MAAM,WAAW,GAAG,kBAAkB,CAAC,SAAS,CAAC,CAAC;IAClD,MAAM,MAAM,GAAG,YAAY,CAAC,WAAW,EAAE;QACvC,MAAM,EAAE,UAAU;QAClB,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,SAAS,EAAE,kBAAkB;QAC7B,QAAQ,EAAE,SAAS;KACpB,CAAC,CAAC;IAEH,MAAM,MAAM,GAAG,YAAY,CAAC,OAAO,CAAC,MAAM,EAAE;QAC1C,MAAM,EAAE,QAAQ;QAChB,SAAS,EAAE,OAAO,CAAC,SAAS;KAC7B,CAAC,CAAC;IAEH,MAAM,QAAQ,GAAG,cAAc,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,kBAAkB,EAAE,CAAC,CAAC;IAC3E,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC;IAC/B,QAAQ,CAAC,UAAU,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;IAErD,MAAM,QAAQ,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC;IAC1B,QAAQ,CAAC,IAAI,EAAE,CAAC;IAChB,cAAc,EAAE,CAAC;AACnB,CAAC"}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
export interface TopCommandOptions {
|
|
2
|
+
by?: string | string[];
|
|
3
|
+
lines?: string | number;
|
|
4
|
+
n?: string | number;
|
|
5
|
+
order?: "asc" | "desc";
|
|
6
|
+
smallest?: boolean;
|
|
7
|
+
nulls?: "first" | "last";
|
|
8
|
+
ignoreCase?: boolean;
|
|
9
|
+
natural?: boolean;
|
|
10
|
+
from?: string;
|
|
11
|
+
to?: string;
|
|
12
|
+
sheet?: string;
|
|
13
|
+
delimiter?: string;
|
|
14
|
+
batchSize?: string | number;
|
|
15
|
+
quiet?: boolean;
|
|
16
|
+
noProgress?: boolean;
|
|
17
|
+
}
|
|
18
|
+
export declare function topCommand(inputPath?: string, options?: TopCommandOptions): Promise<void>;
|
|
19
|
+
//# sourceMappingURL=top.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"top.d.ts","sourceRoot":"","sources":["../../../src/cli/commands/top.ts"],"names":[],"mappings":"AASA,MAAM,WAAW,iBAAiB;IAChC,EAAE,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IACvB,KAAK,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IACxB,CAAC,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IACpB,KAAK,CAAC,EAAE,KAAK,GAAG,MAAM,CAAC;IACvB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,KAAK,CAAC,EAAE,OAAO,GAAG,MAAM,CAAC;IACzB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IAC5B,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,UAAU,CAAC,EAAE,OAAO,CAAC;CACtB;AAED,wBAAsB,UAAU,CAC9B,SAAS,SAAM,EACf,OAAO,GAAE,iBAAsB,GAC9B,OAAO,CAAC,IAAI,CAAC,CAoDf"}
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
import { InvalidArgumentError } from "../../core/errors.js";
|
|
2
|
+
import { createPipeline } from "../../core/pipeline.js";
|
|
3
|
+
import { createReader, inferFormatFromPath as inferReaderFormat } from "../../readers/index.js";
|
|
4
|
+
import { createWriter } from "../../writers/index.js";
|
|
5
|
+
import { topRows } from "../../transforms/top.js";
|
|
6
|
+
import { openReadableStream } from "../../utils/compression.js";
|
|
7
|
+
import { logMemoryDebug } from "../../utils/formatting.js";
|
|
8
|
+
import { ProgressReporter } from "../../utils/progress.js";
|
|
9
|
+
export async function topCommand(inputPath = "-", options = {}) {
|
|
10
|
+
if (!options.by) {
|
|
11
|
+
throw new InvalidArgumentError("The --by option is required for top (e.g. rowpipe top sales.csv --by revenue -n 10)");
|
|
12
|
+
}
|
|
13
|
+
const count = Number(options.n ?? options.lines ?? 10) || 10;
|
|
14
|
+
const effectiveBatchSize = Number(options.batchSize) || 1000;
|
|
15
|
+
const progress = new ProgressReporter(options);
|
|
16
|
+
let fromFormat = options.from?.toLowerCase();
|
|
17
|
+
if (!fromFormat && inputPath !== "-") {
|
|
18
|
+
fromFormat = inferReaderFormat(inputPath) ?? undefined;
|
|
19
|
+
}
|
|
20
|
+
if (!fromFormat) {
|
|
21
|
+
fromFormat = "csv";
|
|
22
|
+
}
|
|
23
|
+
const toFormat = options.to?.toLowerCase() || (fromFormat === "xlsx" ? "csv" : fromFormat);
|
|
24
|
+
const inputStream = openReadableStream(inputPath);
|
|
25
|
+
const reader = createReader(inputStream, {
|
|
26
|
+
format: fromFormat,
|
|
27
|
+
sheet: options.sheet,
|
|
28
|
+
delimiter: options.delimiter,
|
|
29
|
+
batchSize: effectiveBatchSize,
|
|
30
|
+
filePath: inputPath,
|
|
31
|
+
});
|
|
32
|
+
const writer = createWriter(process.stdout, {
|
|
33
|
+
format: toFormat,
|
|
34
|
+
delimiter: options.delimiter,
|
|
35
|
+
});
|
|
36
|
+
const pipeline = createPipeline(reader, { batchSize: effectiveBatchSize });
|
|
37
|
+
pipeline.pipe(topRows({
|
|
38
|
+
by: options.by,
|
|
39
|
+
count,
|
|
40
|
+
order: options.order,
|
|
41
|
+
smallest: options.smallest,
|
|
42
|
+
nulls: options.nulls,
|
|
43
|
+
ignoreCase: options.ignoreCase,
|
|
44
|
+
natural: options.natural,
|
|
45
|
+
}));
|
|
46
|
+
pipeline.onProgress((info) => progress.update(info));
|
|
47
|
+
await pipeline.to(writer);
|
|
48
|
+
progress.done();
|
|
49
|
+
logMemoryDebug();
|
|
50
|
+
}
|
|
51
|
+
//# sourceMappingURL=top.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"top.js","sourceRoot":"","sources":["../../../src/cli/commands/top.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,oBAAoB,EAAE,MAAM,sBAAsB,CAAC;AAC5D,OAAO,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAC;AACxD,OAAO,EAAE,YAAY,EAAE,mBAAmB,IAAI,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAChG,OAAO,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AACtD,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAClD,OAAO,EAAE,kBAAkB,EAAE,MAAM,4BAA4B,CAAC;AAChE,OAAO,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAC3D,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAoB3D,MAAM,CAAC,KAAK,UAAU,UAAU,CAC9B,SAAS,GAAG,GAAG,EACf,UAA6B,EAAE;IAE/B,IAAI,CAAC,OAAO,CAAC,EAAE,EAAE,CAAC;QAChB,MAAM,IAAI,oBAAoB,CAC5B,qFAAqF,CACtF,CAAC;IACJ,CAAC;IAED,MAAM,KAAK,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,OAAO,CAAC,KAAK,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;IAC7D,MAAM,kBAAkB,GAAG,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,IAAI,IAAI,CAAC;IAC7D,MAAM,QAAQ,GAAG,IAAI,gBAAgB,CAAC,OAAO,CAAC,CAAC;IAE/C,IAAI,UAAU,GAAG,OAAO,CAAC,IAAI,EAAE,WAAW,EAAE,CAAC;IAC7C,IAAI,CAAC,UAAU,IAAI,SAAS,KAAK,GAAG,EAAE,CAAC;QACrC,UAAU,GAAG,iBAAiB,CAAC,SAAS,CAAC,IAAI,SAAS,CAAC;IACzD,CAAC;IACD,IAAI,CAAC,UAAU,EAAE,CAAC;QAChB,UAAU,GAAG,KAAK,CAAC;IACrB,CAAC;IAED,MAAM,QAAQ,GAAG,OAAO,CAAC,EAAE,EAAE,WAAW,EAAE,IAAI,CAAC,UAAU,KAAK,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC;IAE3F,MAAM,WAAW,GAAG,kBAAkB,CAAC,SAAS,CAAC,CAAC;IAClD,MAAM,MAAM,GAAG,YAAY,CAAC,WAAW,EAAE;QACvC,MAAM,EAAE,UAAU;QAClB,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,SAAS,EAAE,kBAAkB;QAC7B,QAAQ,EAAE,SAAS;KACpB,CAAC,CAAC;IAEH,MAAM,MAAM,GAAG,YAAY,CAAC,OAAO,CAAC,MAAM,EAAE;QAC1C,MAAM,EAAE,QAAQ;QAChB,SAAS,EAAE,OAAO,CAAC,SAAS;KAC7B,CAAC,CAAC;IAEH,MAAM,QAAQ,GAAG,cAAc,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,kBAAkB,EAAE,CAAC,CAAC;IAC3E,QAAQ,CAAC,IAAI,CACX,OAAO,CAAC;QACN,EAAE,EAAE,OAAO,CAAC,EAAE;QACd,KAAK;QACL,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,QAAQ,EAAE,OAAO,CAAC,QAAQ;QAC1B,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,UAAU,EAAE,OAAO,CAAC,UAAU;QAC9B,OAAO,EAAE,OAAO,CAAC,OAAO;KACzB,CAAC,CACH,CAAC;IACF,QAAQ,CAAC,UAAU,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;IAErD,MAAM,QAAQ,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC;IAC1B,QAAQ,CAAC,IAAI,EAAE,CAAC;IAChB,cAAc,EAAE,CAAC;AACnB,CAAC"}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
export interface UniqueCommandOptions {
|
|
2
|
+
by?: string | string[];
|
|
3
|
+
keep?: "first" | "last";
|
|
4
|
+
memoryLimit?: string;
|
|
5
|
+
tempDir?: string;
|
|
6
|
+
from?: string;
|
|
7
|
+
to?: string;
|
|
8
|
+
sheet?: string;
|
|
9
|
+
delimiter?: string;
|
|
10
|
+
batchSize?: string | number;
|
|
11
|
+
quiet?: boolean;
|
|
12
|
+
noProgress?: boolean;
|
|
13
|
+
}
|
|
14
|
+
export declare function uniqueCommand(inputPath?: string, options?: UniqueCommandOptions): Promise<void>;
|
|
15
|
+
//# sourceMappingURL=unique.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"unique.d.ts","sourceRoot":"","sources":["../../../src/cli/commands/unique.ts"],"names":[],"mappings":"AAQA,MAAM,WAAW,oBAAoB;IACnC,EAAE,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IACvB,IAAI,CAAC,EAAE,OAAO,GAAG,MAAM,CAAC;IACxB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IAC5B,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,UAAU,CAAC,EAAE,OAAO,CAAC;CACtB;AAED,wBAAsB,aAAa,CACjC,SAAS,SAAM,EACf,OAAO,GAAE,oBAAyB,GACjC,OAAO,CAAC,IAAI,CAAC,CA2Cf"}
|