rowpipe 2.1.0 → 2.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +183 -5
- package/dist/analytics/profiler.d.ts +56 -0
- package/dist/analytics/profiler.d.ts.map +1 -0
- package/dist/analytics/profiler.js +277 -0
- package/dist/analytics/profiler.js.map +1 -0
- package/dist/analytics/schema-inference.d.ts.map +1 -1
- package/dist/analytics/schema-inference.js +14 -0
- package/dist/analytics/schema-inference.js.map +1 -1
- package/dist/cli/commands/clean.d.ts +15 -0
- package/dist/cli/commands/clean.d.ts.map +1 -0
- package/dist/cli/commands/clean.js +108 -0
- package/dist/cli/commands/clean.js.map +1 -0
- package/dist/cli/commands/convert.d.ts +6 -0
- package/dist/cli/commands/convert.d.ts.map +1 -1
- package/dist/cli/commands/convert.js +6 -6
- package/dist/cli/commands/convert.js.map +1 -1
- package/dist/cli/commands/db.d.ts +35 -0
- package/dist/cli/commands/db.d.ts.map +1 -0
- package/dist/cli/commands/db.js +173 -0
- package/dist/cli/commands/db.js.map +1 -0
- package/dist/cli/commands/explain.d.ts +8 -1
- package/dist/cli/commands/explain.d.ts.map +1 -1
- package/dist/cli/commands/explain.js +89 -2
- package/dist/cli/commands/explain.js.map +1 -1
- package/dist/cli/commands/join.d.ts +23 -0
- package/dist/cli/commands/join.d.ts.map +1 -0
- package/dist/cli/commands/join.js +97 -0
- package/dist/cli/commands/join.js.map +1 -0
- package/dist/cli/commands/pipeline.d.ts +20 -0
- package/dist/cli/commands/pipeline.d.ts.map +1 -1
- package/dist/cli/commands/pipeline.js +53 -12
- package/dist/cli/commands/pipeline.js.map +1 -1
- package/dist/cli/commands/profile.d.ts +22 -0
- package/dist/cli/commands/profile.d.ts.map +1 -0
- package/dist/cli/commands/profile.js +53 -0
- package/dist/cli/commands/profile.js.map +1 -0
- package/dist/cli/commands/view.d.ts +12 -0
- package/dist/cli/commands/view.d.ts.map +1 -0
- package/dist/cli/commands/view.js +52 -0
- package/dist/cli/commands/view.js.map +1 -0
- package/dist/cli/commands/window.d.ts +10 -0
- package/dist/cli/commands/window.d.ts.map +1 -0
- package/dist/cli/commands/window.js +87 -0
- package/dist/cli/commands/window.js.map +1 -0
- package/dist/cli/index.js +201 -5
- package/dist/cli/index.js.map +1 -1
- package/dist/core/batch.d.ts +1 -1
- package/dist/core/batch.d.ts.map +1 -1
- package/dist/core/batch.js.map +1 -1
- package/dist/core/pipeline.d.ts +2 -2
- package/dist/core/pipeline.d.ts.map +1 -1
- package/dist/core/pipeline.js +5 -2
- package/dist/core/pipeline.js.map +1 -1
- package/dist/core/types.d.ts +21 -1
- package/dist/core/types.d.ts.map +1 -1
- package/dist/db/adapters/index.d.ts +6 -0
- package/dist/db/adapters/index.d.ts.map +1 -0
- package/dist/db/adapters/index.js +21 -0
- package/dist/db/adapters/index.js.map +1 -0
- package/dist/db/adapters/mysql.d.ts +16 -0
- package/dist/db/adapters/mysql.d.ts.map +1 -0
- package/dist/db/adapters/mysql.js +230 -0
- package/dist/db/adapters/mysql.js.map +1 -0
- package/dist/db/adapters/postgres.d.ts +17 -0
- package/dist/db/adapters/postgres.d.ts.map +1 -0
- package/dist/db/adapters/postgres.js +239 -0
- package/dist/db/adapters/postgres.js.map +1 -0
- package/dist/db/adapters/sqlite.d.ts +19 -0
- package/dist/db/adapters/sqlite.d.ts.map +1 -0
- package/dist/db/adapters/sqlite.js +241 -0
- package/dist/db/adapters/sqlite.js.map +1 -0
- package/dist/db/index.d.ts +11 -0
- package/dist/db/index.d.ts.map +1 -0
- package/dist/db/index.js +11 -0
- package/dist/db/index.js.map +1 -0
- package/dist/db/mapping.d.ts +45 -0
- package/dist/db/mapping.d.ts.map +1 -0
- package/dist/db/mapping.js +222 -0
- package/dist/db/mapping.js.map +1 -0
- package/dist/db/pushdown.d.ts +25 -0
- package/dist/db/pushdown.d.ts.map +1 -0
- package/dist/db/pushdown.js +165 -0
- package/dist/db/pushdown.js.map +1 -0
- package/dist/db/sink.d.ts +15 -0
- package/dist/db/sink.d.ts.map +1 -0
- package/dist/db/sink.js +47 -0
- package/dist/db/sink.js.map +1 -0
- package/dist/db/source.d.ts +14 -0
- package/dist/db/source.d.ts.map +1 -0
- package/dist/db/source.js +69 -0
- package/dist/db/source.js.map +1 -0
- package/dist/db/types.d.ts +80 -0
- package/dist/db/types.d.ts.map +1 -0
- package/dist/db/types.js +2 -0
- package/dist/db/types.js.map +1 -0
- package/dist/db/url.d.ts +15 -0
- package/dist/db/url.d.ts.map +1 -0
- package/dist/db/url.js +145 -0
- package/dist/db/url.js.map +1 -0
- package/dist/diff/hash.d.ts.map +1 -1
- package/dist/diff/hash.js +4 -3
- package/dist/diff/hash.js.map +1 -1
- package/dist/diff/key.d.ts.map +1 -1
- package/dist/diff/key.js +8 -1
- package/dist/diff/key.js.map +1 -1
- package/dist/diff/reporter.d.ts.map +1 -1
- package/dist/diff/reporter.js +5 -3
- package/dist/diff/reporter.js.map +1 -1
- package/dist/diff/storage/disk-index.d.ts.map +1 -1
- package/dist/diff/storage/disk-index.js +2 -1
- package/dist/diff/storage/disk-index.js.map +1 -1
- package/dist/diff/storage/memory-index.d.ts.map +1 -1
- package/dist/diff/storage/memory-index.js +3 -2
- package/dist/diff/storage/memory-index.js.map +1 -1
- package/dist/index.d.ts +8 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +9 -1
- package/dist/index.js.map +1 -1
- package/dist/planner/executor.d.ts.map +1 -1
- package/dist/planner/executor.js +7 -0
- package/dist/planner/executor.js.map +1 -1
- package/dist/planner/types.d.ts +4 -0
- package/dist/planner/types.d.ts.map +1 -1
- package/dist/readers/csv.d.ts.map +1 -1
- package/dist/readers/csv.js +1 -2
- package/dist/readers/csv.js.map +1 -1
- package/dist/readers/index.d.ts +5 -1
- package/dist/readers/index.d.ts.map +1 -1
- package/dist/readers/index.js +43 -0
- package/dist/readers/index.js.map +1 -1
- package/dist/readers/json.d.ts.map +1 -1
- package/dist/readers/json.js +1 -2
- package/dist/readers/json.js.map +1 -1
- package/dist/readers/jsonl.d.ts.map +1 -1
- package/dist/readers/jsonl.js +1 -2
- package/dist/readers/jsonl.js.map +1 -1
- package/dist/readers/multi-file.d.ts +29 -0
- package/dist/readers/multi-file.d.ts.map +1 -0
- package/dist/readers/multi-file.js +169 -0
- package/dist/readers/multi-file.js.map +1 -0
- package/dist/transforms/clean.d.ts +18 -0
- package/dist/transforms/clean.d.ts.map +1 -0
- package/dist/transforms/clean.js +118 -0
- package/dist/transforms/clean.js.map +1 -0
- package/dist/transforms/expression.d.ts.map +1 -1
- package/dist/transforms/expression.js +12 -0
- package/dist/transforms/expression.js.map +1 -1
- package/dist/transforms/jit-compiler.d.ts +50 -0
- package/dist/transforms/jit-compiler.d.ts.map +1 -0
- package/dist/transforms/jit-compiler.js +275 -0
- package/dist/transforms/jit-compiler.js.map +1 -0
- package/dist/transforms/join/index-storage.d.ts +36 -0
- package/dist/transforms/join/index-storage.d.ts.map +1 -0
- package/dist/transforms/join/index-storage.js +176 -0
- package/dist/transforms/join/index-storage.js.map +1 -0
- package/dist/transforms/join/index.d.ts +4 -0
- package/dist/transforms/join/index.d.ts.map +1 -0
- package/dist/transforms/join/index.js +4 -0
- package/dist/transforms/join/index.js.map +1 -0
- package/dist/transforms/join/join.d.ts +15 -0
- package/dist/transforms/join/join.d.ts.map +1 -0
- package/dist/transforms/join/join.js +216 -0
- package/dist/transforms/join/join.js.map +1 -0
- package/dist/transforms/join/types.d.ts +29 -0
- package/dist/transforms/join/types.d.ts.map +1 -0
- package/dist/transforms/join/types.js +2 -0
- package/dist/transforms/join/types.js.map +1 -0
- package/dist/transforms/window.d.ts +24 -0
- package/dist/transforms/window.d.ts.map +1 -0
- package/dist/transforms/window.js +317 -0
- package/dist/transforms/window.js.map +1 -0
- package/dist/ui/colors.d.ts +57 -0
- package/dist/ui/colors.d.ts.map +1 -0
- package/dist/ui/colors.js +94 -0
- package/dist/ui/colors.js.map +1 -0
- package/dist/ui/viewer.d.ts +67 -0
- package/dist/ui/viewer.d.ts.map +1 -0
- package/dist/ui/viewer.js +673 -0
- package/dist/ui/viewer.js.map +1 -0
- package/dist/utils/compression.d.ts +68 -19
- package/dist/utils/compression.d.ts.map +1 -1
- package/dist/utils/compression.js +222 -34
- package/dist/utils/compression.js.map +1 -1
- package/dist/utils/formatting.d.ts +5 -0
- package/dist/utils/formatting.d.ts.map +1 -1
- package/dist/utils/formatting.js +15 -0
- package/dist/utils/formatting.js.map +1 -1
- package/dist/writers/csv.d.ts.map +1 -1
- package/dist/writers/csv.js +13 -14
- package/dist/writers/csv.js.map +1 -1
- package/dist/writers/index.d.ts +1 -0
- package/dist/writers/index.d.ts.map +1 -1
- package/dist/writers/index.js +35 -0
- package/dist/writers/index.js.map +1 -1
- package/dist/writers/json.d.ts.map +1 -1
- package/dist/writers/json.js +6 -5
- package/dist/writers/json.js.map +1 -1
- package/dist/writers/jsonl.d.ts.map +1 -1
- package/dist/writers/jsonl.js +6 -5
- package/dist/writers/jsonl.js.map +1 -1
- package/dist/writers/markdown.d.ts.map +1 -1
- package/dist/writers/markdown.js +5 -4
- package/dist/writers/markdown.js.map +1 -1
- package/dist/writers/parquet.d.ts +1 -1
- package/dist/writers/parquet.d.ts.map +1 -1
- package/package.json +10 -2
- package/skills/rowpipe/SKILL.md +8 -4
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
import { cleanRows } from "../../transforms/clean.js";
|
|
2
|
+
import { uniqueRows } from "../../transforms/unique.js";
|
|
3
|
+
import { createReader, inferFormatFromPath as inferReaderFormat } from "../../readers/index.js";
|
|
4
|
+
import { createWriter, inferFormatFromPath as inferWriterFormat } from "../../writers/index.js";
|
|
5
|
+
import { openReadableStream } from "../../utils/compression.js";
|
|
6
|
+
import { logMemoryDebug } from "../../utils/formatting.js";
|
|
7
|
+
import { ProgressReporter } from "../../utils/progress.js";
|
|
8
|
+
import { executePlannedPipeline, optimizePipeline } from "../../planner/index.js";
|
|
9
|
+
import { buildOperationsFromOptions } from "./pipeline.js";
|
|
10
|
+
/**
|
|
11
|
+
* Handles `rowpipe clean [input] [options]`
|
|
12
|
+
*/
|
|
13
|
+
export async function cleanCommand(inputPath = "-", options = {}) {
|
|
14
|
+
const progress = new ProgressReporter(options);
|
|
15
|
+
const effectiveBatchSize = Number(options.batchSize) || 1000;
|
|
16
|
+
let fromFormat = options.from?.toLowerCase();
|
|
17
|
+
if (!fromFormat && inputPath !== "-") {
|
|
18
|
+
fromFormat = inferReaderFormat(inputPath) ?? undefined;
|
|
19
|
+
}
|
|
20
|
+
if (!fromFormat) {
|
|
21
|
+
fromFormat = "csv";
|
|
22
|
+
}
|
|
23
|
+
let toFormat = options.to?.toLowerCase();
|
|
24
|
+
if (options.json) {
|
|
25
|
+
toFormat = "json";
|
|
26
|
+
}
|
|
27
|
+
else if (!toFormat && options.output && options.output !== "-") {
|
|
28
|
+
toFormat = inferWriterFormat(options.output) ?? undefined;
|
|
29
|
+
}
|
|
30
|
+
if (!toFormat) {
|
|
31
|
+
toFormat = fromFormat === "xlsx" ? "csv" : fromFormat;
|
|
32
|
+
}
|
|
33
|
+
const readerInput = inputPath === "-" ? openReadableStream("-", options) : inputPath;
|
|
34
|
+
const reader = createReader(readerInput, {
|
|
35
|
+
format: fromFormat,
|
|
36
|
+
sheet: options.sheet,
|
|
37
|
+
delimiter: options.delimiter,
|
|
38
|
+
filePath: inputPath,
|
|
39
|
+
...options,
|
|
40
|
+
batchSize: effectiveBatchSize,
|
|
41
|
+
});
|
|
42
|
+
// Parse fill-nulls specs (e.g. --fill-nulls 'revenue=0' or --fill-nulls '0')
|
|
43
|
+
let fillNullsParsed = undefined;
|
|
44
|
+
if (options.fillNulls) {
|
|
45
|
+
const rawList = Array.isArray(options.fillNulls) ? options.fillNulls : [options.fillNulls];
|
|
46
|
+
const fillMap = {};
|
|
47
|
+
let isGlobal = false;
|
|
48
|
+
let globalVal = undefined;
|
|
49
|
+
for (const item of rawList) {
|
|
50
|
+
const eqIdx = item.indexOf("=");
|
|
51
|
+
if (eqIdx > 0) {
|
|
52
|
+
const col = item.slice(0, eqIdx).trim();
|
|
53
|
+
const valStr = item.slice(eqIdx + 1).trim();
|
|
54
|
+
try {
|
|
55
|
+
fillMap[col] = JSON.parse(valStr);
|
|
56
|
+
}
|
|
57
|
+
catch {
|
|
58
|
+
fillMap[col] = valStr;
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
else {
|
|
62
|
+
isGlobal = true;
|
|
63
|
+
try {
|
|
64
|
+
globalVal = JSON.parse(item);
|
|
65
|
+
}
|
|
66
|
+
catch {
|
|
67
|
+
globalVal = item;
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
fillNullsParsed = isGlobal ? globalVal : fillMap;
|
|
72
|
+
}
|
|
73
|
+
const nullVals = options.nullValues ? options.nullValues.split(",").map((s) => s.trim()) : undefined;
|
|
74
|
+
const cleanTransform = cleanRows({
|
|
75
|
+
trim: options.trim !== false,
|
|
76
|
+
nullValues: nullVals,
|
|
77
|
+
fillNulls: fillNullsParsed,
|
|
78
|
+
coerce: options.coerce === true,
|
|
79
|
+
stripChars: options.stripChars,
|
|
80
|
+
case: options.case,
|
|
81
|
+
dropEmptyRows: options.dropEmptyRows,
|
|
82
|
+
});
|
|
83
|
+
const rawOps = buildOperationsFromOptions(options);
|
|
84
|
+
const plan = optimizePipeline(rawOps);
|
|
85
|
+
const writer = createWriter(options.output || "-", {
|
|
86
|
+
format: toFormat,
|
|
87
|
+
delimiter: options.delimiter,
|
|
88
|
+
...options,
|
|
89
|
+
});
|
|
90
|
+
const pipeline = executePlannedPipeline(reader, plan, { batchSize: effectiveBatchSize });
|
|
91
|
+
pipeline.pipe(cleanTransform);
|
|
92
|
+
if (options.unique !== undefined && options.unique !== false) {
|
|
93
|
+
pipeline.pipe(uniqueRows({ by: typeof options.unique === "string" ? options.unique.split(",") : undefined }));
|
|
94
|
+
}
|
|
95
|
+
pipeline.onProgress((info) => progress.update(info));
|
|
96
|
+
try {
|
|
97
|
+
await pipeline.to(writer);
|
|
98
|
+
}
|
|
99
|
+
finally {
|
|
100
|
+
if (reader.close)
|
|
101
|
+
await reader.close();
|
|
102
|
+
if (writer.close)
|
|
103
|
+
await writer.close();
|
|
104
|
+
}
|
|
105
|
+
progress.done();
|
|
106
|
+
logMemoryDebug();
|
|
107
|
+
}
|
|
108
|
+
//# sourceMappingURL=clean.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"clean.js","sourceRoot":"","sources":["../../../src/cli/commands/clean.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,2BAA2B,CAAC;AACtD,OAAO,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAC;AAExD,OAAO,EAAE,YAAY,EAAE,mBAAmB,IAAI,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAChG,OAAO,EAAE,YAAY,EAAE,mBAAmB,IAAI,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAChG,OAAO,EAAE,kBAAkB,EAAsB,MAAM,4BAA4B,CAAC;AACpF,OAAO,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAC3D,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAC3D,OAAO,EAAE,sBAAsB,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAClF,OAAO,EAAE,0BAA0B,EAA6B,MAAM,eAAe,CAAC;AAYtF;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,SAAS,GAAG,GAAG,EACf,UAA+B,EAAE;IAEjC,MAAM,QAAQ,GAAG,IAAI,gBAAgB,CAAC,OAAO,CAAC,CAAC;IAC/C,MAAM,kBAAkB,GAAG,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,IAAI,IAAI,CAAC;IAE7D,IAAI,UAAU,GAAG,OAAO,CAAC,IAAI,EAAE,WAAW,EAAE,CAAC;IAC7C,IAAI,CAAC,UAAU,IAAI,SAAS,KAAK,GAAG,EAAE,CAAC;QACrC,UAAU,GAAG,iBAAiB,CAAC,SAAS,CAAC,IAAI,SAAS,CAAC;IACzD,CAAC;IACD,IAAI,CAAC,UAAU,EAAE,CAAC;QAChB,UAAU,GAAG,KAAK,CAAC;IACrB,CAAC;IAED,IAAI,QAAQ,GAAG,OAAO,CAAC,EAAE,EAAE,WAAW,EAAE,CAAC;IACzC,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;QACjB,QAAQ,GAAG,MAAM,CAAC;IACpB,CAAC;SAAM,IAAI,CAAC,QAAQ,IAAI,OAAO,CAAC,MAAM,IAAI,OAAO,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;QACjE,QAAQ,GAAG,iBAAiB,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,SAAS,CAAC;IAC5D,CAAC;IACD,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,QAAQ,GAAG,UAAU,KAAK,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,UAAU,CAAC;IACxD,CAAC;IAED,MAAM,WAAW,GAAG,SAAS,KAAK,GAAG,CAAC,CAAC,CAAC,kBAAkB,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IACrF,MAAM,MAAM,GAAG,YAAY,CAAC,WAAW,EAAE;QACvC,MAAM,EAAE,UAAU;QAClB,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,QAAQ,EAAE,SAAS;QACnB,GAAG,OAAO;QACV,SAAS,EAAE,kBAAkB;KAC9B,CAAC,CAAC;IAEH,6EAA6E;IAC7E,IAAI,eAAe,GAAsC,SAAS,CAAC;IACnE,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;QACtB,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QAC3F,MAAM,OAAO,GAA4B,EAAE,CAAC;QAC5C,IAAI,QAAQ,GAAG,KAAK,CAAC;QACrB,IAAI,SAAS,GAAY,SAAS,CAAC;QAEnC,KAAK,MAAM,IAAI,IAAI,OAAO,EAAE,CAAC;YAC3B,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;YAChC,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;gBACd,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC;gBACxC,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;gBAC5C,IAAI,CAAC;oBACH,OAAO,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;gBACpC,CAAC;gBAAC,MAAM,CAAC;oBACP,OAAO,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC;gBACxB,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,QAAQ,GAAG,IAAI,CAAC;gBAChB,IAAI,CAAC;oBACH,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBAC/B,CAAC;gBAAC,MAAM,CAAC;oBACP,SAAS,GAAG,IAAI,CAAC;gBACnB,CAAC;YACH,CAAC;QACH,CAAC;QACD,eAAe,GAAG,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC;IACnD,CAAC;IAED,MAAM,QAAQ,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAErG,MAAM,cAAc,GAAG,SAAS,CAAC;QAC/B,IAAI,EAAE,OAAO,CAAC,IAAI,KAAK,KAAK;QAC5B,UAAU,EAAE,QAAQ;QACpB,SAAS,EAAE,eAAe;QAC1B,MAAM,EAAE,OAAO,CAAC,MAAM,KAAK,IAAI;QAC/B,UAAU,EAAE,OAAO,CAAC,UAAU;QAC9B,IAAI,EAAE,OAAO,CAAC,IAAI;QAClB,aAAa,EAAE,OAAO,CAAC,aAAa;KACrC,CAAC,CAAC;IAEH,MAAM,MAAM,GAAG,0BAA0B,CAAC,OAAO,CAAC,CAAC;IACnD,MAAM,IAAI,GAAG,gBAAgB,CAAC,MAAM,CAAC,CAAC;IAEtC,MAAM,MAAM,GAAG,YAAY,CAAC,OAAO,CAAC,MAAM,IAAI,GAAG,EAAE;QACjD,MAAM,EAAE,QAAQ;QAChB,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,GAAG,OAAO;KACX,CAAC,CAAC;IAEH,MAAM,QAAQ,GAAG,sBAAsB,CAAC,MAAM,EAAE,IAAI,EAAE,EAAE,SAAS,EAAE,kBAAkB,EAAE,CAAC,CAAC;IACzF,QAAQ,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;IAE9B,IAAI,OAAO,CAAC,MAAM,KAAK,SAAS,IAAI,OAAO,CAAC,MAAM,KAAK,KAAK,EAAE,CAAC;QAC7D,QAAQ,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,EAAE,EAAE,OAAO,OAAO,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;IAChH,CAAC;IAED,QAAQ,CAAC,UAAU,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;IAErD,IAAI,CAAC;QACH,MAAM,QAAQ,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC;IAC5B,CAAC;YAAS,CAAC;QACT,IAAI,MAAM,CAAC,KAAK;YAAE,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;QACvC,IAAI,MAAM,CAAC,KAAK;YAAE,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;IACzC,CAAC;IAED,QAAQ,CAAC,IAAI,EAAE,CAAC;IAChB,cAAc,EAAE,CAAC;AACnB,CAAC"}
|
|
@@ -7,6 +7,12 @@ export interface ConvertCommandOptions {
|
|
|
7
7
|
path?: string;
|
|
8
8
|
delimiter?: string;
|
|
9
9
|
header?: boolean;
|
|
10
|
+
gzip?: boolean;
|
|
11
|
+
brotli?: boolean;
|
|
12
|
+
zstd?: boolean;
|
|
13
|
+
deflate?: boolean;
|
|
14
|
+
compress?: string;
|
|
15
|
+
compression?: string;
|
|
10
16
|
batchSize?: string | number;
|
|
11
17
|
quiet?: boolean;
|
|
12
18
|
noProgress?: boolean;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"convert.d.ts","sourceRoot":"","sources":["../../../src/cli/commands/convert.ts"],"names":[],"mappings":"AAUA,MAAM,WAAW,qBAAqB;IACpC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IAC5B,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,UAAU,CAAC,EAAE,OAAO,CAAC;CACtB;AAED,wBAAsB,cAAc,CAClC,SAAS,SAAM,EACf,UAAU,CAAC,EAAE,MAAM,EACnB,OAAO,GAAE,qBAA0B,GAClC,OAAO,CAAC,IAAI,CAAC,CAmGf"}
|
|
1
|
+
{"version":3,"file":"convert.d.ts","sourceRoot":"","sources":["../../../src/cli/commands/convert.ts"],"names":[],"mappings":"AAUA,MAAM,WAAW,qBAAqB;IACpC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IAC5B,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,UAAU,CAAC,EAAE,OAAO,CAAC;CACtB;AAED,wBAAsB,cAAc,CAClC,SAAS,SAAM,EACf,UAAU,CAAC,EAAE,MAAM,EACnB,OAAO,GAAE,qBAA0B,GAClC,OAAO,CAAC,IAAI,CAAC,CAmGf"}
|
|
@@ -4,7 +4,6 @@ import { createPipeline } from "../../core/pipeline.js";
|
|
|
4
4
|
import { createReader, inferFormatFromPath as inferReaderFormat } from "../../readers/index.js";
|
|
5
5
|
import { XLSXReader } from "../../readers/xlsx.js";
|
|
6
6
|
import { createWriter, inferFormatFromPath as inferWriterFormat } from "../../writers/index.js";
|
|
7
|
-
import { openReadableStream, openWritableStream } from "../../utils/compression.js";
|
|
8
7
|
import { formatNumber, logMemoryDebug } from "../../utils/formatting.js";
|
|
9
8
|
import { ProgressReporter } from "../../utils/progress.js";
|
|
10
9
|
export async function convertCommand(inputPath = "-", outputPath, options = {}) {
|
|
@@ -43,6 +42,7 @@ export async function convertCommand(inputPath = "-", outputPath, options = {})
|
|
|
43
42
|
format: toFormat,
|
|
44
43
|
delimiter: options.delimiter,
|
|
45
44
|
header: options.header,
|
|
45
|
+
...options,
|
|
46
46
|
});
|
|
47
47
|
const pipeline = createPipeline(sheetReader, { batchSize: effectiveBatchSize });
|
|
48
48
|
await pipeline.to(sheetWriter);
|
|
@@ -63,22 +63,22 @@ export async function convertCommand(inputPath = "-", outputPath, options = {})
|
|
|
63
63
|
}
|
|
64
64
|
const progress = new ProgressReporter(options);
|
|
65
65
|
const effectiveOutput = outputPath || "-";
|
|
66
|
-
const
|
|
67
|
-
const outputStream = openWritableStream(effectiveOutput);
|
|
68
|
-
const reader = createReader(inputStream, {
|
|
66
|
+
const reader = createReader(inputPath, {
|
|
69
67
|
format: fromFormat,
|
|
70
68
|
sheet: options.sheet,
|
|
71
69
|
path: options.path,
|
|
72
70
|
delimiter: options.delimiter,
|
|
73
71
|
header: options.header,
|
|
74
|
-
batchSize: effectiveBatchSize,
|
|
75
72
|
filePath: inputPath,
|
|
73
|
+
...options,
|
|
74
|
+
batchSize: effectiveBatchSize,
|
|
76
75
|
});
|
|
77
|
-
const writer = createWriter(
|
|
76
|
+
const writer = createWriter(effectiveOutput, {
|
|
78
77
|
format: toFormat,
|
|
79
78
|
delimiter: options.delimiter,
|
|
80
79
|
header: options.header,
|
|
81
80
|
sheet: options.sheet,
|
|
81
|
+
...options,
|
|
82
82
|
});
|
|
83
83
|
const pipeline = createPipeline(reader, { batchSize: effectiveBatchSize });
|
|
84
84
|
pipeline.onProgress((info) => progress.update(info));
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"convert.js","sourceRoot":"","sources":["../../../src/cli/commands/convert.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AACzC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAC;AACxD,OAAO,EAAE,YAAY,EAAE,mBAAmB,IAAI,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAChG,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AACnD,OAAO,EAAE,YAAY,EAAE,mBAAmB,IAAI,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;
|
|
1
|
+
{"version":3,"file":"convert.js","sourceRoot":"","sources":["../../../src/cli/commands/convert.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AACzC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAC;AACxD,OAAO,EAAE,YAAY,EAAE,mBAAmB,IAAI,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAChG,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AACnD,OAAO,EAAE,YAAY,EAAE,mBAAmB,IAAI,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAEhG,OAAO,EAAE,YAAY,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AACzE,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAsB3D,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,SAAS,GAAG,GAAG,EACf,UAAmB,EACnB,UAAiC,EAAE;IAEnC,MAAM,kBAAkB,GAAG,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,IAAI,IAAI,CAAC;IAE7D,yBAAyB;IACzB,IAAI,UAAU,GAAG,OAAO,CAAC,IAAI,EAAE,WAAW,EAAE,CAAC;IAC7C,IAAI,CAAC,UAAU,IAAI,SAAS,KAAK,GAAG,EAAE,CAAC;QACrC,UAAU,GAAG,iBAAiB,CAAC,SAAS,CAAC,IAAI,SAAS,CAAC;IACzD,CAAC;IACD,IAAI,CAAC,UAAU,EAAE,CAAC;QAChB,UAAU,GAAG,KAAK,CAAC;IACrB,CAAC;IAED,yDAAyD;IACzD,IAAI,OAAO,CAAC,SAAS,IAAI,UAAU,KAAK,MAAM,IAAI,SAAS,KAAK,GAAG,EAAE,CAAC;QACpE,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,UAAU,IAAI,IAAI,CAAC;QACpD,MAAM,QAAQ,GAAG,CAAC,OAAO,CAAC,EAAE,EAAE,WAAW,EAAE,IAAI,KAAK,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QAEzE,MAAM,KAAK,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAEzC,MAAM,UAAU,GAAG,IAAI,UAAU,CAAC,SAAS,CAAC,CAAC;QAC7C,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,eAAe,EAAE,CAAC;QAElD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACxB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,gCAAgC,CAAC,CAAC;YACvD,OAAO;QACT,CAAC;QAED,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,aAAa,MAAM,CAAC,MAAM,eAAe,MAAM,QAAQ,CAAC,CAAC;QAE9E,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,MAAM,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC;YAC9B,MAAM,cAAc,GAAG,GAAG,SAAS,IAAI,QAAQ,EAAE,CAAC;YAClD,MAAM,cAAc,GAAG,IAAI,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC;YAEpD,MAAM,WAAW,GAAG,YAAY,CAAC,SAAS,EAAE;gBAC1C,MAAM,EAAE,MAAM;gBACd,KAAK,EAAE,SAAS;gBAChB,SAAS,EAAE,kBAAkB;gBAC7B,QAAQ,EAAE,SAAS;aACpB,CAAC,CAAC;YAEH,MAAM,WAAW,GAAG,YAAY,CAAC,cAAc,EAAE;gBAC/C,MAAM,EAAE,QAAQ;gBAChB,SAAS,EAAE,OAAO,CAAC,SAAS;gBAC5B,MAAM,EAAE,OAAO,CAAC,MAAM;gBACtB,GAAG,OAAO;aACX,CAAC,CAAC;YAEH,MAAM,QAAQ,GAAG,cAAc,CAAC,WAAW,EAAE,EAAE,SAAS,EAAE,kBAAkB,EAAE,CAAC,CAAC;YAChF,MAAM,QAAQ,CAAC,EAAE,CAAC,WAAW,CAAC,CAAC;YAE/B,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;YACpD,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,cAAc,SAAS,QAAQ,cAAc,KAAK,YAAY,CAAC,QAAQ,CAAC,UAAU,CACnF,CAAC;QACJ,CAAC;QAED,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;QAChC,cAAc,EAAE,CAAC;QACjB,OAAO;IACT,CAAC;IAED,0BAA0B;IAC1B,IAAI,QAAQ,GAAG,OAAO,CAAC,EAAE,EAAE,WAAW,EAAE,CAAC;IACzC,IAAI,CAAC,QAAQ,IAAI,UAAU,IAAI,UAAU,KAAK,GAAG,EAAE,CAAC;QAClD,QAAQ,GAAG,iBAAiB,CAAC,UAAU,CAAC,IAAI,SAAS,CAAC;IACxD,CAAC;IACD,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,QAAQ,GAAG,KAAK,CAAC;IACnB,CAAC;IAED,MAAM,QAAQ,GAAG,IAAI,gBAAgB,CAAC,OAAO,CAAC,CAAC;IAC/C,MAAM,eAAe,GAAG,UAAU,IAAI,GAAG,CAAC;IAE1C,MAAM,MAAM,GAAG,YAAY,CAAC,SAAS,EAAE;QACrC,MAAM,EAAE,UAAU;QAClB,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,IAAI,EAAE,OAAO,CAAC,IAAI;QAClB,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,QAAQ,EAAE,SAAS;QACnB,GAAG,OAAO;QACV,SAAS,EAAE,kBAAkB;KAC9B,CAAC,CAAC;IAEH,MAAM,MAAM,GAAG,YAAY,CAAC,eAAe,EAAE;QAC3C,MAAM,EAAE,QAAQ;QAChB,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,GAAG,OAAO;KACX,CAAC,CAAC;IAEH,MAAM,QAAQ,GAAG,cAAc,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,kBAAkB,EAAE,CAAC,CAAC;IAC3E,QAAQ,CAAC,UAAU,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;IAErD,MAAM,QAAQ,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC;IAC1B,QAAQ,CAAC,IAAI,EAAE,CAAC;IAChB,cAAc,EAAE,CAAC;AACnB,CAAC"}
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import { UnifiedPipelineCliOptions } from "./pipeline.js";
|
|
2
|
+
export interface DbCommandOptions extends UnifiedPipelineCliOptions {
|
|
3
|
+
table?: string;
|
|
4
|
+
query?: string;
|
|
5
|
+
params?: string | string[];
|
|
6
|
+
param?: string | string[];
|
|
7
|
+
where?: string;
|
|
8
|
+
tables?: boolean;
|
|
9
|
+
schema?: string | boolean;
|
|
10
|
+
toDb?: string;
|
|
11
|
+
toTable?: string;
|
|
12
|
+
transaction?: boolean;
|
|
13
|
+
createTable?: boolean;
|
|
14
|
+
upsert?: boolean;
|
|
15
|
+
conflict?: string;
|
|
16
|
+
truncate?: boolean;
|
|
17
|
+
dryRun?: boolean;
|
|
18
|
+
}
|
|
19
|
+
/**
|
|
20
|
+
* Main handler for `rowpipe db <url> [options]`.
|
|
21
|
+
*/
|
|
22
|
+
export declare function dbCommand(dbUrl: string, options?: DbCommandOptions): Promise<void>;
|
|
23
|
+
/**
|
|
24
|
+
* List all tables in the specified database.
|
|
25
|
+
*/
|
|
26
|
+
export declare function dbTablesCommand(dbUrl: string, options?: {
|
|
27
|
+
json?: boolean;
|
|
28
|
+
}): Promise<void>;
|
|
29
|
+
/**
|
|
30
|
+
* Inspect the schema of a specific table in the database.
|
|
31
|
+
*/
|
|
32
|
+
export declare function dbSchemaCommand(dbUrl: string, table: string, options?: {
|
|
33
|
+
json?: boolean;
|
|
34
|
+
}): Promise<void>;
|
|
35
|
+
//# sourceMappingURL=db.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"db.d.ts","sourceRoot":"","sources":["../../../src/cli/commands/db.ts"],"names":[],"mappings":"AAUA,OAAO,EAA8B,yBAAyB,EAAE,MAAM,eAAe,CAAC;AAGtF,MAAM,WAAW,gBAAiB,SAAQ,yBAAyB;IACjE,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IAC3B,KAAK,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IAC1B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC;IAC1B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAED;;GAEG;AACH,wBAAsB,SAAS,CAC7B,KAAK,EAAE,MAAM,EACb,OAAO,GAAE,gBAAqB,GAC7B,OAAO,CAAC,IAAI,CAAC,CAiIf;AAED;;GAEG;AACH,wBAAsB,eAAe,CACnC,KAAK,EAAE,MAAM,EACb,OAAO,GAAE;IAAE,IAAI,CAAC,EAAE,OAAO,CAAA;CAAO,GAC/B,OAAO,CAAC,IAAI,CAAC,CAmBf;AAED;;GAEG;AACH,wBAAsB,eAAe,CACnC,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,MAAM,EACb,OAAO,GAAE;IAAE,IAAI,CAAC,EAAE,OAAO,CAAA;CAAO,GAC/B,OAAO,CAAC,IAAI,CAAC,CAwBf"}
|
|
@@ -0,0 +1,173 @@
|
|
|
1
|
+
import { DatabaseReader } from "../../db/source.js";
|
|
2
|
+
import { DatabaseWriter } from "../../db/sink.js";
|
|
3
|
+
import { createDatabaseAdapter } from "../../db/adapters/index.js";
|
|
4
|
+
import { parseDatabaseUrl } from "../../db/url.js";
|
|
5
|
+
import { analyzeDatabasePushdown } from "../../db/pushdown.js";
|
|
6
|
+
import { createWriter, inferFormatFromPath as inferWriterFormat } from "../../writers/index.js";
|
|
7
|
+
import { openWritableStream } from "../../utils/compression.js";
|
|
8
|
+
import { formatTable, logMemoryDebug, safeJsonStringify } from "../../utils/formatting.js";
|
|
9
|
+
import { ProgressReporter } from "../../utils/progress.js";
|
|
10
|
+
import { executePlannedPipeline, optimizePipeline } from "../../planner/index.js";
|
|
11
|
+
import { buildOperationsFromOptions } from "./pipeline.js";
|
|
12
|
+
import { InvalidArgumentError } from "../../core/errors.js";
|
|
13
|
+
/**
|
|
14
|
+
* Main handler for `rowpipe db <url> [options]`.
|
|
15
|
+
*/
|
|
16
|
+
export async function dbCommand(dbUrl, options = {}) {
|
|
17
|
+
if (!dbUrl || dbUrl === "-") {
|
|
18
|
+
throw new InvalidArgumentError("Database connection URL is required (e.g. postgres://..., mysql://..., sqlite://...)");
|
|
19
|
+
}
|
|
20
|
+
// Handle table list introspection
|
|
21
|
+
if (options.tables) {
|
|
22
|
+
await dbTablesCommand(dbUrl, options);
|
|
23
|
+
return;
|
|
24
|
+
}
|
|
25
|
+
// Handle schema inspection
|
|
26
|
+
if (options.schema) {
|
|
27
|
+
const table = typeof options.schema === "string" ? options.schema : options.table;
|
|
28
|
+
if (!table) {
|
|
29
|
+
throw new InvalidArgumentError("Specify table to inspect schema (e.g. rowpipe db <url> --schema <table_name>)");
|
|
30
|
+
}
|
|
31
|
+
await dbSchemaCommand(dbUrl, table, options);
|
|
32
|
+
return;
|
|
33
|
+
}
|
|
34
|
+
const effectiveBatchSize = Number(options.batchSize) || 1000;
|
|
35
|
+
const progress = new ProgressReporter(options);
|
|
36
|
+
// Parse query parameters
|
|
37
|
+
let queryParams = [];
|
|
38
|
+
if (options.params) {
|
|
39
|
+
const raw = Array.isArray(options.params) ? options.params.join("") : options.params;
|
|
40
|
+
try {
|
|
41
|
+
queryParams = JSON.parse(raw);
|
|
42
|
+
}
|
|
43
|
+
catch {
|
|
44
|
+
queryParams = [raw];
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
else if (options.param) {
|
|
48
|
+
queryParams = Array.isArray(options.param) ? options.param : [options.param];
|
|
49
|
+
}
|
|
50
|
+
const rawOperations = buildOperationsFromOptions(options);
|
|
51
|
+
const parsedConfig = parseDatabaseUrl(dbUrl);
|
|
52
|
+
let finalQuery = options.query;
|
|
53
|
+
let remainingOps = rawOperations;
|
|
54
|
+
// Pushdown analysis when using --table
|
|
55
|
+
if (options.table && !options.query) {
|
|
56
|
+
const analysis = analyzeDatabasePushdown(options.table, parsedConfig.dialect, rawOperations, options.where);
|
|
57
|
+
finalQuery = analysis.generatedQuery;
|
|
58
|
+
remainingOps = analysis.remainingOperations;
|
|
59
|
+
}
|
|
60
|
+
const optimizedPlan = optimizePipeline(remainingOps);
|
|
61
|
+
const reader = new DatabaseReader(parsedConfig, {
|
|
62
|
+
table: options.table,
|
|
63
|
+
query: finalQuery,
|
|
64
|
+
params: queryParams,
|
|
65
|
+
batchSize: effectiveBatchSize,
|
|
66
|
+
});
|
|
67
|
+
// Check if output is another database sink
|
|
68
|
+
if (options.toDb) {
|
|
69
|
+
const targetTable = options.toTable || options.table;
|
|
70
|
+
if (!targetTable) {
|
|
71
|
+
throw new InvalidArgumentError("Specify target table when writing to database (e.g. --to-table <name>)");
|
|
72
|
+
}
|
|
73
|
+
const conflictCols = options.conflict
|
|
74
|
+
? options.conflict.split(",").map((s) => s.trim()).filter(Boolean)
|
|
75
|
+
: undefined;
|
|
76
|
+
const writer = new DatabaseWriter(options.toDb, {
|
|
77
|
+
table: targetTable,
|
|
78
|
+
transaction: options.transaction,
|
|
79
|
+
createTable: options.createTable,
|
|
80
|
+
upsert: options.upsert,
|
|
81
|
+
conflictColumns: conflictCols,
|
|
82
|
+
truncate: options.truncate,
|
|
83
|
+
dryRun: options.dryRun,
|
|
84
|
+
});
|
|
85
|
+
const pipeline = executePlannedPipeline(reader, optimizedPlan, { batchSize: effectiveBatchSize });
|
|
86
|
+
pipeline.onProgress((info) => progress.update(info));
|
|
87
|
+
try {
|
|
88
|
+
await pipeline.to(writer);
|
|
89
|
+
}
|
|
90
|
+
finally {
|
|
91
|
+
await reader.close();
|
|
92
|
+
await writer.close();
|
|
93
|
+
}
|
|
94
|
+
progress.done();
|
|
95
|
+
logMemoryDebug();
|
|
96
|
+
return;
|
|
97
|
+
}
|
|
98
|
+
// Standard File / Stdout Writer
|
|
99
|
+
let toFormat = options.to?.toLowerCase();
|
|
100
|
+
if (options.json) {
|
|
101
|
+
toFormat = "json";
|
|
102
|
+
}
|
|
103
|
+
else if (!toFormat && options.output && options.output !== "-") {
|
|
104
|
+
toFormat = inferWriterFormat(options.output) ?? undefined;
|
|
105
|
+
}
|
|
106
|
+
if (!toFormat) {
|
|
107
|
+
toFormat = "csv";
|
|
108
|
+
}
|
|
109
|
+
const effectiveOutput = options.output || "-";
|
|
110
|
+
const outputStream = openWritableStream(effectiveOutput);
|
|
111
|
+
const writer = createWriter(outputStream, {
|
|
112
|
+
format: toFormat,
|
|
113
|
+
delimiter: options.delimiter,
|
|
114
|
+
});
|
|
115
|
+
const pipeline = executePlannedPipeline(reader, optimizedPlan, { batchSize: effectiveBatchSize });
|
|
116
|
+
pipeline.onProgress((info) => progress.update(info));
|
|
117
|
+
try {
|
|
118
|
+
await pipeline.to(writer);
|
|
119
|
+
}
|
|
120
|
+
finally {
|
|
121
|
+
await reader.close();
|
|
122
|
+
}
|
|
123
|
+
progress.done();
|
|
124
|
+
logMemoryDebug();
|
|
125
|
+
}
|
|
126
|
+
/**
|
|
127
|
+
* List all tables in the specified database.
|
|
128
|
+
*/
|
|
129
|
+
export async function dbTablesCommand(dbUrl, options = {}) {
|
|
130
|
+
const adapter = createDatabaseAdapter(dbUrl);
|
|
131
|
+
try {
|
|
132
|
+
const tables = await adapter.getTables();
|
|
133
|
+
if (options.json) {
|
|
134
|
+
process.stdout.write(safeJsonStringify(tables, 2) + "\n");
|
|
135
|
+
return;
|
|
136
|
+
}
|
|
137
|
+
if (tables.length === 0) {
|
|
138
|
+
process.stdout.write("(no tables found)\n");
|
|
139
|
+
return;
|
|
140
|
+
}
|
|
141
|
+
process.stdout.write(tables.join("\n") + "\n");
|
|
142
|
+
}
|
|
143
|
+
finally {
|
|
144
|
+
await adapter.close();
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
/**
|
|
148
|
+
* Inspect the schema of a specific table in the database.
|
|
149
|
+
*/
|
|
150
|
+
export async function dbSchemaCommand(dbUrl, table, options = {}) {
|
|
151
|
+
const adapter = createDatabaseAdapter(dbUrl);
|
|
152
|
+
try {
|
|
153
|
+
const schema = await adapter.getTableSchema(table);
|
|
154
|
+
if (options.json) {
|
|
155
|
+
process.stdout.write(safeJsonStringify(schema, 2) + "\n");
|
|
156
|
+
return;
|
|
157
|
+
}
|
|
158
|
+
const headers = ["COLUMN", "DATABASE TYPE", "ROWPIPE TYPE", "NULLABLE", "PRIMARY KEY"];
|
|
159
|
+
const rows = schema.columns.map((c) => [
|
|
160
|
+
c.name,
|
|
161
|
+
c.dbType,
|
|
162
|
+
c.rowpipeType,
|
|
163
|
+
c.nullable ? "YES" : "NO",
|
|
164
|
+
c.isPrimaryKey ? "YES" : "NO",
|
|
165
|
+
]);
|
|
166
|
+
process.stdout.write(`\nTable: ${table} (${adapter.dialect})\n\n`);
|
|
167
|
+
process.stdout.write(formatTable(headers, rows) + "\n");
|
|
168
|
+
}
|
|
169
|
+
finally {
|
|
170
|
+
await adapter.close();
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
//# sourceMappingURL=db.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"db.js","sourceRoot":"","sources":["../../../src/cli/commands/db.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACpD,OAAO,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAClD,OAAO,EAAE,qBAAqB,EAAE,MAAM,4BAA4B,CAAC;AACnE,OAAO,EAAiB,gBAAgB,EAA4B,MAAM,iBAAiB,CAAC;AAC5F,OAAO,EAAE,uBAAuB,EAAE,MAAM,sBAAsB,CAAC;AAC/D,OAAO,EAAE,YAAY,EAAE,mBAAmB,IAAI,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAChG,OAAO,EAAE,kBAAkB,EAAE,MAAM,4BAA4B,CAAC;AAChE,OAAO,EAAE,WAAW,EAAE,cAAc,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAC3F,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAC3D,OAAO,EAAE,sBAAsB,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAClF,OAAO,EAAE,0BAA0B,EAA6B,MAAM,eAAe,CAAC;AACtF,OAAO,EAAE,oBAAoB,EAAE,MAAM,sBAAsB,CAAC;AAoB5D;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,KAAa,EACb,UAA4B,EAAE;IAE9B,IAAI,CAAC,KAAK,IAAI,KAAK,KAAK,GAAG,EAAE,CAAC;QAC5B,MAAM,IAAI,oBAAoB,CAAC,sFAAsF,CAAC,CAAC;IACzH,CAAC;IAED,kCAAkC;IAClC,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;QACnB,MAAM,eAAe,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;QACtC,OAAO;IACT,CAAC;IAED,2BAA2B;IAC3B,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;QACnB,MAAM,KAAK,GAAG,OAAO,OAAO,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC;QAClF,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,MAAM,IAAI,oBAAoB,CAAC,+EAA+E,CAAC,CAAC;QAClH,CAAC;QACD,MAAM,eAAe,CAAC,KAAK,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC;QAC7C,OAAO;IACT,CAAC;IAED,MAAM,kBAAkB,GAAG,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,IAAI,IAAI,CAAC;IAC7D,MAAM,QAAQ,GAAG,IAAI,gBAAgB,CAAC,OAAO,CAAC,CAAC;IAE/C,yBAAyB;IACzB,IAAI,WAAW,GAAc,EAAE,CAAC;IAChC,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;QACnB,MAAM,GAAG,GAAG,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC;QACrF,IAAI,CAAC;YACH,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAChC,CAAC;QAAC,MAAM,CAAC;YACP,WAAW,GAAG,CAAC,GAAG,CAAC,CAAC;QACtB,CAAC;IACH,CAAC;SAAM,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;QACzB,WAAW,GAAG,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IAC/E,CAAC;IAED,MAAM,aAAa,GAAG,0BAA0B,CAAC,OAAO,CAAC,CAAC;IAC1D,MAAM,YAAY,GAAG,gBAAgB,CAAC,KAAK,CAAC,CAAC;IAE7C,IAAI,UAAU,GAAuB,OAAO,CAAC,KAAK,CAAC;IACnD,IAAI,YAAY,GAAG,aAAa,CAAC;IAEjC,uCAAuC;IACvC,IAAI,OAAO,CAAC,KAAK,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;QACpC,MAAM,QAAQ,GAAG,uBAAuB,CACtC,OAAO,CAAC,KAAK,EACb,YAAY,CAAC,OAAO,EACpB,aAAa,EACb,OAAO,CAAC,KAAK,CACd,CAAC;QACF,UAAU,GAAG,QAAQ,CAAC,cAAc,CAAC;QACrC,YAAY,GAAG,QAAQ,CAAC,mBAAmB,CAAC;IAC9C,CAAC;IAED,MAAM,aAAa,GAAG,gBAAgB,CAAC,YAAY,CAAC,CAAC;IAErD,MAAM,MAAM,GAAG,IAAI,cAAc,CAAC,YAAY,EAAE;QAC9C,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,KAAK,EAAE,UAAU;QACjB,MAAM,EAAE,WAAW;QACnB,SAAS,EAAE,kBAAkB;KAC9B,CAAC,CAAC;IAEH,2CAA2C;IAC3C,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;QACjB,MAAM,WAAW,GAAG,OAAO,CAAC,OAAO,IAAI,OAAO,CAAC,KAAK,CAAC;QACrD,IAAI,CAAC,WAAW,EAAE,CAAC;YACjB,MAAM,IAAI,oBAAoB,CAAC,wEAAwE,CAAC,CAAC;QAC3G,CAAC;QAED,MAAM,YAAY,GAAG,OAAO,CAAC,QAAQ;YACnC,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC;YAClE,CAAC,CAAC,SAAS,CAAC;QAEd,MAAM,MAAM,GAAG,IAAI,cAAc,CAAC,OAAO,CAAC,IAAI,EAAE;YAC9C,KAAK,EAAE,WAAW;YAClB,WAAW,EAAE,OAAO,CAAC,WAAW;YAChC,WAAW,EAAE,OAAO,CAAC,WAAW;YAChC,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,eAAe,EAAE,YAAY;YAC7B,QAAQ,EAAE,OAAO,CAAC,QAAQ;YAC1B,MAAM,EAAE,OAAO,CAAC,MAAM;SACvB,CAAC,CAAC;QAEH,MAAM,QAAQ,GAAG,sBAAsB,CAAC,MAAM,EAAE,aAAa,EAAE,EAAE,SAAS,EAAE,kBAAkB,EAAE,CAAC,CAAC;QAClG,QAAQ,CAAC,UAAU,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;QAErD,IAAI,CAAC;YACH,MAAM,QAAQ,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC;QAC5B,CAAC;gBAAS,CAAC;YACT,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;YACrB,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;QACvB,CAAC;QAED,QAAQ,CAAC,IAAI,EAAE,CAAC;QAChB,cAAc,EAAE,CAAC;QACjB,OAAO;IACT,CAAC;IAED,gCAAgC;IAChC,IAAI,QAAQ,GAAG,OAAO,CAAC,EAAE,EAAE,WAAW,EAAE,CAAC;IACzC,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;QACjB,QAAQ,GAAG,MAAM,CAAC;IACpB,CAAC;SAAM,IAAI,CAAC,QAAQ,IAAI,OAAO,CAAC,MAAM,IAAI,OAAO,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;QACjE,QAAQ,GAAG,iBAAiB,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,SAAS,CAAC;IAC5D,CAAC;IACD,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,QAAQ,GAAG,KAAK,CAAC;IACnB,CAAC;IAED,MAAM,eAAe,GAAG,OAAO,CAAC,MAAM,IAAI,GAAG,CAAC;IAC9C,MAAM,YAAY,GAAG,kBAAkB,CAAC,eAAe,CAAC,CAAC;IACzD,MAAM,MAAM,GAAG,YAAY,CAAC,YAAY,EAAE;QACxC,MAAM,EAAE,QAAQ;QAChB,SAAS,EAAE,OAAO,CAAC,SAAS;KAC7B,CAAC,CAAC;IAEH,MAAM,QAAQ,GAAG,sBAAsB,CAAC,MAAM,EAAE,aAAa,EAAE,EAAE,SAAS,EAAE,kBAAkB,EAAE,CAAC,CAAC;IAClG,QAAQ,CAAC,UAAU,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;IAErD,IAAI,CAAC;QACH,MAAM,QAAQ,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC;IAC5B,CAAC;YAAS,CAAC;QACT,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;IACvB,CAAC;IAED,QAAQ,CAAC,IAAI,EAAE,CAAC;IAChB,cAAc,EAAE,CAAC;AACnB,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,KAAa,EACb,UAA8B,EAAE;IAEhC,MAAM,OAAO,GAAG,qBAAqB,CAAC,KAAK,CAAC,CAAC;IAC7C,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,SAAS,EAAE,CAAC;QAEzC,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;YACjB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,iBAAiB,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC;YAC1D,OAAO;QACT,CAAC;QAED,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACxB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAC;YAC5C,OAAO;QACT,CAAC;QAED,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC;IACjD,CAAC;YAAS,CAAC;QACT,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;IACxB,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,KAAa,EACb,KAAa,EACb,UAA8B,EAAE;IAEhC,MAAM,OAAO,GAAG,qBAAqB,CAAC,KAAK,CAAC,CAAC;IAC7C,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC;QAEnD,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;YACjB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,iBAAiB,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC;YAC1D,OAAO;QACT,CAAC;QAED,MAAM,OAAO,GAAG,CAAC,QAAQ,EAAE,eAAe,EAAE,cAAc,EAAE,UAAU,EAAE,aAAa,CAAC,CAAC;QACvF,MAAM,IAAI,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;YACrC,CAAC,CAAC,IAAI;YACN,CAAC,CAAC,MAAM;YACR,CAAC,CAAC,WAAW;YACb,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI;YACzB,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI;SAC9B,CAAC,CAAC;QAEH,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,YAAY,KAAK,KAAK,OAAO,CAAC,OAAO,OAAO,CAAC,CAAC;QACnE,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,WAAW,CAAC,OAAO,EAAE,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC;IAC1D,CAAC;YAAS,CAAC;QACT,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;IACxB,CAAC;AACH,CAAC"}
|
|
@@ -1,3 +1,10 @@
|
|
|
1
1
|
import { UnifiedPipelineCliOptions } from "./pipeline.js";
|
|
2
|
-
export
|
|
2
|
+
export interface ExplainCliOptions extends UnifiedPipelineCliOptions {
|
|
3
|
+
table?: string;
|
|
4
|
+
query?: string;
|
|
5
|
+
where?: string;
|
|
6
|
+
toDb?: string;
|
|
7
|
+
toTable?: string;
|
|
8
|
+
}
|
|
9
|
+
export declare function explainCommand(inputOrArgs?: string | string[], options?: ExplainCliOptions): Promise<void>;
|
|
3
10
|
//# sourceMappingURL=explain.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"explain.d.ts","sourceRoot":"","sources":["../../../src/cli/commands/explain.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"explain.d.ts","sourceRoot":"","sources":["../../../src/cli/commands/explain.ts"],"names":[],"mappings":"AAGA,OAAO,EAA8B,yBAAyB,EAAE,MAAM,eAAe,CAAC;AAEtF,MAAM,WAAW,iBAAkB,SAAQ,yBAAyB;IAClE,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,wBAAsB,cAAc,CAClC,WAAW,GAAE,MAAM,GAAG,MAAM,EAAQ,EACpC,OAAO,GAAE,iBAAsB,GAC9B,OAAO,CAAC,IAAI,CAAC,CAiGf"}
|
|
@@ -1,10 +1,97 @@
|
|
|
1
1
|
import { formatExecutionPlan, optimizePipeline } from "../../planner/index.js";
|
|
2
|
+
import { isDatabaseUrl, parseDatabaseUrl, sanitizeConnectionString } from "../../db/url.js";
|
|
3
|
+
import { analyzeDatabasePushdown } from "../../db/pushdown.js";
|
|
2
4
|
import { buildOperationsFromOptions } from "./pipeline.js";
|
|
3
|
-
export async function explainCommand(
|
|
5
|
+
export async function explainCommand(inputOrArgs = "-", options = {}) {
|
|
6
|
+
let inputPath = "-";
|
|
7
|
+
if (Array.isArray(inputOrArgs)) {
|
|
8
|
+
if (inputOrArgs[0] === "db") {
|
|
9
|
+
inputPath = inputOrArgs[1] || "-";
|
|
10
|
+
}
|
|
11
|
+
else {
|
|
12
|
+
inputPath = inputOrArgs[0] || "-";
|
|
13
|
+
}
|
|
14
|
+
}
|
|
15
|
+
else {
|
|
16
|
+
inputPath = inputOrArgs;
|
|
17
|
+
}
|
|
4
18
|
const operations = buildOperationsFromOptions(options);
|
|
19
|
+
// Check if input is a database URL or table mode
|
|
20
|
+
if (isDatabaseUrl(inputPath) || options.table || options.query) {
|
|
21
|
+
let sanitizedDb = "Database";
|
|
22
|
+
let dialect = "sqlite";
|
|
23
|
+
if (isDatabaseUrl(inputPath)) {
|
|
24
|
+
const cfg = parseDatabaseUrl(inputPath);
|
|
25
|
+
sanitizedDb = cfg.sanitizedUrl;
|
|
26
|
+
dialect = cfg.dialect;
|
|
27
|
+
}
|
|
28
|
+
if (options.table && !options.query) {
|
|
29
|
+
const analysis = analyzeDatabasePushdown(options.table, dialect, operations, options.where);
|
|
30
|
+
const optimizedPlan = optimizePipeline(analysis.remainingOperations);
|
|
31
|
+
let out = "\n======================================================\n";
|
|
32
|
+
out += " Rowpipe Execution Plan \n";
|
|
33
|
+
out += "======================================================\n\n";
|
|
34
|
+
out += `DatabaseSource (${sanitizedDb})\n`;
|
|
35
|
+
out += ` Table: ${options.table}\n`;
|
|
36
|
+
out += ` Dialect: ${dialect}\n\n`;
|
|
37
|
+
out += "Remote Database Pushdown:\n";
|
|
38
|
+
out += ` SQL: ${analysis.generatedQuery}\n`;
|
|
39
|
+
if (analysis.pushedSummary.length > 0) {
|
|
40
|
+
for (const s of analysis.pushedSummary) {
|
|
41
|
+
out += ` • ${s}\n`;
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
else {
|
|
45
|
+
out += " • SELECT *\n";
|
|
46
|
+
}
|
|
47
|
+
out += "\nLocal Stream Operations:\n";
|
|
48
|
+
if (optimizedPlan.operations.length === 0) {
|
|
49
|
+
out += " (none - full pushdown achieved!)\n";
|
|
50
|
+
}
|
|
51
|
+
else {
|
|
52
|
+
for (const op of optimizedPlan.operations) {
|
|
53
|
+
out += ` • ${op.type}\n`;
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
out += "\nOutput Target:\n";
|
|
57
|
+
if (options.toDb) {
|
|
58
|
+
out += ` DatabaseSink (${sanitizeConnectionString(options.toDb)}) -> Table: ${options.toTable || options.table}\n`;
|
|
59
|
+
}
|
|
60
|
+
else if (options.output) {
|
|
61
|
+
out += ` Writer (${options.output})\n`;
|
|
62
|
+
}
|
|
63
|
+
else {
|
|
64
|
+
out += " Stdout Writer\n";
|
|
65
|
+
}
|
|
66
|
+
out += "\n------------------------------------------------------\n";
|
|
67
|
+
if (optimizedPlan.optimizationsApplied.length > 0) {
|
|
68
|
+
out += "Local Optimizations Applied:\n";
|
|
69
|
+
for (const opt of optimizedPlan.optimizationsApplied) {
|
|
70
|
+
out += ` • ${opt}\n`;
|
|
71
|
+
}
|
|
72
|
+
out += "------------------------------------------------------\n";
|
|
73
|
+
}
|
|
74
|
+
out += "Memory Classification:\n";
|
|
75
|
+
if (optimizedPlan.memoryClassification === "streaming") {
|
|
76
|
+
out += " Fully Streaming (Cursor-based bounded memory)\n";
|
|
77
|
+
}
|
|
78
|
+
else {
|
|
79
|
+
out += ` ${optimizedPlan.memoryClassification}\n`;
|
|
80
|
+
}
|
|
81
|
+
out += "======================================================\n\n";
|
|
82
|
+
process.stdout.write(out);
|
|
83
|
+
return;
|
|
84
|
+
}
|
|
85
|
+
}
|
|
5
86
|
const plan = optimizePipeline(operations);
|
|
6
87
|
const inputDesc = inputPath === "-" ? "Stdin Stream" : `Reader (${inputPath})`;
|
|
7
|
-
|
|
88
|
+
let outputDesc = "Stdout Writer";
|
|
89
|
+
if (options.toDb) {
|
|
90
|
+
outputDesc = `DatabaseSink (${sanitizeConnectionString(options.toDb)}) -> Table: ${options.toTable || "target"}`;
|
|
91
|
+
}
|
|
92
|
+
else if (options.output) {
|
|
93
|
+
outputDesc = `Writer (${options.output})`;
|
|
94
|
+
}
|
|
8
95
|
const formatted = formatExecutionPlan(plan, inputDesc, outputDesc);
|
|
9
96
|
process.stdout.write(formatted);
|
|
10
97
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"explain.js","sourceRoot":"","sources":["../../../src/cli/commands/explain.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,mBAAmB,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAC/E,OAAO,EAAE,0BAA0B,EAA6B,MAAM,eAAe,CAAC;
|
|
1
|
+
{"version":3,"file":"explain.js","sourceRoot":"","sources":["../../../src/cli/commands/explain.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,mBAAmB,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAC/E,OAAO,EAAE,aAAa,EAAE,gBAAgB,EAAE,wBAAwB,EAAE,MAAM,iBAAiB,CAAC;AAC5F,OAAO,EAAE,uBAAuB,EAAE,MAAM,sBAAsB,CAAC;AAC/D,OAAO,EAAE,0BAA0B,EAA6B,MAAM,eAAe,CAAC;AAUtF,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,cAAiC,GAAG,EACpC,UAA6B,EAAE;IAE/B,IAAI,SAAS,GAAG,GAAG,CAAC;IACpB,IAAI,KAAK,CAAC,OAAO,CAAC,WAAW,CAAC,EAAE,CAAC;QAC/B,IAAI,WAAW,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;YAC5B,SAAS,GAAG,WAAW,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC;QACpC,CAAC;aAAM,CAAC;YACN,SAAS,GAAG,WAAW,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC;QACpC,CAAC;IACH,CAAC;SAAM,CAAC;QACN,SAAS,GAAG,WAAW,CAAC;IAC1B,CAAC;IAED,MAAM,UAAU,GAAG,0BAA0B,CAAC,OAAO,CAAC,CAAC;IAEvD,iDAAiD;IACjD,IAAI,aAAa,CAAC,SAAS,CAAC,IAAI,OAAO,CAAC,KAAK,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;QAC/D,IAAI,WAAW,GAAG,UAAU,CAAC;QAC7B,IAAI,OAAO,GAAQ,QAAQ,CAAC;QAC5B,IAAI,aAAa,CAAC,SAAS,CAAC,EAAE,CAAC;YAC7B,MAAM,GAAG,GAAG,gBAAgB,CAAC,SAAS,CAAC,CAAC;YACxC,WAAW,GAAG,GAAG,CAAC,YAAY,CAAC;YAC/B,OAAO,GAAG,GAAG,CAAC,OAAO,CAAC;QACxB,CAAC;QAED,IAAI,OAAO,CAAC,KAAK,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;YACpC,MAAM,QAAQ,GAAG,uBAAuB,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,EAAE,UAAU,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;YAC5F,MAAM,aAAa,GAAG,gBAAgB,CAAC,QAAQ,CAAC,mBAAmB,CAAC,CAAC;YAErE,IAAI,GAAG,GAAG,4DAA4D,CAAC;YACvE,GAAG,IAAI,0DAA0D,CAAC;YAClE,GAAG,IAAI,4DAA4D,CAAC;YAEpE,GAAG,IAAI,mBAAmB,WAAW,KAAK,CAAC;YAC3C,GAAG,IAAI,YAAY,OAAO,CAAC,KAAK,IAAI,CAAC;YACrC,GAAG,IAAI,cAAc,OAAO,MAAM,CAAC;YAEnC,GAAG,IAAI,6BAA6B,CAAC;YACrC,GAAG,IAAI,UAAU,QAAQ,CAAC,cAAc,IAAI,CAAC;YAC7C,IAAI,QAAQ,CAAC,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACtC,KAAK,MAAM,CAAC,IAAI,QAAQ,CAAC,aAAa,EAAE,CAAC;oBACvC,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;gBACtB,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,GAAG,IAAI,gBAAgB,CAAC;YAC1B,CAAC;YAED,GAAG,IAAI,8BAA8B,CAAC;YACtC,IAAI,aAAa,CAAC,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC1C,GAAG,IAAI,sCAAsC,CAAC;YAChD,CAAC;iBAAM,CAAC;gBACN,KAAK,MAAM,EAAE,IAAI,aAAa,CAAC,UAAU,EAAE,CAAC;oBAC1C,GAAG,IAAI,OAAO,EAAE,CAAC,IAAI,IAAI,CAAC;gBAC5B,CAAC;YACH,CAAC;YAED,GAAG,IAAI,oBAAoB,CAAC;YAC5B,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;gBACjB,GAAG,IAAI,mBAAmB,wBAAwB,CAAC,OAAO,CAAC,IAAI,CAAC,eAAe,OAAO,CAAC,OAAO,IAAI,OAAO,CAAC,KAAK,IAAI,CAAC;YACtH,CAAC;iBAAM,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;gBAC1B,GAAG,IAAI,aAAa,OAAO,CAAC,MAAM,KAAK,CAAC;YAC1C,CAAC;iBAAM,CAAC;gBACN,GAAG,IAAI,mBAAmB,CAAC;YAC7B,CAAC;YAED,GAAG,IAAI,4DAA4D,CAAC;YACpE,IAAI,aAAa,CAAC,oBAAoB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAClD,GAAG,IAAI,gCAAgC,CAAC;gBACxC,KAAK,MAAM,GAAG,IAAI,aAAa,CAAC,oBAAoB,EAAE,CAAC;oBACrD,GAAG,IAAI,OAAO,GAAG,IAAI,CAAC;gBACxB,CAAC;gBACD,GAAG,IAAI,0DAA0D,CAAC;YACpE,CAAC;YAED,GAAG,IAAI,0BAA0B,CAAC;YAClC,IAAI,aAAa,CAAC,oBAAoB,KAAK,WAAW,EAAE,CAAC;gBACvD,GAAG,IAAI,mDAAmD,CAAC;YAC7D,CAAC;iBAAM,CAAC;gBACN,GAAG,IAAI,KAAK,aAAa,CAAC,oBAAoB,IAAI,CAAC;YACrD,CAAC;YACD,GAAG,IAAI,4DAA4D,CAAC;YAEpE,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YAC1B,OAAO;QACT,CAAC;IACH,CAAC;IAED,MAAM,IAAI,GAAG,gBAAgB,CAAC,UAAU,CAAC,CAAC;IAC1C,MAAM,SAAS,GAAG,SAAS,KAAK,GAAG,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,WAAW,SAAS,GAAG,CAAC;IAC/E,IAAI,UAAU,GAAG,eAAe,CAAC;IACjC,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;QACjB,UAAU,GAAG,iBAAiB,wBAAwB,CAAC,OAAO,CAAC,IAAI,CAAC,eAAe,OAAO,CAAC,OAAO,IAAI,QAAQ,EAAE,CAAC;IACnH,CAAC;SAAM,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;QAC1B,UAAU,GAAG,WAAW,OAAO,CAAC,MAAM,GAAG,CAAC;IAC5C,CAAC;IAED,MAAM,SAAS,GAAG,mBAAmB,CAAC,IAAI,EAAE,SAAS,EAAE,UAAU,CAAC,CAAC;IACnE,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;AAClC,CAAC"}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import type { JoinType } from "../../transforms/join/types.js";
|
|
2
|
+
import { UnifiedPipelineCliOptions } from "./pipeline.js";
|
|
3
|
+
export interface JoinCommandOptions extends UnifiedPipelineCliOptions {
|
|
4
|
+
on?: string | string[];
|
|
5
|
+
leftKey?: string | string[];
|
|
6
|
+
rightKey?: string | string[];
|
|
7
|
+
type?: JoinType;
|
|
8
|
+
prefixLeft?: string;
|
|
9
|
+
suffixLeft?: string;
|
|
10
|
+
prefixRight?: string;
|
|
11
|
+
suffixRight?: string;
|
|
12
|
+
fromLeft?: string;
|
|
13
|
+
fromRight?: string;
|
|
14
|
+
leftSheet?: string;
|
|
15
|
+
rightSheet?: string;
|
|
16
|
+
leftDelimiter?: string;
|
|
17
|
+
rightDelimiter?: string;
|
|
18
|
+
}
|
|
19
|
+
/**
|
|
20
|
+
* Handles `rowpipe join <left> <right> [options]`
|
|
21
|
+
*/
|
|
22
|
+
export declare function joinCommand(leftPath: string, rightPath: string, options?: JoinCommandOptions): Promise<void>;
|
|
23
|
+
//# sourceMappingURL=join.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"join.d.ts","sourceRoot":"","sources":["../../../src/cli/commands/join.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,gCAAgC,CAAC;AAK/D,OAAO,EAA8B,yBAAyB,EAAE,MAAM,eAAe,CAAC;AAGtF,MAAM,WAAW,kBAAmB,SAAQ,yBAAyB;IACnE,EAAE,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IACvB,OAAO,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IAC5B,QAAQ,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IAC7B,IAAI,CAAC,EAAE,QAAQ,CAAC;IAChB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAED;;GAEG;AACH,wBAAsB,WAAW,CAC/B,QAAQ,EAAE,MAAM,EAChB,SAAS,EAAE,MAAM,EACjB,OAAO,GAAE,kBAAuB,GAC/B,OAAO,CAAC,IAAI,CAAC,CA4Ff"}
|