rowpipe 2.1.0 → 2.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (207) hide show
  1. package/README.md +183 -5
  2. package/dist/analytics/profiler.d.ts +56 -0
  3. package/dist/analytics/profiler.d.ts.map +1 -0
  4. package/dist/analytics/profiler.js +277 -0
  5. package/dist/analytics/profiler.js.map +1 -0
  6. package/dist/analytics/schema-inference.d.ts.map +1 -1
  7. package/dist/analytics/schema-inference.js +14 -0
  8. package/dist/analytics/schema-inference.js.map +1 -1
  9. package/dist/cli/commands/clean.d.ts +15 -0
  10. package/dist/cli/commands/clean.d.ts.map +1 -0
  11. package/dist/cli/commands/clean.js +108 -0
  12. package/dist/cli/commands/clean.js.map +1 -0
  13. package/dist/cli/commands/convert.d.ts +6 -0
  14. package/dist/cli/commands/convert.d.ts.map +1 -1
  15. package/dist/cli/commands/convert.js +6 -6
  16. package/dist/cli/commands/convert.js.map +1 -1
  17. package/dist/cli/commands/db.d.ts +35 -0
  18. package/dist/cli/commands/db.d.ts.map +1 -0
  19. package/dist/cli/commands/db.js +173 -0
  20. package/dist/cli/commands/db.js.map +1 -0
  21. package/dist/cli/commands/explain.d.ts +8 -1
  22. package/dist/cli/commands/explain.d.ts.map +1 -1
  23. package/dist/cli/commands/explain.js +89 -2
  24. package/dist/cli/commands/explain.js.map +1 -1
  25. package/dist/cli/commands/join.d.ts +23 -0
  26. package/dist/cli/commands/join.d.ts.map +1 -0
  27. package/dist/cli/commands/join.js +97 -0
  28. package/dist/cli/commands/join.js.map +1 -0
  29. package/dist/cli/commands/pipeline.d.ts +20 -0
  30. package/dist/cli/commands/pipeline.d.ts.map +1 -1
  31. package/dist/cli/commands/pipeline.js +53 -12
  32. package/dist/cli/commands/pipeline.js.map +1 -1
  33. package/dist/cli/commands/profile.d.ts +22 -0
  34. package/dist/cli/commands/profile.d.ts.map +1 -0
  35. package/dist/cli/commands/profile.js +53 -0
  36. package/dist/cli/commands/profile.js.map +1 -0
  37. package/dist/cli/commands/view.d.ts +12 -0
  38. package/dist/cli/commands/view.d.ts.map +1 -0
  39. package/dist/cli/commands/view.js +52 -0
  40. package/dist/cli/commands/view.js.map +1 -0
  41. package/dist/cli/commands/window.d.ts +10 -0
  42. package/dist/cli/commands/window.d.ts.map +1 -0
  43. package/dist/cli/commands/window.js +87 -0
  44. package/dist/cli/commands/window.js.map +1 -0
  45. package/dist/cli/index.js +201 -5
  46. package/dist/cli/index.js.map +1 -1
  47. package/dist/core/batch.d.ts +1 -1
  48. package/dist/core/batch.d.ts.map +1 -1
  49. package/dist/core/batch.js.map +1 -1
  50. package/dist/core/pipeline.d.ts +2 -2
  51. package/dist/core/pipeline.d.ts.map +1 -1
  52. package/dist/core/pipeline.js +5 -2
  53. package/dist/core/pipeline.js.map +1 -1
  54. package/dist/core/types.d.ts +21 -1
  55. package/dist/core/types.d.ts.map +1 -1
  56. package/dist/db/adapters/index.d.ts +6 -0
  57. package/dist/db/adapters/index.d.ts.map +1 -0
  58. package/dist/db/adapters/index.js +21 -0
  59. package/dist/db/adapters/index.js.map +1 -0
  60. package/dist/db/adapters/mysql.d.ts +16 -0
  61. package/dist/db/adapters/mysql.d.ts.map +1 -0
  62. package/dist/db/adapters/mysql.js +230 -0
  63. package/dist/db/adapters/mysql.js.map +1 -0
  64. package/dist/db/adapters/postgres.d.ts +17 -0
  65. package/dist/db/adapters/postgres.d.ts.map +1 -0
  66. package/dist/db/adapters/postgres.js +239 -0
  67. package/dist/db/adapters/postgres.js.map +1 -0
  68. package/dist/db/adapters/sqlite.d.ts +18 -0
  69. package/dist/db/adapters/sqlite.d.ts.map +1 -0
  70. package/dist/db/adapters/sqlite.js +210 -0
  71. package/dist/db/adapters/sqlite.js.map +1 -0
  72. package/dist/db/index.d.ts +11 -0
  73. package/dist/db/index.d.ts.map +1 -0
  74. package/dist/db/index.js +11 -0
  75. package/dist/db/index.js.map +1 -0
  76. package/dist/db/mapping.d.ts +45 -0
  77. package/dist/db/mapping.d.ts.map +1 -0
  78. package/dist/db/mapping.js +222 -0
  79. package/dist/db/mapping.js.map +1 -0
  80. package/dist/db/pushdown.d.ts +25 -0
  81. package/dist/db/pushdown.d.ts.map +1 -0
  82. package/dist/db/pushdown.js +165 -0
  83. package/dist/db/pushdown.js.map +1 -0
  84. package/dist/db/sink.d.ts +15 -0
  85. package/dist/db/sink.d.ts.map +1 -0
  86. package/dist/db/sink.js +47 -0
  87. package/dist/db/sink.js.map +1 -0
  88. package/dist/db/source.d.ts +14 -0
  89. package/dist/db/source.d.ts.map +1 -0
  90. package/dist/db/source.js +69 -0
  91. package/dist/db/source.js.map +1 -0
  92. package/dist/db/types.d.ts +80 -0
  93. package/dist/db/types.d.ts.map +1 -0
  94. package/dist/db/types.js +2 -0
  95. package/dist/db/types.js.map +1 -0
  96. package/dist/db/url.d.ts +15 -0
  97. package/dist/db/url.d.ts.map +1 -0
  98. package/dist/db/url.js +145 -0
  99. package/dist/db/url.js.map +1 -0
  100. package/dist/diff/hash.d.ts.map +1 -1
  101. package/dist/diff/hash.js +4 -3
  102. package/dist/diff/hash.js.map +1 -1
  103. package/dist/diff/key.d.ts.map +1 -1
  104. package/dist/diff/key.js +8 -1
  105. package/dist/diff/key.js.map +1 -1
  106. package/dist/diff/reporter.d.ts.map +1 -1
  107. package/dist/diff/reporter.js +5 -3
  108. package/dist/diff/reporter.js.map +1 -1
  109. package/dist/diff/storage/disk-index.d.ts.map +1 -1
  110. package/dist/diff/storage/disk-index.js +2 -1
  111. package/dist/diff/storage/disk-index.js.map +1 -1
  112. package/dist/diff/storage/memory-index.d.ts.map +1 -1
  113. package/dist/diff/storage/memory-index.js +3 -2
  114. package/dist/diff/storage/memory-index.js.map +1 -1
  115. package/dist/index.d.ts +8 -1
  116. package/dist/index.d.ts.map +1 -1
  117. package/dist/index.js +9 -1
  118. package/dist/index.js.map +1 -1
  119. package/dist/planner/executor.d.ts.map +1 -1
  120. package/dist/planner/executor.js +7 -0
  121. package/dist/planner/executor.js.map +1 -1
  122. package/dist/planner/types.d.ts +4 -0
  123. package/dist/planner/types.d.ts.map +1 -1
  124. package/dist/readers/csv.d.ts.map +1 -1
  125. package/dist/readers/csv.js +1 -2
  126. package/dist/readers/csv.js.map +1 -1
  127. package/dist/readers/index.d.ts +5 -1
  128. package/dist/readers/index.d.ts.map +1 -1
  129. package/dist/readers/index.js +43 -0
  130. package/dist/readers/index.js.map +1 -1
  131. package/dist/readers/json.d.ts.map +1 -1
  132. package/dist/readers/json.js +1 -2
  133. package/dist/readers/json.js.map +1 -1
  134. package/dist/readers/jsonl.d.ts.map +1 -1
  135. package/dist/readers/jsonl.js +1 -2
  136. package/dist/readers/jsonl.js.map +1 -1
  137. package/dist/readers/multi-file.d.ts +29 -0
  138. package/dist/readers/multi-file.d.ts.map +1 -0
  139. package/dist/readers/multi-file.js +169 -0
  140. package/dist/readers/multi-file.js.map +1 -0
  141. package/dist/transforms/clean.d.ts +18 -0
  142. package/dist/transforms/clean.d.ts.map +1 -0
  143. package/dist/transforms/clean.js +118 -0
  144. package/dist/transforms/clean.js.map +1 -0
  145. package/dist/transforms/expression.d.ts.map +1 -1
  146. package/dist/transforms/expression.js +12 -0
  147. package/dist/transforms/expression.js.map +1 -1
  148. package/dist/transforms/jit-compiler.d.ts +50 -0
  149. package/dist/transforms/jit-compiler.d.ts.map +1 -0
  150. package/dist/transforms/jit-compiler.js +275 -0
  151. package/dist/transforms/jit-compiler.js.map +1 -0
  152. package/dist/transforms/join/index-storage.d.ts +33 -0
  153. package/dist/transforms/join/index-storage.d.ts.map +1 -0
  154. package/dist/transforms/join/index-storage.js +139 -0
  155. package/dist/transforms/join/index-storage.js.map +1 -0
  156. package/dist/transforms/join/index.d.ts +4 -0
  157. package/dist/transforms/join/index.d.ts.map +1 -0
  158. package/dist/transforms/join/index.js +4 -0
  159. package/dist/transforms/join/index.js.map +1 -0
  160. package/dist/transforms/join/join.d.ts +15 -0
  161. package/dist/transforms/join/join.d.ts.map +1 -0
  162. package/dist/transforms/join/join.js +216 -0
  163. package/dist/transforms/join/join.js.map +1 -0
  164. package/dist/transforms/join/types.d.ts +29 -0
  165. package/dist/transforms/join/types.d.ts.map +1 -0
  166. package/dist/transforms/join/types.js +2 -0
  167. package/dist/transforms/join/types.js.map +1 -0
  168. package/dist/transforms/window.d.ts +24 -0
  169. package/dist/transforms/window.d.ts.map +1 -0
  170. package/dist/transforms/window.js +317 -0
  171. package/dist/transforms/window.js.map +1 -0
  172. package/dist/ui/colors.d.ts +57 -0
  173. package/dist/ui/colors.d.ts.map +1 -0
  174. package/dist/ui/colors.js +94 -0
  175. package/dist/ui/colors.js.map +1 -0
  176. package/dist/ui/viewer.d.ts +67 -0
  177. package/dist/ui/viewer.d.ts.map +1 -0
  178. package/dist/ui/viewer.js +673 -0
  179. package/dist/ui/viewer.js.map +1 -0
  180. package/dist/utils/compression.d.ts +68 -19
  181. package/dist/utils/compression.d.ts.map +1 -1
  182. package/dist/utils/compression.js +222 -34
  183. package/dist/utils/compression.js.map +1 -1
  184. package/dist/utils/formatting.d.ts +5 -0
  185. package/dist/utils/formatting.d.ts.map +1 -1
  186. package/dist/utils/formatting.js +15 -0
  187. package/dist/utils/formatting.js.map +1 -1
  188. package/dist/writers/csv.d.ts.map +1 -1
  189. package/dist/writers/csv.js +13 -14
  190. package/dist/writers/csv.js.map +1 -1
  191. package/dist/writers/index.d.ts +1 -0
  192. package/dist/writers/index.d.ts.map +1 -1
  193. package/dist/writers/index.js +35 -0
  194. package/dist/writers/index.js.map +1 -1
  195. package/dist/writers/json.d.ts.map +1 -1
  196. package/dist/writers/json.js +6 -5
  197. package/dist/writers/json.js.map +1 -1
  198. package/dist/writers/jsonl.d.ts.map +1 -1
  199. package/dist/writers/jsonl.js +6 -5
  200. package/dist/writers/jsonl.js.map +1 -1
  201. package/dist/writers/markdown.d.ts.map +1 -1
  202. package/dist/writers/markdown.js +5 -4
  203. package/dist/writers/markdown.js.map +1 -1
  204. package/dist/writers/parquet.d.ts +1 -1
  205. package/dist/writers/parquet.d.ts.map +1 -1
  206. package/package.json +10 -2
  207. package/skills/rowpipe/SKILL.md +8 -4
@@ -0,0 +1,108 @@
1
+ import { cleanRows } from "../../transforms/clean.js";
2
+ import { uniqueRows } from "../../transforms/unique.js";
3
+ import { createReader, inferFormatFromPath as inferReaderFormat } from "../../readers/index.js";
4
+ import { createWriter, inferFormatFromPath as inferWriterFormat } from "../../writers/index.js";
5
+ import { openReadableStream } from "../../utils/compression.js";
6
+ import { logMemoryDebug } from "../../utils/formatting.js";
7
+ import { ProgressReporter } from "../../utils/progress.js";
8
+ import { executePlannedPipeline, optimizePipeline } from "../../planner/index.js";
9
+ import { buildOperationsFromOptions } from "./pipeline.js";
10
+ /**
11
+ * Handles `rowpipe clean [input] [options]`
12
+ */
13
+ export async function cleanCommand(inputPath = "-", options = {}) {
14
+ const progress = new ProgressReporter(options);
15
+ const effectiveBatchSize = Number(options.batchSize) || 1000;
16
+ let fromFormat = options.from?.toLowerCase();
17
+ if (!fromFormat && inputPath !== "-") {
18
+ fromFormat = inferReaderFormat(inputPath) ?? undefined;
19
+ }
20
+ if (!fromFormat) {
21
+ fromFormat = "csv";
22
+ }
23
+ let toFormat = options.to?.toLowerCase();
24
+ if (options.json) {
25
+ toFormat = "json";
26
+ }
27
+ else if (!toFormat && options.output && options.output !== "-") {
28
+ toFormat = inferWriterFormat(options.output) ?? undefined;
29
+ }
30
+ if (!toFormat) {
31
+ toFormat = fromFormat === "xlsx" ? "csv" : fromFormat;
32
+ }
33
+ const readerInput = inputPath === "-" ? openReadableStream("-", options) : inputPath;
34
+ const reader = createReader(readerInput, {
35
+ format: fromFormat,
36
+ sheet: options.sheet,
37
+ delimiter: options.delimiter,
38
+ filePath: inputPath,
39
+ ...options,
40
+ batchSize: effectiveBatchSize,
41
+ });
42
+ // Parse fill-nulls specs (e.g. --fill-nulls 'revenue=0' or --fill-nulls '0')
43
+ let fillNullsParsed = undefined;
44
+ if (options.fillNulls) {
45
+ const rawList = Array.isArray(options.fillNulls) ? options.fillNulls : [options.fillNulls];
46
+ const fillMap = {};
47
+ let isGlobal = false;
48
+ let globalVal = undefined;
49
+ for (const item of rawList) {
50
+ const eqIdx = item.indexOf("=");
51
+ if (eqIdx > 0) {
52
+ const col = item.slice(0, eqIdx).trim();
53
+ const valStr = item.slice(eqIdx + 1).trim();
54
+ try {
55
+ fillMap[col] = JSON.parse(valStr);
56
+ }
57
+ catch {
58
+ fillMap[col] = valStr;
59
+ }
60
+ }
61
+ else {
62
+ isGlobal = true;
63
+ try {
64
+ globalVal = JSON.parse(item);
65
+ }
66
+ catch {
67
+ globalVal = item;
68
+ }
69
+ }
70
+ }
71
+ fillNullsParsed = isGlobal ? globalVal : fillMap;
72
+ }
73
+ const nullVals = options.nullValues ? options.nullValues.split(",").map((s) => s.trim()) : undefined;
74
+ const cleanTransform = cleanRows({
75
+ trim: options.trim !== false,
76
+ nullValues: nullVals,
77
+ fillNulls: fillNullsParsed,
78
+ coerce: options.coerce === true,
79
+ stripChars: options.stripChars,
80
+ case: options.case,
81
+ dropEmptyRows: options.dropEmptyRows,
82
+ });
83
+ const rawOps = buildOperationsFromOptions(options);
84
+ const plan = optimizePipeline(rawOps);
85
+ const writer = createWriter(options.output || "-", {
86
+ format: toFormat,
87
+ delimiter: options.delimiter,
88
+ ...options,
89
+ });
90
+ const pipeline = executePlannedPipeline(reader, plan, { batchSize: effectiveBatchSize });
91
+ pipeline.pipe(cleanTransform);
92
+ if (options.unique !== undefined && options.unique !== false) {
93
+ pipeline.pipe(uniqueRows({ by: typeof options.unique === "string" ? options.unique.split(",") : undefined }));
94
+ }
95
+ pipeline.onProgress((info) => progress.update(info));
96
+ try {
97
+ await pipeline.to(writer);
98
+ }
99
+ finally {
100
+ if (reader.close)
101
+ await reader.close();
102
+ if (writer.close)
103
+ await writer.close();
104
+ }
105
+ progress.done();
106
+ logMemoryDebug();
107
+ }
108
+ //# sourceMappingURL=clean.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"clean.js","sourceRoot":"","sources":["../../../src/cli/commands/clean.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,2BAA2B,CAAC;AACtD,OAAO,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAC;AAExD,OAAO,EAAE,YAAY,EAAE,mBAAmB,IAAI,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAChG,OAAO,EAAE,YAAY,EAAE,mBAAmB,IAAI,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAChG,OAAO,EAAE,kBAAkB,EAAsB,MAAM,4BAA4B,CAAC;AACpF,OAAO,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAC3D,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAC3D,OAAO,EAAE,sBAAsB,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAClF,OAAO,EAAE,0BAA0B,EAA6B,MAAM,eAAe,CAAC;AAYtF;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,SAAS,GAAG,GAAG,EACf,UAA+B,EAAE;IAEjC,MAAM,QAAQ,GAAG,IAAI,gBAAgB,CAAC,OAAO,CAAC,CAAC;IAC/C,MAAM,kBAAkB,GAAG,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,IAAI,IAAI,CAAC;IAE7D,IAAI,UAAU,GAAG,OAAO,CAAC,IAAI,EAAE,WAAW,EAAE,CAAC;IAC7C,IAAI,CAAC,UAAU,IAAI,SAAS,KAAK,GAAG,EAAE,CAAC;QACrC,UAAU,GAAG,iBAAiB,CAAC,SAAS,CAAC,IAAI,SAAS,CAAC;IACzD,CAAC;IACD,IAAI,CAAC,UAAU,EAAE,CAAC;QAChB,UAAU,GAAG,KAAK,CAAC;IACrB,CAAC;IAED,IAAI,QAAQ,GAAG,OAAO,CAAC,EAAE,EAAE,WAAW,EAAE,CAAC;IACzC,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;QACjB,QAAQ,GAAG,MAAM,CAAC;IACpB,CAAC;SAAM,IAAI,CAAC,QAAQ,IAAI,OAAO,CAAC,MAAM,IAAI,OAAO,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;QACjE,QAAQ,GAAG,iBAAiB,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,SAAS,CAAC;IAC5D,CAAC;IACD,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,QAAQ,GAAG,UAAU,KAAK,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,UAAU,CAAC;IACxD,CAAC;IAED,MAAM,WAAW,GAAG,SAAS,KAAK,GAAG,CAAC,CAAC,CAAC,kBAAkB,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IACrF,MAAM,MAAM,GAAG,YAAY,CAAC,WAAW,EAAE;QACvC,MAAM,EAAE,UAAU;QAClB,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,QAAQ,EAAE,SAAS;QACnB,GAAG,OAAO;QACV,SAAS,EAAE,kBAAkB;KAC9B,CAAC,CAAC;IAEH,6EAA6E;IAC7E,IAAI,eAAe,GAAsC,SAAS,CAAC;IACnE,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;QACtB,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QAC3F,MAAM,OAAO,GAA4B,EAAE,CAAC;QAC5C,IAAI,QAAQ,GAAG,KAAK,CAAC;QACrB,IAAI,SAAS,GAAY,SAAS,CAAC;QAEnC,KAAK,MAAM,IAAI,IAAI,OAAO,EAAE,CAAC;YAC3B,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;YAChC,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;gBACd,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC;gBACxC,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;gBAC5C,IAAI,CAAC;oBACH,OAAO,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;gBACpC,CAAC;gBAAC,MAAM,CAAC;oBACP,OAAO,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC;gBACxB,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,QAAQ,GAAG,IAAI,CAAC;gBAChB,IAAI,CAAC;oBACH,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBAC/B,CAAC;gBAAC,MAAM,CAAC;oBACP,SAAS,GAAG,IAAI,CAAC;gBACnB,CAAC;YACH,CAAC;QACH,CAAC;QACD,eAAe,GAAG,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC;IACnD,CAAC;IAED,MAAM,QAAQ,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAErG,MAAM,cAAc,GAAG,SAAS,CAAC;QAC/B,IAAI,EAAE,OAAO,CAAC,IAAI,KAAK,KAAK;QAC5B,UAAU,EAAE,QAAQ;QACpB,SAAS,EAAE,eAAe;QAC1B,MAAM,EAAE,OAAO,CAAC,MAAM,KAAK,IAAI;QAC/B,UAAU,EAAE,OAAO,CAAC,UAAU;QAC9B,IAAI,EAAE,OAAO,CAAC,IAAI;QAClB,aAAa,EAAE,OAAO,CAAC,aAAa;KACrC,CAAC,CAAC;IAEH,MAAM,MAAM,GAAG,0BAA0B,CAAC,OAAO,CAAC,CAAC;IACnD,MAAM,IAAI,GAAG,gBAAgB,CAAC,MAAM,CAAC,CAAC;IAEtC,MAAM,MAAM,GAAG,YAAY,CAAC,OAAO,CAAC,MAAM,IAAI,GAAG,EAAE;QACjD,MAAM,EAAE,QAAQ;QAChB,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,GAAG,OAAO;KACX,CAAC,CAAC;IAEH,MAAM,QAAQ,GAAG,sBAAsB,CAAC,MAAM,EAAE,IAAI,EAAE,EAAE,SAAS,EAAE,kBAAkB,EAAE,CAAC,CAAC;IACzF,QAAQ,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;IAE9B,IAAI,OAAO,CAAC,MAAM,KAAK,SAAS,IAAI,OAAO,CAAC,MAAM,KAAK,KAAK,EAAE,CAAC;QAC7D,QAAQ,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,EAAE,EAAE,OAAO,OAAO,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;IAChH,CAAC;IAED,QAAQ,CAAC,UAAU,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;IAErD,IAAI,CAAC;QACH,MAAM,QAAQ,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC;IAC5B,CAAC;YAAS,CAAC;QACT,IAAI,MAAM,CAAC,KAAK;YAAE,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;QACvC,IAAI,MAAM,CAAC,KAAK;YAAE,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;IACzC,CAAC;IAED,QAAQ,CAAC,IAAI,EAAE,CAAC;IAChB,cAAc,EAAE,CAAC;AACnB,CAAC"}
@@ -7,6 +7,12 @@ export interface ConvertCommandOptions {
7
7
  path?: string;
8
8
  delimiter?: string;
9
9
  header?: boolean;
10
+ gzip?: boolean;
11
+ brotli?: boolean;
12
+ zstd?: boolean;
13
+ deflate?: boolean;
14
+ compress?: string;
15
+ compression?: string;
10
16
  batchSize?: string | number;
11
17
  quiet?: boolean;
12
18
  noProgress?: boolean;
@@ -1 +1 @@
1
- {"version":3,"file":"convert.d.ts","sourceRoot":"","sources":["../../../src/cli/commands/convert.ts"],"names":[],"mappings":"AAUA,MAAM,WAAW,qBAAqB;IACpC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IAC5B,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,UAAU,CAAC,EAAE,OAAO,CAAC;CACtB;AAED,wBAAsB,cAAc,CAClC,SAAS,SAAM,EACf,UAAU,CAAC,EAAE,MAAM,EACnB,OAAO,GAAE,qBAA0B,GAClC,OAAO,CAAC,IAAI,CAAC,CAmGf"}
1
+ {"version":3,"file":"convert.d.ts","sourceRoot":"","sources":["../../../src/cli/commands/convert.ts"],"names":[],"mappings":"AAUA,MAAM,WAAW,qBAAqB;IACpC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IAC5B,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,UAAU,CAAC,EAAE,OAAO,CAAC;CACtB;AAED,wBAAsB,cAAc,CAClC,SAAS,SAAM,EACf,UAAU,CAAC,EAAE,MAAM,EACnB,OAAO,GAAE,qBAA0B,GAClC,OAAO,CAAC,IAAI,CAAC,CAmGf"}
@@ -4,7 +4,6 @@ import { createPipeline } from "../../core/pipeline.js";
4
4
  import { createReader, inferFormatFromPath as inferReaderFormat } from "../../readers/index.js";
5
5
  import { XLSXReader } from "../../readers/xlsx.js";
6
6
  import { createWriter, inferFormatFromPath as inferWriterFormat } from "../../writers/index.js";
7
- import { openReadableStream, openWritableStream } from "../../utils/compression.js";
8
7
  import { formatNumber, logMemoryDebug } from "../../utils/formatting.js";
9
8
  import { ProgressReporter } from "../../utils/progress.js";
10
9
  export async function convertCommand(inputPath = "-", outputPath, options = {}) {
@@ -43,6 +42,7 @@ export async function convertCommand(inputPath = "-", outputPath, options = {})
43
42
  format: toFormat,
44
43
  delimiter: options.delimiter,
45
44
  header: options.header,
45
+ ...options,
46
46
  });
47
47
  const pipeline = createPipeline(sheetReader, { batchSize: effectiveBatchSize });
48
48
  await pipeline.to(sheetWriter);
@@ -63,22 +63,22 @@ export async function convertCommand(inputPath = "-", outputPath, options = {})
63
63
  }
64
64
  const progress = new ProgressReporter(options);
65
65
  const effectiveOutput = outputPath || "-";
66
- const inputStream = openReadableStream(inputPath);
67
- const outputStream = openWritableStream(effectiveOutput);
68
- const reader = createReader(inputStream, {
66
+ const reader = createReader(inputPath, {
69
67
  format: fromFormat,
70
68
  sheet: options.sheet,
71
69
  path: options.path,
72
70
  delimiter: options.delimiter,
73
71
  header: options.header,
74
- batchSize: effectiveBatchSize,
75
72
  filePath: inputPath,
73
+ ...options,
74
+ batchSize: effectiveBatchSize,
76
75
  });
77
- const writer = createWriter(outputStream, {
76
+ const writer = createWriter(effectiveOutput, {
78
77
  format: toFormat,
79
78
  delimiter: options.delimiter,
80
79
  header: options.header,
81
80
  sheet: options.sheet,
81
+ ...options,
82
82
  });
83
83
  const pipeline = createPipeline(reader, { batchSize: effectiveBatchSize });
84
84
  pipeline.onProgress((info) => progress.update(info));
@@ -1 +1 @@
1
- {"version":3,"file":"convert.js","sourceRoot":"","sources":["../../../src/cli/commands/convert.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AACzC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAC;AACxD,OAAO,EAAE,YAAY,EAAE,mBAAmB,IAAI,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAChG,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AACnD,OAAO,EAAE,YAAY,EAAE,mBAAmB,IAAI,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAChG,OAAO,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,MAAM,4BAA4B,CAAC;AACpF,OAAO,EAAE,YAAY,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AACzE,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAgB3D,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,SAAS,GAAG,GAAG,EACf,UAAmB,EACnB,UAAiC,EAAE;IAEnC,MAAM,kBAAkB,GAAG,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,IAAI,IAAI,CAAC;IAE7D,yBAAyB;IACzB,IAAI,UAAU,GAAG,OAAO,CAAC,IAAI,EAAE,WAAW,EAAE,CAAC;IAC7C,IAAI,CAAC,UAAU,IAAI,SAAS,KAAK,GAAG,EAAE,CAAC;QACrC,UAAU,GAAG,iBAAiB,CAAC,SAAS,CAAC,IAAI,SAAS,CAAC;IACzD,CAAC;IACD,IAAI,CAAC,UAAU,EAAE,CAAC;QAChB,UAAU,GAAG,KAAK,CAAC;IACrB,CAAC;IAED,yDAAyD;IACzD,IAAI,OAAO,CAAC,SAAS,IAAI,UAAU,KAAK,MAAM,IAAI,SAAS,KAAK,GAAG,EAAE,CAAC;QACpE,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,UAAU,IAAI,IAAI,CAAC;QACpD,MAAM,QAAQ,GAAG,CAAC,OAAO,CAAC,EAAE,EAAE,WAAW,EAAE,IAAI,KAAK,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QAEzE,MAAM,KAAK,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAEzC,MAAM,UAAU,GAAG,IAAI,UAAU,CAAC,SAAS,CAAC,CAAC;QAC7C,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,eAAe,EAAE,CAAC;QAElD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACxB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,gCAAgC,CAAC,CAAC;YACvD,OAAO;QACT,CAAC;QAED,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,aAAa,MAAM,CAAC,MAAM,eAAe,MAAM,QAAQ,CAAC,CAAC;QAE9E,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,MAAM,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC;YAC9B,MAAM,cAAc,GAAG,GAAG,SAAS,IAAI,QAAQ,EAAE,CAAC;YAClD,MAAM,cAAc,GAAG,IAAI,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC;YAEpD,MAAM,WAAW,GAAG,YAAY,CAAC,SAAS,EAAE;gBAC1C,MAAM,EAAE,MAAM;gBACd,KAAK,EAAE,SAAS;gBAChB,SAAS,EAAE,kBAAkB;gBAC7B,QAAQ,EAAE,SAAS;aACpB,CAAC,CAAC;YAEH,MAAM,WAAW,GAAG,YAAY,CAAC,cAAc,EAAE;gBAC/C,MAAM,EAAE,QAAQ;gBAChB,SAAS,EAAE,OAAO,CAAC,SAAS;gBAC5B,MAAM,EAAE,OAAO,CAAC,MAAM;aACvB,CAAC,CAAC;YAEH,MAAM,QAAQ,GAAG,cAAc,CAAC,WAAW,EAAE,EAAE,SAAS,EAAE,kBAAkB,EAAE,CAAC,CAAC;YAChF,MAAM,QAAQ,CAAC,EAAE,CAAC,WAAW,CAAC,CAAC;YAE/B,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;YACpD,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,cAAc,SAAS,QAAQ,cAAc,KAAK,YAAY,CAAC,QAAQ,CAAC,UAAU,CACnF,CAAC;QACJ,CAAC;QAED,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;QAChC,cAAc,EAAE,CAAC;QACjB,OAAO;IACT,CAAC;IAED,0BAA0B;IAC1B,IAAI,QAAQ,GAAG,OAAO,CAAC,EAAE,EAAE,WAAW,EAAE,CAAC;IACzC,IAAI,CAAC,QAAQ,IAAI,UAAU,IAAI,UAAU,KAAK,GAAG,EAAE,CAAC;QAClD,QAAQ,GAAG,iBAAiB,CAAC,UAAU,CAAC,IAAI,SAAS,CAAC;IACxD,CAAC;IACD,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,QAAQ,GAAG,KAAK,CAAC;IACnB,CAAC;IAED,MAAM,QAAQ,GAAG,IAAI,gBAAgB,CAAC,OAAO,CAAC,CAAC;IAC/C,MAAM,eAAe,GAAG,UAAU,IAAI,GAAG,CAAC;IAE1C,MAAM,WAAW,GAAG,kBAAkB,CAAC,SAAS,CAAC,CAAC;IAClD,MAAM,YAAY,GAAG,kBAAkB,CAAC,eAAe,CAAC,CAAC;IAEzD,MAAM,MAAM,GAAG,YAAY,CAAC,WAAW,EAAE;QACvC,MAAM,EAAE,UAAU;QAClB,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,IAAI,EAAE,OAAO,CAAC,IAAI;QAClB,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,SAAS,EAAE,kBAAkB;QAC7B,QAAQ,EAAE,SAAS;KACpB,CAAC,CAAC;IAEH,MAAM,MAAM,GAAG,YAAY,CAAC,YAAY,EAAE;QACxC,MAAM,EAAE,QAAQ;QAChB,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,KAAK,EAAE,OAAO,CAAC,KAAK;KACrB,CAAC,CAAC;IAEH,MAAM,QAAQ,GAAG,cAAc,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,kBAAkB,EAAE,CAAC,CAAC;IAC3E,QAAQ,CAAC,UAAU,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;IAErD,MAAM,QAAQ,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC;IAC1B,QAAQ,CAAC,IAAI,EAAE,CAAC;IAChB,cAAc,EAAE,CAAC;AACnB,CAAC"}
1
+ {"version":3,"file":"convert.js","sourceRoot":"","sources":["../../../src/cli/commands/convert.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AACzC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAC;AACxD,OAAO,EAAE,YAAY,EAAE,mBAAmB,IAAI,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAChG,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AACnD,OAAO,EAAE,YAAY,EAAE,mBAAmB,IAAI,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAEhG,OAAO,EAAE,YAAY,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AACzE,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAsB3D,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,SAAS,GAAG,GAAG,EACf,UAAmB,EACnB,UAAiC,EAAE;IAEnC,MAAM,kBAAkB,GAAG,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,IAAI,IAAI,CAAC;IAE7D,yBAAyB;IACzB,IAAI,UAAU,GAAG,OAAO,CAAC,IAAI,EAAE,WAAW,EAAE,CAAC;IAC7C,IAAI,CAAC,UAAU,IAAI,SAAS,KAAK,GAAG,EAAE,CAAC;QACrC,UAAU,GAAG,iBAAiB,CAAC,SAAS,CAAC,IAAI,SAAS,CAAC;IACzD,CAAC;IACD,IAAI,CAAC,UAAU,EAAE,CAAC;QAChB,UAAU,GAAG,KAAK,CAAC;IACrB,CAAC;IAED,yDAAyD;IACzD,IAAI,OAAO,CAAC,SAAS,IAAI,UAAU,KAAK,MAAM,IAAI,SAAS,KAAK,GAAG,EAAE,CAAC;QACpE,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,UAAU,IAAI,IAAI,CAAC;QACpD,MAAM,QAAQ,GAAG,CAAC,OAAO,CAAC,EAAE,EAAE,WAAW,EAAE,IAAI,KAAK,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QAEzE,MAAM,KAAK,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAEzC,MAAM,UAAU,GAAG,IAAI,UAAU,CAAC,SAAS,CAAC,CAAC;QAC7C,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,eAAe,EAAE,CAAC;QAElD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACxB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,gCAAgC,CAAC,CAAC;YACvD,OAAO;QACT,CAAC;QAED,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,aAAa,MAAM,CAAC,MAAM,eAAe,MAAM,QAAQ,CAAC,CAAC;QAE9E,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,MAAM,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC;YAC9B,MAAM,cAAc,GAAG,GAAG,SAAS,IAAI,QAAQ,EAAE,CAAC;YAClD,MAAM,cAAc,GAAG,IAAI,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC;YAEpD,MAAM,WAAW,GAAG,YAAY,CAAC,SAAS,EAAE;gBAC1C,MAAM,EAAE,MAAM;gBACd,KAAK,EAAE,SAAS;gBAChB,SAAS,EAAE,kBAAkB;gBAC7B,QAAQ,EAAE,SAAS;aACpB,CAAC,CAAC;YAEH,MAAM,WAAW,GAAG,YAAY,CAAC,cAAc,EAAE;gBAC/C,MAAM,EAAE,QAAQ;gBAChB,SAAS,EAAE,OAAO,CAAC,SAAS;gBAC5B,MAAM,EAAE,OAAO,CAAC,MAAM;gBACtB,GAAG,OAAO;aACX,CAAC,CAAC;YAEH,MAAM,QAAQ,GAAG,cAAc,CAAC,WAAW,EAAE,EAAE,SAAS,EAAE,kBAAkB,EAAE,CAAC,CAAC;YAChF,MAAM,QAAQ,CAAC,EAAE,CAAC,WAAW,CAAC,CAAC;YAE/B,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;YACpD,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,cAAc,SAAS,QAAQ,cAAc,KAAK,YAAY,CAAC,QAAQ,CAAC,UAAU,CACnF,CAAC;QACJ,CAAC;QAED,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;QAChC,cAAc,EAAE,CAAC;QACjB,OAAO;IACT,CAAC;IAED,0BAA0B;IAC1B,IAAI,QAAQ,GAAG,OAAO,CAAC,EAAE,EAAE,WAAW,EAAE,CAAC;IACzC,IAAI,CAAC,QAAQ,IAAI,UAAU,IAAI,UAAU,KAAK,GAAG,EAAE,CAAC;QAClD,QAAQ,GAAG,iBAAiB,CAAC,UAAU,CAAC,IAAI,SAAS,CAAC;IACxD,CAAC;IACD,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,QAAQ,GAAG,KAAK,CAAC;IACnB,CAAC;IAED,MAAM,QAAQ,GAAG,IAAI,gBAAgB,CAAC,OAAO,CAAC,CAAC;IAC/C,MAAM,eAAe,GAAG,UAAU,IAAI,GAAG,CAAC;IAE1C,MAAM,MAAM,GAAG,YAAY,CAAC,SAAS,EAAE;QACrC,MAAM,EAAE,UAAU;QAClB,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,IAAI,EAAE,OAAO,CAAC,IAAI;QAClB,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,QAAQ,EAAE,SAAS;QACnB,GAAG,OAAO;QACV,SAAS,EAAE,kBAAkB;KAC9B,CAAC,CAAC;IAEH,MAAM,MAAM,GAAG,YAAY,CAAC,eAAe,EAAE;QAC3C,MAAM,EAAE,QAAQ;QAChB,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,GAAG,OAAO;KACX,CAAC,CAAC;IAEH,MAAM,QAAQ,GAAG,cAAc,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,kBAAkB,EAAE,CAAC,CAAC;IAC3E,QAAQ,CAAC,UAAU,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;IAErD,MAAM,QAAQ,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC;IAC1B,QAAQ,CAAC,IAAI,EAAE,CAAC;IAChB,cAAc,EAAE,CAAC;AACnB,CAAC"}
@@ -0,0 +1,35 @@
1
+ import { UnifiedPipelineCliOptions } from "./pipeline.js";
2
+ export interface DbCommandOptions extends UnifiedPipelineCliOptions {
3
+ table?: string;
4
+ query?: string;
5
+ params?: string | string[];
6
+ param?: string | string[];
7
+ where?: string;
8
+ tables?: boolean;
9
+ schema?: string | boolean;
10
+ toDb?: string;
11
+ toTable?: string;
12
+ transaction?: boolean;
13
+ createTable?: boolean;
14
+ upsert?: boolean;
15
+ conflict?: string;
16
+ truncate?: boolean;
17
+ dryRun?: boolean;
18
+ }
19
+ /**
20
+ * Main handler for `rowpipe db <url> [options]`.
21
+ */
22
+ export declare function dbCommand(dbUrl: string, options?: DbCommandOptions): Promise<void>;
23
+ /**
24
+ * List all tables in the specified database.
25
+ */
26
+ export declare function dbTablesCommand(dbUrl: string, options?: {
27
+ json?: boolean;
28
+ }): Promise<void>;
29
+ /**
30
+ * Inspect the schema of a specific table in the database.
31
+ */
32
+ export declare function dbSchemaCommand(dbUrl: string, table: string, options?: {
33
+ json?: boolean;
34
+ }): Promise<void>;
35
+ //# sourceMappingURL=db.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"db.d.ts","sourceRoot":"","sources":["../../../src/cli/commands/db.ts"],"names":[],"mappings":"AAUA,OAAO,EAA8B,yBAAyB,EAAE,MAAM,eAAe,CAAC;AAGtF,MAAM,WAAW,gBAAiB,SAAQ,yBAAyB;IACjE,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IAC3B,KAAK,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IAC1B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC;IAC1B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAED;;GAEG;AACH,wBAAsB,SAAS,CAC7B,KAAK,EAAE,MAAM,EACb,OAAO,GAAE,gBAAqB,GAC7B,OAAO,CAAC,IAAI,CAAC,CAiIf;AAED;;GAEG;AACH,wBAAsB,eAAe,CACnC,KAAK,EAAE,MAAM,EACb,OAAO,GAAE;IAAE,IAAI,CAAC,EAAE,OAAO,CAAA;CAAO,GAC/B,OAAO,CAAC,IAAI,CAAC,CAmBf;AAED;;GAEG;AACH,wBAAsB,eAAe,CACnC,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,MAAM,EACb,OAAO,GAAE;IAAE,IAAI,CAAC,EAAE,OAAO,CAAA;CAAO,GAC/B,OAAO,CAAC,IAAI,CAAC,CAwBf"}
@@ -0,0 +1,173 @@
1
+ import { DatabaseReader } from "../../db/source.js";
2
+ import { DatabaseWriter } from "../../db/sink.js";
3
+ import { createDatabaseAdapter } from "../../db/adapters/index.js";
4
+ import { parseDatabaseUrl } from "../../db/url.js";
5
+ import { analyzeDatabasePushdown } from "../../db/pushdown.js";
6
+ import { createWriter, inferFormatFromPath as inferWriterFormat } from "../../writers/index.js";
7
+ import { openWritableStream } from "../../utils/compression.js";
8
+ import { formatTable, logMemoryDebug, safeJsonStringify } from "../../utils/formatting.js";
9
+ import { ProgressReporter } from "../../utils/progress.js";
10
+ import { executePlannedPipeline, optimizePipeline } from "../../planner/index.js";
11
+ import { buildOperationsFromOptions } from "./pipeline.js";
12
+ import { InvalidArgumentError } from "../../core/errors.js";
13
+ /**
14
+ * Main handler for `rowpipe db <url> [options]`.
15
+ */
16
+ export async function dbCommand(dbUrl, options = {}) {
17
+ if (!dbUrl || dbUrl === "-") {
18
+ throw new InvalidArgumentError("Database connection URL is required (e.g. postgres://..., mysql://..., sqlite://...)");
19
+ }
20
+ // Handle table list introspection
21
+ if (options.tables) {
22
+ await dbTablesCommand(dbUrl, options);
23
+ return;
24
+ }
25
+ // Handle schema inspection
26
+ if (options.schema) {
27
+ const table = typeof options.schema === "string" ? options.schema : options.table;
28
+ if (!table) {
29
+ throw new InvalidArgumentError("Specify table to inspect schema (e.g. rowpipe db <url> --schema <table_name>)");
30
+ }
31
+ await dbSchemaCommand(dbUrl, table, options);
32
+ return;
33
+ }
34
+ const effectiveBatchSize = Number(options.batchSize) || 1000;
35
+ const progress = new ProgressReporter(options);
36
+ // Parse query parameters
37
+ let queryParams = [];
38
+ if (options.params) {
39
+ const raw = Array.isArray(options.params) ? options.params.join("") : options.params;
40
+ try {
41
+ queryParams = JSON.parse(raw);
42
+ }
43
+ catch {
44
+ queryParams = [raw];
45
+ }
46
+ }
47
+ else if (options.param) {
48
+ queryParams = Array.isArray(options.param) ? options.param : [options.param];
49
+ }
50
+ const rawOperations = buildOperationsFromOptions(options);
51
+ const parsedConfig = parseDatabaseUrl(dbUrl);
52
+ let finalQuery = options.query;
53
+ let remainingOps = rawOperations;
54
+ // Pushdown analysis when using --table
55
+ if (options.table && !options.query) {
56
+ const analysis = analyzeDatabasePushdown(options.table, parsedConfig.dialect, rawOperations, options.where);
57
+ finalQuery = analysis.generatedQuery;
58
+ remainingOps = analysis.remainingOperations;
59
+ }
60
+ const optimizedPlan = optimizePipeline(remainingOps);
61
+ const reader = new DatabaseReader(parsedConfig, {
62
+ table: options.table,
63
+ query: finalQuery,
64
+ params: queryParams,
65
+ batchSize: effectiveBatchSize,
66
+ });
67
+ // Check if output is another database sink
68
+ if (options.toDb) {
69
+ const targetTable = options.toTable || options.table;
70
+ if (!targetTable) {
71
+ throw new InvalidArgumentError("Specify target table when writing to database (e.g. --to-table <name>)");
72
+ }
73
+ const conflictCols = options.conflict
74
+ ? options.conflict.split(",").map((s) => s.trim()).filter(Boolean)
75
+ : undefined;
76
+ const writer = new DatabaseWriter(options.toDb, {
77
+ table: targetTable,
78
+ transaction: options.transaction,
79
+ createTable: options.createTable,
80
+ upsert: options.upsert,
81
+ conflictColumns: conflictCols,
82
+ truncate: options.truncate,
83
+ dryRun: options.dryRun,
84
+ });
85
+ const pipeline = executePlannedPipeline(reader, optimizedPlan, { batchSize: effectiveBatchSize });
86
+ pipeline.onProgress((info) => progress.update(info));
87
+ try {
88
+ await pipeline.to(writer);
89
+ }
90
+ finally {
91
+ await reader.close();
92
+ await writer.close();
93
+ }
94
+ progress.done();
95
+ logMemoryDebug();
96
+ return;
97
+ }
98
+ // Standard File / Stdout Writer
99
+ let toFormat = options.to?.toLowerCase();
100
+ if (options.json) {
101
+ toFormat = "json";
102
+ }
103
+ else if (!toFormat && options.output && options.output !== "-") {
104
+ toFormat = inferWriterFormat(options.output) ?? undefined;
105
+ }
106
+ if (!toFormat) {
107
+ toFormat = "csv";
108
+ }
109
+ const effectiveOutput = options.output || "-";
110
+ const outputStream = openWritableStream(effectiveOutput);
111
+ const writer = createWriter(outputStream, {
112
+ format: toFormat,
113
+ delimiter: options.delimiter,
114
+ });
115
+ const pipeline = executePlannedPipeline(reader, optimizedPlan, { batchSize: effectiveBatchSize });
116
+ pipeline.onProgress((info) => progress.update(info));
117
+ try {
118
+ await pipeline.to(writer);
119
+ }
120
+ finally {
121
+ await reader.close();
122
+ }
123
+ progress.done();
124
+ logMemoryDebug();
125
+ }
126
+ /**
127
+ * List all tables in the specified database.
128
+ */
129
+ export async function dbTablesCommand(dbUrl, options = {}) {
130
+ const adapter = createDatabaseAdapter(dbUrl);
131
+ try {
132
+ const tables = await adapter.getTables();
133
+ if (options.json) {
134
+ process.stdout.write(safeJsonStringify(tables, 2) + "\n");
135
+ return;
136
+ }
137
+ if (tables.length === 0) {
138
+ process.stdout.write("(no tables found)\n");
139
+ return;
140
+ }
141
+ process.stdout.write(tables.join("\n") + "\n");
142
+ }
143
+ finally {
144
+ await adapter.close();
145
+ }
146
+ }
147
+ /**
148
+ * Inspect the schema of a specific table in the database.
149
+ */
150
+ export async function dbSchemaCommand(dbUrl, table, options = {}) {
151
+ const adapter = createDatabaseAdapter(dbUrl);
152
+ try {
153
+ const schema = await adapter.getTableSchema(table);
154
+ if (options.json) {
155
+ process.stdout.write(safeJsonStringify(schema, 2) + "\n");
156
+ return;
157
+ }
158
+ const headers = ["COLUMN", "DATABASE TYPE", "ROWPIPE TYPE", "NULLABLE", "PRIMARY KEY"];
159
+ const rows = schema.columns.map((c) => [
160
+ c.name,
161
+ c.dbType,
162
+ c.rowpipeType,
163
+ c.nullable ? "YES" : "NO",
164
+ c.isPrimaryKey ? "YES" : "NO",
165
+ ]);
166
+ process.stdout.write(`\nTable: ${table} (${adapter.dialect})\n\n`);
167
+ process.stdout.write(formatTable(headers, rows) + "\n");
168
+ }
169
+ finally {
170
+ await adapter.close();
171
+ }
172
+ }
173
+ //# sourceMappingURL=db.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"db.js","sourceRoot":"","sources":["../../../src/cli/commands/db.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACpD,OAAO,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAClD,OAAO,EAAE,qBAAqB,EAAE,MAAM,4BAA4B,CAAC;AACnE,OAAO,EAAiB,gBAAgB,EAA4B,MAAM,iBAAiB,CAAC;AAC5F,OAAO,EAAE,uBAAuB,EAAE,MAAM,sBAAsB,CAAC;AAC/D,OAAO,EAAE,YAAY,EAAE,mBAAmB,IAAI,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAChG,OAAO,EAAE,kBAAkB,EAAE,MAAM,4BAA4B,CAAC;AAChE,OAAO,EAAE,WAAW,EAAE,cAAc,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAC3F,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAC3D,OAAO,EAAE,sBAAsB,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAClF,OAAO,EAAE,0BAA0B,EAA6B,MAAM,eAAe,CAAC;AACtF,OAAO,EAAE,oBAAoB,EAAE,MAAM,sBAAsB,CAAC;AAoB5D;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,KAAa,EACb,UAA4B,EAAE;IAE9B,IAAI,CAAC,KAAK,IAAI,KAAK,KAAK,GAAG,EAAE,CAAC;QAC5B,MAAM,IAAI,oBAAoB,CAAC,sFAAsF,CAAC,CAAC;IACzH,CAAC;IAED,kCAAkC;IAClC,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;QACnB,MAAM,eAAe,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;QACtC,OAAO;IACT,CAAC;IAED,2BAA2B;IAC3B,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;QACnB,MAAM,KAAK,GAAG,OAAO,OAAO,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC;QAClF,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,MAAM,IAAI,oBAAoB,CAAC,+EAA+E,CAAC,CAAC;QAClH,CAAC;QACD,MAAM,eAAe,CAAC,KAAK,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC;QAC7C,OAAO;IACT,CAAC;IAED,MAAM,kBAAkB,GAAG,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,IAAI,IAAI,CAAC;IAC7D,MAAM,QAAQ,GAAG,IAAI,gBAAgB,CAAC,OAAO,CAAC,CAAC;IAE/C,yBAAyB;IACzB,IAAI,WAAW,GAAc,EAAE,CAAC;IAChC,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;QACnB,MAAM,GAAG,GAAG,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC;QACrF,IAAI,CAAC;YACH,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAChC,CAAC;QAAC,MAAM,CAAC;YACP,WAAW,GAAG,CAAC,GAAG,CAAC,CAAC;QACtB,CAAC;IACH,CAAC;SAAM,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;QACzB,WAAW,GAAG,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IAC/E,CAAC;IAED,MAAM,aAAa,GAAG,0BAA0B,CAAC,OAAO,CAAC,CAAC;IAC1D,MAAM,YAAY,GAAG,gBAAgB,CAAC,KAAK,CAAC,CAAC;IAE7C,IAAI,UAAU,GAAuB,OAAO,CAAC,KAAK,CAAC;IACnD,IAAI,YAAY,GAAG,aAAa,CAAC;IAEjC,uCAAuC;IACvC,IAAI,OAAO,CAAC,KAAK,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;QACpC,MAAM,QAAQ,GAAG,uBAAuB,CACtC,OAAO,CAAC,KAAK,EACb,YAAY,CAAC,OAAO,EACpB,aAAa,EACb,OAAO,CAAC,KAAK,CACd,CAAC;QACF,UAAU,GAAG,QAAQ,CAAC,cAAc,CAAC;QACrC,YAAY,GAAG,QAAQ,CAAC,mBAAmB,CAAC;IAC9C,CAAC;IAED,MAAM,aAAa,GAAG,gBAAgB,CAAC,YAAY,CAAC,CAAC;IAErD,MAAM,MAAM,GAAG,IAAI,cAAc,CAAC,YAAY,EAAE;QAC9C,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,KAAK,EAAE,UAAU;QACjB,MAAM,EAAE,WAAW;QACnB,SAAS,EAAE,kBAAkB;KAC9B,CAAC,CAAC;IAEH,2CAA2C;IAC3C,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;QACjB,MAAM,WAAW,GAAG,OAAO,CAAC,OAAO,IAAI,OAAO,CAAC,KAAK,CAAC;QACrD,IAAI,CAAC,WAAW,EAAE,CAAC;YACjB,MAAM,IAAI,oBAAoB,CAAC,wEAAwE,CAAC,CAAC;QAC3G,CAAC;QAED,MAAM,YAAY,GAAG,OAAO,CAAC,QAAQ;YACnC,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC;YAClE,CAAC,CAAC,SAAS,CAAC;QAEd,MAAM,MAAM,GAAG,IAAI,cAAc,CAAC,OAAO,CAAC,IAAI,EAAE;YAC9C,KAAK,EAAE,WAAW;YAClB,WAAW,EAAE,OAAO,CAAC,WAAW;YAChC,WAAW,EAAE,OAAO,CAAC,WAAW;YAChC,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,eAAe,EAAE,YAAY;YAC7B,QAAQ,EAAE,OAAO,CAAC,QAAQ;YAC1B,MAAM,EAAE,OAAO,CAAC,MAAM;SACvB,CAAC,CAAC;QAEH,MAAM,QAAQ,GAAG,sBAAsB,CAAC,MAAM,EAAE,aAAa,EAAE,EAAE,SAAS,EAAE,kBAAkB,EAAE,CAAC,CAAC;QAClG,QAAQ,CAAC,UAAU,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;QAErD,IAAI,CAAC;YACH,MAAM,QAAQ,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC;QAC5B,CAAC;gBAAS,CAAC;YACT,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;YACrB,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;QACvB,CAAC;QAED,QAAQ,CAAC,IAAI,EAAE,CAAC;QAChB,cAAc,EAAE,CAAC;QACjB,OAAO;IACT,CAAC;IAED,gCAAgC;IAChC,IAAI,QAAQ,GAAG,OAAO,CAAC,EAAE,EAAE,WAAW,EAAE,CAAC;IACzC,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;QACjB,QAAQ,GAAG,MAAM,CAAC;IACpB,CAAC;SAAM,IAAI,CAAC,QAAQ,IAAI,OAAO,CAAC,MAAM,IAAI,OAAO,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;QACjE,QAAQ,GAAG,iBAAiB,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,SAAS,CAAC;IAC5D,CAAC;IACD,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,QAAQ,GAAG,KAAK,CAAC;IACnB,CAAC;IAED,MAAM,eAAe,GAAG,OAAO,CAAC,MAAM,IAAI,GAAG,CAAC;IAC9C,MAAM,YAAY,GAAG,kBAAkB,CAAC,eAAe,CAAC,CAAC;IACzD,MAAM,MAAM,GAAG,YAAY,CAAC,YAAY,EAAE;QACxC,MAAM,EAAE,QAAQ;QAChB,SAAS,EAAE,OAAO,CAAC,SAAS;KAC7B,CAAC,CAAC;IAEH,MAAM,QAAQ,GAAG,sBAAsB,CAAC,MAAM,EAAE,aAAa,EAAE,EAAE,SAAS,EAAE,kBAAkB,EAAE,CAAC,CAAC;IAClG,QAAQ,CAAC,UAAU,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;IAErD,IAAI,CAAC;QACH,MAAM,QAAQ,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC;IAC5B,CAAC;YAAS,CAAC;QACT,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;IACvB,CAAC;IAED,QAAQ,CAAC,IAAI,EAAE,CAAC;IAChB,cAAc,EAAE,CAAC;AACnB,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,KAAa,EACb,UAA8B,EAAE;IAEhC,MAAM,OAAO,GAAG,qBAAqB,CAAC,KAAK,CAAC,CAAC;IAC7C,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,SAAS,EAAE,CAAC;QAEzC,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;YACjB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,iBAAiB,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC;YAC1D,OAAO;QACT,CAAC;QAED,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACxB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAC;YAC5C,OAAO;QACT,CAAC;QAED,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC;IACjD,CAAC;YAAS,CAAC;QACT,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;IACxB,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,KAAa,EACb,KAAa,EACb,UAA8B,EAAE;IAEhC,MAAM,OAAO,GAAG,qBAAqB,CAAC,KAAK,CAAC,CAAC;IAC7C,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC;QAEnD,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;YACjB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,iBAAiB,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC;YAC1D,OAAO;QACT,CAAC;QAED,MAAM,OAAO,GAAG,CAAC,QAAQ,EAAE,eAAe,EAAE,cAAc,EAAE,UAAU,EAAE,aAAa,CAAC,CAAC;QACvF,MAAM,IAAI,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;YACrC,CAAC,CAAC,IAAI;YACN,CAAC,CAAC,MAAM;YACR,CAAC,CAAC,WAAW;YACb,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI;YACzB,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI;SAC9B,CAAC,CAAC;QAEH,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,YAAY,KAAK,KAAK,OAAO,CAAC,OAAO,OAAO,CAAC,CAAC;QACnE,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,WAAW,CAAC,OAAO,EAAE,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC;IAC1D,CAAC;YAAS,CAAC;QACT,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;IACxB,CAAC;AACH,CAAC"}
@@ -1,3 +1,10 @@
1
1
  import { UnifiedPipelineCliOptions } from "./pipeline.js";
2
- export declare function explainCommand(inputPath?: string, options?: UnifiedPipelineCliOptions): Promise<void>;
2
+ export interface ExplainCliOptions extends UnifiedPipelineCliOptions {
3
+ table?: string;
4
+ query?: string;
5
+ where?: string;
6
+ toDb?: string;
7
+ toTable?: string;
8
+ }
9
+ export declare function explainCommand(inputOrArgs?: string | string[], options?: ExplainCliOptions): Promise<void>;
3
10
  //# sourceMappingURL=explain.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"explain.d.ts","sourceRoot":"","sources":["../../../src/cli/commands/explain.ts"],"names":[],"mappings":"AACA,OAAO,EAA8B,yBAAyB,EAAE,MAAM,eAAe,CAAC;AAEtF,wBAAsB,cAAc,CAClC,SAAS,SAAM,EACf,OAAO,GAAE,yBAA8B,GACtC,OAAO,CAAC,IAAI,CAAC,CASf"}
1
+ {"version":3,"file":"explain.d.ts","sourceRoot":"","sources":["../../../src/cli/commands/explain.ts"],"names":[],"mappings":"AAGA,OAAO,EAA8B,yBAAyB,EAAE,MAAM,eAAe,CAAC;AAEtF,MAAM,WAAW,iBAAkB,SAAQ,yBAAyB;IAClE,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,wBAAsB,cAAc,CAClC,WAAW,GAAE,MAAM,GAAG,MAAM,EAAQ,EACpC,OAAO,GAAE,iBAAsB,GAC9B,OAAO,CAAC,IAAI,CAAC,CAiGf"}
@@ -1,10 +1,97 @@
1
1
  import { formatExecutionPlan, optimizePipeline } from "../../planner/index.js";
2
+ import { isDatabaseUrl, parseDatabaseUrl, sanitizeConnectionString } from "../../db/url.js";
3
+ import { analyzeDatabasePushdown } from "../../db/pushdown.js";
2
4
  import { buildOperationsFromOptions } from "./pipeline.js";
3
- export async function explainCommand(inputPath = "-", options = {}) {
5
+ export async function explainCommand(inputOrArgs = "-", options = {}) {
6
+ let inputPath = "-";
7
+ if (Array.isArray(inputOrArgs)) {
8
+ if (inputOrArgs[0] === "db") {
9
+ inputPath = inputOrArgs[1] || "-";
10
+ }
11
+ else {
12
+ inputPath = inputOrArgs[0] || "-";
13
+ }
14
+ }
15
+ else {
16
+ inputPath = inputOrArgs;
17
+ }
4
18
  const operations = buildOperationsFromOptions(options);
19
+ // Check if input is a database URL or table mode
20
+ if (isDatabaseUrl(inputPath) || options.table || options.query) {
21
+ let sanitizedDb = "Database";
22
+ let dialect = "sqlite";
23
+ if (isDatabaseUrl(inputPath)) {
24
+ const cfg = parseDatabaseUrl(inputPath);
25
+ sanitizedDb = cfg.sanitizedUrl;
26
+ dialect = cfg.dialect;
27
+ }
28
+ if (options.table && !options.query) {
29
+ const analysis = analyzeDatabasePushdown(options.table, dialect, operations, options.where);
30
+ const optimizedPlan = optimizePipeline(analysis.remainingOperations);
31
+ let out = "\n======================================================\n";
32
+ out += " Rowpipe Execution Plan \n";
33
+ out += "======================================================\n\n";
34
+ out += `DatabaseSource (${sanitizedDb})\n`;
35
+ out += ` Table: ${options.table}\n`;
36
+ out += ` Dialect: ${dialect}\n\n`;
37
+ out += "Remote Database Pushdown:\n";
38
+ out += ` SQL: ${analysis.generatedQuery}\n`;
39
+ if (analysis.pushedSummary.length > 0) {
40
+ for (const s of analysis.pushedSummary) {
41
+ out += ` • ${s}\n`;
42
+ }
43
+ }
44
+ else {
45
+ out += " • SELECT *\n";
46
+ }
47
+ out += "\nLocal Stream Operations:\n";
48
+ if (optimizedPlan.operations.length === 0) {
49
+ out += " (none - full pushdown achieved!)\n";
50
+ }
51
+ else {
52
+ for (const op of optimizedPlan.operations) {
53
+ out += ` • ${op.type}\n`;
54
+ }
55
+ }
56
+ out += "\nOutput Target:\n";
57
+ if (options.toDb) {
58
+ out += ` DatabaseSink (${sanitizeConnectionString(options.toDb)}) -> Table: ${options.toTable || options.table}\n`;
59
+ }
60
+ else if (options.output) {
61
+ out += ` Writer (${options.output})\n`;
62
+ }
63
+ else {
64
+ out += " Stdout Writer\n";
65
+ }
66
+ out += "\n------------------------------------------------------\n";
67
+ if (optimizedPlan.optimizationsApplied.length > 0) {
68
+ out += "Local Optimizations Applied:\n";
69
+ for (const opt of optimizedPlan.optimizationsApplied) {
70
+ out += ` • ${opt}\n`;
71
+ }
72
+ out += "------------------------------------------------------\n";
73
+ }
74
+ out += "Memory Classification:\n";
75
+ if (optimizedPlan.memoryClassification === "streaming") {
76
+ out += " Fully Streaming (Cursor-based bounded memory)\n";
77
+ }
78
+ else {
79
+ out += ` ${optimizedPlan.memoryClassification}\n`;
80
+ }
81
+ out += "======================================================\n\n";
82
+ process.stdout.write(out);
83
+ return;
84
+ }
85
+ }
5
86
  const plan = optimizePipeline(operations);
6
87
  const inputDesc = inputPath === "-" ? "Stdin Stream" : `Reader (${inputPath})`;
7
- const outputDesc = options.output ? `Writer (${options.output})` : "Stdout Writer";
88
+ let outputDesc = "Stdout Writer";
89
+ if (options.toDb) {
90
+ outputDesc = `DatabaseSink (${sanitizeConnectionString(options.toDb)}) -> Table: ${options.toTable || "target"}`;
91
+ }
92
+ else if (options.output) {
93
+ outputDesc = `Writer (${options.output})`;
94
+ }
8
95
  const formatted = formatExecutionPlan(plan, inputDesc, outputDesc);
9
96
  process.stdout.write(formatted);
10
97
  }
@@ -1 +1 @@
1
- {"version":3,"file":"explain.js","sourceRoot":"","sources":["../../../src/cli/commands/explain.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,mBAAmB,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAC/E,OAAO,EAAE,0BAA0B,EAA6B,MAAM,eAAe,CAAC;AAEtF,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,SAAS,GAAG,GAAG,EACf,UAAqC,EAAE;IAEvC,MAAM,UAAU,GAAG,0BAA0B,CAAC,OAAO,CAAC,CAAC;IACvD,MAAM,IAAI,GAAG,gBAAgB,CAAC,UAAU,CAAC,CAAC;IAE1C,MAAM,SAAS,GAAG,SAAS,KAAK,GAAG,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,WAAW,SAAS,GAAG,CAAC;IAC/E,MAAM,UAAU,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,WAAW,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,eAAe,CAAC;IAEnF,MAAM,SAAS,GAAG,mBAAmB,CAAC,IAAI,EAAE,SAAS,EAAE,UAAU,CAAC,CAAC;IACnE,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;AAClC,CAAC"}
1
+ {"version":3,"file":"explain.js","sourceRoot":"","sources":["../../../src/cli/commands/explain.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,mBAAmB,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAC/E,OAAO,EAAE,aAAa,EAAE,gBAAgB,EAAE,wBAAwB,EAAE,MAAM,iBAAiB,CAAC;AAC5F,OAAO,EAAE,uBAAuB,EAAE,MAAM,sBAAsB,CAAC;AAC/D,OAAO,EAAE,0BAA0B,EAA6B,MAAM,eAAe,CAAC;AAUtF,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,cAAiC,GAAG,EACpC,UAA6B,EAAE;IAE/B,IAAI,SAAS,GAAG,GAAG,CAAC;IACpB,IAAI,KAAK,CAAC,OAAO,CAAC,WAAW,CAAC,EAAE,CAAC;QAC/B,IAAI,WAAW,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;YAC5B,SAAS,GAAG,WAAW,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC;QACpC,CAAC;aAAM,CAAC;YACN,SAAS,GAAG,WAAW,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC;QACpC,CAAC;IACH,CAAC;SAAM,CAAC;QACN,SAAS,GAAG,WAAW,CAAC;IAC1B,CAAC;IAED,MAAM,UAAU,GAAG,0BAA0B,CAAC,OAAO,CAAC,CAAC;IAEvD,iDAAiD;IACjD,IAAI,aAAa,CAAC,SAAS,CAAC,IAAI,OAAO,CAAC,KAAK,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;QAC/D,IAAI,WAAW,GAAG,UAAU,CAAC;QAC7B,IAAI,OAAO,GAAQ,QAAQ,CAAC;QAC5B,IAAI,aAAa,CAAC,SAAS,CAAC,EAAE,CAAC;YAC7B,MAAM,GAAG,GAAG,gBAAgB,CAAC,SAAS,CAAC,CAAC;YACxC,WAAW,GAAG,GAAG,CAAC,YAAY,CAAC;YAC/B,OAAO,GAAG,GAAG,CAAC,OAAO,CAAC;QACxB,CAAC;QAED,IAAI,OAAO,CAAC,KAAK,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;YACpC,MAAM,QAAQ,GAAG,uBAAuB,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,EAAE,UAAU,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;YAC5F,MAAM,aAAa,GAAG,gBAAgB,CAAC,QAAQ,CAAC,mBAAmB,CAAC,CAAC;YAErE,IAAI,GAAG,GAAG,4DAA4D,CAAC;YACvE,GAAG,IAAI,0DAA0D,CAAC;YAClE,GAAG,IAAI,4DAA4D,CAAC;YAEpE,GAAG,IAAI,mBAAmB,WAAW,KAAK,CAAC;YAC3C,GAAG,IAAI,YAAY,OAAO,CAAC,KAAK,IAAI,CAAC;YACrC,GAAG,IAAI,cAAc,OAAO,MAAM,CAAC;YAEnC,GAAG,IAAI,6BAA6B,CAAC;YACrC,GAAG,IAAI,UAAU,QAAQ,CAAC,cAAc,IAAI,CAAC;YAC7C,IAAI,QAAQ,CAAC,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACtC,KAAK,MAAM,CAAC,IAAI,QAAQ,CAAC,aAAa,EAAE,CAAC;oBACvC,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;gBACtB,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,GAAG,IAAI,gBAAgB,CAAC;YAC1B,CAAC;YAED,GAAG,IAAI,8BAA8B,CAAC;YACtC,IAAI,aAAa,CAAC,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC1C,GAAG,IAAI,sCAAsC,CAAC;YAChD,CAAC;iBAAM,CAAC;gBACN,KAAK,MAAM,EAAE,IAAI,aAAa,CAAC,UAAU,EAAE,CAAC;oBAC1C,GAAG,IAAI,OAAO,EAAE,CAAC,IAAI,IAAI,CAAC;gBAC5B,CAAC;YACH,CAAC;YAED,GAAG,IAAI,oBAAoB,CAAC;YAC5B,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;gBACjB,GAAG,IAAI,mBAAmB,wBAAwB,CAAC,OAAO,CAAC,IAAI,CAAC,eAAe,OAAO,CAAC,OAAO,IAAI,OAAO,CAAC,KAAK,IAAI,CAAC;YACtH,CAAC;iBAAM,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;gBAC1B,GAAG,IAAI,aAAa,OAAO,CAAC,MAAM,KAAK,CAAC;YAC1C,CAAC;iBAAM,CAAC;gBACN,GAAG,IAAI,mBAAmB,CAAC;YAC7B,CAAC;YAED,GAAG,IAAI,4DAA4D,CAAC;YACpE,IAAI,aAAa,CAAC,oBAAoB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAClD,GAAG,IAAI,gCAAgC,CAAC;gBACxC,KAAK,MAAM,GAAG,IAAI,aAAa,CAAC,oBAAoB,EAAE,CAAC;oBACrD,GAAG,IAAI,OAAO,GAAG,IAAI,CAAC;gBACxB,CAAC;gBACD,GAAG,IAAI,0DAA0D,CAAC;YACpE,CAAC;YAED,GAAG,IAAI,0BAA0B,CAAC;YAClC,IAAI,aAAa,CAAC,oBAAoB,KAAK,WAAW,EAAE,CAAC;gBACvD,GAAG,IAAI,mDAAmD,CAAC;YAC7D,CAAC;iBAAM,CAAC;gBACN,GAAG,IAAI,KAAK,aAAa,CAAC,oBAAoB,IAAI,CAAC;YACrD,CAAC;YACD,GAAG,IAAI,4DAA4D,CAAC;YAEpE,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YAC1B,OAAO;QACT,CAAC;IACH,CAAC;IAED,MAAM,IAAI,GAAG,gBAAgB,CAAC,UAAU,CAAC,CAAC;IAC1C,MAAM,SAAS,GAAG,SAAS,KAAK,GAAG,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,WAAW,SAAS,GAAG,CAAC;IAC/E,IAAI,UAAU,GAAG,eAAe,CAAC;IACjC,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;QACjB,UAAU,GAAG,iBAAiB,wBAAwB,CAAC,OAAO,CAAC,IAAI,CAAC,eAAe,OAAO,CAAC,OAAO,IAAI,QAAQ,EAAE,CAAC;IACnH,CAAC;SAAM,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;QAC1B,UAAU,GAAG,WAAW,OAAO,CAAC,MAAM,GAAG,CAAC;IAC5C,CAAC;IAED,MAAM,SAAS,GAAG,mBAAmB,CAAC,IAAI,EAAE,SAAS,EAAE,UAAU,CAAC,CAAC;IACnE,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;AAClC,CAAC"}
@@ -0,0 +1,23 @@
1
+ import type { JoinType } from "../../transforms/join/types.js";
2
+ import { UnifiedPipelineCliOptions } from "./pipeline.js";
3
+ export interface JoinCommandOptions extends UnifiedPipelineCliOptions {
4
+ on?: string | string[];
5
+ leftKey?: string | string[];
6
+ rightKey?: string | string[];
7
+ type?: JoinType;
8
+ prefixLeft?: string;
9
+ suffixLeft?: string;
10
+ prefixRight?: string;
11
+ suffixRight?: string;
12
+ fromLeft?: string;
13
+ fromRight?: string;
14
+ leftSheet?: string;
15
+ rightSheet?: string;
16
+ leftDelimiter?: string;
17
+ rightDelimiter?: string;
18
+ }
19
+ /**
20
+ * Handles `rowpipe join <left> <right> [options]`
21
+ */
22
+ export declare function joinCommand(leftPath: string, rightPath: string, options?: JoinCommandOptions): Promise<void>;
23
+ //# sourceMappingURL=join.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"join.d.ts","sourceRoot":"","sources":["../../../src/cli/commands/join.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,gCAAgC,CAAC;AAK/D,OAAO,EAA8B,yBAAyB,EAAE,MAAM,eAAe,CAAC;AAGtF,MAAM,WAAW,kBAAmB,SAAQ,yBAAyB;IACnE,EAAE,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IACvB,OAAO,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IAC5B,QAAQ,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IAC7B,IAAI,CAAC,EAAE,QAAQ,CAAC;IAChB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAED;;GAEG;AACH,wBAAsB,WAAW,CAC/B,QAAQ,EAAE,MAAM,EAChB,SAAS,EAAE,MAAM,EACjB,OAAO,GAAE,kBAAuB,GAC/B,OAAO,CAAC,IAAI,CAAC,CA4Ff"}